diff options
Diffstat (limited to 'drivers/block/mtip32xx')
-rw-r--r-- | drivers/block/mtip32xx/Kconfig | 9 | ||||
-rw-r--r-- | drivers/block/mtip32xx/Makefile | 5 | ||||
-rw-r--r-- | drivers/block/mtip32xx/mtip32xx.c | 3651 | ||||
-rw-r--r-- | drivers/block/mtip32xx/mtip32xx.h | 423 |
4 files changed, 4088 insertions, 0 deletions
diff --git a/drivers/block/mtip32xx/Kconfig b/drivers/block/mtip32xx/Kconfig new file mode 100644 index 000000000000..b5dd14e072f2 --- /dev/null +++ b/drivers/block/mtip32xx/Kconfig @@ -0,0 +1,9 @@ +# +# mtip32xx device driver configuration +# + +config BLK_DEV_PCIESSD_MTIP32XX + tristate "Block Device Driver for Micron PCIe SSDs" + depends on HOTPLUG_PCI_PCIE + help + This enables the block driver for Micron PCIe SSDs. diff --git a/drivers/block/mtip32xx/Makefile b/drivers/block/mtip32xx/Makefile new file mode 100644 index 000000000000..4fbef8c8329b --- /dev/null +++ b/drivers/block/mtip32xx/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for Block device driver for Micron PCIe SSD +# + +obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx.o diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c new file mode 100644 index 000000000000..b74eab70c3d0 --- /dev/null +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -0,0 +1,3651 @@ +/* + * Driver for the Micron P320 SSD + * Copyright (C) 2011 Micron Technology, Inc. + * + * Portions of this code were derived from works subjected to the + * following copyright: + * Copyright (C) 2009 Integrated Device Technology, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/ata.h> +#include <linux/delay.h> +#include <linux/hdreg.h> +#include <linux/uaccess.h> +#include <linux/random.h> +#include <linux/smp.h> +#include <linux/compat.h> +#include <linux/fs.h> +#include <linux/module.h> +#include <linux/genhd.h> +#include <linux/blkdev.h> +#include <linux/bio.h> +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/kthread.h> +#include <../drivers/ata/ahci.h> +#include "mtip32xx.h" + +#define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) +#define HW_CMD_TBL_SZ (AHCI_CMD_TBL_HDR_SZ + (MTIP_MAX_SG * 16)) +#define HW_CMD_TBL_AR_SZ (HW_CMD_TBL_SZ * MTIP_MAX_COMMAND_SLOTS) +#define HW_PORT_PRIV_DMA_SZ \ + (HW_CMD_SLOT_SZ + HW_CMD_TBL_AR_SZ + AHCI_RX_FIS_SZ) + +#define HOST_HSORG 0xFC +#define HSORG_DISABLE_SLOTGRP_INTR (1<<24) +#define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) +#define HSORG_HWREV 0xFF00 +#define HSORG_STYLE 0x8 +#define HSORG_SLOTGROUPS 0x7 + +#define PORT_COMMAND_ISSUE 0x38 +#define PORT_SDBV 0x7C + +#define PORT_OFFSET 0x100 +#define PORT_MEM_SIZE 0x80 + +#define PORT_IRQ_ERR \ + (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ + PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ + PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ + PORT_IRQ_OVERFLOW) +#define PORT_IRQ_LEGACY \ + (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) +#define PORT_IRQ_HANDLED \ + (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ + PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ + PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) +#define DEF_PORT_IRQ \ + (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) + +/* product numbers */ +#define MTIP_PRODUCT_UNKNOWN 0x00 +#define MTIP_PRODUCT_ASICFPGA 0x11 + +/* Device instance number, incremented each time a device is probed. */ +static int instance; + +/* + * Global variable used to hold the major block device number + * allocated in mtip_init(). + */ +static int mtip_major; + +static DEFINE_SPINLOCK(rssd_index_lock); +static DEFINE_IDA(rssd_index_ida); + +static int mtip_block_initialize(struct driver_data *dd); + +#ifdef CONFIG_COMPAT +struct mtip_compat_ide_task_request_s { + __u8 io_ports[8]; + __u8 hob_ports[8]; + ide_reg_valid_t out_flags; + ide_reg_valid_t in_flags; + int data_phase; + int req_cmd; + compat_ulong_t out_size; + compat_ulong_t in_size; +}; +#endif + +/* + * This function check_for_surprise_removal is called + * while card is removed from the system and it will + * read the vendor id from the configration space + * + * @pdev Pointer to the pci_dev structure. + * + * return value + * true if device removed, else false + */ +static bool mtip_check_surprise_removal(struct pci_dev *pdev) +{ + u16 vendor_id = 0; + + /* Read the vendorID from the configuration space */ + pci_read_config_word(pdev, 0x00, &vendor_id); + if (vendor_id == 0xFFFF) + return true; /* device removed */ + + return false; /* device present */ +} + +/* + * This function is called for clean the pending command in the + * command slot during the surprise removal of device and return + * error to the upper layer. + * + * @dd Pointer to the DRIVER_DATA structure. + * + * return value + * None + */ +static void mtip_command_cleanup(struct driver_data *dd) +{ + int group = 0, commandslot = 0, commandindex = 0; + struct mtip_cmd *command; + struct mtip_port *port = dd->port; + + for (group = 0; group < 4; group++) { + for (commandslot = 0; commandslot < 32; commandslot++) { + if (!(port->allocated[group] & (1 << commandslot))) + continue; + + commandindex = group << 5 | commandslot; + command = &port->commands[commandindex]; + + if (atomic_read(&command->active) + && (command->async_callback)) { + command->async_callback(command->async_data, + -ENODEV); + command->async_callback = NULL; + command->async_data = NULL; + } + + dma_unmap_sg(&port->dd->pdev->dev, + command->sg, + command->scatter_ents, + command->direction); + } + } + + up(&port->cmd_slot); + + atomic_set(&dd->drv_cleanup_done, true); +} + +/* + * Obtain an empty command slot. + * + * This function needs to be reentrant since it could be called + * at the same time on multiple CPUs. The allocation of the + * command slot must be atomic. + * + * @port Pointer to the port data structure. + * + * return value + * >= 0 Index of command slot obtained. + * -1 No command slots available. + */ +static int get_slot(struct mtip_port *port) +{ + int slot, i; + unsigned int num_command_slots = port->dd->slot_groups * 32; + + /* + * Try 10 times, because there is a small race here. + * that's ok, because it's still cheaper than a lock. + * + * Race: Since this section is not protected by lock, same bit + * could be chosen by different process contexts running in + * different processor. So instead of costly lock, we are going + * with loop. + */ + for (i = 0; i < 10; i++) { + slot = find_next_zero_bit(port->allocated, + num_command_slots, 1); + if ((slot < num_command_slots) && + (!test_and_set_bit(slot, port->allocated))) + return slot; + } + dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n"); + + if (mtip_check_surprise_removal(port->dd->pdev)) { + /* Device not present, clean outstanding commands */ + mtip_command_cleanup(port->dd); + } + return -1; +} + +/* + * Release a command slot. + * + * @port Pointer to the port data structure. + * @tag Tag of command to release + * + * return value + * None + */ +static inline void release_slot(struct mtip_port *port, int tag) +{ + smp_mb__before_clear_bit(); + clear_bit(tag, port->allocated); + smp_mb__after_clear_bit(); +} + +/* + * Reset the HBA (without sleeping) + * + * Just like hba_reset, except does not call sleep, so can be + * run from interrupt/tasklet context. + * + * @dd Pointer to the driver data structure. + * + * return value + * 0 The reset was successful. + * -1 The HBA Reset bit did not clear. + */ +static int hba_reset_nosleep(struct driver_data *dd) +{ + unsigned long timeout; + + /* Chip quirk: quiesce any chip function */ + mdelay(10); + + /* Set the reset bit */ + writel(HOST_RESET, dd->mmio + HOST_CTL); + + /* Flush */ + readl(dd->mmio + HOST_CTL); + + /* + * Wait 10ms then spin for up to 1 second + * waiting for reset acknowledgement + */ + timeout = jiffies + msecs_to_jiffies(1000); + mdelay(10); + while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) + && time_before(jiffies, timeout)) + mdelay(1); + + if (readl(dd->mmio + HOST_CTL) & HOST_RESET) + return -1; + + return 0; +} + +/* + * Issue a command to the hardware. + * + * Set the appropriate bit in the s_active and Command Issue hardware + * registers, causing hardware command processing to begin. + * + * @port Pointer to the port structure. + * @tag The tag of the command to be issued. + * + * return value + * None + */ +static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) +{ + unsigned long flags = 0; + + atomic_set(&port->commands[tag].active, 1); + + spin_lock_irqsave(&port->cmd_issue_lock, flags); + + writel((1 << MTIP_TAG_BIT(tag)), + port->s_active[MTIP_TAG_INDEX(tag)]); + writel((1 << MTIP_TAG_BIT(tag)), + port->cmd_issue[MTIP_TAG_INDEX(tag)]); + + spin_unlock_irqrestore(&port->cmd_issue_lock, flags); +} + +/* + * Enable/disable the reception of FIS + * + * @port Pointer to the port data structure + * @enable 1 to enable, 0 to disable + * + * return value + * Previous state: 1 enabled, 0 disabled + */ +static int mtip_enable_fis(struct mtip_port *port, int enable) +{ + u32 tmp; + + /* enable FIS reception */ + tmp = readl(port->mmio + PORT_CMD); + if (enable) + writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); + else + writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); + + /* Flush */ + readl(port->mmio + PORT_CMD); + + return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); +} + +/* + * Enable/disable the DMA engine + * + * @port Pointer to the port data structure + * @enable 1 to enable, 0 to disable + * + * return value + * Previous state: 1 enabled, 0 disabled. + */ +static int mtip_enable_engine(struct mtip_port *port, int enable) +{ + u32 tmp; + + /* enable FIS reception */ + tmp = readl(port->mmio + PORT_CMD); + if (enable) + writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); + else + writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); + + readl(port->mmio + PORT_CMD); + return (((tmp & PORT_CMD_START) == PORT_CMD_START)); +} + +/* + * Enables the port DMA engine and FIS reception. + * + * return value + * None + */ +static inline void mtip_start_port(struct mtip_port *port) +{ + /* Enable FIS reception */ + mtip_enable_fis(port, 1); + + /* Enable the DMA engine */ + mtip_enable_engine(port, 1); +} + +/* + * Deinitialize a port by disabling port interrupts, the DMA engine, + * and FIS reception. + * + * @port Pointer to the port structure + * + * return value + * None + */ +static inline void mtip_deinit_port(struct mtip_port *port) +{ + /* Disable interrupts on this port */ + writel(0, port->mmio + PORT_IRQ_MASK); + + /* Disable the DMA engine */ + mtip_enable_engine(port, 0); + + /* Disable FIS reception */ + mtip_enable_fis(port, 0); +} + +/* + * Initialize a port. + * + * This function deinitializes the port by calling mtip_deinit_port() and + * then initializes it by setting the command header and RX FIS addresses, + * clearing the SError register and any pending port interrupts before + * re-enabling the default set of port interrupts. + * + * @port Pointer to the port structure. + * + * return value + * None + */ +static void mtip_init_port(struct mtip_port *port) +{ + int i; + mtip_deinit_port(port); + + /* Program the command list base and FIS base addresses */ + if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { + writel((port->command_list_dma >> 16) >> 16, + port->mmio + PORT_LST_ADDR_HI); + writel((port->rxfis_dma >> 16) >> 16, + port->mmio + PORT_FIS_ADDR_HI); + } + + writel(port->command_list_dma & 0xFFFFFFFF, + port->mmio + PORT_LST_ADDR); + writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); + + /* Clear SError */ + writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); + + /* reset the completed registers.*/ + for (i = 0; i < port->dd->slot_groups; i++) + writel(0xFFFFFFFF, port->completed[i]); + + /* Clear any pending interrupts for this port */ + writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); + + /* Enable port interrupts */ + writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); +} + +/* + * Restart a port + * + * @port Pointer to the port data structure. + * + * return value + * None + */ +static void mtip_restart_port(struct mtip_port *port) +{ + unsigned long timeout; + + /* Disable the DMA engine */ + mtip_enable_engine(port, 0); + + /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ + timeout = jiffies + msecs_to_jiffies(500); + while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) + && time_before(jiffies, timeout)) + ; + + /* + * Chip quirk: escalate to hba reset if + * PxCMD.CR not clear after 500 ms + */ + if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { + dev_warn(&port->dd->pdev->dev, + "PxCMD.CR not clear, escalating reset\n"); + + if (hba_reset_nosleep(port->dd)) + dev_err(&port->dd->pdev->dev, + "HBA reset escalation failed.\n"); + + /* 30 ms delay before com reset to quiesce chip */ + mdelay(30); + } + + dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); + + /* Set PxSCTL.DET */ + writel(readl(port->mmio + PORT_SCR_CTL) | + 1, port->mmio + PORT_SCR_CTL); + readl(port->mmio + PORT_SCR_CTL); + + /* Wait 1 ms to quiesce chip function */ + timeout = jiffies + msecs_to_jiffies(1); + while (time_before(jiffies, timeout)) + ; + + /* Clear PxSCTL.DET */ + writel(readl(port->mmio + PORT_SCR_CTL) & ~1, + port->mmio + PORT_SCR_CTL); + readl(port->mmio + PORT_SCR_CTL); + + /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ + timeout = jiffies + msecs_to_jiffies(500); + while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) + && time_before(jiffies, timeout)) + ; + + if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) + dev_warn(&port->dd->pdev->dev, + "COM reset failed\n"); + + /* Clear SError, the PxSERR.DIAG.x should be set so clear it */ + writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); + + /* Enable the DMA engine */ + mtip_enable_engine(port, 1); +} + +/* + * Called periodically to see if any read/write commands are + * taking too long to complete. + * + * @data Pointer to the PORT data structure. + * + * return value + * None + */ +static void mtip_timeout_function(unsigned long int data) +{ + struct mtip_port *port = (struct mtip_port *) data; + struct host_to_dev_fis *fis; + struct mtip_cmd *command; + int tag, cmdto_cnt = 0; + unsigned int bit, group; + unsigned int num_command_slots = port->dd->slot_groups * 32; + + if (unlikely(!port)) + return; + + if (atomic_read(&port->dd->resumeflag) == true) { + mod_timer(&port->cmd_timer, + jiffies + msecs_to_jiffies(30000)); + return; + } + + for (tag = 0; tag < num_command_slots; tag++) { + /* + * Skip internal command slot as it has + * its own timeout mechanism + */ + if (tag == MTIP_TAG_INTERNAL) + continue; + + if (atomic_read(&port->commands[tag].active) && + (time_after(jiffies, port->commands[tag].comp_time))) { + group = tag >> 5; + bit = tag & 0x1F; + + command = &port->commands[tag]; + fis = (struct host_to_dev_fis *) command->command; + + dev_warn(&port->dd->pdev->dev, + "Timeout for command tag %d\n", tag); + + cmdto_cnt++; + if (cmdto_cnt == 1) + set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags); + + /* + * Clear the completed bit. This should prevent + * any interrupt handlers from trying to retire + * the command. + */ + writel(1 << bit, port->completed[group]); + + /* Call the async completion callback. */ + if (likely(command->async_callback)) + command->async_callback(command->async_data, + -EIO); + command->async_callback = NULL; + command->comp_func = NULL; + + /* Unmap the DMA scatter list entries */ + dma_unmap_sg(&port->dd->pdev->dev, + command->sg, + command->scatter_ents, + command->direction); + + /* + * Clear the allocated bit and active tag for the + * command. + */ + atomic_set(&port->commands[tag].active, 0); + release_slot(port, tag); + + up(&port->cmd_slot); + } + } + + if (cmdto_cnt) { + dev_warn(&port->dd->pdev->dev, + "%d commands timed out: restarting port", + cmdto_cnt); + mtip_restart_port(port); + clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags); + wake_up_interruptible(&port->svc_wait); + } + + /* Restart the timer */ + mod_timer(&port->cmd_timer, + jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD)); +} + +/* + * IO completion function. + * + * This completion function is called by the driver ISR when a + * command that was issued by the kernel completes. It first calls the + * asynchronous completion function which normally calls back into the block + * layer passing the asynchronous callback data, then unmaps the + * scatter list associated with the completed command, and finally + * clears the allocated bit associated with the completed command. + * + * @port Pointer to the port data structure. + * @tag Tag of the command. + * @data Pointer to driver_data. + * @status Completion status. + * + * return value + * None + */ +static void mtip_async_complete(struct mtip_port *port, + int tag, + void *data, + int status) +{ + struct mtip_cmd *command; + struct driver_data *dd = data; + int cb_status = status ? -EIO : 0; + + if (unlikely(!dd) || unlikely(!port)) + return; + + command = &port->commands[tag]; + + if (unlikely(status == PORT_IRQ_TF_ERR)) { + dev_warn(&port->dd->pdev->dev, + "Command tag %d failed due to TFE\n", tag); + } + + /* Upper layer callback */ + if (likely(command->async_callback)) + command->async_callback(command->async_data, cb_status); + + command->async_callback = NULL; + command->comp_func = NULL; + + /* Unmap the DMA scatter list entries */ + dma_unmap_sg(&dd->pdev->dev, + command->sg, + command->scatter_ents, + command->direction); + + /* Clear the allocated and active bits for the command */ + atomic_set(&port->commands[tag].active, 0); + release_slot(port, tag); + + up(&port->cmd_slot); +} + +/* + * Internal command completion callback function. + * + * This function is normally called by the driver ISR when an internal + * command completed. This function signals the command completion by + * calling complete(). + * + * @port Pointer to the port data structure. + * @tag Tag of the command that has completed. + * @data Pointer to a completion structure. + * @status Completion status. + * + * return value + * None + */ +static void mtip_completion(struct mtip_port *port, + int tag, + void *data, + int status) +{ + struct mtip_cmd *command = &port->commands[tag]; + struct completion *waiting = data; + if (unlikely(status == PORT_IRQ_TF_ERR)) + dev_warn(&port->dd->pdev->dev, + "Internal command %d completed with TFE\n", tag); + + command->async_callback = NULL; + command->comp_func = NULL; + + complete(waiting); +} + +/* + * Helper function for tag logging + */ +static void print_tags(struct driver_data *dd, + char *msg, + unsigned long *tagbits) +{ + unsigned int tag, count = 0; + + for (tag = 0; tag < (dd->slot_groups) * 32; tag++) { + if (test_bit(tag, tagbits)) + count++; + } + if (count) + dev_info(&dd->pdev->dev, "%s [%i tags]\n", msg, count); +} + +/* + * Handle an error. + * + * @dd Pointer to the DRIVER_DATA structure. + * + * return value + * None + */ +static void mtip_handle_tfe(struct driver_data *dd) +{ + int group, tag, bit, reissue; + struct mtip_port *port; + struct mtip_cmd *command; + u32 completed; + struct host_to_dev_fis *fis; + unsigned long tagaccum[SLOTBITS_IN_LONGS]; + + dev_warn(&dd->pdev->dev, "Taskfile error\n"); + + port = dd->port; + + /* Stop the timer to prevent command timeouts. */ + del_timer(&port->cmd_timer); + + /* Set eh_active */ + set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags); + + /* Loop through all the groups */ + for (group = 0; group < dd->slot_groups; group++) { + completed = readl(port->completed[group]); + + /* clear completed status register in the hardware.*/ + writel(completed, port->completed[group]); + + /* clear the tag accumulator */ + memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); + + /* Process successfully completed commands */ + for (bit = 0; bit < 32 && completed; bit++) { + if (!(completed & (1<<bit))) + continue; + tag = (group << 5) + bit; + + /* Skip the internal command slot */ + if (tag == MTIP_TAG_INTERNAL) + continue; + + command = &port->commands[tag]; + if (likely(command->comp_func)) { + set_bit(tag, tagaccum); + atomic_set(&port->commands[tag].active, 0); + command->comp_func(port, + tag, + command->comp_data, + 0); + } else { + dev_err(&port->dd->pdev->dev, + "Missing completion func for tag %d", + tag); + if (mtip_check_surprise_removal(dd->pdev)) { + mtip_command_cleanup(dd); + /* don't proceed further */ + return; + } + } + } + } + print_tags(dd, "TFE tags completed:", tagaccum); + + /* Restart the port */ + mdelay(20); + mtip_restart_port(port); + + /* clear the tag accumulator */ + memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); + + /* Loop through all the groups */ + for (group = 0; group < dd->slot_groups; group++) { + for (bit = 0; bit < 32; bit++) { + reissue = 1; + tag = (group << 5) + bit; + + /* If the active bit is set re-issue the command */ + if (atomic_read(&port->commands[tag].active) == 0) + continue; + + fis = (struct host_to_dev_fis *) + port->commands[tag].command; + + /* Should re-issue? */ + if (tag == MTIP_TAG_INTERNAL || + fis->command == ATA_CMD_SET_FEATURES) + reissue = 0; + + /* + * First check if this command has + * exceeded its retries. + */ + if (reissue && + (port->commands[tag].retries-- > 0)) { + + set_bit(tag, tagaccum); + + /* Update the timeout value. */ + port->commands[tag].comp_time = + jiffies + msecs_to_jiffies( + MTIP_NCQ_COMMAND_TIMEOUT_MS); + /* Re-issue the command. */ + mtip_issue_ncq_command(port, tag); + + continue; + } + + /* Retire a command that will not be reissued */ + dev_warn(&port->dd->pdev->dev, + "retiring tag %d\n", tag); + atomic_set(&port->commands[tag].active, 0); + + if (port->commands[tag].comp_func) + port->commands[tag].comp_func( + port, + tag, + port->commands[tag].comp_data, + PORT_IRQ_TF_ERR); + else + dev_warn(&port->dd->pdev->dev, + "Bad completion for tag %d\n", + tag); + } + } + print_tags(dd, "TFE tags reissued:", tagaccum); + + /* clear eh_active */ + clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags); + wake_up_interruptible(&port->svc_wait); + + mod_timer(&port->cmd_timer, + jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD)); +} + +/* + * Handle a set device bits interrupt + */ +static inline void mtip_process_sdbf(struct driver_data *dd) +{ + struct mtip_port *port = dd->port; + int group, tag, bit; + u32 completed; + struct mtip_cmd *command; + + /* walk all bits in all slot groups */ + for (group = 0; group < dd->slot_groups; group++) { + completed = readl(port->completed[group]); + + /* clear completed status register in the hardware.*/ + writel(completed, port->completed[group]); + + /* Process completed commands. */ + for (bit = 0; + (bit < 32) && completed; + bit++, completed >>= 1) { + if (completed & 0x01) { + tag = (group << 5) | bit; + + /* skip internal command slot. */ + if (unlikely(tag == MTIP_TAG_INTERNAL)) + continue; + + command = &port->commands[tag]; + /* make internal callback */ + if (likely(command->comp_func)) { + command->comp_func( + port, + tag, + command->comp_data, + 0); + } else { + dev_warn(&dd->pdev->dev, + "Null completion " + "for tag %d", + tag); + + if (mtip_check_surprise_removal( + dd->pdev)) { + mtip_command_cleanup(dd); + return; + } + } + } + } + } +} + +/* + * Process legacy pio and d2h interrupts + */ +static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) +{ + struct mtip_port *port = dd->port; + struct mtip_cmd *cmd = &port->commands[MTIP_TAG_INTERNAL]; + + if (test_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags) && + (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) + & (1 << MTIP_TAG_INTERNAL))) { + if (cmd->comp_func) { + cmd->comp_func(port, + MTIP_TAG_INTERNAL, + cmd->comp_data, + 0); + return; + } + } + + dev_warn(&dd->pdev->dev, "IRQ status 0x%x ignored.\n", port_stat); + + return; +} + +/* + * Demux and handle errors + */ +static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) +{ + if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) + mtip_handle_tfe(dd); + + if (unlikely(port_stat & PORT_IRQ_CONNECT)) { + dev_warn(&dd->pdev->dev, + "Clearing PxSERR.DIAG.x\n"); + writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); + } + + if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { + dev_warn(&dd->pdev->dev, + "Clearing PxSERR.DIAG.n\n"); + writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); + } + + if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { + dev_warn(&dd->pdev->dev, + "Port stat errors %x unhandled\n", + (port_stat & ~PORT_IRQ_HANDLED)); + } +} + +static inline irqreturn_t mtip_handle_irq(struct driver_data *data) +{ + struct driver_data *dd = (struct driver_data *) data; + struct mtip_port *port = dd->port; + u32 hba_stat, port_stat; + int rv = IRQ_NONE; + + hba_stat = readl(dd->mmio + HOST_IRQ_STAT); + if (hba_stat) { + rv = IRQ_HANDLED; + + /* Acknowledge the interrupt status on the port.*/ + port_stat = readl(port->mmio + PORT_IRQ_STAT); + writel(port_stat, port->mmio + PORT_IRQ_STAT); + + /* Demux port status */ + if (likely(port_stat & PORT_IRQ_SDB_FIS)) + mtip_process_sdbf(dd); + + if (unlikely(port_stat & PORT_IRQ_ERR)) { + if (unlikely(mtip_check_surprise_removal(dd->pdev))) { + mtip_command_cleanup(dd); + /* don't proceed further */ + return IRQ_HANDLED; + } + + mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); + } + + if (unlikely(port_stat & PORT_IRQ_LEGACY)) + mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); + } + + /* acknowledge interrupt */ + writel(hba_stat, dd->mmio + HOST_IRQ_STAT); + + return rv; +} + +/* + * Wrapper for mtip_handle_irq + * (ignores return code) + */ +static void mtip_tasklet(unsigned long data) +{ + mtip_handle_irq((struct driver_data *) data); +} + +/* + * HBA interrupt subroutine. + * + * @irq IRQ number. + * @instance Pointer to the driver data structure. + * + * return value + * IRQ_HANDLED A HBA interrupt was pending and handled. + * IRQ_NONE This interrupt was not for the HBA. + */ +static irqreturn_t mtip_irq_handler(int irq, void *instance) +{ + struct driver_data *dd = instance; + tasklet_schedule(&dd->tasklet); + return IRQ_HANDLED; +} + +static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) +{ + atomic_set(&port->commands[tag].active, 1); + writel(1 << MTIP_TAG_BIT(tag), + port->cmd_issue[MTIP_TAG_INDEX(tag)]); +} + +/* + * Wait for port to quiesce + * + * @port Pointer to port data structure + * @timeout Max duration to wait (ms) + * + * return value + * 0 Success + * -EBUSY Commands still active + */ +static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) +{ + unsigned long to; + unsigned int n; + unsigned int active = 1; + + to = jiffies + msecs_to_jiffies(timeout); + do { + if (test_bit(MTIP_FLAG_SVC_THD_ACTIVE_BIT, &port->flags) && + test_bit(MTIP_FLAG_ISSUE_CMDS_BIT, &port->flags)) { + msleep(20); + continue; /* svc thd is actively issuing commands */ + } + /* + * Ignore s_active bit 0 of array element 0. + * This bit will always be set + */ + active = readl(port->s_active[0]) & 0xFFFFFFFE; + for (n = 1; n < port->dd->slot_groups; n++) + active |= readl(port->s_active[n]); + + if (!active) + break; + + msleep(20); + } while (time_before(jiffies, to)); + + return active ? -EBUSY : 0; +} + +/* + * Execute an internal command and wait for the completion. + * + * @port Pointer to the port data structure. + * @fis Pointer to the FIS that describes the command. + * @fis_len Length in WORDS of the FIS. + * @buffer DMA accessible for command data. + * @buf_len Length, in bytes, of the data buffer. + * @opts Command header options, excluding the FIS length + * and the number of PRD entries. + * @timeout Time in ms to wait for the command to complete. + * + * return value + * 0 Command completed successfully. + * -EFAULT The buffer address is not correctly aligned. + * -EBUSY Internal command or other IO in progress. + * -EAGAIN Time out waiting for command to complete. + */ +static int mtip_exec_internal_command(struct mtip_port *port, + void *fis, + int fis_len, + dma_addr_t buffer, + int buf_len, + u32 opts, + gfp_t atomic, + unsigned long timeout) +{ + struct mtip_cmd_sg *command_sg; + DECLARE_COMPLETION_ONSTACK(wait); + int rv = 0; + struct mtip_cmd *int_cmd = &port->commands[MTIP_TAG_INTERNAL]; + + /* Make sure the buffer is 8 byte aligned. This is asic specific. */ + if (buffer & 0x00000007) { + dev_err(&port->dd->pdev->dev, + "SG buffer is not 8 byte aligned\n"); + return -EFAULT; + } + + /* Only one internal command should be running at a time */ + if (test_and_set_bit(MTIP_TAG_INTERNAL, port->allocated)) { + dev_warn(&port->dd->pdev->dev, + "Internal command already active\n"); + return -EBUSY; + } + set_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags); + + if (atomic == GFP_KERNEL) { + /* wait for io to complete if non atomic */ + if (mtip_quiesce_io(port, 5000) < 0) { + dev_warn(&port->dd->pdev->dev, + "Failed to quiesce IO\n"); + release_slot(port, MTIP_TAG_INTERNAL); + clear_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags); + wake_up_interruptible(&port->svc_wait); + return -EBUSY; + } + + /* Set the completion function and data for the command. */ + int_cmd->comp_data = &wait; + int_cmd->comp_func = mtip_completion; + + } else { + /* Clear completion - we're going to poll */ + int_cmd->comp_data = NULL; + int_cmd->comp_func = NULL; + } + + /* Copy the command to the command table */ + memcpy(int_cmd->command, fis, fis_len*4); + + /* Populate the SG list */ + int_cmd->command_header->opts = + __force_bit2int cpu_to_le32(opts | fis_len); + if (buf_len) { + command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ; + + command_sg->info = + __force_bit2int cpu_to_le32((buf_len-1) & 0x3FFFFF); + command_sg->dba = + __force_bit2int cpu_to_le32(buffer & 0xFFFFFFFF); + command_sg->dba_upper = + __force_bit2int cpu_to_le32((buffer >> 16) >> 16); + + int_cmd->command_header->opts |= + __force_bit2int cpu_to_le32((1 << 16)); + } + + /* Populate the command header */ + int_cmd->command_header->byte_count = 0; + + /* Issue the command to the hardware */ + mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); + + /* Poll if atomic, wait_for_completion otherwise */ + if (atomic == GFP_KERNEL) { + /* Wait for the command to complete or timeout. */ + if (wait_for_completion_timeout( + & |