Lines Matching refs:dd
101 static int mtip_block_initialize(struct driver_data *dd);
148 static void mtip_command_cleanup(struct driver_data *dd) in mtip_command_cleanup() argument
152 struct mtip_port *port = dd->port; in mtip_command_cleanup()
176 dma_unmap_sg(&port->dd->pdev->dev, in mtip_command_cleanup()
185 set_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag); in mtip_command_cleanup()
205 unsigned int num_command_slots = port->dd->slot_groups * 32; in get_slot()
223 dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n"); in get_slot()
225 if (mtip_check_surprise_removal(port->dd->pdev)) { in get_slot()
227 mtip_command_cleanup(port->dd); in get_slot()
257 static int mtip_hba_reset(struct driver_data *dd) in mtip_hba_reset() argument
262 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
265 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
271 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
274 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
277 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
419 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
434 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
441 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
442 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
469 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
477 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
480 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
481 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
488 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
500 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
514 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
518 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
526 static int mtip_device_reset(struct driver_data *dd) in mtip_device_reset() argument
530 if (mtip_check_surprise_removal(dd->pdev)) in mtip_device_reset()
533 if (mtip_hba_reset(dd) < 0) in mtip_device_reset()
537 mtip_init_port(dd->port); in mtip_device_reset()
538 mtip_start_port(dd->port); in mtip_device_reset()
541 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
542 dd->mmio + HOST_CTL); in mtip_device_reset()
549 static void print_tags(struct driver_data *dd, in print_tags() argument
561 dev_warn(&dd->pdev->dev, in print_tags()
587 if (test_bit(MTIP_DDF_RESUME_BIT, &port->dd->dd_flag)) { in mtip_timeout_function()
594 num_command_slots = port->dd->slot_groups * 32; in mtip_timeout_function()
632 dma_unmap_sg(&port->dd->pdev->dev, in mtip_timeout_function()
649 print_tags(port->dd, "timed out", tagaccum, cmdto_cnt); in mtip_timeout_function()
651 mtip_device_reset(port->dd); in mtip_timeout_function()
701 struct driver_data *dd = data; in mtip_async_complete() local
704 if (unlikely(!dd) || unlikely(!port)) in mtip_async_complete()
710 dev_warn(&port->dd->pdev->dev, in mtip_async_complete()
722 dma_unmap_sg(&dd->pdev->dev, in mtip_async_complete()
760 dev_warn(&port->dd->pdev->dev, in mtip_completion()
789 static void mtip_handle_tfe(struct driver_data *dd) in mtip_handle_tfe() argument
802 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
804 port = dd->port; in mtip_handle_tfe()
827 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
853 dev_err(&port->dd->pdev->dev, in mtip_handle_tfe()
856 if (mtip_check_surprise_removal(dd->pdev)) { in mtip_handle_tfe()
857 mtip_command_cleanup(dd); in mtip_handle_tfe()
865 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
872 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
873 dd->port->log_buf, in mtip_handle_tfe()
874 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
876 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
880 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
882 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
884 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
889 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
891 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
896 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
907 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
927 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
959 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
970 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
975 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
992 struct driver_data *dd = port->dd; in mtip_workq_sdbfx() local
1021 dev_warn(&dd->pdev->dev, in mtip_workq_sdbfx()
1027 dd->pdev)) { in mtip_workq_sdbfx()
1028 mtip_command_cleanup(dd); in mtip_workq_sdbfx()
1037 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
1038 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
1044 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) in mtip_process_legacy() argument
1046 struct mtip_port *port = dd->port; in mtip_process_legacy()
1067 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) in mtip_process_errors() argument
1070 mtip_handle_tfe(dd); in mtip_process_errors()
1073 dev_warn(&dd->pdev->dev, in mtip_process_errors()
1075 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
1079 dev_warn(&dd->pdev->dev, in mtip_process_errors()
1081 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
1085 dev_warn(&dd->pdev->dev, in mtip_process_errors()
1093 struct driver_data *dd = (struct driver_data *) data; in mtip_handle_irq() local
1094 struct mtip_port *port = dd->port; in mtip_handle_irq()
1100 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
1111 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
1116 twork = &dd->work[i]; in mtip_handle_irq()
1122 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
1125 twork = &dd->work[i]; in mtip_handle_irq()
1129 dd->isr_workq, in mtip_handle_irq()
1133 if (likely(dd->work[0].completed)) in mtip_handle_irq()
1135 dd->work[0].completed); in mtip_handle_irq()
1147 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_handle_irq()
1148 mtip_command_cleanup(dd); in mtip_handle_irq()
1153 &dd->dd_flag)) in mtip_handle_irq()
1156 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); in mtip_handle_irq()
1160 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); in mtip_handle_irq()
1165 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
1182 struct driver_data *dd = instance; in mtip_irq_handler() local
1184 return mtip_handle_irq(dd); in mtip_irq_handler()
1204 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1211 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
1254 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_quiesce_io()
1261 for (n = 1; n < port->dd->slot_groups; n++) in mtip_quiesce_io()
1305 struct driver_data *dd = port->dd; in mtip_exec_internal_command() local
1309 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
1322 dev_warn(&dd->pdev->dev, in mtip_exec_internal_command()
1336 dev_warn(&dd->pdev->dev, in mtip_exec_internal_command()
1387 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1393 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1397 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1401 if (mtip_check_surprise_removal(dd->pdev) || in mtip_exec_internal_command()
1403 &dd->dd_flag)) { in mtip_exec_internal_command()
1404 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1410 mtip_device_reset(dd); /* recover from timeout issue */ in mtip_exec_internal_command()
1422 if (mtip_check_surprise_removal(dd->pdev)) { in mtip_exec_internal_command()
1428 &dd->dd_flag)) { in mtip_exec_internal_command()
1437 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1440 mtip_device_reset(dd); in mtip_exec_internal_command()
1445 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_exec_internal_command()
1448 dd->mmio + HOST_IRQ_STAT); in mtip_exec_internal_command()
1457 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1458 mtip_device_reset(dd); in mtip_exec_internal_command()
1520 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1569 port->dd->trim_supp = true; in mtip_get_identify()
1572 port->dd->trim_supp = false; in mtip_get_identify()
1624 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1725 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1729 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1733 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1740 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1752 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1772 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, in mtip_send_trim() argument
1781 if (!len || dd->trim_supp == false) in mtip_send_trim()
1794 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, in mtip_send_trim()
1822 if (mtip_exec_internal_command(dd->port, in mtip_send_trim()
1832 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); in mtip_send_trim()
1846 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) in mtip_hw_get_capacity() argument
1848 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1877 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1881 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1885 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1887 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1888 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1893 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1905 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1918 static inline void fill_command_sg(struct driver_data *dd, in fill_command_sg() argument
1932 dev_err(&dd->pdev->dev, in fill_command_sg()
2031 buf = dmam_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
2036 dev_err(&port->dd->pdev->dev, in exec_drive_command()
2108 dmam_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
2159 static void mtip_set_timeout(struct driver_data *dd, in mtip_set_timeout() argument
2170 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
2172 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
2194 static int exec_drive_taskfile(struct driver_data *dd, in exec_drive_taskfile() argument
2235 outbuf_dma = pci_map_single(dd->pdev, in exec_drive_taskfile()
2257 inbuf_dma = pci_map_single(dd->pdev, in exec_drive_taskfile()
2271 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2274 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
2277 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
2319 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
2347 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
2356 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
2368 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
2371 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
2374 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
2380 pci_unmap_single(dd->pdev, inbuf_dma, in exec_drive_taskfile()
2383 pci_unmap_single(dd->pdev, outbuf_dma, in exec_drive_taskfile()
2431 pci_unmap_single(dd->pdev, inbuf_dma, in exec_drive_taskfile()
2434 pci_unmap_single(dd->pdev, outbuf_dma, in exec_drive_taskfile()
2459 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, in mtip_hw_ioctl() argument
2465 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
2481 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
2505 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2526 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_hw_ioctl()
2563 static void mtip_hw_submit_io(struct driver_data *dd, sector_t sector, in mtip_hw_submit_io() argument
2568 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2574 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2605 fill_command_sg(dd, command, nents); in mtip_hw_submit_io()
2620 command->comp_data = dd; in mtip_hw_submit_io()
2656 static void mtip_hw_release_scatterlist(struct driver_data *dd, int tag, in mtip_hw_release_scatterlist() argument
2659 struct semaphore *sem = unaligned ? &dd->port->cmd_slot_unal : in mtip_hw_release_scatterlist()
2660 &dd->port->cmd_slot; in mtip_hw_release_scatterlist()
2661 release_slot(dd->port, tag); in mtip_hw_release_scatterlist()
2676 static struct scatterlist *mtip_hw_get_scatterlist(struct driver_data *dd, in mtip_hw_get_scatterlist() argument
2679 struct semaphore *sem = unaligned ? &dd->port->cmd_slot_unal : in mtip_hw_get_scatterlist()
2680 &dd->port->cmd_slot; in mtip_hw_get_scatterlist()
2688 *tag = get_slot(dd->port); in mtip_hw_get_scatterlist()
2690 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_get_scatterlist()
2699 return dd->port->commands[*tag].sg; in mtip_hw_get_scatterlist()
2716 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status() local
2719 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2721 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2736 struct driver_data *dd, *tmp; in show_device_status() local
2743 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { in show_device_status()
2744 if (dd->pdev) { in show_device_status()
2745 if (dd->port && in show_device_status()
2746 dd->port->identify && in show_device_status()
2747 dd->port->identify_valid) { in show_device_status()
2749 (char *) (dd->port->identify + 10), 21); in show_device_status()
2750 status = *(dd->port->identify + 141); in show_device_status()
2756 if (dd->port && in show_device_status()
2757 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2760 dev_name(&dd->pdev->dev), in show_device_status()
2766 dev_name(&dd->pdev->dev), in show_device_status()
2773 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { in show_device_status()
2774 if (dd->pdev) { in show_device_status()
2775 if (dd->port && in show_device_status()
2776 dd->port->identify && in show_device_status()
2777 dd->port->identify_valid) { in show_device_status()
2779 (char *) (dd->port->identify+10), 21); in show_device_status()
2780 status = *(dd->port->identify + 141); in show_device_status()
2786 if (dd->port && in show_device_status()
2787 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2790 dev_name(&dd->pdev->dev), in show_device_status()
2796 dev_name(&dd->pdev->dev), in show_device_status()
2828 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers() local
2839 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2841 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2846 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2848 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2853 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2855 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2859 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2861 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2866 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2869 dd->port->allocated[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2871 group_allocated = dd->port->allocated[n]; in mtip_hw_read_registers()
2878 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2881 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2883 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2899 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags() local
2907 dd->port->flags); in mtip_hw_read_flags()
2909 dd->dd_flag); in mtip_hw_read_flags()
2950 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) in mtip_hw_sysfs_init() argument
2952 if (!kobj || !dd) in mtip_hw_sysfs_init()
2956 dev_warn(&dd->pdev->dev, in mtip_hw_sysfs_init()
2971 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) in mtip_hw_sysfs_exit() argument
2973 if (!kobj || !dd) in mtip_hw_sysfs_exit()
2981 static int mtip_hw_debugfs_init(struct driver_data *dd) in mtip_hw_debugfs_init() argument
2986 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2987 if (IS_ERR_OR_NULL(dd->dfs_node)) { in mtip_hw_debugfs_init()
2988 dev_warn(&dd->pdev->dev, in mtip_hw_debugfs_init()
2990 dd->disk->disk_name); in mtip_hw_debugfs_init()
2991 dd->dfs_node = NULL; in mtip_hw_debugfs_init()
2995 debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2997 debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd, in mtip_hw_debugfs_init()
3003 static void mtip_hw_debugfs_exit(struct driver_data *dd) in mtip_hw_debugfs_exit() argument
3005 if (dd->dfs_node) in mtip_hw_debugfs_exit()
3006 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
3018 static inline void hba_setup(struct driver_data *dd) in hba_setup() argument
3021 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
3027 dd->mmio + HOST_HSORG); in hba_setup()
3030 static int mtip_device_unaligned_constrained(struct driver_data *dd) in mtip_device_unaligned_constrained() argument
3032 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
3045 static void mtip_detect_product(struct driver_data *dd) in mtip_detect_product() argument
3057 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
3059 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
3060 dd->slot_groups = 1; in mtip_detect_product()
3063 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
3066 dev_info(&dd->pdev->dev, in mtip_detect_product()
3074 dev_warn(&dd->pdev->dev, in mtip_detect_product()
3079 dd->slot_groups = slotgroups; in mtip_detect_product()
3083 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
3095 static int mtip_ftl_rebuild_poll(struct driver_data *dd) in mtip_ftl_rebuild_poll() argument
3099 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
3107 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
3109 if (mtip_check_surprise_removal(dd->pdev)) in mtip_ftl_rebuild_poll()
3112 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
3115 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
3120 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
3126 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
3129 mtip_block_initialize(dd); in mtip_ftl_rebuild_poll()
3136 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
3153 struct driver_data *dd = (struct driver_data *)data; in mtip_service_thread() local
3155 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
3156 struct mtip_port *port = dd->port; in mtip_service_thread()
3170 &dd->dd_flag))) in mtip_service_thread()
3204 if (!mtip_ftl_rebuild_poll(dd)) in mtip_service_thread()
3206 &dd->dd_flag); in mtip_service_thread()
3225 static int mtip_hw_init(struct driver_data *dd) in mtip_hw_init() argument
3234 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; in mtip_hw_init()
3236 mtip_detect_product(dd); in mtip_hw_init()
3237 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
3241 num_command_slots = dd->slot_groups * 32; in mtip_hw_init()
3243 hba_setup(dd); in mtip_hw_init()
3245 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
3246 dd->numa_node); in mtip_hw_init()
3247 if (!dd->port) { in mtip_hw_init()
3248 dev_err(&dd->pdev->dev, in mtip_hw_init()
3255 dd->work[i].port = dd->port; in mtip_hw_init()
3258 if (mtip_device_unaligned_constrained(dd)) in mtip_hw_init()
3259 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
3261 dd->unal_qdepth = 0; in mtip_hw_init()
3264 sema_init(&dd->port->cmd_slot, num_command_slots - 1 - dd->unal_qdepth); in mtip_hw_init()
3265 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
3269 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
3272 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
3273 dd->port->dd = dd; in mtip_hw_init()
3276 dd->port->command_list = in mtip_hw_init()
3277 dmam_alloc_coherent(&dd->pdev->dev, in mtip_hw_init()
3279 &dd->port->command_list_dma, in mtip_hw_init()
3281 if (!dd->port->command_list) { in mtip_hw_init()
3282 dev_err(&dd->pdev->dev, in mtip_hw_init()
3289 memset(dd->port->command_list, in mtip_hw_init()
3294 dd->port->rxfis = dd->port->command_list + HW_CMD_SLOT_SZ; in mtip_hw_init()
3295 dd->port->rxfis_dma = dd->port->command_list_dma + HW_CMD_SLOT_SZ; in mtip_hw_init()
3298 dd->port->command_table = dd->port->rxfis + AHCI_RX_FIS_SZ; in mtip_hw_init()
3299 dd->port->command_tbl_dma = dd->port->rxfis_dma + AHCI_RX_FIS_SZ; in mtip_hw_init()
3302 dd->port->identify = dd->port->command_table + in mtip_hw_init()
3304 dd->port->identify_dma = dd->port->command_tbl_dma + in mtip_hw_init()
3308 dd->port->sector_buffer = (void *) dd->port->identify + ATA_SECT_SIZE; in mtip_hw_init()
3309 dd->port->sector_buffer_dma = dd->port->identify_dma + ATA_SECT_SIZE; in mtip_hw_init()
3312 dd->port->log_buf = (void *)dd->port->sector_buffer + ATA_SECT_SIZE; in mtip_hw_init()
3313 dd->port->log_buf_dma = dd->port->sector_buffer_dma + ATA_SECT_SIZE; in mtip_hw_init()
3316 dd->port->smart_buf = (void *)dd->port->log_buf + ATA_SECT_SIZE; in mtip_hw_init()
3317 dd->port->smart_buf_dma = dd->port->log_buf_dma + ATA_SECT_SIZE; in mtip_hw_init()
3322 dd->port->commands[i].command_header = in mtip_hw_init()
3323 dd->port->command_list + in mtip_hw_init()
3325 dd->port->commands[i].command_header_dma = in mtip_hw_init()
3326 dd->port->command_list_dma + in mtip_hw_init()
3329 dd->port->commands[i].command = in mtip_hw_init()
3330 dd->port->command_table + (HW_CMD_TBL_SZ * i); in mtip_hw_init()
3331 dd->port->commands[i].command_dma = in mtip_hw_init()
3332 dd->port->command_tbl_dma + (HW_CMD_TBL_SZ * i); in mtip_hw_init()
3334 if (readl(dd->mmio + HOST_CAP) & HOST_CAP_64) in mtip_hw_init()
3335 dd->port->commands[i].command_header->ctbau = in mtip_hw_init()
3337 (dd->port->commands[i].command_dma >> 16) >> 16); in mtip_hw_init()
3338 dd->port->commands[i].command_header->ctba = in mtip_hw_init()
3340 dd->port->commands[i].command_dma & 0xFFFFFFFF); in mtip_hw_init()
3347 sg_init_table(dd->port->commands[i].sg, MTIP_MAX_SG); in mtip_hw_init()
3350 atomic_set(&dd->port->commands[i].active, 0); in mtip_hw_init()
3354 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
3355 dd->port->s_active[i] = in mtip_hw_init()
3356 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
3357 dd->port->cmd_issue[i] = in mtip_hw_init()
3358 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
3359 dd->port->completed[i] = in mtip_hw_init()
3360 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
3365 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
3369 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_hw_init()
3371 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3377 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
3379 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3387 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
3388 if (mtip_hba_reset(dd) < 0) { in mtip_hw_init()
3389 dev_err(&dd->pdev->dev, in mtip_hw_init()
3396 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
3397 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
3400 mtip_init_port(dd->port); in mtip_hw_init()
3401 mtip_start_port(dd->port); in mtip_hw_init()
3404 rv = devm_request_irq(&dd->pdev->dev, in mtip_hw_init()
3405 dd->pdev->irq, in mtip_hw_init()
3408 dev_driver_string(&dd->pdev->dev), in mtip_hw_init()
3409 dd); in mtip_hw_init()
3412 dev_err(&dd->pdev->dev, in mtip_hw_init()
3413 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
3416 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
3419 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
3420 dd->mmio + HOST_CTL); in mtip_hw_init()
3422 init_timer(&dd->port->cmd_timer); in mtip_hw_init()
3423 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
3425 dd->port->cmd_timer.data = (unsigned long int) dd->port; in mtip_hw_init()
3426 dd->port->cmd_timer.function = mtip_timeout_function; in mtip_hw_init()
3427 mod_timer(&dd->port->cmd_timer, in mtip_hw_init()
3431 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
3436 if (mtip_get_identify(dd->port, NULL) < 0) { in mtip_hw_init()
3441 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_init()
3443 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_init()
3446 mtip_dump_identify(dd->port); in mtip_hw_init()
3449 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_init()
3450 dd->port->log_buf, in mtip_hw_init()
3451 dd->port->log_buf_dma, 1); in mtip_hw_init()
3453 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3457 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_init()
3459 dev_info(&dd->pdev->dev, in mtip_hw_init()
3461 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_init()
3464 dev_info(&dd->pdev->dev, in mtip_hw_init()
3466 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_init()
3469 dev_info(&dd->pdev->dev, in mtip_hw_init()
3477 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_init()
3478 dev_warn(&dd->pdev->dev, in mtip_hw_init()
3481 dev_info(&dd->pdev->dev, in mtip_hw_init()
3487 del_timer_sync(&dd->port->cmd_timer); in mtip_hw_init()
3490 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
3491 dd->mmio + HOST_CTL); in mtip_hw_init()
3494 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
3495 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); in mtip_hw_init()
3498 mtip_deinit_port(dd->port); in mtip_hw_init()
3501 dmam_free_coherent(&dd->pdev->dev, in mtip_hw_init()
3503 dd->port->command_list, in mtip_hw_init()
3504 dd->port->command_list_dma); in mtip_hw_init()
3507 kfree(dd->port); in mtip_hw_init()
3520 static int mtip_hw_exit(struct driver_data *dd) in mtip_hw_exit() argument
3526 if (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) { in mtip_hw_exit()
3528 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) in mtip_hw_exit()
3529 if (mtip_standby_immediate(dd->port)) in mtip_hw_exit()
3530 dev_warn(&dd->pdev->dev, in mtip_hw_exit()
3534 mtip_deinit_port(dd->port); in mtip_hw_exit()
3537 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
3538 dd->mmio + HOST_CTL); in mtip_hw_exit()
3541 del_timer_sync(&dd->port->cmd_timer); in mtip_hw_exit()
3544 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
3545 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); in mtip_hw_exit()
3548 dmam_free_coherent(&dd->pdev->dev, in mtip_hw_exit()
3550 dd->port->command_list, in mtip_hw_exit()
3551 dd->port->command_list_dma); in mtip_hw_exit()
3553 kfree(dd->port); in mtip_hw_exit()
3569 static int mtip_hw_shutdown(struct driver_data *dd) in mtip_hw_shutdown() argument
3575 mtip_standby_immediate(dd->port); in mtip_hw_shutdown()
3592 static int mtip_hw_suspend(struct driver_data *dd) in mtip_hw_suspend() argument
3598 if (mtip_standby_immediate(dd->port) != 0) { in mtip_hw_suspend()
3599 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
3605 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
3606 dd->mmio + HOST_CTL); in mtip_hw_suspend()
3607 mtip_deinit_port(dd->port); in mtip_hw_suspend()
3624 static int mtip_hw_resume(struct driver_data *dd) in mtip_hw_resume() argument
3627 hba_setup(dd); in mtip_hw_resume()
3630 if (mtip_hba_reset(dd) != 0) { in mtip_hw_resume()
3631 dev_err(&dd->pdev->dev, in mtip_hw_resume()
3640 mtip_init_port(dd->port); in mtip_hw_resume()
3641 mtip_start_port(dd->port); in mtip_hw_resume()
3644 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
3645 dd->mmio + HOST_CTL); in mtip_hw_resume()
3699 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl() local
3704 if (!dd) in mtip_block_ioctl()
3707 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3714 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_ioctl()
3737 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl() local
3742 if (!dd) in mtip_block_compat_ioctl()
3745 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3774 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_block_compat_ioctl()
3791 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_compat_ioctl()
3817 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo() local
3820 if (!dd) in mtip_block_getgeo()
3823 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_getgeo()
3824 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3864 struct driver_data *dd = queue->queuedata; in mtip_make_request() local
3870 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { in mtip_make_request()
3872 &dd->dd_flag))) { in mtip_make_request()
3876 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { in mtip_make_request()
3881 &dd->dd_flag) && in mtip_make_request()
3886 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))) { in mtip_make_request()
3893 bio_endio(bio, mtip_send_trim(dd, bio->bi_sector, in mtip_make_request()
3905 dd->unal_qdepth) { in mtip_make_request()
3912 sg = mtip_hw_get_scatterlist(dd, &tag, unaligned); in mtip_make_request()
3917 dev_warn(&dd->pdev->dev, in mtip_make_request()
3920 mtip_hw_release_scatterlist(dd, tag, unaligned); in mtip_make_request()
3934 mtip_hw_submit_io(dd, in mtip_make_request()
3958 static int mtip_block_initialize(struct driver_data *dd) in mtip_block_initialize() argument
3966 if (dd->disk) in mtip_block_initialize()
3970 wait_for_rebuild = mtip_hw_init(dd); in mtip_block_initialize()
3972 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3978 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); in mtip_block_initialize()
3979 if (dd->disk == NULL) { in mtip_block_initialize()
3980 dev_err(&dd->pdev->dev, in mtip_block_initialize()
4001 dd->disk->disk_name, in mtip_block_initialize()
4006 dd->disk->driverfs_dev = &dd->pdev->dev; in mtip_block_initialize()
4007 dd->disk->major = dd->major; in mtip_block_initialize()
4008 dd->disk->first_minor = dd->instance * MTIP_MAX_MINORS; in mtip_block_initialize()
4009 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
4010 dd->disk->private_data = dd; in mtip_block_initialize()
4011 dd->index = index; in mtip_block_initialize()
4022 dd->queue = blk_alloc_queue_node(GFP_KERNEL, dd->numa_node); in mtip_block_initialize()
4023 if (dd->queue == NULL) { in mtip_block_initialize()
4024 dev_err(&dd->pdev->dev, in mtip_block_initialize()
4031 blk_queue_make_request(dd->queue, mtip_make_request); in mtip_block_initialize()
4033 dd->disk->queue = dd->queue; in mtip_block_initialize()
4034 dd->queue->queuedata = dd; in mtip_block_initialize()
4037 set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags); in mtip_block_initialize()
4038 clear_bit(QUEUE_FLAG_ADD_RANDOM, &dd->queue->queue_flags); in mtip_block_initialize()
4039 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); in mtip_block_initialize()
4040 blk_queue_physical_block_size(dd->queue, 4096); in mtip_block_initialize()
4041 blk_queue_max_hw_sectors(dd->queue, 0xffff); in mtip_block_initialize()
4042 blk_queue_max_segment_size(dd->queue, 0x400000); in mtip_block_initialize()
4043 blk_queue_io_min(dd->queue, 4096); in mtip_block_initialize()
4049 blk_queue_flush(dd->queue, 0); in mtip_block_initialize()
4052 if (dd->trim_supp == true) { in mtip_block_initialize()
4053 set_bit(QUEUE_FLAG_DISCARD, &dd->queue->queue_flags); in mtip_block_initialize()
4054 dd->queue->limits.discard_granularity = 4096; in mtip_block_initialize()
4055 blk_queue_max_discard_sectors(dd->queue, in mtip_block_initialize()
4057 dd->queue->limits.discard_zeroes_data = 0; in mtip_block_initialize()
4061 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_initialize()
4062 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
4067 set_capacity(dd->disk, capacity); in mtip_block_initialize()
4070 add_disk(dd->disk); in mtip_block_initialize()
4076 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); in mtip_block_initialize()
4078 mtip_hw_sysfs_init(dd, kobj); in mtip_block_initialize()
4081 mtip_hw_debugfs_init(dd); in mtip_block_initialize()
4083 if (dd->mtip_svc_handler) { in mtip_block_initialize()
4084 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
4090 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
4091 dd, dd->numa_node, thd_name); in mtip_block_initialize()
4093 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
4094 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
4095 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
4099 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
4106 mtip_hw_debugfs_exit(dd); in mtip_block_initialize()
4109 del_gendisk(dd->disk); in mtip_block_initialize()
4112 blk_cleanup_queue(dd->queue); in mtip_block_initialize()
4121 put_disk(dd->disk); in mtip_block_initialize()
4124 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
4140 static int mtip_block_remove(struct driver_data *dd) in mtip_block_remove() argument
4144 if (dd->mtip_svc_handler) { in mtip_block_remove()
4145 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_block_remove()
4146 wake_up_interruptible(&dd->port->svc_wait); in mtip_block_remove()
4147 kthread_stop(dd->mtip_svc_handler); in mtip_block_remove()
4151 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { in mtip_block_remove()
4152 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); in mtip_block_remove()
4154 mtip_hw_sysfs_exit(dd, kobj); in mtip_block_remove()
4158 mtip_hw_debugfs_exit(dd); in mtip_block_remove()
4164 if (dd->disk) { in mtip_block_remove()
4165 if (dd->disk->queue) in mtip_block_remove()
4166 del_gendisk(dd->disk); in mtip_block_remove()
4168 put_disk(dd->disk); in mtip_block_remove()
4172 ida_remove(&rssd_index_ida, dd->index); in mtip_block_remove()
4175 blk_cleanup_queue(dd->queue); in mtip_block_remove()
4176 dd->disk = NULL; in mtip_block_remove()
4177 dd->queue = NULL; in mtip_block_remove()
4180 mtip_hw_exit(dd); in mtip_block_remove()
4197 static int mtip_block_shutdown(struct driver_data *dd) in mtip_block_shutdown() argument
4200 if (dd->disk) { in mtip_block_shutdown()
4201 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
4202 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
4204 if (dd->disk->queue) { in mtip_block_shutdown()
4205 del_gendisk(dd->disk); in mtip_block_shutdown()
4206 blk_cleanup_queue(dd->queue); in mtip_block_shutdown()
4208 put_disk(dd->disk); in mtip_block_shutdown()
4209 dd->disk = NULL; in mtip_block_shutdown()
4210 dd->queue = NULL; in mtip_block_shutdown()
4214 ida_remove(&rssd_index_ida, dd->index); in mtip_block_shutdown()
4217 mtip_hw_shutdown(dd); in mtip_block_shutdown()
4221 static int mtip_block_suspend(struct driver_data *dd) in mtip_block_suspend() argument
4223 dev_info(&dd->pdev->dev, in mtip_block_suspend()
4224 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
4225 mtip_hw_suspend(dd); in mtip_block_suspend()
4229 static int mtip_block_resume(struct driver_data *dd) in mtip_block_resume() argument
4231 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
4232 dd->disk->disk_name); in mtip_block_resume()
4233 mtip_hw_resume(dd); in mtip_block_resume()
4300 struct driver_data *dd = NULL; in mtip_pci_probe() local
4320 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); in mtip_pci_probe()
4321 if (dd == NULL) { in mtip_pci_probe()
4328 pci_set_drvdata(pdev, dd); in mtip_pci_probe()
4358 dd->major = mtip_major; in mtip_pci_probe()
4359 dd->instance = instance; in mtip_pci_probe()
4360 dd->pdev = pdev; in mtip_pci_probe()
4361 dd->numa_node = my_node; in mtip_pci_probe()
4363 INIT_LIST_HEAD(&dd->online_list); in mtip_pci_probe()
4364 INIT_LIST_HEAD(&dd->remove_list); in mtip_pci_probe()
4366 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
4367 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
4369 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
4370 if (!dd->isr_workq) { in mtip_pci_probe()
4371 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
4378 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
4387 dd->numa_node, in mtip_pci_probe()
4389 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
4394 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4396 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
4399 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
4400 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4401 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
4402 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
4403 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
4404 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
4405 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
4406 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
4412 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
4421 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
4422 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
4423 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
4424 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
4425 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
4426 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
4427 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
4428 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
4439 rv = mtip_block_initialize(dd); in mtip_pci_probe()
4452 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
4458 list_add(&dd->online_list, &online_list); in mtip_pci_probe()
4465 if (dd->isr_workq) { in mtip_pci_probe()
4466 flush_workqueue(dd->isr_workq); in mtip_pci_probe()
4467 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
4468 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
4469 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
4470 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
4476 kfree(dd); in mtip_pci_probe()
4492 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_remove() local
4496 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
4499 list_del_init(&dd->online_list); in mtip_pci_remove()
4500 list_add(&dd->remove_list, &removing_list); in mtip_pci_remove()
4504 while (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) { in mtip_pci_remove()
4509 mtip_command_cleanup(dd); in mtip_pci_remove()
4516 mtip_block_remove(dd); in mtip_pci_remove()
4518 if (dd->isr_workq) { in mtip_pci_remove()
4519 flush_workqueue(dd->isr_workq); in mtip_pci_remove()
4520 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
4521 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
4522 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
4523 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
4529 list_del_init(&dd->remove_list); in mtip_pci_remove()
4532 kfree(dd); in mtip_pci_remove()
4546 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_suspend() local
4548 if (!dd) { in mtip_pci_suspend()
4554 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
4557 rv = mtip_block_suspend(dd); in mtip_pci_suspend()
4587 struct driver_data *dd; in mtip_pci_resume() local
4589 dd = pci_get_drvdata(pdev); in mtip_pci_resume()
4590 if (!dd) { in mtip_pci_resume()
4615 rv = mtip_block_resume(dd); in mtip_pci_resume()
4620 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
4633 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_shutdown() local
4634 if (dd) in mtip_pci_shutdown()
4635 mtip_block_shutdown(dd); in mtip_pci_shutdown()