55 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
57 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info);
58 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
60 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
63 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
66 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
70 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
71 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
72 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info);
73 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,
75 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
76 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
77 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
214 static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_offline()  argument
216 	return !ctrl_info->controller_online;  in pqi_ctrl_offline()
219 static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)  in pqi_check_ctrl_health()  argument
221 	if (ctrl_info->controller_online)  in pqi_check_ctrl_health()
222 		if (!sis_is_firmware_running(ctrl_info))  in pqi_check_ctrl_health()
223 			pqi_take_ctrl_offline(ctrl_info);  in pqi_check_ctrl_health()
232 	struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_mode()  argument
234 	return sis_read_driver_scratch(ctrl_info);  in pqi_get_ctrl_mode()
237 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_save_ctrl_mode()  argument
240 	sis_write_driver_scratch(ctrl_info, mode);  in pqi_save_ctrl_mode()
243 static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_device_reset()  argument
245 	ctrl_info->block_device_reset = true;  in pqi_ctrl_block_device_reset()
248 static inline bool pqi_device_reset_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_device_reset_blocked()  argument
250 	return ctrl_info->block_device_reset;  in pqi_device_reset_blocked()
253 static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_blocked()  argument
255 	return ctrl_info->block_requests;  in pqi_ctrl_blocked()
258 static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_requests()  argument
260 	ctrl_info->block_requests = true;  in pqi_ctrl_block_requests()
261 	scsi_block_requests(ctrl_info->scsi_host);  in pqi_ctrl_block_requests()
264 static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_requests()  argument
266 	ctrl_info->block_requests = false;  in pqi_ctrl_unblock_requests()
267 	wake_up_all(&ctrl_info->block_requests_wait);  in pqi_ctrl_unblock_requests()
268 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_ctrl_unblock_requests()
269 	scsi_unblock_requests(ctrl_info->scsi_host);  in pqi_ctrl_unblock_requests()
272 static unsigned long pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_if_ctrl_blocked()  argument
277 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_wait_if_ctrl_blocked()
280 	atomic_inc(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
283 		wait_event(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
284 			!pqi_ctrl_blocked(ctrl_info));  in pqi_wait_if_ctrl_blocked()
290 			wait_event_timeout(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
291 				!pqi_ctrl_blocked(ctrl_info),  in pqi_wait_if_ctrl_blocked()
296 	atomic_dec(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
301 static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_until_quiesced()  argument
303 	while (atomic_read(&ctrl_info->num_busy_threads) >  in pqi_ctrl_wait_until_quiesced()
304 		atomic_read(&ctrl_info->num_blocked_threads))  in pqi_ctrl_wait_until_quiesced()
328 static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_start()  argument
330 	ctrl_info->in_ofa = true;  in pqi_ctrl_ofa_start()
333 static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_done()  argument
335 	ctrl_info->in_ofa = false;  in pqi_ctrl_ofa_done()
338 static inline bool pqi_ctrl_in_ofa(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_in_ofa()  argument
340 	return ctrl_info->in_ofa;  in pqi_ctrl_in_ofa()
348 static inline bool pqi_device_in_remove(struct pqi_ctrl_info *ctrl_info,  in pqi_device_in_remove()  argument
351 	return device->in_remove && !ctrl_info->in_shutdown;  in pqi_device_in_remove()
354 static inline void pqi_ctrl_shutdown_start(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_shutdown_start()  argument
356 	ctrl_info->in_shutdown = true;  in pqi_ctrl_shutdown_start()
359 static inline bool pqi_ctrl_in_shutdown(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_in_shutdown()  argument
361 	return ctrl_info->in_shutdown;  in pqi_ctrl_in_shutdown()
365 	struct pqi_ctrl_info *ctrl_info, unsigned long delay)  in pqi_schedule_rescan_worker_with_delay()  argument
367 	if (pqi_ctrl_offline(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
369 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
372 	schedule_delayed_work(&ctrl_info->rescan_work, delay);  in pqi_schedule_rescan_worker_with_delay()
375 static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker()  argument
377 	pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);  in pqi_schedule_rescan_worker()
383 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker_delayed()  argument
385 	pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);  in pqi_schedule_rescan_worker_delayed()
388 static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_rescan_worker()  argument
390 	cancel_delayed_work_sync(&ctrl_info->rescan_work);  in pqi_cancel_rescan_worker()
393 static inline void pqi_cancel_event_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_event_worker()  argument
395 	cancel_work_sync(&ctrl_info->event_work);  in pqi_cancel_event_worker()
398 static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)  in pqi_read_heartbeat_counter()  argument
400 	if (!ctrl_info->heartbeat_counter)  in pqi_read_heartbeat_counter()
403 	return readl(ctrl_info->heartbeat_counter);  in pqi_read_heartbeat_counter()
406 static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)  in pqi_read_soft_reset_status()  argument
408 	if (!ctrl_info->soft_reset_status)  in pqi_read_soft_reset_status()
411 	return readb(ctrl_info->soft_reset_status);  in pqi_read_soft_reset_status()
414 static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info,  in pqi_clear_soft_reset_status()  argument
419 	if (!ctrl_info->soft_reset_status)  in pqi_clear_soft_reset_status()
422 	status = pqi_read_soft_reset_status(ctrl_info);  in pqi_clear_soft_reset_status()
424 	writeb(status, ctrl_info->soft_reset_status);  in pqi_clear_soft_reset_status()
464 static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_path_request()  argument
545 		dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);  in pqi_build_raid_path_request()
564 	return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],  in pqi_build_raid_path_request()
577 	struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_request()  argument
580 	u16 i = ctrl_info->next_io_request_slot;	/* benignly racy */  in pqi_alloc_io_request()
583 		io_request = &ctrl_info->io_request_pool[i];  in pqi_alloc_io_request()
587 		i = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
591 	ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
603 static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_send_scsi_raid_request()  argument
611 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_send_scsi_raid_request()
617 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_send_scsi_raid_request()
620 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_send_scsi_raid_request()
627 static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_request()  argument
630 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_request()
634 static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_with_error()  argument
638 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_with_error()
642 static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_controller()  argument
645 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,  in pqi_identify_controller()
649 static inline int pqi_sense_subsystem_info(struct  pqi_ctrl_info *ctrl_info,  in pqi_sense_subsystem_info()  argument
652 	return pqi_send_ctrl_raid_request(ctrl_info,  in pqi_sense_subsystem_info()
657 static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_inquiry()  argument
660 	return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,  in pqi_scsi_inquiry()
664 static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_physical_device()  argument
673 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_identify_physical_device()
683 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_identify_physical_device()
686 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_identify_physical_device()
691 static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,  in pqi_flush_cache()  argument
701 	if (pqi_ctrl_offline(ctrl_info))  in pqi_flush_cache()
710 	rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,  in pqi_flush_cache()
718 int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,  in pqi_csmi_smp_passthru()  argument
722 	return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,  in pqi_csmi_smp_passthru()
728 static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)  in pqi_set_diag_rescan()  argument
737 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,  in pqi_set_diag_rescan()
744 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,  in pqi_set_diag_rescan()
753 static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,  in pqi_write_host_wellness()  argument
756 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,  in pqi_write_host_wellness()
774 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_driver_version_to_host_wellness()  argument
802 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_driver_version_to_host_wellness()
823 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_current_time_to_host_wellness()  argument
865 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_current_time_to_host_wellness()
877 	struct pqi_ctrl_info *ctrl_info;  in pqi_update_time_worker()  local
879 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_update_time_worker()
882 	if (pqi_ctrl_offline(ctrl_info))  in pqi_update_time_worker()
885 	rc = pqi_write_current_time_to_host_wellness(ctrl_info);  in pqi_update_time_worker()
887 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_time_worker()
890 	schedule_delayed_work(&ctrl_info->update_time_work,  in pqi_update_time_worker()
895 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_update_time_worker()  argument
897 	schedule_delayed_work(&ctrl_info->update_time_work, 0);  in pqi_schedule_update_time_worker()
901 	struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_update_time_worker()  argument
903 	cancel_delayed_work_sync(&ctrl_info->update_time_work);  in pqi_cancel_update_time_worker()
906 static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_luns()  argument
909 	return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer,  in pqi_report_luns()
913 static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_phys_logical_luns()  argument
929 	rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,  in pqi_report_phys_logical_luns()
950 	rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);  in pqi_report_phys_logical_luns()
976 static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_phys_luns()  argument
979 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,  in pqi_report_phys_luns()
983 static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_logical_luns()  argument
986 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);  in pqi_report_logical_luns()
989 static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_lists()  argument
1000 	rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);  in pqi_get_device_lists()
1002 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1005 	rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);  in pqi_get_device_lists()
1007 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1098 static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_level()  argument
1109 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_level()
1122 static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_validate_raid_map()  argument
1162 	dev_warn(&ctrl_info->pci_dev->dev,  in pqi_validate_raid_map()
1170 static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_map()  argument
1181 	rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1198 		rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1206 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_get_raid_map()
1215 	rc = pqi_validate_raid_map(ctrl_info, device, raid_map);  in pqi_get_raid_map()
1229 static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_bypass_status()  argument
1240 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_bypass_status()
1254 		pqi_get_raid_map(ctrl_info, device) == 0)  in pqi_get_raid_bypass_status()
1265 static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_volume_status()  argument
1279 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_volume_status()
1303 static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_physical_device_info()  argument
1311 	rc = pqi_identify_physical_device(ctrl_info, device,  in pqi_get_physical_device_info()
1342 static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_logical_device_info()  argument
1353 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);  in pqi_get_logical_device_info()
1370 			pqi_get_raid_level(ctrl_info, device);  in pqi_get_logical_device_info()
1371 			pqi_get_raid_bypass_status(ctrl_info, device);  in pqi_get_logical_device_info()
1372 			pqi_get_volume_status(ctrl_info, device);  in pqi_get_logical_device_info()
1382 static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_info()  argument
1392 		rc = pqi_get_logical_device_info(ctrl_info, device);  in pqi_get_device_info()
1394 		rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);  in pqi_get_device_info()
1399 static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_show_volume_status()  argument
1496 	dev_info(&ctrl_info->pci_dev->dev,  in pqi_show_volume_status()
1498 		ctrl_info->scsi_host->host_no,  in pqi_show_volume_status()
1504 	struct pqi_ctrl_info *ctrl_info;  in pqi_rescan_worker()  local
1506 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_rescan_worker()
1509 	pqi_scan_scsi_devices(ctrl_info);  in pqi_rescan_worker()
1512 static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,  in pqi_add_device()  argument
1518 		rc = scsi_add_device(ctrl_info->scsi_host, device->bus,  in pqi_add_device()
1521 		rc = pqi_add_sas_device(ctrl_info->sas_host, device);  in pqi_add_device()
1528 static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,  in pqi_remove_device()  argument
1535 	rc = pqi_device_wait_for_pending_io(ctrl_info, device, PQI_PENDING_IO_TIMEOUT_SECS);  in pqi_remove_device()
1537 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_remove_device()
1539 			ctrl_info->scsi_host->host_no, device->bus,  in pqi_remove_device()
1551 static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,  in pqi_find_scsi_dev()  argument
1556 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)  in pqi_find_scsi_dev()
1582 static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_find_entry()  argument
1587 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {  in pqi_scsi_find_entry()
1612 static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,  in pqi_dev_info()  argument
1619 		"%d:%d:", ctrl_info->scsi_host->host_no, device->bus);  in pqi_dev_info()
1668 	dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);  in pqi_dev_info()
1744 static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,  in pqi_fixup_botched_add()  argument
1749 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1751 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1765 static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,  in pqi_update_device_list()  argument
1785 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1788 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)  in pqi_update_device_list()
1794 		find_result = pqi_scsi_find_entry(ctrl_info, device,  in pqi_update_device_list()
1825 	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1841 			&ctrl_info->scsi_device_list);  in pqi_update_device_list()
1847 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1849 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_update_device_list()
1850 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_update_device_list()
1855 			pqi_dev_info(ctrl_info, "offline", device);  in pqi_update_device_list()
1856 			pqi_show_volume_status(ctrl_info, device);  in pqi_update_device_list()
1860 			pqi_remove_device(ctrl_info, device);  in pqi_update_device_list()
1863 				pqi_dev_info(ctrl_info, "removed", device);  in pqi_update_device_list()
1872 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1891 			rc = pqi_add_device(ctrl_info, device);  in pqi_update_device_list()
1893 				pqi_dev_info(ctrl_info, "added", device);  in pqi_update_device_list()
1895 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_device_list()
1897 					ctrl_info->scsi_host->host_no,  in pqi_update_device_list()
1900 				pqi_fixup_botched_add(ctrl_info, device);  in pqi_update_device_list()
1953 static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_update_scsi_devices()  argument
1977 	rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);  in pqi_update_scsi_devices()
2004 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2031 		dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);  in pqi_update_scsi_devices()
2039 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2094 		rc = pqi_get_device_info(ctrl_info, device, id_phys);  in pqi_update_scsi_devices()
2096 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2102 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2107 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2137 	pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);  in pqi_update_scsi_devices()
2156 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_scan_scsi_devices()  argument
2160 	if (pqi_ctrl_offline(ctrl_info))  in pqi_scan_scsi_devices()
2163 	if (!mutex_trylock(&ctrl_info->scan_mutex)) {  in pqi_scan_scsi_devices()
2164 		pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2167 		rc = pqi_update_scsi_devices(ctrl_info);  in pqi_scan_scsi_devices()
2169 			pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2170 		mutex_unlock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
2178 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_start()  local
2180 	ctrl_info = shost_to_hba(shost);  in pqi_scan_start()
2181 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_scan_start()
2184 	pqi_scan_scsi_devices(ctrl_info);  in pqi_scan_start()
2192 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_finished()  local
2194 	ctrl_info = shost_priv(shost);  in pqi_scan_finished()
2196 	return !mutex_is_locked(&ctrl_info->scan_mutex);  in pqi_scan_finished()
2199 static void pqi_wait_until_scan_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_scan_finished()  argument
2201 	mutex_lock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2202 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2205 static void pqi_wait_until_lun_reset_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_lun_reset_finished()  argument
2207 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2208 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2211 static void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_ofa_finished()  argument
2213 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
2214 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
2244 static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_bypass_submit_scsi_cmd()  argument
2566 	return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,  in pqi_raid_bypass_submit_scsi_cmd()
2584 static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_mode_ready()  argument
2591 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_mode_ready()
2600 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2612 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2624 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2645 	struct pqi_ctrl_info *ctrl_info;  in pqi_take_device_offline()  local
2653 	ctrl_info = shost_to_hba(sdev->host);  in pqi_take_device_offline()
2654 	pqi_schedule_rescan_worker(ctrl_info);  in pqi_take_device_offline()
2655 	dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",  in pqi_take_device_offline()
2656 		path, ctrl_info->scsi_host->host_no, device->bus,  in pqi_take_device_offline()
2731 			struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);  in pqi_process_raid_io_error()  local
2738 						ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
2746 						sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
2885 static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info)  in pqi_invalid_response()  argument
2887 	pqi_take_ctrl_offline(ctrl_info);  in pqi_invalid_response()
2890 static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)  in pqi_process_io_intr()  argument
2904 		if (oq_pi >= ctrl_info->num_elements_per_oq) {  in pqi_process_io_intr()
2905 			pqi_invalid_response(ctrl_info);  in pqi_process_io_intr()
2906 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
2908 				oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);  in pqi_process_io_intr()
2919 		if (request_id >= ctrl_info->max_io_slots) {  in pqi_process_io_intr()
2920 			pqi_invalid_response(ctrl_info);  in pqi_process_io_intr()
2921 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
2923 				request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);  in pqi_process_io_intr()
2927 		io_request = &ctrl_info->io_request_pool[request_id];  in pqi_process_io_intr()
2929 			pqi_invalid_response(ctrl_info);  in pqi_process_io_intr()
2930 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
2961 			io_request->error_info = ctrl_info->error_buffer +  in pqi_process_io_intr()
2967 			pqi_invalid_response(ctrl_info);  in pqi_process_io_intr()
2968 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
2980 		oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;  in pqi_process_io_intr()
3004 static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,  in pqi_send_event_ack()  argument
3013 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
3023 			ctrl_info->num_elements_per_iq))  in pqi_send_event_ack()
3029 		if (pqi_ctrl_offline(ctrl_info))  in pqi_send_event_ack()
3038 	iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;  in pqi_send_event_ack()
3050 static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,  in pqi_acknowledge_event()  argument
3064 	pqi_send_event_ack(ctrl_info, &request, sizeof(request));  in pqi_acknowledge_event()
3071 	struct pqi_ctrl_info *ctrl_info)  in pqi_poll_for_soft_reset_status()  argument
3079 		status = pqi_read_soft_reset_status(ctrl_info);  in pqi_poll_for_soft_reset_status()
3087 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_soft_reset_status()
3092 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_soft_reset_status()
3099 static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_process_soft_reset()  argument
3107 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3108 			"resetting controller %u\n", ctrl_info->ctrl_id);  in pqi_process_soft_reset()
3109 		sis_soft_reset(ctrl_info);  in pqi_process_soft_reset()
3112 		rc = pqi_ofa_ctrl_restart(ctrl_info);  in pqi_process_soft_reset()
3113 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3114 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3116 			ctrl_info->ctrl_id, rc == 0 ? "SUCCESS" : "FAILED");  in pqi_process_soft_reset()
3119 		pqi_ofa_ctrl_unquiesce(ctrl_info);  in pqi_process_soft_reset()
3120 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3122 			ctrl_info->ctrl_id, "ABORTED");  in pqi_process_soft_reset()
3125 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3126 		pqi_take_ctrl_offline(ctrl_info);  in pqi_process_soft_reset()
3131 static void pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_process_event()  argument
3139 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_ofa_process_event()
3142 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3144 			ctrl_info->ctrl_id);  in pqi_ofa_process_event()
3145 		pqi_ofa_ctrl_quiesce(ctrl_info);  in pqi_ofa_process_event()
3146 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3147 		if (ctrl_info->soft_reset_handshake_supported) {  in pqi_ofa_process_event()
3148 			status = pqi_poll_for_soft_reset_status(ctrl_info);  in pqi_ofa_process_event()
3149 			pqi_process_soft_reset(ctrl_info, status);  in pqi_ofa_process_event()
3151 			pqi_process_soft_reset(ctrl_info,  in pqi_ofa_process_event()
3156 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3157 		pqi_ofa_setup_host_buffer(ctrl_info,  in pqi_ofa_process_event()
3159 		pqi_ofa_host_memory_update(ctrl_info);  in pqi_ofa_process_event()
3161 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_ofa_process_event()
3162 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3163 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3165 			ctrl_info->ctrl_id, event->ofa_cancel_reason);  in pqi_ofa_process_event()
3168 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_ofa_process_event()
3174 	struct pqi_ctrl_info *ctrl_info;  in pqi_event_worker()  local
3177 	ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);  in pqi_event_worker()
3179 	pqi_ctrl_busy(ctrl_info);  in pqi_event_worker()
3180 	pqi_wait_if_ctrl_blocked(ctrl_info, NO_TIMEOUT);  in pqi_event_worker()
3181 	if (pqi_ctrl_offline(ctrl_info))  in pqi_event_worker()
3184 	pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_event_worker()
3186 	event = ctrl_info->events;  in pqi_event_worker()
3191 				pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
3192 				pqi_ofa_process_event(ctrl_info, event);  in pqi_event_worker()
3195 			pqi_acknowledge_event(ctrl_info, event);  in pqi_event_worker()
3201 	pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
3210 	struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t,  in pqi_heartbeat_timer_handler()  local
3213 	pqi_check_ctrl_health(ctrl_info);  in pqi_heartbeat_timer_handler()
3214 	if (pqi_ctrl_offline(ctrl_info))  in pqi_heartbeat_timer_handler()
3217 	num_interrupts = atomic_read(&ctrl_info->num_interrupts);  in pqi_heartbeat_timer_handler()
3218 	heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);  in pqi_heartbeat_timer_handler()
3220 	if (num_interrupts == ctrl_info->previous_num_interrupts) {  in pqi_heartbeat_timer_handler()
3221 		if (heartbeat_count == ctrl_info->previous_heartbeat_count) {  in pqi_heartbeat_timer_handler()
3222 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_heartbeat_timer_handler()
3225 			pqi_take_ctrl_offline(ctrl_info);  in pqi_heartbeat_timer_handler()
3229 		ctrl_info->previous_num_interrupts = num_interrupts;  in pqi_heartbeat_timer_handler()
3232 	ctrl_info->previous_heartbeat_count = heartbeat_count;  in pqi_heartbeat_timer_handler()
3233 	mod_timer(&ctrl_info->heartbeat_timer,  in pqi_heartbeat_timer_handler()
3237 static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_start_heartbeat_timer()  argument
3239 	if (!ctrl_info->heartbeat_counter)  in pqi_start_heartbeat_timer()
3242 	ctrl_info->previous_num_interrupts =  in pqi_start_heartbeat_timer()
3243 		atomic_read(&ctrl_info->num_interrupts);  in pqi_start_heartbeat_timer()
3244 	ctrl_info->previous_heartbeat_count =  in pqi_start_heartbeat_timer()
3245 		pqi_read_heartbeat_counter(ctrl_info);  in pqi_start_heartbeat_timer()
3247 	ctrl_info->heartbeat_timer.expires =  in pqi_start_heartbeat_timer()
3249 	add_timer(&ctrl_info->heartbeat_timer);  in pqi_start_heartbeat_timer()
3252 static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_stop_heartbeat_timer()  argument
3254 	del_timer_sync(&ctrl_info->heartbeat_timer);  in pqi_stop_heartbeat_timer()
3291 static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)  in pqi_process_event_intr()  argument
3301 	event_queue = &ctrl_info->event_queue;  in pqi_process_event_intr()
3308 			pqi_invalid_response(ctrl_info);  in pqi_process_event_intr()
3309 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_event_intr()
3325 			event = &ctrl_info->events[event_index];  in pqi_process_event_intr()
3340 		schedule_work(&ctrl_info->event_work);  in pqi_process_event_intr()
3348 static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_legacy_intx()  argument
3355 	pqi_registers = ctrl_info->pqi_registers;  in pqi_configure_legacy_intx()
3367 static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_change_irq_mode()  argument
3370 	switch (ctrl_info->irq_mode) {  in pqi_change_irq_mode()
3376 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3377 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3386 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3387 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3392 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3399 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3402 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3403 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3411 	ctrl_info->irq_mode = new_mode;  in pqi_change_irq_mode()
3416 static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)  in pqi_is_valid_irq()  argument
3421 	switch (ctrl_info->irq_mode) {  in pqi_is_valid_irq()
3427 			readl(&ctrl_info->pqi_registers->legacy_intx_status);  in pqi_is_valid_irq()
3444 	struct pqi_ctrl_info *ctrl_info;  in pqi_irq_handler()  local
3450 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3452 	if (!pqi_is_valid_irq(ctrl_info))  in pqi_irq_handler()
3455 	num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
3459 	if (irq == ctrl_info->event_irq) {  in pqi_irq_handler()
3460 		num_events_handled = pqi_process_event_intr(ctrl_info);  in pqi_irq_handler()
3468 		atomic_inc(&ctrl_info->num_interrupts);  in pqi_irq_handler()
3470 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
3471 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
3477 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_request_irqs()  argument
3479 	struct pci_dev *pci_dev = ctrl_info->pci_dev;  in pqi_request_irqs()
3483 	ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);  in pqi_request_irqs()
3485 	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {  in pqi_request_irqs()
3487 			DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);  in pqi_request_irqs()
3494 		ctrl_info->num_msix_vectors_initialized++;  in pqi_request_irqs()
3500 static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_free_irqs()  argument
3504 	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)  in pqi_free_irqs()
3505 		free_irq(pci_irq_vector(ctrl_info->pci_dev, i),  in pqi_free_irqs()
3506 			&ctrl_info->queue_groups[i]);  in pqi_free_irqs()
3508 	ctrl_info->num_msix_vectors_initialized = 0;  in pqi_free_irqs()
3511 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_msix_interrupts()  argument
3515 	num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,  in pqi_enable_msix_interrupts()
3516 			PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,  in pqi_enable_msix_interrupts()
3519 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_enable_msix_interrupts()
3525 	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;  in pqi_enable_msix_interrupts()
3526 	ctrl_info->irq_mode = IRQ_MODE_MSIX;  in pqi_enable_msix_interrupts()
3530 static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_msix_interrupts()  argument
3532 	if (ctrl_info->num_msix_vectors_enabled) {  in pqi_disable_msix_interrupts()
3533 		pci_free_irq_vectors(ctrl_info->pci_dev);  in pqi_disable_msix_interrupts()
3534 		ctrl_info->num_msix_vectors_enabled = 0;  in pqi_disable_msix_interrupts()
3538 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_operational_queues()  argument
3554 		ctrl_info->num_elements_per_iq;  in pqi_alloc_operational_queues()
3557 		ctrl_info->num_elements_per_oq;  in pqi_alloc_operational_queues()
3558 	num_inbound_queues = ctrl_info->num_queue_groups * 2;  in pqi_alloc_operational_queues()
3559 	num_outbound_queues = ctrl_info->num_queue_groups;  in pqi_alloc_operational_queues()
3560 	num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;  in pqi_alloc_operational_queues()
3592 	ctrl_info->queue_memory_base =  in pqi_alloc_operational_queues()
3593 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_operational_queues()
3594 				   &ctrl_info->queue_memory_base_dma_handle,  in pqi_alloc_operational_queues()
3597 	if (!ctrl_info->queue_memory_base)  in pqi_alloc_operational_queues()
3600 	ctrl_info->queue_memory_length = alloc_length;  in pqi_alloc_operational_queues()
3602 	element_array = PTR_ALIGN(ctrl_info->queue_memory_base,  in pqi_alloc_operational_queues()
3605 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3606 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3609 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3610 				(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3616 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3617 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3623 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3624 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3627 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3628 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3634 	ctrl_info->event_queue.oq_element_array = element_array;  in pqi_alloc_operational_queues()
3635 	ctrl_info->event_queue.oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
3636 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3637 		(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3644 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3645 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3648 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3650 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3656 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3658 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3664 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3666 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3672 	ctrl_info->event_queue.oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
3673 	ctrl_info->event_queue.oq_pi_bus_addr =  in pqi_alloc_operational_queues()
3674 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3676 		(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3681 static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_init_operational_queues()  argument
3691 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3692 		ctrl_info->queue_groups[i].ctrl_info = ctrl_info;  in pqi_init_operational_queues()
3699 	ctrl_info->event_queue.oq_id = next_oq_id++;  in pqi_init_operational_queues()
3700 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3701 		ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3702 		ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3703 		ctrl_info->queue_groups[i].oq_id = next_oq_id++;  in pqi_init_operational_queues()
3710 	ctrl_info->event_queue.int_msg_num = 0;  in pqi_init_operational_queues()
3711 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3712 		ctrl_info->queue_groups[i].int_msg_num = i;  in pqi_init_operational_queues()
3714 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3715 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);  in pqi_init_operational_queues()
3716 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);  in pqi_init_operational_queues()
3717 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);  in pqi_init_operational_queues()
3718 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);  in pqi_init_operational_queues()
3722 static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_admin_queues()  argument
3731 	ctrl_info->admin_queue_memory_base =  in pqi_alloc_admin_queues()
3732 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_admin_queues()
3733 				   &ctrl_info->admin_queue_memory_base_dma_handle,  in pqi_alloc_admin_queues()
3736 	if (!ctrl_info->admin_queue_memory_base)  in pqi_alloc_admin_queues()
3739 	ctrl_info->admin_queue_memory_length = alloc_length;  in pqi_alloc_admin_queues()
3741 	admin_queues = &ctrl_info->admin_queues;  in pqi_alloc_admin_queues()
3742 	admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,  in pqi_alloc_admin_queues()
3753 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3755 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3757 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3759 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3761 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3763 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3765 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3767 		(void __iomem *)ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3775 static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_admin_queues()  argument
3783 	pqi_registers = ctrl_info->pqi_registers;  in pqi_create_admin_queues()
3784 	admin_queues = &ctrl_info->admin_queues;  in pqi_create_admin_queues()
3817 	admin_queues->iq_pi = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3820 	admin_queues->oq_ci = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3827 static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request()  argument
3834 	admin_queues = &ctrl_info->admin_queues;  in pqi_submit_admin_request()
3854 static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,  in pqi_poll_for_admin_response()  argument
3862 	admin_queues = &ctrl_info->admin_queues;  in pqi_poll_for_admin_response()
3872 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_admin_response()
3876 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_admin_response()
3891 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,  in pqi_start_io()  argument
3930 			ctrl_info->num_elements_per_iq))  in pqi_start_io()
3940 			ctrl_info->num_elements_per_iq - iq_pi;  in pqi_start_io()
3954 			ctrl_info->num_elements_per_iq;  in pqi_start_io()
3973 static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_completion_io()  argument
3985 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_completion_io()
3986 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_completion_io()
4026 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_raid_request_synchronous()  argument
4043 		if (down_interruptible(&ctrl_info->sync_request_sem))  in pqi_submit_raid_request_synchronous()
4047 			down(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4050 			if (down_timeout(&ctrl_info->sync_request_sem,  in pqi_submit_raid_request_synchronous()
4063 	pqi_ctrl_busy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4064 	timeout_msecs = pqi_wait_if_ctrl_blocked(ctrl_info, timeout_msecs);  in pqi_submit_raid_request_synchronous()
4066 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4071 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_submit_raid_request_synchronous()
4072 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4077 	atomic_inc(&ctrl_info->sync_cmds_outstanding);  in pqi_submit_raid_request_synchronous()
4079 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_submit_raid_request_synchronous()
4095 	pqi_start_io(ctrl_info,  in pqi_submit_raid_request_synchronous()
4096 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_submit_raid_request_synchronous()
4099 	pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4102 		pqi_wait_for_completion_io(ctrl_info, &wait);  in pqi_submit_raid_request_synchronous()
4106 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_submit_raid_request_synchronous()
4125 	atomic_dec(&ctrl_info->sync_cmds_outstanding);  in pqi_submit_raid_request_synchronous()
4127 	up(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4152 	struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request_synchronous()  argument
4158 	pqi_submit_admin_request(ctrl_info, request);  in pqi_submit_admin_request_synchronous()
4160 	rc = pqi_poll_for_admin_response(ctrl_info, response);  in pqi_submit_admin_request_synchronous()
4169 static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_report_device_capability()  argument
4191 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_report_device_capability()
4198 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_report_device_capability()
4201 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_report_device_capability()
4213 	ctrl_info->max_inbound_queues =  in pqi_report_device_capability()
4215 	ctrl_info->max_elements_per_iq =  in pqi_report_device_capability()
4217 	ctrl_info->max_iq_element_length =  in pqi_report_device_capability()
4220 	ctrl_info->max_outbound_queues =  in pqi_report_device_capability()
4222 	ctrl_info->max_elements_per_oq =  in pqi_report_device_capability()
4224 	ctrl_info->max_oq_element_length =  in pqi_report_device_capability()
4231 	ctrl_info->max_inbound_iu_length_per_firmware =  in pqi_report_device_capability()
4234 	ctrl_info->inbound_spanning_supported =  in pqi_report_device_capability()
4236 	ctrl_info->outbound_spanning_supported =  in pqi_report_device_capability()
4245 static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_validate_device_capability()  argument
4247 	if (ctrl_info->max_iq_element_length <  in pqi_validate_device_capability()
4249 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4251 			ctrl_info->max_iq_element_length,  in pqi_validate_device_capability()
4256 	if (ctrl_info->max_oq_element_length <  in pqi_validate_device_capability()
4258 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4260 			ctrl_info->max_oq_element_length,  in pqi_validate_device_capability()
4265 	if (ctrl_info->max_inbound_iu_length_per_firmware <  in pqi_validate_device_capability()
4267 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4269 			ctrl_info->max_inbound_iu_length_per_firmware,  in pqi_validate_device_capability()
4274 	if (!ctrl_info->inbound_spanning_supported) {  in pqi_validate_device_capability()
4275 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4280 	if (ctrl_info->outbound_spanning_supported) {  in pqi_validate_device_capability()
4281 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4289 static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)  in pqi_create_event_queue()  argument
4296 	event_queue = &ctrl_info->event_queue;  in pqi_create_event_queue()
4321 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_event_queue()
4326 	event_queue->oq_ci = ctrl_info->iomem_base +  in pqi_create_event_queue()
4334 static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,  in pqi_create_queue_group()  argument
4342 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
4360 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4366 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4369 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4374 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4395 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4401 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4404 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4409 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4429 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4432 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4451 	put_unaligned_le16(ctrl_info->num_elements_per_oq,  in pqi_create_queue_group()
4459 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4462 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4467 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
4475 static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_queues()  argument
4480 	rc = pqi_create_event_queue(ctrl_info);  in pqi_create_queues()
4482 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4487 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_create_queues()
4488 		rc = pqi_create_queue_group(ctrl_info, i);  in pqi_create_queues()
4490 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4492 				i, ctrl_info->num_queue_groups);  in pqi_create_queues()
4504 static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_events()  argument
4527 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4534 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_configure_events()
4537 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4548 			put_unaligned_le16(ctrl_info->event_queue.oq_id,  in pqi_configure_events()
4563 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4570 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_configure_events()
4573 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4583 static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_events()  argument
4585 	return pqi_configure_events(ctrl_info, true);  in pqi_enable_events()
4588 static inline int pqi_disable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_events()  argument
4590 	return pqi_configure_events(ctrl_info, false);  in pqi_disable_events()
4593 static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_free_all_io_requests()  argument
4600 	if (!ctrl_info->io_request_pool)  in pqi_free_all_io_requests()
4603 	dev = &ctrl_info->pci_dev->dev;  in pqi_free_all_io_requests()
4604 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_free_all_io_requests()
4605 	io_request = ctrl_info->io_request_pool;  in pqi_free_all_io_requests()
4607 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_free_all_io_requests()
4617 	kfree(ctrl_info->io_request_pool);  in pqi_free_all_io_requests()
4618 	ctrl_info->io_request_pool = NULL;  in pqi_free_all_io_requests()
4621 static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_error_buffer()  argument
4624 	ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_error_buffer()
4625 				     ctrl_info->error_buffer_length,  in pqi_alloc_error_buffer()
4626 				     &ctrl_info->error_buffer_dma_handle,  in pqi_alloc_error_buffer()
4628 	if (!ctrl_info->error_buffer)  in pqi_alloc_error_buffer()
4634 static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_resources()  argument
4643 	ctrl_info->io_request_pool =  in pqi_alloc_io_resources()
4644 		kcalloc(ctrl_info->max_io_slots,  in pqi_alloc_io_resources()
4645 			sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);  in pqi_alloc_io_resources()
4647 	if (!ctrl_info->io_request_pool) {  in pqi_alloc_io_resources()
4648 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4653 	dev = &ctrl_info->pci_dev->dev;  in pqi_alloc_io_resources()
4654 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_alloc_io_resources()
4655 	io_request = ctrl_info->io_request_pool;  in pqi_alloc_io_resources()
4657 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_alloc_io_resources()
4659 			kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);  in pqi_alloc_io_resources()
4662 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4672 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4687 	pqi_free_all_io_requests(ctrl_info);  in pqi_alloc_io_resources()
4697 static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_io_resources()  argument
4702 	ctrl_info->scsi_ml_can_queue =  in pqi_calculate_io_resources()
4703 		ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;  in pqi_calculate_io_resources()
4704 	ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;  in pqi_calculate_io_resources()
4706 	ctrl_info->error_buffer_length =  in pqi_calculate_io_resources()
4707 		ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;  in pqi_calculate_io_resources()
4710 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4713 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4721 	max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);  in pqi_calculate_io_resources()
4725 	ctrl_info->sg_chain_buffer_length =  in pqi_calculate_io_resources()
4728 	ctrl_info->sg_tablesize = max_sg_entries;  in pqi_calculate_io_resources()
4729 	ctrl_info->max_sectors = max_transfer_size / 512;  in pqi_calculate_io_resources()
4732 static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_queue_resources()  argument
4744 		max_queue_groups = min(ctrl_info->max_inbound_queues / 2,  in pqi_calculate_queue_resources()
4745 			ctrl_info->max_outbound_queues - 1);  in pqi_calculate_queue_resources()
4749 		num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);  in pqi_calculate_queue_resources()
4753 	ctrl_info->num_queue_groups = num_queue_groups;  in pqi_calculate_queue_resources()
4754 	ctrl_info->max_hw_queue_index = num_queue_groups - 1;  in pqi_calculate_queue_resources()
4760 	ctrl_info->max_inbound_iu_length =  in pqi_calculate_queue_resources()
4761 		(ctrl_info->max_inbound_iu_length_per_firmware /  in pqi_calculate_queue_resources()
4766 		(ctrl_info->max_inbound_iu_length /  in pqi_calculate_queue_resources()
4773 		ctrl_info->max_elements_per_iq);  in pqi_calculate_queue_resources()
4777 		ctrl_info->max_elements_per_oq);  in pqi_calculate_queue_resources()
4779 	ctrl_info->num_elements_per_iq = num_elements_per_iq;  in pqi_calculate_queue_resources()
4780 	ctrl_info->num_elements_per_oq = num_elements_per_oq;  in pqi_calculate_queue_resources()
4782 	ctrl_info->max_sg_per_iu =  in pqi_calculate_queue_resources()
4783 		((ctrl_info->max_inbound_iu_length -  in pqi_calculate_queue_resources()
4800 static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_sg_list()  argument
4825 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_raid_sg_list()
4864 static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_sg_list()  argument
4890 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_aio_sg_list()
4941 	struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,  in pqi_raid_submit_scsi_cmd_with_io_request()  argument
5013 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_raid_submit_scsi_cmd_with_io_request()
5019 	rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
5025 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
5030 static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_submit_scsi_cmd()  argument
5036 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_raid_submit_scsi_cmd()
5038 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_raid_submit_scsi_cmd()
5042 static inline void pqi_schedule_bypass_retry(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_bypass_retry()  argument
5044 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_schedule_bypass_retry()
5045 		schedule_work(&ctrl_info->raid_bypass_retry_work);  in pqi_schedule_bypass_retry()
5052 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_needed()  local
5067 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_raid_bypass_retry_needed()
5068 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_bypass_retry_needed()
5075 	struct pqi_ctrl_info *ctrl_info,  in pqi_add_to_raid_bypass_retry_list()  argument
5080 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
5083 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
5086 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
5087 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
5103 	struct pqi_ctrl_info *ctrl_info;  in pqi_queue_raid_bypass_retry()  local
5108 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_queue_raid_bypass_retry()
5110 	pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request, false);  in pqi_queue_raid_bypass_retry()
5111 	pqi_schedule_bypass_retry(ctrl_info);  in pqi_queue_raid_bypass_retry()
5118 	struct pqi_ctrl_info *ctrl_info;  in pqi_retry_raid_bypass()  local
5130 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_retry_raid_bypass()
5135 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_retry_raid_bypass()
5140 	struct pqi_ctrl_info *ctrl_info)  in pqi_next_queued_raid_bypass_request()  argument
5145 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
5147 		&ctrl_info->raid_bypass_retry_list,  in pqi_next_queued_raid_bypass_request()
5151 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
5156 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_retry_raid_bypass_requests()  argument
5161 	pqi_ctrl_busy(ctrl_info);  in pqi_retry_raid_bypass_requests()
5164 		if (pqi_ctrl_blocked(ctrl_info))  in pqi_retry_raid_bypass_requests()
5166 		io_request = pqi_next_queued_raid_bypass_request(ctrl_info);  in pqi_retry_raid_bypass_requests()
5171 			pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request,  in pqi_retry_raid_bypass_requests()
5173 			pqi_schedule_bypass_retry(ctrl_info);  in pqi_retry_raid_bypass_requests()
5178 	pqi_ctrl_unbusy(ctrl_info);  in pqi_retry_raid_bypass_requests()
5183 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_worker()  local
5185 	ctrl_info = container_of(work, struct pqi_ctrl_info,  in pqi_raid_bypass_retry_worker()
5187 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_raid_bypass_retry_worker()
5191 	struct pqi_ctrl_info *ctrl_info)  in pqi_clear_all_queued_raid_bypass_retries()  argument
5195 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
5196 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_clear_all_queued_raid_bypass_retries()
5197 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
5217 static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_scsi_cmd()  argument
5221 	return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,  in pqi_aio_submit_scsi_cmd()
5225 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_io()  argument
5234 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_aio_submit_io()
5268 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_aio_submit_io()
5284 	rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);  in pqi_aio_submit_io()
5290 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
5295 static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,  in pqi_get_hw_queue()  argument
5301 	if (hw_queue > ctrl_info->max_hw_queue_index)  in pqi_get_hw_queue()
5334 	struct pqi_ctrl_info *ctrl_info;  in pqi_scsi_queue_command()  local
5341 	ctrl_info = shost_to_hba(shost);  in pqi_scsi_queue_command()
5351 	if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(ctrl_info,  in pqi_scsi_queue_command()
5358 	pqi_ctrl_busy(ctrl_info);  in pqi_scsi_queue_command()
5359 	if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device) ||  in pqi_scsi_queue_command()
5360 	    pqi_ctrl_in_ofa(ctrl_info) || pqi_ctrl_in_shutdown(ctrl_info)) {  in pqi_scsi_queue_command()
5371 	hw_queue = pqi_get_hw_queue(ctrl_info, scmd);  in pqi_scsi_queue_command()
5372 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
5378 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,  in pqi_scsi_queue_command()
5386 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5389 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5391 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5395 	pqi_ctrl_unbusy(ctrl_info);  in pqi_scsi_queue_command()
5402 static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_until_queued_io_drained()  argument
5419 			pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_queued_io_drained()
5420 			if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_queued_io_drained()
5429 static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_inbound_queues_empty()  argument
5438 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_wait_until_inbound_queues_empty()
5439 		queue_group = &ctrl_info->queue_groups[i];  in pqi_wait_until_inbound_queues_empty()
5441 		rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);  in pqi_wait_until_inbound_queues_empty()
5452 				pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
5453 				if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_inbound_queues_empty()
5463 static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,  in pqi_fail_io_queued_for_device()  argument
5475 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_device()
5476 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
5506 static void pqi_fail_io_queued_for_all_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_io_queued_for_all_devices()  argument
5516 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_all_devices()
5517 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_all_devices()
5544 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_device_wait_for_pending_io()  argument
5552 		pqi_check_ctrl_health(ctrl_info);  in pqi_device_wait_for_pending_io()
5553 		if (pqi_ctrl_offline(ctrl_info))  in pqi_device_wait_for_pending_io()
5557 				dev_err(&ctrl_info->pci_dev->dev,  in pqi_device_wait_for_pending_io()
5568 static int pqi_ctrl_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_ctrl_wait_for_pending_io()  argument
5580 		spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ctrl_wait_for_pending_io()
5581 		list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_ctrl_wait_for_pending_io()
5588 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_ctrl_wait_for_pending_io()
5594 		pqi_check_ctrl_health(ctrl_info);  in pqi_ctrl_wait_for_pending_io()
5595 		if (pqi_ctrl_offline(ctrl_info))  in pqi_ctrl_wait_for_pending_io()
5600 				dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_wait_for_pending_io()
5611 static int pqi_ctrl_wait_for_pending_sync_cmds(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_for_pending_sync_cmds()  argument
5613 	while (atomic_read(&ctrl_info->sync_cmds_outstanding)) {  in pqi_ctrl_wait_for_pending_sync_cmds()
5614 		pqi_check_ctrl_health(ctrl_info);  in pqi_ctrl_wait_for_pending_sync_cmds()
5615 		if (pqi_ctrl_offline(ctrl_info))  in pqi_ctrl_wait_for_pending_sync_cmds()
5634 static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_lun_reset_completion()  argument
5646 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_lun_reset_completion()
5647 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_lun_reset_completion()
5656 static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_lun_reset()  argument
5664 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_lun_reset()
5678 	if (ctrl_info->tmf_iu_timeout_supported)  in pqi_lun_reset()
5682 	pqi_start_io(ctrl_info,  in pqi_lun_reset()
5683 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_lun_reset()
5686 	rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);  in pqi_lun_reset()
5701 static int _pqi_device_reset(struct pqi_ctrl_info *ctrl_info,  in _pqi_device_reset()  argument
5709 		rc = pqi_lun_reset(ctrl_info, device);  in _pqi_device_reset()
5717 	rc |= pqi_device_wait_for_pending_io(ctrl_info, device, timeout_secs);  in _pqi_device_reset()
5722 static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_device_reset()  argument
5727 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset()
5729 	pqi_ctrl_block_requests(ctrl_info);  in pqi_device_reset()
5730 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_device_reset()
5731 	pqi_fail_io_queued_for_device(ctrl_info, device);  in pqi_device_reset()
5732 	rc = pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_device_reset()
5734 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_device_reset()
5739 		rc = _pqi_device_reset(ctrl_info, device);  in pqi_device_reset()
5743 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset()
5752 	struct pqi_ctrl_info *ctrl_info;  in pqi_eh_device_reset_handler()  local
5756 	ctrl_info = shost_to_hba(shost);  in pqi_eh_device_reset_handler()
5759 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5763 	pqi_check_ctrl_health(ctrl_info);  in pqi_eh_device_reset_handler()
5764 	if (pqi_ctrl_offline(ctrl_info) ||  in pqi_eh_device_reset_handler()
5765 		pqi_device_reset_blocked(ctrl_info)) {  in pqi_eh_device_reset_handler()
5770 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_eh_device_reset_handler()
5772 	atomic_inc(&ctrl_info->sync_cmds_outstanding);  in pqi_eh_device_reset_handler()
5773 	rc = pqi_device_reset(ctrl_info, device);  in pqi_eh_device_reset_handler()
5774 	atomic_dec(&ctrl_info->sync_cmds_outstanding);  in pqi_eh_device_reset_handler()
5777 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5789 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_alloc()  local
5793 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_alloc()
5795 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5800 		device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);  in pqi_slave_alloc()
5807 		device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),  in pqi_slave_alloc()
5825 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5832 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_map_queues()  local
5835 					ctrl_info->pci_dev, 0);  in pqi_map_queues()
5852 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_destroy()  local
5854 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_destroy()
5856 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_destroy()
5865 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_destroy()
5868 		pqi_dev_info(ctrl_info, "removed", device);  in pqi_slave_destroy()
5873 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_getpciinfo_ioctl()  argument
5883 	pci_dev = ctrl_info->pci_dev;  in pqi_getpciinfo_ioctl()
5983 static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_passthru_ioctl()  argument
5994 	if (pqi_ctrl_offline(ctrl_info))  in pqi_passthru_ioctl()
6065 		rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_passthru_ioctl()
6076 	if (ctrl_info->raid_iu_timeout_supported)  in pqi_passthru_ioctl()
6079 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_passthru_ioctl()
6083 		pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_passthru_ioctl()
6128 	struct pqi_ctrl_info *ctrl_info;  in pqi_ioctl()  local
6130 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ioctl()
6132 	if (pqi_ctrl_in_ofa(ctrl_info) || pqi_ctrl_in_shutdown(ctrl_info))  in pqi_ioctl()
6139 		rc = pqi_scan_scsi_devices(ctrl_info);  in pqi_ioctl()
6142 		rc = pqi_getpciinfo_ioctl(ctrl_info, arg);  in pqi_ioctl()
6148 		rc = pqi_passthru_ioctl(ctrl_info, arg);  in pqi_ioctl()
6162 	struct pqi_ctrl_info *ctrl_info;  in pqi_firmware_version_show()  local
6165 	ctrl_info = shost_to_hba(shost);  in pqi_firmware_version_show()
6167 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);  in pqi_firmware_version_show()
6181 	struct pqi_ctrl_info *ctrl_info;  in pqi_serial_number_show()  local
6184 	ctrl_info = shost_to_hba(shost);  in pqi_serial_number_show()
6186 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);  in pqi_serial_number_show()
6193 	struct pqi_ctrl_info *ctrl_info;  in pqi_model_show()  local
6196 	ctrl_info = shost_to_hba(shost);  in pqi_model_show()
6198 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);  in pqi_model_show()
6205 	struct pqi_ctrl_info *ctrl_info;  in pqi_vendor_show()  local
6208 	ctrl_info = shost_to_hba(shost);  in pqi_vendor_show()
6210 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);  in pqi_vendor_show()
6286 	struct pqi_ctrl_info *ctrl_info;  in pqi_unique_id_show()  local
6293 	ctrl_info = shost_to_hba(sdev->host);  in pqi_unique_id_show()
6295 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
6299 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
6310 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
6323 	struct pqi_ctrl_info *ctrl_info;  in pqi_lunid_show()  local
6330 	ctrl_info = shost_to_hba(sdev->host);  in pqi_lunid_show()
6332 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
6336 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
6342 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
6352 	struct pqi_ctrl_info *ctrl_info;  in pqi_path_info_show()  local
6365 	ctrl_info = shost_to_hba(sdev->host);  in pqi_path_info_show()
6367 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
6371 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
6388 					ctrl_info->scsi_host->host_no,  in pqi_path_info_show()
6427 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
6435 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_address_show()  local
6442 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_address_show()
6444 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
6448 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
6454 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
6462 	struct pqi_ctrl_info *ctrl_info;  in pqi_ssd_smart_path_enabled_show()  local
6468 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ssd_smart_path_enabled_show()
6470 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
6474 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
6482 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
6490 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_level_show()  local
6497 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_level_show()
6499 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
6503 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
6512 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
6520 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_cnt_show()  local
6527 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_bypass_cnt_show()
6529 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
6533 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
6539 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
6581 static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_register_scsi()  argument
6586 	shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));  in pqi_register_scsi()
6588 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6590 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6601 	shost->max_sectors = ctrl_info->max_sectors;  in pqi_register_scsi()
6602 	shost->can_queue = ctrl_info->scsi_ml_can_queue;  in pqi_register_scsi()
6604 	shost->sg_tablesize = ctrl_info->sg_tablesize;  in pqi_register_scsi()
6606 	shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);  in pqi_register_scsi()
6608 	shost->nr_hw_queues = ctrl_info->num_queue_groups;  in pqi_register_scsi()
6610 	shost->hostdata[0] = (unsigned long)ctrl_info;  in pqi_register_scsi()
6612 	rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);  in pqi_register_scsi()
6614 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6616 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6620 	rc = pqi_add_sas_host(shost, ctrl_info);  in pqi_register_scsi()
6622 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6624 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6628 	ctrl_info->scsi_host = shost;  in pqi_register_scsi()
6640 static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_unregister_scsi()  argument
6644 	pqi_delete_sas_host(ctrl_info);  in pqi_unregister_scsi()
6646 	shost = ctrl_info->scsi_host;  in pqi_unregister_scsi()
6654 static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_reset_completion()  argument
6662 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_reset_completion()
6671 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_pqi_reset_completion()
6672 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_pqi_reset_completion()
6685 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_reset()  argument
6690 	if (ctrl_info->pqi_reset_quiesce_supported) {  in pqi_reset()
6691 		rc = sis_pqi_reset_quiesce(ctrl_info);  in pqi_reset()
6693 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
6704 	writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);  in pqi_reset()
6706 	rc = pqi_wait_for_pqi_reset_completion(ctrl_info);  in pqi_reset()
6708 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
6714 static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_serial_number()  argument
6723 	rc = pqi_sense_subsystem_info(ctrl_info, sense_info);  in pqi_get_ctrl_serial_number()
6727 	memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,  in pqi_get_ctrl_serial_number()
6729 	ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';  in pqi_get_ctrl_serial_number()
6737 static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_product_details()  argument
6746 	rc = pqi_identify_controller(ctrl_info, identify);  in pqi_get_ctrl_product_details()
6750 	memcpy(ctrl_info->firmware_version, identify->firmware_version,  in pqi_get_ctrl_product_details()
6752 	ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';  in pqi_get_ctrl_product_details()
6753 	snprintf(ctrl_info->firmware_version +  in pqi_get_ctrl_product_details()
6754 		strlen(ctrl_info->firmware_version),  in pqi_get_ctrl_product_details()
6755 		sizeof(ctrl_info->firmware_version),  in pqi_get_ctrl_product_details()
6758 	memcpy(ctrl_info->model, identify->product_id,  in pqi_get_ctrl_product_details()
6760 	ctrl_info->model[sizeof(identify->product_id)] = '\0';  in pqi_get_ctrl_product_details()
6762 	memcpy(ctrl_info->vendor, identify->vendor_id,  in pqi_get_ctrl_product_details()
6764 	ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';  in pqi_get_ctrl_product_details()
6773 	struct pqi_ctrl_info *ctrl_info;  member
6826 static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,  in pqi_config_table_update()  argument
6843 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_config_table_update()
6847 static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,  in pqi_enable_firmware_features()  argument
6863 	return pqi_config_table_update(ctrl_info,  in pqi_enable_firmware_features()
6873 	void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
6877 static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_status()  argument
6881 		dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",  in pqi_firmware_feature_status()
6887 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_firmware_feature_status()
6892 	dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",  in pqi_firmware_feature_status()
6896 static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,  in pqi_ctrl_update_feature_flags()  argument
6901 		ctrl_info->soft_reset_handshake_supported =  in pqi_ctrl_update_feature_flags()
6905 		ctrl_info->raid_iu_timeout_supported =  in pqi_ctrl_update_feature_flags()
6909 		ctrl_info->tmf_iu_timeout_supported =  in pqi_ctrl_update_feature_flags()
6914 	pqi_firmware_feature_status(ctrl_info, firmware_feature);  in pqi_ctrl_update_feature_flags()
6917 static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_update()  argument
6921 		firmware_feature->feature_status(ctrl_info, firmware_feature);  in pqi_firmware_feature_update()
6958 	struct pqi_ctrl_info *ctrl_info;  in pqi_process_firmware_features()  local
6964 	ctrl_info = section_info->ctrl_info;  in pqi_process_firmware_features()
6975 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
6990 	rc = pqi_enable_firmware_features(ctrl_info, firmware_features,  in pqi_process_firmware_features()
6993 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_firmware_features()
6998 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
7012 		pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
7036 static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)  in pqi_process_config_table()  argument
7045 	table_length = ctrl_info->config_table_length;  in pqi_process_config_table()
7051 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
7060 	table_iomem_addr = ctrl_info->iomem_base +  in pqi_process_config_table()
7061 		ctrl_info->config_table_offset;  in pqi_process_config_table()
7064 	section_info.ctrl_info = ctrl_info;  in pqi_process_config_table()
7082 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
7085 				ctrl_info->heartbeat_counter =  in pqi_process_config_table()
7093 			ctrl_info->soft_reset_status =  in pqi_process_config_table()
7112 static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_revert_to_sis_mode()  argument
7116 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);  in pqi_revert_to_sis_mode()
7117 	rc = pqi_reset(ctrl_info);  in pqi_revert_to_sis_mode()
7120 	rc = sis_reenable_sis_mode(ctrl_info);  in pqi_revert_to_sis_mode()
7122 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_revert_to_sis_mode()
7126 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_revert_to_sis_mode()
7136 static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_force_sis_mode()  argument
7138 	if (!sis_is_firmware_running(ctrl_info))  in pqi_force_sis_mode()
7141 	if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)  in pqi_force_sis_mode()
7144 	if (sis_is_kernel_up(ctrl_info)) {  in pqi_force_sis_mode()
7145 		pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_force_sis_mode()
7149 	return pqi_revert_to_sis_mode(ctrl_info);  in pqi_force_sis_mode()
7154 static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init()  argument
7159 		sis_soft_reset(ctrl_info);  in pqi_ctrl_init()
7162 		rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init()
7171 	rc = sis_wait_for_ctrl_ready(ctrl_info);  in pqi_ctrl_init()
7179 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init()
7181 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7186 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init()
7188 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7194 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
7196 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
7199 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
7201 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
7205 	pqi_calculate_io_resources(ctrl_info);  in pqi_ctrl_init()
7207 	rc = pqi_alloc_error_buffer(ctrl_info);  in pqi_ctrl_init()
7209 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7219 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init()
7221 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7227 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init()
7229 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7235 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init()
7236 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init()
7238 	rc = pqi_alloc_admin_queues(ctrl_info);  in pqi_ctrl_init()
7240 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7245 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init()
7247 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7252 	rc = pqi_report_device_capability(ctrl_info);  in pqi_ctrl_init()
7254 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7259 	rc = pqi_validate_device_capability(ctrl_info);  in pqi_ctrl_init()
7263 	pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
7265 	rc = pqi_enable_msix_interrupts(ctrl_info);  in pqi_ctrl_init()
7269 	if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {  in pqi_ctrl_init()
7270 		ctrl_info->max_msix_vectors =  in pqi_ctrl_init()
7271 			ctrl_info->num_msix_vectors_enabled;  in pqi_ctrl_init()
7272 		pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
7275 	rc = pqi_alloc_io_resources(ctrl_info);  in pqi_ctrl_init()
7279 	rc = pqi_alloc_operational_queues(ctrl_info);  in pqi_ctrl_init()
7281 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7286 	pqi_init_operational_queues(ctrl_info);  in pqi_ctrl_init()
7288 	rc = pqi_request_irqs(ctrl_info);  in pqi_ctrl_init()
7292 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init()
7296 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init()
7298 	ctrl_info->controller_online = true;  in pqi_ctrl_init()
7300 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init()
7304 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init()
7306 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init()
7308 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7314 	rc = pqi_register_scsi(ctrl_info);  in pqi_ctrl_init()
7318 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init()
7320 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7325 	rc = pqi_get_ctrl_serial_number(ctrl_info);  in pqi_ctrl_init()
7327 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7332 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init()
7334 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7339 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init()
7341 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7346 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init()
7348 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init()
7353 static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_reinit_queues()  argument
7359 	admin_queues = &ctrl_info->admin_queues;  in pqi_reinit_queues()
7364 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_reinit_queues()
7365 		ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;  in pqi_reinit_queues()
7366 		ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;  in pqi_reinit_queues()
7367 		ctrl_info->queue_groups[i].oq_ci_copy = 0;  in pqi_reinit_queues()
7369 		writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);  in pqi_reinit_queues()
7370 		writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);  in pqi_reinit_queues()
7371 		writel(0, ctrl_info->queue_groups[i].oq_pi);  in pqi_reinit_queues()
7374 	event_queue = &ctrl_info->event_queue;  in pqi_reinit_queues()
7379 static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init_resume()  argument
7383 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init_resume()
7391 	rc = sis_wait_for_ctrl_ready_resume(ctrl_info);  in pqi_ctrl_init_resume()
7399 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init_resume()
7401 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7406 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init_resume()
7408 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7418 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init_resume()
7420 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7426 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init_resume()
7428 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7434 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init_resume()
7435 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init_resume()
7437 	pqi_reinit_queues(ctrl_info);  in pqi_ctrl_init_resume()
7439 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init_resume()
7441 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7446 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init_resume()
7450 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init_resume()
7452 	ctrl_info->controller_online = true;  in pqi_ctrl_init_resume()
7453 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ctrl_init_resume()
7455 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init_resume()
7459 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init_resume()
7461 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init_resume()
7463 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7468 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init_resume()
7470 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7475 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init_resume()
7477 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7482 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init_resume()
7484 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7489 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init_resume()
7491 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init_resume()
7507 static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_pci_init()  argument
7512 	rc = pci_enable_device(ctrl_info->pci_dev);  in pqi_pci_init()
7514 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7524 	rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);  in pqi_pci_init()
7526 		dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");  in pqi_pci_init()
7530 	rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);  in pqi_pci_init()
7532 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7537 	ctrl_info->iomem_base = ioremap(pci_resource_start(  in pqi_pci_init()
7538 		ctrl_info->pci_dev, 0),  in pqi_pci_init()
7540 	if (!ctrl_info->iomem_base) {  in pqi_pci_init()
7541 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7550 	rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,  in pqi_pci_init()
7553 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7559 	pci_set_master(ctrl_info->pci_dev);  in pqi_pci_init()
7561 	ctrl_info->registers = ctrl_info->iomem_base;  in pqi_pci_init()
7562 	ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;  in pqi_pci_init()
7564 	pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);  in pqi_pci_init()
7569 	pci_release_regions(ctrl_info->pci_dev);  in pqi_pci_init()
7571 	pci_disable_device(ctrl_info->pci_dev);  in pqi_pci_init()
7576 static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_cleanup_pci_init()  argument
7578 	iounmap(ctrl_info->iomem_base);  in pqi_cleanup_pci_init()
7579 	pci_release_regions(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
7580 	if (pci_is_enabled(ctrl_info->pci_dev))  in pqi_cleanup_pci_init()
7581 		pci_disable_device(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
7582 	pci_set_drvdata(ctrl_info->pci_dev, NULL);  in pqi_cleanup_pci_init()
7587 	struct pqi_ctrl_info *ctrl_info;  in pqi_alloc_ctrl_info()  local
7589 	ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),  in pqi_alloc_ctrl_info()
7591 	if (!ctrl_info)  in pqi_alloc_ctrl_info()
7594 	mutex_init(&ctrl_info->scan_mutex);  in pqi_alloc_ctrl_info()
7595 	mutex_init(&ctrl_info->lun_reset_mutex);  in pqi_alloc_ctrl_info()
7596 	mutex_init(&ctrl_info->ofa_mutex);  in pqi_alloc_ctrl_info()
7598 	INIT_LIST_HEAD(&ctrl_info->scsi_device_list);  in pqi_alloc_ctrl_info()
7599 	spin_lock_init(&ctrl_info->scsi_device_list_lock);  in pqi_alloc_ctrl_info()
7601 	INIT_WORK(&ctrl_info->event_work, pqi_event_worker);  in pqi_alloc_ctrl_info()
7602 	atomic_set(&ctrl_info->num_interrupts, 0);  in pqi_alloc_ctrl_info()
7603 	atomic_set(&ctrl_info->sync_cmds_outstanding, 0);  in pqi_alloc_ctrl_info()
7605 	INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);  in pqi_alloc_ctrl_info()
7606 	INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);  in pqi_alloc_ctrl_info()
7608 	timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);  in pqi_alloc_ctrl_info()
7609 	INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);  in pqi_alloc_ctrl_info()
7611 	sema_init(&ctrl_info->sync_request_sem,  in pqi_alloc_ctrl_info()
7613 	init_waitqueue_head(&ctrl_info->block_requests_wait);  in pqi_alloc_ctrl_info()
7615 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_alloc_ctrl_info()
7616 	spin_lock_init(&ctrl_info->raid_bypass_retry_list_lock);  in pqi_alloc_ctrl_info()
7617 	INIT_WORK(&ctrl_info->raid_bypass_retry_work,  in pqi_alloc_ctrl_info()
7620 	ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;  in pqi_alloc_ctrl_info()
7621 	ctrl_info->irq_mode = IRQ_MODE_NONE;  in pqi_alloc_ctrl_info()
7622 	ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;  in pqi_alloc_ctrl_info()
7624 	return ctrl_info;  in pqi_alloc_ctrl_info()
7627 static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_info()  argument
7629 	kfree(ctrl_info);  in pqi_free_ctrl_info()
7632 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_free_interrupts()  argument
7634 	pqi_free_irqs(ctrl_info);  in pqi_free_interrupts()
7635 	pqi_disable_msix_interrupts(ctrl_info);  in pqi_free_interrupts()
7638 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_resources()  argument
7640 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_free_ctrl_resources()
7641 	pqi_free_interrupts(ctrl_info);  in pqi_free_ctrl_resources()
7642 	if (ctrl_info->queue_memory_base)  in pqi_free_ctrl_resources()
7643 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7644 			ctrl_info->queue_memory_length,  in pqi_free_ctrl_resources()
7645 			ctrl_info->queue_memory_base,  in pqi_free_ctrl_resources()
7646 			ctrl_info->queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
7647 	if (ctrl_info->admin_queue_memory_base)  in pqi_free_ctrl_resources()
7648 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7649 			ctrl_info->admin_queue_memory_length,  in pqi_free_ctrl_resources()
7650 			ctrl_info->admin_queue_memory_base,  in pqi_free_ctrl_resources()
7651 			ctrl_info->admin_queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
7652 	pqi_free_all_io_requests(ctrl_info);  in pqi_free_ctrl_resources()
7653 	if (ctrl_info->error_buffer)  in pqi_free_ctrl_resources()
7654 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7655 			ctrl_info->error_buffer_length,  in pqi_free_ctrl_resources()
7656 			ctrl_info->error_buffer,  in pqi_free_ctrl_resources()
7657 			ctrl_info->error_buffer_dma_handle);  in pqi_free_ctrl_resources()
7658 	if (ctrl_info->iomem_base)  in pqi_free_ctrl_resources()
7659 		pqi_cleanup_pci_init(ctrl_info);  in pqi_free_ctrl_resources()
7660 	pqi_free_ctrl_info(ctrl_info);  in pqi_free_ctrl_resources()
7663 static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_ctrl()  argument
7665 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_remove_ctrl()
7666 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_remove_ctrl()
7667 	pqi_unregister_scsi(ctrl_info);  in pqi_remove_ctrl()
7668 	if (ctrl_info->pqi_mode_enabled)  in pqi_remove_ctrl()
7669 		pqi_revert_to_sis_mode(ctrl_info);  in pqi_remove_ctrl()
7670 	pqi_free_ctrl_resources(ctrl_info);  in pqi_remove_ctrl()
7673 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_quiesce()  argument
7675 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7676 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7677 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7678 	pqi_wait_until_scan_finished(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7679 	pqi_ctrl_ofa_start(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7680 	pqi_ctrl_block_requests(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7681 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7682 	pqi_ctrl_wait_for_pending_io(ctrl_info, PQI_PENDING_IO_TIMEOUT_SECS);  in pqi_ofa_ctrl_quiesce()
7683 	pqi_fail_io_queued_for_all_devices(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7684 	pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7685 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7686 	ctrl_info->pqi_mode_enabled = false;  in pqi_ofa_ctrl_quiesce()
7687 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_ofa_ctrl_quiesce()
7690 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_unquiesce()  argument
7692 	pqi_ofa_free_host_buffer(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7693 	ctrl_info->pqi_mode_enabled = true;  in pqi_ofa_ctrl_unquiesce()
7694 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ofa_ctrl_unquiesce()
7695 	ctrl_info->controller_online = true;  in pqi_ofa_ctrl_unquiesce()
7696 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7697 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7698 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7699 	pqi_clear_soft_reset_status(ctrl_info,  in pqi_ofa_ctrl_unquiesce()
7701 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7704 static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_alloc_mem()  argument
7714 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_alloc_mem()
7719 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_alloc_mem()
7724 	ctrl_info->pqi_ofa_chunk_virt_addr =  in pqi_ofa_alloc_mem()
7726 	if (!ctrl_info->pqi_ofa_chunk_virt_addr)  in pqi_ofa_alloc_mem()
7732 		ctrl_info->pqi_ofa_chunk_virt_addr[i] =  in pqi_ofa_alloc_mem()
7736 		if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])  in pqi_ofa_alloc_mem()
7757 				ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_alloc_mem()
7760 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_alloc_mem()
7767 static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_alloc_host_buffer()  argument
7774 			ctrl_info->pqi_ofa_mem_virt_addr->bytes_allocated);  in pqi_ofa_alloc_host_buffer()
7778 		if (!pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_sz))  in pqi_ofa_alloc_host_buffer()
7784 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_setup_host_buffer()  argument
7790 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_setup_host_buffer()
7793 					    &ctrl_info->pqi_ofa_mem_dma_handle,  in pqi_ofa_setup_host_buffer()
7804 	ctrl_info->pqi_ofa_mem_virt_addr = pqi_ofa_memory;  in pqi_ofa_setup_host_buffer()
7806 	if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {  in pqi_ofa_setup_host_buffer()
7814 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_free_host_buffer()  argument
7820 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_free_host_buffer()
7832 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_ofa_free_host_buffer()
7834 			ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_free_host_buffer()
7837 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_free_host_buffer()
7840 	dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_ofa_free_host_buffer()
7842 			ctrl_info->pqi_ofa_mem_dma_handle);  in pqi_ofa_free_host_buffer()
7843 	ctrl_info->pqi_ofa_mem_virt_addr = NULL;  in pqi_ofa_free_host_buffer()
7846 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_host_memory_update()  argument
7854 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_host_memory_update()
7867 		put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,  in pqi_ofa_host_memory_update()
7874 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_ofa_host_memory_update()
7878 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_restart()  argument
7881 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_ofa_ctrl_restart()
7904 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_all_outstanding_requests()  argument
7910 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_fail_all_outstanding_requests()
7911 		io_request = &ctrl_info->io_request_pool[i];  in pqi_fail_all_outstanding_requests()
7929 static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline_deferred()  argument
7932 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7933 	pqi_free_interrupts(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7934 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7935 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7936 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7937 	pqi_fail_all_outstanding_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7938 	pqi_clear_all_queued_raid_bypass_retries(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7939 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7944 	struct pqi_ctrl_info *ctrl_info;  in pqi_ctrl_offline_worker()  local
7946 	ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);  in pqi_ctrl_offline_worker()
7947 	pqi_take_ctrl_offline_deferred(ctrl_info);  in pqi_ctrl_offline_worker()
7950 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline()  argument
7952 	if (!ctrl_info->controller_online)  in pqi_take_ctrl_offline()
7955 	ctrl_info->controller_online = false;  in pqi_take_ctrl_offline()
7956 	ctrl_info->pqi_mode_enabled = false;  in pqi_take_ctrl_offline()
7957 	pqi_ctrl_block_requests(ctrl_info);  in pqi_take_ctrl_offline()
7959 		sis_shutdown_ctrl(ctrl_info);  in pqi_take_ctrl_offline()
7960 	pci_disable_device(ctrl_info->pci_dev);  in pqi_take_ctrl_offline()
7961 	dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");  in pqi_take_ctrl_offline()
7962 	schedule_work(&ctrl_info->ctrl_offline_work);  in pqi_take_ctrl_offline()
7983 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_probe()  local
8007 	ctrl_info = pqi_alloc_ctrl_info(node);  in pqi_pci_probe()
8008 	if (!ctrl_info) {  in pqi_pci_probe()
8014 	ctrl_info->pci_dev = pci_dev;  in pqi_pci_probe()
8016 	rc = pqi_pci_init(ctrl_info);  in pqi_pci_probe()
8020 	rc = pqi_ctrl_init(ctrl_info);  in pqi_pci_probe()
8027 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_probe()
8034 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_remove()  local
8036 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_pci_remove()
8037 	if (!ctrl_info)  in pqi_pci_remove()
8040 	ctrl_info->in_shutdown = true;  in pqi_pci_remove()
8042 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_remove()
8045 static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)  in pqi_crash_if_pending_command()  argument
8051 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_crash_if_pending_command()
8052 		io_request = &ctrl_info->io_request_pool[i];  in pqi_crash_if_pending_command()
8064 	struct pqi_ctrl_info *ctrl_info;  in pqi_shutdown()  local
8066 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_shutdown()
8067 	if (!ctrl_info) {  in pqi_shutdown()
8073 	pqi_disable_events(ctrl_info);  in pqi_shutdown()
8074 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_shutdown()
8075 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_shutdown()
8076 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_shutdown()
8077 	pqi_cancel_event_worker(ctrl_info);  in pqi_shutdown()
8079 	pqi_ctrl_shutdown_start(ctrl_info);  in pqi_shutdown()
8080 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_shutdown()
8082 	rc = pqi_ctrl_wait_for_pending_io(ctrl_info, NO_TIMEOUT);  in pqi_shutdown()
8089 	pqi_ctrl_block_device_reset(ctrl_info);  in pqi_shutdown()
8090 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_shutdown()
8096 	rc = pqi_flush_cache(ctrl_info, SHUTDOWN);  in pqi_shutdown()
8101 	pqi_ctrl_block_requests(ctrl_info);  in pqi_shutdown()
8103 	rc = pqi_ctrl_wait_for_pending_sync_cmds(ctrl_info);  in pqi_shutdown()
8110 	pqi_crash_if_pending_command(ctrl_info);  in pqi_shutdown()
8111 	pqi_reset(ctrl_info);  in pqi_shutdown()
8140 	struct pqi_ctrl_info *ctrl_info;  in pqi_suspend()  local
8142 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_suspend()
8144 	pqi_disable_events(ctrl_info);  in pqi_suspend()
8145 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_suspend()
8146 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_suspend()
8147 	pqi_wait_until_scan_finished(ctrl_info);  in pqi_suspend()
8148 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_suspend()
8149 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_suspend()
8150 	pqi_flush_cache(ctrl_info, SUSPEND);  in pqi_suspend()
8151 	pqi_ctrl_block_requests(ctrl_info);  in pqi_suspend()
8152 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_suspend()
8153 	pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_suspend()
8154 	pqi_ctrl_wait_for_pending_io(ctrl_info, NO_TIMEOUT);  in pqi_suspend()
8155 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_suspend()
8163 	ctrl_info->controller_online = false;  in pqi_suspend()
8164 	ctrl_info->pqi_mode_enabled = false;  in pqi_suspend()
8172 	struct pqi_ctrl_info *ctrl_info;  in pqi_resume()  local
8174 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_resume()
8177 		ctrl_info->max_hw_queue_index = 0;  in pqi_resume()
8178 		pqi_free_interrupts(ctrl_info);  in pqi_resume()
8179 		pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);  in pqi_resume()
8182 			&ctrl_info->queue_groups[0]);  in pqi_resume()
8184 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_resume()
8189 		pqi_start_heartbeat_timer(ctrl_info);  in pqi_resume()
8190 		pqi_ctrl_unblock_requests(ctrl_info);  in pqi_resume()
8197 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_resume()