1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * Copyright (C) 2017-2023 Broadcom Inc.
6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
11 #include <linux/io-64-nonatomic-lo-hi.h>
23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
46 	pi = op_req_q->pi;  in mpi3mr_check_req_qfull()
47 	ci = READ_ONCE(op_req_q->ci);  in mpi3mr_check_req_qfull()
48 	max_entries = op_req_q->num_requests;  in mpi3mr_check_req_qfull()
50 	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))  in mpi3mr_check_req_qfull()
60 	max_vectors = mrioc->intr_info_count;  in mpi3mr_sync_irqs()
63 		synchronize_irq(pci_irq_vector(mrioc->pdev, i));  in mpi3mr_sync_irqs()
68 	mrioc->intr_enabled = 0;  in mpi3mr_ioc_disable_intr()
74 	mrioc->intr_enabled = 1;  in mpi3mr_ioc_enable_intr()
83 	if (!mrioc->intr_info)  in mpi3mr_cleanup_isr()
86 	for (i = 0; i < mrioc->intr_info_count; i++)  in mpi3mr_cleanup_isr()
87 		free_irq(pci_irq_vector(mrioc->pdev, i),  in mpi3mr_cleanup_isr()
88 		    (mrioc->intr_info + i));  in mpi3mr_cleanup_isr()
90 	kfree(mrioc->intr_info);  in mpi3mr_cleanup_isr()
91 	mrioc->intr_info = NULL;  in mpi3mr_cleanup_isr()
92 	mrioc->intr_info_count = 0;  in mpi3mr_cleanup_isr()
93 	mrioc->is_intr_info_set = false;  in mpi3mr_cleanup_isr()
94 	pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_cleanup_isr()
102 	sgel->flags = flags;  in mpi3mr_add_sg_single()
103 	sgel->length = cpu_to_le32(length);  in mpi3mr_add_sg_single()
104 	sgel->address = cpu_to_le64(dma_addr);  in mpi3mr_add_sg_single()
111 	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);  in mpi3mr_build_zero_len_sge()
120 	if ((phys_addr < mrioc->reply_buf_dma) ||  in mpi3mr_get_reply_virt_addr()
121 	    (phys_addr > mrioc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
124 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);  in mpi3mr_get_reply_virt_addr()
133 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);  in mpi3mr_get_sensebuf_virt_addr()
142 	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
143 	old_idx  =  mrioc->reply_free_queue_host_index;  in mpi3mr_repost_reply_buf()
144 	mrioc->reply_free_queue_host_index = (  in mpi3mr_repost_reply_buf()
145 	    (mrioc->reply_free_queue_host_index ==  in mpi3mr_repost_reply_buf()
146 	    (mrioc->reply_free_qsz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
147 	    (mrioc->reply_free_queue_host_index + 1));  in mpi3mr_repost_reply_buf()
148 	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);  in mpi3mr_repost_reply_buf()
149 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_repost_reply_buf()
150 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_repost_reply_buf()
151 	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
160 	spin_lock_irqsave(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
161 	old_idx  =  mrioc->sbq_host_index;  in mpi3mr_repost_sense_buf()
162 	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==  in mpi3mr_repost_sense_buf()
163 	    (mrioc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
164 	    (mrioc->sbq_host_index + 1));  in mpi3mr_repost_sense_buf()
165 	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);  in mpi3mr_repost_sense_buf()
166 	writel(mrioc->sbq_host_index,  in mpi3mr_repost_sense_buf()
167 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_repost_sense_buf()
168 	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
177 	event = event_reply->event;  in mpi3mr_print_event_data()
198 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
200 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
206 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
208 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
214 		    (struct mpi3_event_data_device_status_change *)event_reply->event_data;  in mpi3mr_print_event_data()
216 		    event_data->dev_handle, event_data->reason_code);  in mpi3mr_print_event_data()
222 		    (struct mpi3_event_data_sas_discovery *)event_reply->event_data;  in mpi3mr_print_event_data()
224 		    (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?  in mpi3mr_print_event_data()
226 		    le32_to_cpu(event_data->discovery_status));  in mpi3mr_print_event_data()
265 		    (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;  in mpi3mr_print_event_data()
267 		    (event_data->reason_code ==  in mpi3mr_print_event_data()
269 		if (event_data->enumeration_status)  in mpi3mr_print_event_data()
271 			    le32_to_cpu(event_data->enumeration_status));  in mpi3mr_print_event_data()
291 	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);  in mpi3mr_handle_events()
304 		return &mrioc->init_cmds;  in mpi3mr_get_drv_cmd()
306 		return &mrioc->cfg_cmds;  in mpi3mr_get_drv_cmd()
308 		return &mrioc->bsg_cmds;  in mpi3mr_get_drv_cmd()
310 		return &mrioc->host_tm_cmds;  in mpi3mr_get_drv_cmd()
312 		return &mrioc->pel_abort_cmd;  in mpi3mr_get_drv_cmd()
314 		return &mrioc->pel_cmds;  in mpi3mr_get_drv_cmd()
316 		return &mrioc->transport_cmds;  in mpi3mr_get_drv_cmd()
318 		if (def_reply && def_reply->function ==  in mpi3mr_get_drv_cmd()
327 		idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;  in mpi3mr_get_drv_cmd()
328 		return &mrioc->dev_rmhs_cmds[idx];  in mpi3mr_get_drv_cmd()
333 		idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;  in mpi3mr_get_drv_cmd()
334 		return &mrioc->evtack_cmds[idx];  in mpi3mr_get_drv_cmd()
355 	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_desc()
360 		host_tag = le16_to_cpu(status_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
361 		ioc_status = le16_to_cpu(status_desc->ioc_status);  in mpi3mr_process_admin_reply_desc()
364 			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
369 		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);  in mpi3mr_process_admin_reply_desc()
373 		host_tag = le16_to_cpu(def_reply->host_tag);  in mpi3mr_process_admin_reply_desc()
374 		ioc_status = le16_to_cpu(def_reply->ioc_status);  in mpi3mr_process_admin_reply_desc()
377 			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
379 		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {  in mpi3mr_process_admin_reply_desc()
382 			    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
387 		host_tag = le16_to_cpu(success_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
395 		if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_admin_reply_desc()
396 			cmdptr->state |= MPI3MR_CMD_COMPLETE;  in mpi3mr_process_admin_reply_desc()
397 			cmdptr->ioc_loginfo = ioc_loginfo;  in mpi3mr_process_admin_reply_desc()
398 			cmdptr->ioc_status = ioc_status;  in mpi3mr_process_admin_reply_desc()
399 			cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_process_admin_reply_desc()
401 				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;  in mpi3mr_process_admin_reply_desc()
402 				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,  in mpi3mr_process_admin_reply_desc()
403 				    mrioc->reply_sz);  in mpi3mr_process_admin_reply_desc()
405 			if (sense_buf && cmdptr->sensebuf) {  in mpi3mr_process_admin_reply_desc()
406 				cmdptr->is_sense = 1;  in mpi3mr_process_admin_reply_desc()
407 				memcpy(cmdptr->sensebuf, sense_buf,  in mpi3mr_process_admin_reply_desc()
410 			if (cmdptr->is_waiting) {  in mpi3mr_process_admin_reply_desc()
411 				complete(&cmdptr->done);  in mpi3mr_process_admin_reply_desc()
412 				cmdptr->is_waiting = 0;  in mpi3mr_process_admin_reply_desc()
413 			} else if (cmdptr->callback)  in mpi3mr_process_admin_reply_desc()
414 				cmdptr->callback(mrioc, cmdptr);  in mpi3mr_process_admin_reply_desc()
420 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
425 	u32 exp_phase = mrioc->admin_reply_ephase;  in mpi3mr_process_admin_reply_q()
426 	u32 admin_reply_ci = mrioc->admin_reply_ci;  in mpi3mr_process_admin_reply_q()
431 	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))  in mpi3mr_process_admin_reply_q()
434 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
437 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
439 		atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
444 		if (mrioc->unrecoverable)  in mpi3mr_process_admin_reply_q()
447 		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);  in mpi3mr_process_admin_reply_q()
452 		if (++admin_reply_ci == mrioc->num_admin_replies) {  in mpi3mr_process_admin_reply_q()
457 		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
459 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
464 	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
465 	mrioc->admin_reply_ci = admin_reply_ci;  in mpi3mr_process_admin_reply_q()
466 	mrioc->admin_reply_ephase = exp_phase;  in mpi3mr_process_admin_reply_q()
467 	atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
473  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
474  *	queue's consumer index from operational reply descriptor queue.
476  * @reply_ci: operational reply descriptor's queue consumer index
484 	struct segments *segments = op_reply_q->q_segments;  in mpi3mr_get_reply_desc()
488 	    segments[reply_ci / op_reply_q->segment_qd].segment;  in mpi3mr_get_reply_desc()
490 	    (reply_ci % op_reply_q->segment_qd);  in mpi3mr_get_reply_desc()
495  * mpi3mr_process_op_reply_q - Operational reply queue handler
497  * @op_reply_q: Operational reply queue info
499  * Checks the specific operational reply queue and drains the
517 	reply_qidx = op_reply_q->qid - 1;  in mpi3mr_process_op_reply_q()
519 	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))  in mpi3mr_process_op_reply_q()
522 	exp_phase = op_reply_q->ephase;  in mpi3mr_process_op_reply_q()
523 	reply_ci = op_reply_q->ci;  in mpi3mr_process_op_reply_q()
526 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
528 		atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
533 		if (mrioc->unrecoverable)  in mpi3mr_process_op_reply_q()
536 		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;  in mpi3mr_process_op_reply_q()
537 		op_req_q = &mrioc->req_qinfo[req_q_idx];  in mpi3mr_process_op_reply_q()
539 		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));  in mpi3mr_process_op_reply_q()
542 		atomic_dec(&op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
547 		if (++reply_ci == op_reply_q->num_replies) {  in mpi3mr_process_op_reply_q()
554 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
562 		if (num_op_reply > mrioc->max_host_ios) {  in mpi3mr_process_op_reply_q()
563 			op_reply_q->enable_irq_poll = true;  in mpi3mr_process_op_reply_q()
570 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
571 	op_reply_q->ci = reply_ci;  in mpi3mr_process_op_reply_q()
572 	op_reply_q->ephase = exp_phase;  in mpi3mr_process_op_reply_q()
574 	atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
579  * mpi3mr_blk_mq_poll - Operational reply queue handler
583  * Checks the specific operational reply queue and drains the
595 	mrioc = (struct mpi3mr_ioc *)shost->hostdata;  in mpi3mr_blk_mq_poll()
597 	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||  in mpi3mr_blk_mq_poll()
598 	    mrioc->unrecoverable))  in mpi3mr_blk_mq_poll()
602 			&mrioc->op_reply_qinfo[queue_num]);  in mpi3mr_blk_mq_poll()
617 	mrioc = intr_info->mrioc;  in mpi3mr_isr_primary()
619 	if (!mrioc->intr_enabled)  in mpi3mr_isr_primary()
622 	midx = intr_info->msix_index;  in mpi3mr_isr_primary()
626 	if (intr_info->op_reply_q)  in mpi3mr_isr_primary()
628 		    intr_info->op_reply_q);  in mpi3mr_isr_primary()
653 	if (!intr_info->op_reply_q)  in mpi3mr_isr()
656 	if (!intr_info->op_reply_q->enable_irq_poll ||  in mpi3mr_isr()
657 	    !atomic_read(&intr_info->op_reply_q->pend_ios))  in mpi3mr_isr()
660 	disable_irq_nosync(intr_info->os_irq);  in mpi3mr_isr()
666  * mpi3mr_isr_poll - Reply queue polling routine
682 	if (!intr_info || !intr_info->op_reply_q)  in mpi3mr_isr_poll()
685 	mrioc = intr_info->mrioc;  in mpi3mr_isr_poll()
686 	midx = intr_info->msix_index;  in mpi3mr_isr_poll()
690 		if (!mrioc->intr_enabled || mrioc->unrecoverable)  in mpi3mr_isr_poll()
695 		if (intr_info->op_reply_q)  in mpi3mr_isr_poll()
698 				intr_info->op_reply_q);  in mpi3mr_isr_poll()
702 	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&  in mpi3mr_isr_poll()
703 	    (num_op_reply < mrioc->max_host_ios));  in mpi3mr_isr_poll()
705 	intr_info->op_reply_q->enable_irq_poll = false;  in mpi3mr_isr_poll()
706 	enable_irq(intr_info->os_irq);  in mpi3mr_isr_poll()
714  * mpi3mr_request_irq - Request IRQ and register ISR
720  * Return: 0 on success and non zero on failures.
724 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_request_irq()
725 	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;  in mpi3mr_request_irq()
728 	intr_info->mrioc = mrioc;  in mpi3mr_request_irq()
729 	intr_info->msix_index = index;  in mpi3mr_request_irq()
730 	intr_info->op_reply_q = NULL;  in mpi3mr_request_irq()
732 	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",  in mpi3mr_request_irq()
733 	    mrioc->driver_name, mrioc->id, index);  in mpi3mr_request_irq()
737 	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
740 	    NULL, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
744 		    intr_info->name, pci_irq_vector(pdev, index));  in mpi3mr_request_irq()
748 	intr_info->os_irq = pci_irq_vector(pdev, index);  in mpi3mr_request_irq()
754 	if (!mrioc->requested_poll_qcount)  in mpi3mr_calc_poll_queues()
759 		(mrioc->requested_poll_qcount < max_vectors - 2)) {  in mpi3mr_calc_poll_queues()
762 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
766 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
767 		mrioc->requested_poll_qcount = 0;  in mpi3mr_calc_poll_queues()
772  * mpi3mr_setup_isr - Setup ISR for the controller
778  * Return: 0 on success and non zero on failures.
788 	if (mrioc->is_intr_info_set)  in mpi3mr_setup_isr()
795 		retval = pci_alloc_irq_vectors(mrioc->pdev,  in mpi3mr_setup_isr()
804 		    min_t(int, mrioc->cpu_count + 1 +  in mpi3mr_setup_isr()
805 			mrioc->requested_poll_qcount, mrioc->msix_count);  in mpi3mr_setup_isr()
810 		    "MSI-X vectors supported: %d, no of cores: %d,",  in mpi3mr_setup_isr()
811 		    mrioc->msix_count, mrioc->cpu_count);  in mpi3mr_setup_isr()
813 		    "MSI-x vectors requested: %d poll_queues %d\n",  in mpi3mr_setup_isr()
814 		    max_vectors, mrioc->requested_poll_qcount);  in mpi3mr_setup_isr()
816 		desc.post_vectors = mrioc->requested_poll_qcount;  in mpi3mr_setup_isr()
820 		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,  in mpi3mr_setup_isr()
831 		 * If only one MSI-x is allocated, then MSI-x 0 will be shared  in mpi3mr_setup_isr()
832 		 * between Admin queue and operational queue  in mpi3mr_setup_isr()
835 			mrioc->op_reply_q_offset = 0;  in mpi3mr_setup_isr()
843 		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;  in mpi3mr_setup_isr()
849 	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,  in mpi3mr_setup_isr()
851 	if (!mrioc->intr_info) {  in mpi3mr_setup_isr()
852 		retval = -ENOMEM;  in mpi3mr_setup_isr()
853 		pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_setup_isr()
859 			mrioc->intr_info_count = i;  in mpi3mr_setup_isr()
864 		mrioc->is_intr_info_set = true;  in mpi3mr_setup_isr()
865 	mrioc->intr_info_count = max_vectors;  in mpi3mr_setup_isr()
947  * mpi3mr_reset_rc_name - get reset reason code name
978  * mpi3mr_reset_type_name - get reset type name
1000  * mpi3mr_print_fault_info - Display fault information
1012 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_print_fault_info()
1015 		code = readl(&mrioc->sysif_regs->fault);  in mpi3mr_print_fault_info()
1016 		code1 = readl(&mrioc->sysif_regs->fault_info[0]);  in mpi3mr_print_fault_info()
1017 		code2 = readl(&mrioc->sysif_regs->fault_info[1]);  in mpi3mr_print_fault_info()
1018 		code3 = readl(&mrioc->sysif_regs->fault_info[2]);  in mpi3mr_print_fault_info()
1027  * mpi3mr_get_iocstate - Get IOC State
1040 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_get_iocstate()
1041 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_get_iocstate()
1043 	if (mrioc->unrecoverable)  in mpi3mr_get_iocstate()
1062  * mpi3mr_clear_reset_history - clear reset history
1074 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1076 		writel(ioc_status, &mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1080  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1087  * Return: 0 on success, -1 on failure.
1093 	int retval = -1;  in mpi3mr_issue_and_process_mur()
1096 	if (mrioc->unrecoverable) {  in mpi3mr_issue_and_process_mur()
1101 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_and_process_mur()
1102 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1104 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1108 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_and_process_mur()
1118 	} while (--timeout);  in mpi3mr_issue_and_process_mur()
1120 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1132  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1137  * older values else return -EPERM
1144 	if (mrioc->facts.reply_sz > mrioc->reply_sz) {  in mpi3mr_revalidate_factsdata()
1147 		    mrioc->reply_sz, mrioc->facts.reply_sz);  in mpi3mr_revalidate_factsdata()
1148 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1151 	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {  in mpi3mr_revalidate_factsdata()
1153 		    "cannot reduce number of operational reply queues from %d to %d\n",  in mpi3mr_revalidate_factsdata()
1154 		    mrioc->num_op_reply_q,  in mpi3mr_revalidate_factsdata()
1155 		    mrioc->facts.max_op_reply_q);  in mpi3mr_revalidate_factsdata()
1156 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1159 	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {  in mpi3mr_revalidate_factsdata()
1161 		    "cannot reduce number of operational request queues from %d to %d\n",  in mpi3mr_revalidate_factsdata()
1162 		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);  in mpi3mr_revalidate_factsdata()
1163 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1166 	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))  in mpi3mr_revalidate_factsdata()
1170 			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);  in mpi3mr_revalidate_factsdata()
1172 	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &  in mpi3mr_revalidate_factsdata()
1179 	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {  in mpi3mr_revalidate_factsdata()
1180 		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,  in mpi3mr_revalidate_factsdata()
1185 				mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1186 				mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1187 			return -EPERM;  in mpi3mr_revalidate_factsdata()
1189 		bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_revalidate_factsdata()
1190 		mrioc->removepend_bitmap = removepend_bitmap;  in mpi3mr_revalidate_factsdata()
1193 			 mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1194 			 mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1195 		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_revalidate_factsdata()
1202  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1217 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1218 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1219 	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);  in mpi3mr_bring_ioc_ready()
1224 	mrioc->ready_timeout =  in mpi3mr_bring_ioc_ready()
1228 	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1236 		timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1239 		} while (--timeout);  in mpi3mr_bring_ioc_ready()
1241 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1242 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1245 			retval = -1;  in mpi3mr_bring_ioc_ready()
1271 					readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_bring_ioc_ready()
1275 				if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1276 					mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1281 			} while (--timeout);  in mpi3mr_bring_ioc_ready()
1310 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1312 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1314 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1323 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1324 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1327 			retval = -1;  in mpi3mr_bring_ioc_ready()
1331 	} while (--timeout);  in mpi3mr_bring_ioc_ready()
1343  * mpi3mr_soft_reset_success - Check softreset is success or not
1362  * mpi3mr_diagfault_success - Check diag fault is success or not
1377 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_diagfault_success()
1386  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1398 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1400 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1404  * mpi3mr_issue_reset - Issue reset to the controller
1414  * Return: 0 on success, non-zero on failure.
1419 	int retval = -1;  in mpi3mr_issue_reset()
1427 	if (mrioc->unrecoverable)  in mpi3mr_issue_reset()
1448 			mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1453 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1455 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1457 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1459 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1461 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1463 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1465 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1467 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1473 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_reset()
1475 	    &mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1479 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1481 			    readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1490 		} while (--timeout);  in mpi3mr_issue_reset()
1495 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1501 		} while (--timeout);  in mpi3mr_issue_reset()
1508 	    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1510 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1511 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1517 		mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1522  * mpi3mr_admin_request_post - Post request to admin queue
1532  * Return: 0 on success, non-zero on failure.
1542 	if (mrioc->unrecoverable) {  in mpi3mr_admin_request_post()
1544 		return -EFAULT;  in mpi3mr_admin_request_post()
1547 	spin_lock_irqsave(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1548 	areq_pi = mrioc->admin_req_pi;  in mpi3mr_admin_request_post()
1549 	areq_ci = mrioc->admin_req_ci;  in mpi3mr_admin_request_post()
1550 	max_entries = mrioc->num_admin_req;  in mpi3mr_admin_request_post()
1552 	    (areq_pi == (max_entries - 1)))) {  in mpi3mr_admin_request_post()
1554 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1557 	if (!ignore_reset && mrioc->reset_in_progress) {  in mpi3mr_admin_request_post()
1559 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1562 	areq_entry = (u8 *)mrioc->admin_req_base +  in mpi3mr_admin_request_post()
1569 	mrioc->admin_req_pi = areq_pi;  in mpi3mr_admin_request_post()
1571 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_admin_request_post()
1574 	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1580  * mpi3mr_free_op_req_q_segments - free request memory segments
1582  * @q_idx: operational request queue index
1584  * Free memory segments allocated for operational request queue
1594 	segments = mrioc->req_qinfo[q_idx].q_segments;  in mpi3mr_free_op_req_q_segments()
1598 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_req_q_segments()
1600 		if (mrioc->req_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_req_q_segments()
1601 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1603 			    mrioc->req_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_req_q_segments()
1604 			    mrioc->req_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_req_q_segments()
1605 			mrioc->req_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_req_q_segments()
1608 		size = mrioc->req_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_req_q_segments()
1609 		    mrioc->facts.op_req_sz;  in mpi3mr_free_op_req_q_segments()
1611 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_req_q_segments()
1614 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1618 	kfree(mrioc->req_qinfo[q_idx].q_segments);  in mpi3mr_free_op_req_q_segments()
1619 	mrioc->req_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_req_q_segments()
1620 	mrioc->req_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_req_q_segments()
1624  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1626  * @q_idx: operational reply queue index
1628  * Free memory segments allocated for operational reply queue
1638 	segments = mrioc->op_reply_qinfo[q_idx].q_segments;  in mpi3mr_free_op_reply_q_segments()
1642 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_reply_q_segments()
1644 		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_reply_q_segments()
1645 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1647 			    mrioc->op_reply_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_reply_q_segments()
1648 			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_reply_q_segments()
1649 			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_reply_q_segments()
1652 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_reply_q_segments()
1653 		    mrioc->op_reply_desc_sz;  in mpi3mr_free_op_reply_q_segments()
1655 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_reply_q_segments()
1658 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1663 	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);  in mpi3mr_free_op_reply_q_segments()
1664 	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_reply_q_segments()
1665 	mrioc->op_reply_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_reply_q_segments()
1669  * mpi3mr_delete_op_reply_q - delete operational reply queue
1671  * @qidx: operational reply queue index
1676  * Return:  0 on success, non-zero on failure.
1681 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_delete_op_reply_q()
1685 	reply_qid = op_reply_q->qid;  in mpi3mr_delete_op_reply_q()
1687 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_delete_op_reply_q()
1690 		retval = -1;  in mpi3mr_delete_op_reply_q()
1695 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :  in mpi3mr_delete_op_reply_q()
1696 	    mrioc->active_poll_qcount--;  in mpi3mr_delete_op_reply_q()
1699 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1700 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_q()
1701 		retval = -1;  in mpi3mr_delete_op_reply_q()
1703 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1706 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_q()
1707 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_q()
1708 	mrioc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_q()
1713 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_delete_op_reply_q()
1720 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_delete_op_reply_q()
1722 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_q()
1726 		retval = -1;  in mpi3mr_delete_op_reply_q()
1729 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_q()
1733 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_q()
1734 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_q()
1735 		retval = -1;  in mpi3mr_delete_op_reply_q()
1738 	mrioc->intr_info[midx].op_reply_q = NULL;  in mpi3mr_delete_op_reply_q()
1742 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_q()
1743 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1750  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1754  * Allocate segmented memory pools for operational reply
1757  * Return: 0 on success, non-zero on failure.
1761 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_alloc_op_reply_q_segments()
1766 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_reply_q_segments()
1767 		op_reply_q->segment_qd =  in mpi3mr_alloc_op_reply_q_segments()
1768 		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1772 		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
1773 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,  in mpi3mr_alloc_op_reply_q_segments()
1775 		if (!op_reply_q->q_segment_list)  in mpi3mr_alloc_op_reply_q_segments()
1776 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
1777 		q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;  in mpi3mr_alloc_op_reply_q_segments()
1779 		op_reply_q->segment_qd = op_reply_q->num_replies;  in mpi3mr_alloc_op_reply_q_segments()
1780 		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1783 	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,  in mpi3mr_alloc_op_reply_q_segments()
1784 	    op_reply_q->segment_qd);  in mpi3mr_alloc_op_reply_q_segments()
1786 	op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,  in mpi3mr_alloc_op_reply_q_segments()
1788 	if (!op_reply_q->q_segments)  in mpi3mr_alloc_op_reply_q_segments()
1789 		return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
1791 	segments = op_reply_q->q_segments;  in mpi3mr_alloc_op_reply_q_segments()
1792 	for (i = 0; i < op_reply_q->num_segments; i++) {  in mpi3mr_alloc_op_reply_q_segments()
1794 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
1797 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
1798 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_reply_q_segments()
1807  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1811  * Allocate segmented memory pools for operational request
1814  * Return: 0 on success, non-zero on failure.
1818 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_alloc_op_req_q_segments()
1823 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_req_q_segments()
1824 		op_req_q->segment_qd =  in mpi3mr_alloc_op_req_q_segments()
1825 		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
1829 		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
1830 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,  in mpi3mr_alloc_op_req_q_segments()
1832 		if (!op_req_q->q_segment_list)  in mpi3mr_alloc_op_req_q_segments()
1833 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
1834 		q_segment_list_entry = (u64 *)op_req_q->q_segment_list;  in mpi3mr_alloc_op_req_q_segments()
1837 		op_req_q->segment_qd = op_req_q->num_requests;  in mpi3mr_alloc_op_req_q_segments()
1838 		size = op_req_q->num_requests * mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
1841 	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,  in mpi3mr_alloc_op_req_q_segments()
1842 	    op_req_q->segment_qd);  in mpi3mr_alloc_op_req_q_segments()
1844 	op_req_q->q_segments = kcalloc(op_req_q->num_segments,  in mpi3mr_alloc_op_req_q_segments()
1846 	if (!op_req_q->q_segments)  in mpi3mr_alloc_op_req_q_segments()
1847 		return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
1849 	segments = op_req_q->q_segments;  in mpi3mr_alloc_op_req_q_segments()
1850 	for (i = 0; i < op_req_q->num_segments; i++) {  in mpi3mr_alloc_op_req_q_segments()
1852 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
1855 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
1856 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_req_q_segments()
1865  * mpi3mr_create_op_reply_q - create operational reply queue
1867  * @qidx: operational reply queue index
1872  * Return:  0 on success, non-zero on failure.
1877 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_create_op_reply_q()
1881 	reply_qid = op_reply_q->qid;  in mpi3mr_create_op_reply_q()
1883 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_create_op_reply_q()
1886 		retval = -1;  in mpi3mr_create_op_reply_q()
1894 	op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;  in mpi3mr_create_op_reply_q()
1895 	if ((mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&  in mpi3mr_create_op_reply_q()
1896 		!mrioc->pdev->revision)  in mpi3mr_create_op_reply_q()
1897 		op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;  in mpi3mr_create_op_reply_q()
1898 	op_reply_q->ci = 0;  in mpi3mr_create_op_reply_q()
1899 	op_reply_q->ephase = 1;  in mpi3mr_create_op_reply_q()
1900 	atomic_set(&op_reply_q->pend_ios, 0);  in mpi3mr_create_op_reply_q()
1901 	atomic_set(&op_reply_q->in_use, 0);  in mpi3mr_create_op_reply_q()
1902 	op_reply_q->enable_irq_poll = false;  in mpi3mr_create_op_reply_q()
1904 	if (!op_reply_q->q_segments) {  in mpi3mr_create_op_reply_q()
1913 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
1914 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_q()
1915 		retval = -1;  in mpi3mr_create_op_reply_q()
1919 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_q()
1920 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_q()
1921 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_q()
1926 	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))  in mpi3mr_create_op_reply_q()
1927 		op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;  in mpi3mr_create_op_reply_q()
1929 		op_reply_q->qtype = MPI3MR_POLL_QUEUE;  in mpi3mr_create_op_reply_q()
1931 	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {  in mpi3mr_create_op_reply_q()
1935 			cpu_to_le16(mrioc->intr_info[midx].msix_index);  in mpi3mr_create_op_reply_q()
1937 		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);  in mpi3mr_create_op_reply_q()
1940 		if (!mrioc->active_poll_qcount)  in mpi3mr_create_op_reply_q()
1941 			disable_irq_nosync(pci_irq_vector(mrioc->pdev,  in mpi3mr_create_op_reply_q()
1942 			    mrioc->intr_info_count - 1));  in mpi3mr_create_op_reply_q()
1945 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_reply_q()
1949 		    op_reply_q->q_segment_list_dma);  in mpi3mr_create_op_reply_q()
1952 		    op_reply_q->q_segments[0].segment_dma);  in mpi3mr_create_op_reply_q()
1954 	create_req.size = cpu_to_le16(op_reply_q->num_replies);  in mpi3mr_create_op_reply_q()
1956 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_reply_q()
1963 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_reply_q()
1965 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_q()
1969 		retval = -1;  in mpi3mr_create_op_reply_q()
1972 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_q()
1976 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_q()
1977 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_q()
1978 		retval = -1;  in mpi3mr_create_op_reply_q()
1981 	op_reply_q->qid = reply_qid;  in mpi3mr_create_op_reply_q()
1982 	if (midx < mrioc->intr_info_count)  in mpi3mr_create_op_reply_q()
1983 		mrioc->intr_info[midx].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_q()
1985 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :  in mpi3mr_create_op_reply_q()
1986 	    mrioc->active_poll_qcount++;  in mpi3mr_create_op_reply_q()
1989 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_q()
1990 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
1997  * mpi3mr_create_op_req_q - create operational request queue
1999  * @idx: operational request queue index
2005  * Return:  0 on success, non-zero on failure.
2011 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;  in mpi3mr_create_op_req_q()
2015 	req_qid = op_req_q->qid;  in mpi3mr_create_op_req_q()
2018 		retval = -1;  in mpi3mr_create_op_req_q()
2026 	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;  in mpi3mr_create_op_req_q()
2027 	op_req_q->ci = 0;  in mpi3mr_create_op_req_q()
2028 	op_req_q->pi = 0;  in mpi3mr_create_op_req_q()
2029 	op_req_q->reply_qid = reply_qid;  in mpi3mr_create_op_req_q()
2030 	spin_lock_init(&op_req_q->q_lock);  in mpi3mr_create_op_req_q()
2032 	if (!op_req_q->q_segments) {  in mpi3mr_create_op_req_q()
2041 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2042 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_q()
2043 		retval = -1;  in mpi3mr_create_op_req_q()
2047 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_q()
2048 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_q()
2049 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_q()
2053 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_req_q()
2057 		    op_req_q->q_segment_list_dma);  in mpi3mr_create_op_req_q()
2060 		    op_req_q->q_segments[0].segment_dma);  in mpi3mr_create_op_req_q()
2062 	create_req.size = cpu_to_le16(op_req_q->num_requests);  in mpi3mr_create_op_req_q()
2064 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_req_q()
2071 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_req_q()
2073 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_q()
2077 		retval = -1;  in mpi3mr_create_op_req_q()
2080 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_q()
2084 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_q()
2085 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_q()
2086 		retval = -1;  in mpi3mr_create_op_req_q()
2089 	op_req_q->qid = req_qid;  in mpi3mr_create_op_req_q()
2092 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_q()
2093 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2100  * mpi3mr_create_op_queues - create operational queue pairs
2103  * Allocate memory for operational queue meta data and call
2106  * Return: 0 on success, non-zero on failures.
2113 	num_queues = min_t(int, mrioc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
2114 	    mrioc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
2117 	    mrioc->intr_info_count - mrioc->op_reply_q_offset;  in mpi3mr_create_op_queues()
2118 	if (!mrioc->num_queues)  in mpi3mr_create_op_queues()
2119 		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);  in mpi3mr_create_op_queues()
2124 	num_queues = mrioc->num_op_reply_q ?  in mpi3mr_create_op_queues()
2125 	    mrioc->num_op_reply_q : mrioc->num_queues;  in mpi3mr_create_op_queues()
2126 	ioc_info(mrioc, "trying to create %d operational queue pairs\n",  in mpi3mr_create_op_queues()
2129 	if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2130 		mrioc->req_qinfo = kcalloc(num_queues,  in mpi3mr_create_op_queues()
2132 		if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2133 			retval = -1;  in mpi3mr_create_op_queues()
2137 		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *  in mpi3mr_create_op_queues()
2139 		if (!mrioc->op_reply_qinfo) {  in mpi3mr_create_op_queues()
2140 			retval = -1;  in mpi3mr_create_op_queues()
2145 	if (mrioc->enable_segqueue)  in mpi3mr_create_op_queues()
2147 		    "allocating operational queues through segmented queues\n");  in mpi3mr_create_op_queues()
2155 		    mrioc->op_reply_qinfo[i].qid)) {  in mpi3mr_create_op_queues()
2164 		retval = -1;  in mpi3mr_create_op_queues()
2167 	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;  in mpi3mr_create_op_queues()
2169 	    "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",  in mpi3mr_create_op_queues()
2170 	    mrioc->num_op_reply_q, mrioc->default_qcount,  in mpi3mr_create_op_queues()
2171 	    mrioc->active_poll_qcount);  in mpi3mr_create_op_queues()
2175 	kfree(mrioc->req_qinfo);  in mpi3mr_create_op_queues()
2176 	mrioc->req_qinfo = NULL;  in mpi3mr_create_op_queues()
2178 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_create_op_queues()
2179 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_create_op_queues()
2185  * mpi3mr_op_request_post - Post request to operational queue
2187  * @op_req_q: Operational request queue info
2190  * Post the MPI3 request into operational request queue and
2194  * Return: 0 on success, non-zero on failure.
2204 	u16 req_sz = mrioc->facts.op_req_sz;  in mpi3mr_op_request_post()
2205 	struct segments *segments = op_req_q->q_segments;  in mpi3mr_op_request_post()
2207 	reply_qidx = op_req_q->reply_qid - 1;  in mpi3mr_op_request_post()
2209 	if (mrioc->unrecoverable)  in mpi3mr_op_request_post()
2210 		return -EFAULT;  in mpi3mr_op_request_post()
2212 	spin_lock_irqsave(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2213 	pi = op_req_q->pi;  in mpi3mr_op_request_post()
2214 	max_entries = op_req_q->num_requests;  in mpi3mr_op_request_post()
2218 		    reply_qidx, mrioc->op_reply_q_offset);  in mpi3mr_op_request_post()
2219 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);  in mpi3mr_op_request_post()
2222 			retval = -EAGAIN;  in mpi3mr_op_request_post()
2227 	if (mrioc->reset_in_progress) {  in mpi3mr_op_request_post()
2229 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2233 	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;  in mpi3mr_op_request_post()
2235 	    ((pi % op_req_q->segment_qd) * req_sz);  in mpi3mr_op_request_post()
2242 	op_req_q->pi = pi;  in mpi3mr_op_request_post()
2245 	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)  in mpi3mr_op_request_post()
2247 		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;  in mpi3mr_op_request_post()
2249 	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);  in mpi3mr_op_request_post()
2252 	writel(op_req_q->pi,  in mpi3mr_op_request_post()
2253 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);  in mpi3mr_op_request_post()
2256 	spin_unlock_irqrestore(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2261  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2278 	if (mrioc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
2283 	if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_check_rh_fault_ioc()
2284 		mrioc->unrecoverable = 1;  in mpi3mr_check_rh_fault_ioc()
2289 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_check_rh_fault_ioc()
2300 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_check_rh_fault_ioc()
2304 	} while (--timeout);  in mpi3mr_check_rh_fault_ioc()
2308  * mpi3mr_sync_timestamp - Issue time stamp sync request
2314  * Return: 0 on success, non-zero on failure.
2323 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2324 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_sync_timestamp()
2325 		retval = -1;  in mpi3mr_sync_timestamp()
2327 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2330 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_sync_timestamp()
2331 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_sync_timestamp()
2332 	mrioc->init_cmds.callback = NULL;  in mpi3mr_sync_timestamp()
2339 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_sync_timestamp()
2347 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_sync_timestamp()
2349 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_sync_timestamp()
2351 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_sync_timestamp()
2352 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_sync_timestamp()
2355 		retval = -1;  in mpi3mr_sync_timestamp()
2358 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_sync_timestamp()
2362 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_sync_timestamp()
2363 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_sync_timestamp()
2364 		retval = -1;  in mpi3mr_sync_timestamp()
2369 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_sync_timestamp()
2370 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2377  * mpi3mr_print_pkg_ver - display controller fw package version
2383  * Return: 0 on success and non-zero on failure.
2388 	int retval = -1;  in mpi3mr_print_pkg_ver()
2395 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_print_pkg_ver()
2398 		return -ENOMEM;  in mpi3mr_print_pkg_ver()
2401 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2402 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_pkg_ver()
2404 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2407 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_pkg_ver()
2408 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_print_pkg_ver()
2409 	mrioc->init_cmds.callback = NULL;  in mpi3mr_print_pkg_ver()
2419 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_print_pkg_ver()
2426 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_print_pkg_ver()
2428 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_pkg_ver()
2432 		retval = -1;  in mpi3mr_print_pkg_ver()
2435 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_print_pkg_ver()
2438 		if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {  in mpi3mr_print_pkg_ver()
2440 			    "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",  in mpi3mr_print_pkg_ver()
2441 			    manifest->package_version.gen_major,  in mpi3mr_print_pkg_ver()
2442 			    manifest->package_version.gen_minor,  in mpi3mr_print_pkg_ver()
2443 			    manifest->package_version.phase_major,  in mpi3mr_print_pkg_ver()
2444 			    manifest->package_version.phase_minor,  in mpi3mr_print_pkg_ver()
2445 			    manifest->package_version.customer_id,  in mpi3mr_print_pkg_ver()
2446 			    manifest->package_version.build_num);  in mpi3mr_print_pkg_ver()
2451 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_pkg_ver()
2452 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2456 		dma_free_coherent(&mrioc->pdev->dev, data_len, data,  in mpi3mr_print_pkg_ver()
2462  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2480 	if (mrioc->reset_in_progress)  in mpi3mr_watchdog_work()
2483 	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {  in mpi3mr_watchdog_work()
2485 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2488 	if (mrioc->unrecoverable) {  in mpi3mr_watchdog_work()
2495 	if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {  in mpi3mr_watchdog_work()
2496 		mrioc->ts_update_counter = 0;  in mpi3mr_watchdog_work()
2500 	if ((mrioc->prepare_for_reset) &&  in mpi3mr_watchdog_work()
2501 	    ((mrioc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_work()
2508 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_watchdog_work()
2519 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_watchdog_work()
2520 	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_watchdog_work()
2522 		if (!mrioc->diagsave_timeout) {  in mpi3mr_watchdog_work()
2526 		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_work()
2531 	mrioc->diagsave_timeout = 0;  in mpi3mr_watchdog_work()
2538 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2552 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2553 	if (mrioc->watchdog_work_q)  in mpi3mr_watchdog_work()
2554 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_watchdog_work()
2555 		    &mrioc->watchdog_work,  in mpi3mr_watchdog_work()
2557 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2562  * mpi3mr_start_watchdog - Start watchdog
2572 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2575 	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);  in mpi3mr_start_watchdog()
2576 	snprintf(mrioc->watchdog_work_q_name,  in mpi3mr_start_watchdog()
2577 	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,  in mpi3mr_start_watchdog()
2578 	    mrioc->id);  in mpi3mr_start_watchdog()
2579 	mrioc->watchdog_work_q =  in mpi3mr_start_watchdog()
2580 	    create_singlethread_workqueue(mrioc->watchdog_work_q_name);  in mpi3mr_start_watchdog()
2581 	if (!mrioc->watchdog_work_q) {  in mpi3mr_start_watchdog()
2586 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2587 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_start_watchdog()
2588 		    &mrioc->watchdog_work,  in mpi3mr_start_watchdog()
2593  * mpi3mr_stop_watchdog - Stop watchdog
2606 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2607 	wq = mrioc->watchdog_work_q;  in mpi3mr_stop_watchdog()
2608 	mrioc->watchdog_work_q = NULL;  in mpi3mr_stop_watchdog()
2609 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2611 		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))  in mpi3mr_stop_watchdog()
2618  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2624  * Return: 0 on success, non-zero on failures.
2631 	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2632 	mrioc->num_admin_req = mrioc->admin_req_q_sz /  in mpi3mr_setup_admin_qpair()
2634 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
2636 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2637 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /  in mpi3mr_setup_admin_qpair()
2639 	mrioc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
2640 	mrioc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
2641 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_setup_admin_qpair()
2643 	if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2644 		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2645 		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);  in mpi3mr_setup_admin_qpair()
2647 		if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2648 			retval = -1;  in mpi3mr_setup_admin_qpair()
2652 		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2653 		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2656 		if (!mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2657 			retval = -1;  in mpi3mr_setup_admin_qpair()
2662 	num_admin_entries = (mrioc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
2663 	    (mrioc->num_admin_req);  in mpi3mr_setup_admin_qpair()
2664 	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);  in mpi3mr_setup_admin_qpair()
2665 	mpi3mr_writeq(mrioc->admin_req_dma,  in mpi3mr_setup_admin_qpair()
2666 	    &mrioc->sysif_regs->admin_request_queue_address);  in mpi3mr_setup_admin_qpair()
2667 	mpi3mr_writeq(mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2668 	    &mrioc->sysif_regs->admin_reply_queue_address);  in mpi3mr_setup_admin_qpair()
2669 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_setup_admin_qpair()
2670 	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_setup_admin_qpair()
2675 	if (mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2676 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
2677 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_setup_admin_qpair()
2678 		mrioc->admin_reply_base = NULL;  in mpi3mr_setup_admin_qpair()
2680 	if (mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2681 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
2682 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_setup_admin_qpair()
2683 		mrioc->admin_req_base = NULL;  in mpi3mr_setup_admin_qpair()
2689  * mpi3mr_issue_iocfacts - Send IOC Facts
2696  * Return: 0 on success, non-zero on failures.
2708 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocfacts()
2712 		retval = -1;  in mpi3mr_issue_iocfacts()
2717 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2718 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
2719 		retval = -1;  in mpi3mr_issue_iocfacts()
2721 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2724 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
2725 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
2726 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
2733 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocfacts()
2740 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocfacts()
2742 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
2746 		retval = -1;  in mpi3mr_issue_iocfacts()
2749 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
2753 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
2754 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
2755 		retval = -1;  in mpi3mr_issue_iocfacts()
2761 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
2762 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2766 		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);  in mpi3mr_issue_iocfacts()
2772  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2778  * Return: 0 on success, non-zero on failure.
2782 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_check_reset_dma_mask()
2784 	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);  in mpi3mr_check_reset_dma_mask()
2786 	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))  in mpi3mr_check_reset_dma_mask()
2790 	    mrioc->dma_mask, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
2792 	r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
2798 	mrioc->dma_mask = facts_dma_mask;  in mpi3mr_check_reset_dma_mask()
2803  * mpi3mr_process_factsdata - Process IOC facts data
2817 	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=  in mpi3mr_process_factsdata()
2822 		    le16_to_cpu(facts_data->ioc_facts_data_length) * 4);  in mpi3mr_process_factsdata()
2825 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_process_factsdata()
2828 	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {  in mpi3mr_process_factsdata()
2831 		    req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));  in mpi3mr_process_factsdata()
2834 	memset(&mrioc->facts, 0, sizeof(mrioc->facts));  in mpi3mr_process_factsdata()
2836 	facts_flags = le32_to_cpu(facts_data->flags);  in mpi3mr_process_factsdata()
2837 	mrioc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
2838 	mrioc->op_reply_desc_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
2842 	mrioc->facts.ioc_num = facts_data->ioc_number;  in mpi3mr_process_factsdata()
2843 	mrioc->facts.who_init = facts_data->who_init;  in mpi3mr_process_factsdata()
2844 	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);  in mpi3mr_process_factsdata()
2845 	mrioc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
2847 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
2850 	mrioc->facts.protocol_flags = facts_data->protocol_flags;  in mpi3mr_process_factsdata()
2851 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);  in mpi3mr_process_factsdata()
2852 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);  in mpi3mr_process_factsdata()
2853 	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);  in mpi3mr_process_factsdata()
2854 	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;  in mpi3mr_process_factsdata()
2855 	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);  in mpi3mr_process_factsdata()
2856 	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);  in mpi3mr_process_factsdata()
2857 	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);  in mpi3mr_process_factsdata()
2858 	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);  in mpi3mr_process_factsdata()
2859 	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);  in mpi3mr_process_factsdata()
2860 	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);  in mpi3mr_process_factsdata()
2861 	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);  in mpi3mr_process_factsdata()
2862 	mrioc->facts.max_pcie_switches =  in mpi3mr_process_factsdata()
2863 	    le16_to_cpu(facts_data->max_pcie_switches);  in mpi3mr_process_factsdata()
2864 	mrioc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
2865 	    le16_to_cpu(facts_data->max_sas_expanders);  in mpi3mr_process_factsdata()
2866 	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);  in mpi3mr_process_factsdata()
2867 	mrioc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
2868 	    le16_to_cpu(facts_data->max_sas_initiators);  in mpi3mr_process_factsdata()
2869 	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);  in mpi3mr_process_factsdata()
2870 	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);  in mpi3mr_process_factsdata()
2871 	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);  in mpi3mr_process_factsdata()
2872 	mrioc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
2873 	    le16_to_cpu(facts_data->max_operational_request_queues);  in mpi3mr_process_factsdata()
2874 	mrioc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
2875 	    le16_to_cpu(facts_data->max_operational_reply_queues);  in mpi3mr_process_factsdata()
2876 	mrioc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
2877 	    le32_to_cpu(facts_data->ioc_capabilities);  in mpi3mr_process_factsdata()
2878 	mrioc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
2879 	    le16_to_cpu(facts_data->fw_version.build_num);  in mpi3mr_process_factsdata()
2880 	mrioc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
2881 	    le16_to_cpu(facts_data->fw_version.customer_id);  in mpi3mr_process_factsdata()
2882 	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;  in mpi3mr_process_factsdata()
2883 	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;  in mpi3mr_process_factsdata()
2884 	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;  in mpi3mr_process_factsdata()
2885 	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;  in mpi3mr_process_factsdata()
2886 	mrioc->msix_count = min_t(int, mrioc->msix_count,  in mpi3mr_process_factsdata()
2887 	    mrioc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
2888 	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;  in mpi3mr_process_factsdata()
2889 	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;  in mpi3mr_process_factsdata()
2890 	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;  in mpi3mr_process_factsdata()
2891 	mrioc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
2892 	    le16_to_cpu(facts_data->shutdown_timeout);  in mpi3mr_process_factsdata()
2894 	mrioc->facts.max_dev_per_tg =  in mpi3mr_process_factsdata()
2895 	    facts_data->max_devices_per_throttle_group;  in mpi3mr_process_factsdata()
2896 	mrioc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
2897 	    le16_to_cpu(facts_data->io_throttle_data_length);  in mpi3mr_process_factsdata()
2898 	mrioc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
2899 	    le16_to_cpu(facts_data->max_io_throttle_group);  in mpi3mr_process_factsdata()
2900 	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);  in mpi3mr_process_factsdata()
2901 	mrioc->facts.io_throttle_high =  in mpi3mr_process_factsdata()
2902 	    le16_to_cpu(facts_data->io_throttle_high);  in mpi3mr_process_factsdata()
2904 	if (mrioc->facts.max_data_length ==  in mpi3mr_process_factsdata()
2906 		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;  in mpi3mr_process_factsdata()
2908 		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;  in mpi3mr_process_factsdata()
2910 	if (mrioc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
2911 		mrioc->io_throttle_data_length =  in mpi3mr_process_factsdata()
2912 		    (mrioc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
2915 		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;  in mpi3mr_process_factsdata()
2917 	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
2918 	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
2921 	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
2922 	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);  in mpi3mr_process_factsdata()
2925 	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,  in mpi3mr_process_factsdata()
2926 	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);  in mpi3mr_process_factsdata()
2928 	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
2929 	    mrioc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
2931 	    mrioc->facts.dma_mask, (facts_flags &  in mpi3mr_process_factsdata()
2932 	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);  in mpi3mr_process_factsdata()
2935 	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);  in mpi3mr_process_factsdata()
2938 	   mrioc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
2939 	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
2943  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2949  * Return: 0 on success, non-zero on failures.
2956 	if (mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2959 	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2960 	if (!mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2963 	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2964 	if (!mrioc->bsg_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2967 	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2968 	if (!mrioc->transport_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2972 		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
2974 		if (!mrioc->dev_rmhs_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
2979 		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
2981 		if (!mrioc->evtack_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
2985 	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2986 	if (!mrioc->host_tm_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2989 	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2990 	if (!mrioc->pel_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
2993 	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
2994 	if (!mrioc->pel_abort_cmd.reply)  in mpi3mr_alloc_reply_sense_bufs()
2997 	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_alloc_reply_sense_bufs()
2998 	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,  in mpi3mr_alloc_reply_sense_bufs()
3000 	if (!mrioc->removepend_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3003 	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3004 	if (!mrioc->devrem_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3007 	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,  in mpi3mr_alloc_reply_sense_bufs()
3009 	if (!mrioc->evtack_cmds_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3012 	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;  in mpi3mr_alloc_reply_sense_bufs()
3013 	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3014 	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_alloc_reply_sense_bufs()
3015 	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3018 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpi3mr_alloc_reply_sense_bufs()
3019 	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3020 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_reply_sense_bufs()
3021 	if (!mrioc->reply_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3026 	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3027 	    &mrioc->reply_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3028 	if (!mrioc->reply_buf)  in mpi3mr_alloc_reply_sense_bufs()
3031 	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;  in mpi3mr_alloc_reply_sense_bufs()
3034 	sz = mrioc->reply_free_qsz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3035 	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3036 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3037 	if (!mrioc->reply_free_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3041 	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3042 	    GFP_KERNEL, &mrioc->reply_free_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3043 	if (!mrioc->reply_free_q)  in mpi3mr_alloc_reply_sense_bufs()
3047 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpi3mr_alloc_reply_sense_bufs()
3048 	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3049 	    &mrioc->pdev->dev, sz, 4, 0);  in mpi3mr_alloc_reply_sense_bufs()
3050 	if (!mrioc->sense_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3054 	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3055 	    &mrioc->sense_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3056 	if (!mrioc->sense_buf)  in mpi3mr_alloc_reply_sense_bufs()
3060 	sz = mrioc->sense_buf_q_sz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3061 	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3062 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3063 	if (!mrioc->sense_buf_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3067 	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3068 	    GFP_KERNEL, &mrioc->sense_buf_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3069 	if (!mrioc->sense_buf_q)  in mpi3mr_alloc_reply_sense_bufs()
3075 	retval = -1;  in mpi3mr_alloc_reply_sense_bufs()
3080  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3094 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpimr_initialize_reply_sbuf_queues()
3097 	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,  in mpimr_initialize_reply_sbuf_queues()
3098 	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3099 	sz = mrioc->reply_free_qsz * 8;  in mpimr_initialize_reply_sbuf_queues()
3102 	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3103 	    (unsigned long long)mrioc->reply_free_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3104 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpimr_initialize_reply_sbuf_queues()
3107 	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,  in mpimr_initialize_reply_sbuf_queues()
3108 	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3109 	sz = mrioc->sense_buf_q_sz * 8;  in mpimr_initialize_reply_sbuf_queues()
3112 	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3113 	    (unsigned long long)mrioc->sense_buf_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3116 	for (i = 0, phy_addr = mrioc->reply_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3117 	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)  in mpimr_initialize_reply_sbuf_queues()
3118 		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3119 	mrioc->reply_free_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3122 	for (i = 0, phy_addr = mrioc->sense_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3123 	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)  in mpimr_initialize_reply_sbuf_queues()
3124 		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3125 	mrioc->sense_buf_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3129  * mpi3mr_issue_iocinit - Send IOC Init
3135  * Return: 0 on success, non-zero on failures.
3146 	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocinit()
3149 		retval = -1;  in mpi3mr_issue_iocinit()
3154 	drv_info->information_length = cpu_to_le32(data_len);  in mpi3mr_issue_iocinit()
3155 	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));  in mpi3mr_issue_iocinit()
3156 	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));  in mpi3mr_issue_iocinit()
3157 	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));  in mpi3mr_issue_iocinit()
3158 	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));  in mpi3mr_issue_iocinit()
3159 	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));  in mpi3mr_issue_iocinit()
3160 	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,  in mpi3mr_issue_iocinit()
3161 	    sizeof(drv_info->driver_release_date));  in mpi3mr_issue_iocinit()
3162 	drv_info->driver_capabilities = 0;  in mpi3mr_issue_iocinit()
3163 	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,  in mpi3mr_issue_iocinit()
3164 	    sizeof(mrioc->driver_info));  in mpi3mr_issue_iocinit()
3167 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3168 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
3169 		retval = -1;  in mpi3mr_issue_iocinit()
3171 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3174 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
3175 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
3176 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
3184 	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);  in mpi3mr_issue_iocinit()
3186 	    cpu_to_le64(mrioc->reply_free_q_dma);  in mpi3mr_issue_iocinit()
3189 	    cpu_to_le16(mrioc->sense_buf_q_sz);  in mpi3mr_issue_iocinit()
3191 	    cpu_to_le64(mrioc->sense_buf_q_dma);  in mpi3mr_issue_iocinit()
3197 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocinit()
3204 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocinit()
3206 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
3210 		retval = -1;  in mpi3mr_issue_iocinit()
3213 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
3217 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
3218 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
3219 		retval = -1;  in mpi3mr_issue_iocinit()
3223 	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;  in mpi3mr_issue_iocinit()
3224 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_issue_iocinit()
3225 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_issue_iocinit()
3227 	mrioc->sbq_host_index = mrioc->num_sense_bufs;  in mpi3mr_issue_iocinit()
3228 	writel(mrioc->sbq_host_index,  in mpi3mr_issue_iocinit()
3229 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_issue_iocinit()
3231 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
3232 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3236 		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,  in mpi3mr_issue_iocinit()
3243  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3250  * Return: 0 on success, non-zero on failures.
3263 	mrioc->event_masks[word] &= ~desired_event;  in mpi3mr_unmask_events()
3267  * mpi3mr_issue_event_notification - Send event notification
3273  * Return: 0 on success, non-zero on failures.
3282 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3283 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
3284 		retval = -1;  in mpi3mr_issue_event_notification()
3286 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3289 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
3290 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
3291 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
3296 		    cpu_to_le32(mrioc->event_masks[i]);  in mpi3mr_issue_event_notification()
3297 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_event_notification()
3304 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_event_notification()
3306 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
3310 		retval = -1;  in mpi3mr_issue_event_notification()
3313 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
3317 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
3318 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
3319 		retval = -1;  in mpi3mr_issue_event_notification()
3324 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
3325 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3331  * mpi3mr_process_event_ack - Process event acknowledgment
3339  * Return: 0 on success, non-zero on failures.
3348 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3349 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
3350 		retval = -1;  in mpi3mr_process_event_ack()
3352 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3355 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
3356 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
3357 	mrioc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
3363 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_process_event_ack()
3370 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_process_event_ack()
3372 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
3374 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_process_event_ack()
3377 		retval = -1;  in mpi3mr_process_event_ack()
3380 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
3384 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
3385 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
3386 		retval = -1;  in mpi3mr_process_event_ack()
3391 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
3392 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3398  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3405  * Return: 0 on success, non-zero on failure
3413 	if (mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3416 	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;  in mpi3mr_alloc_chain_bufs()
3424 	mrioc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
3426 	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3427 	if (!mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3430 	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3432 		mrioc->max_sgl_entries = mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3434 	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);  in mpi3mr_alloc_chain_bufs()
3436 			mrioc->max_sgl_entries, sz/1024);  in mpi3mr_alloc_chain_bufs()
3438 	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",  in mpi3mr_alloc_chain_bufs()
3439 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_chain_bufs()
3440 	if (!mrioc->chain_buf_pool) {  in mpi3mr_alloc_chain_bufs()
3446 		mrioc->chain_sgl_list[i].addr =  in mpi3mr_alloc_chain_bufs()
3447 		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_chain_bufs()
3448 		    &mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_alloc_chain_bufs()
3450 		if (!mrioc->chain_sgl_list[i].addr)  in mpi3mr_alloc_chain_bufs()
3453 	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3454 	if (!mrioc->chain_bitmap)  in mpi3mr_alloc_chain_bufs()
3458 	retval = -1;  in mpi3mr_alloc_chain_bufs()
3463  * mpi3mr_port_enable_complete - Mark port enable complete
3475 	drv_cmd->callback = NULL;  in mpi3mr_port_enable_complete()
3476 	mrioc->scan_started = 0;  in mpi3mr_port_enable_complete()
3477 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_port_enable_complete()
3478 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;  in mpi3mr_port_enable_complete()
3480 		mrioc->scan_failed = drv_cmd->ioc_status;  in mpi3mr_port_enable_complete()
3481 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_port_enable_complete()
3485  * mpi3mr_issue_port_enable - Issue Port Enable
3493  * Return: 0 on success, non-zero on failures.
3502 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3503 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3504 		retval = -1;  in mpi3mr_issue_port_enable()
3506 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3509 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3511 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3512 		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3514 		mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3515 		mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3516 		init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_port_enable()
3527 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3531 	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));  in mpi3mr_issue_port_enable()
3532 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3534 		retval = -1;  in mpi3mr_issue_port_enable()
3538 	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);  in mpi3mr_issue_port_enable()
3541 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_port_enable()
3542 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3567  * mpi3mr_print_ioc_info - Display controller information
3582 	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;  in mpi3mr_print_ioc_info()
3584 	switch (mrioc->facts.personality) {  in mpi3mr_print_ioc_info()
3599 	    fwver->gen_major, fwver->gen_minor, fwver->ph_major,  in mpi3mr_print_ioc_info()
3600 	    fwver->ph_minor, fwver->cust_id, fwver->build_num);  in mpi3mr_print_ioc_info()
3603 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
3606 				    sizeof(protocol) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
3614 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
3617 				    sizeof(capabilities) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
3628  * mpi3mr_cleanup_resources - Free PCI resources
3633  * Return: 0 on success and non-zero on failure.
3637 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_cleanup_resources()
3641 	if (mrioc->sysif_regs) {  in mpi3mr_cleanup_resources()
3642 		iounmap((void __iomem *)mrioc->sysif_regs);  in mpi3mr_cleanup_resources()
3643 		mrioc->sysif_regs = NULL;  in mpi3mr_cleanup_resources()
3647 		if (mrioc->bars)  in mpi3mr_cleanup_resources()
3648 			pci_release_selected_regions(pdev, mrioc->bars);  in mpi3mr_cleanup_resources()
3654  * mpi3mr_setup_resources - Enable PCI resources
3657  * Enable PCI device memory, MSI-x registers and set DMA mask.
3659  * Return: 0 on success and non-zero on failure.
3663 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_setup_resources()
3667 	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :  in mpi3mr_setup_resources()
3672 		retval = -ENODEV;  in mpi3mr_setup_resources()
3678 		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");  in mpi3mr_setup_resources()
3679 		retval = -ENODEV;  in mpi3mr_setup_resources()
3682 	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpi3mr_setup_resources()
3684 	if (pci_request_selected_regions(pdev, mrioc->bars,  in mpi3mr_setup_resources()
3685 	    mrioc->driver_name)) {  in mpi3mr_setup_resources()
3687 		retval = -ENODEV;  in mpi3mr_setup_resources()
3693 			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);  in mpi3mr_setup_resources()
3695 			mrioc->sysif_regs =  in mpi3mr_setup_resources()
3696 			    ioremap(mrioc->sysif_regs_phys, memap_sz);  in mpi3mr_setup_resources()
3703 	retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);  in mpi3mr_setup_resources()
3708 			retval = dma_set_mask_and_coherent(&pdev->dev,  in mpi3mr_setup_resources()
3712 			mrioc->dma_mask = 0;  in mpi3mr_setup_resources()
3717 	mrioc->dma_mask = dma_mask;  in mpi3mr_setup_resources()
3719 	if (!mrioc->sysif_regs) {  in mpi3mr_setup_resources()
3722 		retval = -EINVAL;  in mpi3mr_setup_resources()
3727 	mrioc->msix_count = (message_control & 0x3FF) + 1;  in mpi3mr_setup_resources()
3731 	pci_set_drvdata(pdev, mrioc->shost);  in mpi3mr_setup_resources()
3736 	    (unsigned long long)mrioc->sysif_regs_phys,  in mpi3mr_setup_resources()
3737 	    mrioc->sysif_regs, memap_sz);  in mpi3mr_setup_resources()
3738 	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",  in mpi3mr_setup_resources()
3739 	    mrioc->msix_count);  in mpi3mr_setup_resources()
3742 		mrioc->requested_poll_qcount = min_t(int, poll_queues,  in mpi3mr_setup_resources()
3743 				mrioc->msix_count - 2);  in mpi3mr_setup_resources()
3752  * mpi3mr_enable_events - Enable required events
3759  * Return: 0 on success and non-zero on failure.
3767 		mrioc->event_masks[i] = -1;  in mpi3mr_enable_events()
3792  * mpi3mr_init_ioc - Initialize the controller
3798  * registers, create admin and operational reply queue pairs,
3804  * Return: 0 on success and non-zero on failure.
3835 	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;  in mpi3mr_init_ioc()
3836 	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;  in mpi3mr_init_ioc()
3837 	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;  in mpi3mr_init_ioc()
3838 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_init_ioc()
3841 		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,  in mpi3mr_init_ioc()
3844 	if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_init_ioc()
3846 		mrioc->sas_transport_enabled = 1;  in mpi3mr_init_ioc()
3847 		mrioc->scsi_device_channel = 1;  in mpi3mr_init_ioc()
3848 		mrioc->shost->max_channel = 1;  in mpi3mr_init_ioc()
3849 		mrioc->shost->transportt = mpi3mr_transport_template;  in mpi3mr_init_ioc()
3852 	mrioc->reply_sz = mrioc->facts.reply_sz;  in mpi3mr_init_ioc()
3863 	if (!mrioc->cfg_page) {  in mpi3mr_init_ioc()
3865 		mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;  in mpi3mr_init_ioc()
3866 		mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_init_ioc()
3867 		    mrioc->cfg_page_sz, &mrioc->cfg_page_dma, GFP_KERNEL);  in mpi3mr_init_ioc()
3868 		if (!mrioc->cfg_page) {  in mpi3mr_init_ioc()
3869 			retval = -1;  in mpi3mr_init_ioc()
3874 	if (!mrioc->init_cmds.reply) {  in mpi3mr_init_ioc()
3884 	if (!mrioc->chain_sgl_list) {  in mpi3mr_init_ioc()
3908 		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",  in mpi3mr_init_ioc()
3920 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
3922 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_init_ioc()
3923 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_init_ioc()
3924 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_init_ioc()
3926 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
3927 			retval = -ENOMEM;  in mpi3mr_init_ioc()
3932 	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {  in mpi3mr_init_ioc()
3935 		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);  in mpi3mr_init_ioc()
3936 		if (!mrioc->throttle_groups) {  in mpi3mr_init_ioc()
3937 			retval = -1;  in mpi3mr_init_ioc()
3959 	retval = -1;  in mpi3mr_init_ioc()
3964 	mrioc->unrecoverable = 1;  in mpi3mr_init_ioc()
3969  * mpi3mr_reinit_ioc - Re-Initialize the controller
3973  * This the controller re-initialization routine, executed from
3975  * operational reply queue pairs, allocate required memory for
3980  * Return: 0 on success and non-zero on failure.
4044 			ioc_err(mrioc, "failed to re-setup ISR\n");  in mpi3mr_reinit_ioc()
4049 	dprint_reset(mrioc, "creating operational queue pairs\n");  in mpi3mr_reinit_ioc()
4052 		ioc_err(mrioc, "failed to create operational queue pairs\n");  in mpi3mr_reinit_ioc()
4056 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4058 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_reinit_ioc()
4059 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_reinit_ioc()
4060 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_reinit_ioc()
4062 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4063 			retval = -ENOMEM;  in mpi3mr_reinit_ioc()
4068 	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {  in mpi3mr_reinit_ioc()
4070 		    "cannot create minimum number of operational queues expected:%d created:%d\n",  in mpi3mr_reinit_ioc()
4071 		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);  in mpi3mr_reinit_ioc()
4072 		retval = -1;  in mpi3mr_reinit_ioc()
4083 	mrioc->device_refresh_on = 1;  in mpi3mr_reinit_ioc()
4094 		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)  in mpi3mr_reinit_ioc()
4096 		if (!pci_device_is_present(mrioc->pdev))  in mpi3mr_reinit_ioc()
4097 			mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4098 		if (mrioc->unrecoverable) {  in mpi3mr_reinit_ioc()
4099 			retval = -1;  in mpi3mr_reinit_ioc()
4102 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_reinit_ioc()
4106 			mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4107 			mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4108 			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4111 	} while (--pe_timeout);  in mpi3mr_reinit_ioc()
4117 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4118 		mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4119 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4121 	} else if (mrioc->scan_failed) {  in mpi3mr_reinit_ioc()
4124 		    mrioc->scan_failed);  in mpi3mr_reinit_ioc()
4129 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4135 		    (is_resume)?"resume":"re-initialization", retry);  in mpi3mr_reinit_ioc()
4139 	retval = -1;  in mpi3mr_reinit_ioc()
4142 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4145 	mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4150  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4153  * @qidx: Operational reply queue index
4159 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_memset_op_reply_q_buffers()
4163 	if (!op_reply_q->q_segments)  in mpi3mr_memset_op_reply_q_buffers()
4166 	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;  in mpi3mr_memset_op_reply_q_buffers()
4167 	segments = op_reply_q->q_segments;  in mpi3mr_memset_op_reply_q_buffers()
4168 	for (i = 0; i < op_reply_q->num_segments; i++)  in mpi3mr_memset_op_reply_q_buffers()
4173  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4176  * @qidx: Operational request queue index
4182 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_memset_op_req_q_buffers()
4186 	if (!op_req_q->q_segments)  in mpi3mr_memset_op_req_q_buffers()
4189 	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;  in mpi3mr_memset_op_req_q_buffers()
4190 	segments = op_req_q->q_segments;  in mpi3mr_memset_op_req_q_buffers()
4191 	for (i = 0; i < op_req_q->num_segments; i++)  in mpi3mr_memset_op_req_q_buffers()
4196  * mpi3mr_memset_buffers - memset memory for a controller
4210 	mrioc->change_count = 0;  in mpi3mr_memset_buffers()
4211 	mrioc->active_poll_qcount = 0;  in mpi3mr_memset_buffers()
4212 	mrioc->default_qcount = 0;  in mpi3mr_memset_buffers()
4213 	if (mrioc->admin_req_base)  in mpi3mr_memset_buffers()
4214 		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);  in mpi3mr_memset_buffers()
4215 	if (mrioc->admin_reply_base)  in mpi3mr_memset_buffers()
4216 		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
4217 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_memset_buffers()
4219 	if (mrioc->init_cmds.reply) {  in mpi3mr_memset_buffers()
4220 		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));  in mpi3mr_memset_buffers()
4221 		memset(mrioc->bsg_cmds.reply, 0,  in mpi3mr_memset_buffers()
4222 		    sizeof(*mrioc->bsg_cmds.reply));  in mpi3mr_memset_buffers()
4223 		memset(mrioc->host_tm_cmds.reply, 0,  in mpi3mr_memset_buffers()
4224 		    sizeof(*mrioc->host_tm_cmds.reply));  in mpi3mr_memset_buffers()
4225 		memset(mrioc->pel_cmds.reply, 0,  in mpi3mr_memset_buffers()
4226 		    sizeof(*mrioc->pel_cmds.reply));  in mpi3mr_memset_buffers()
4227 		memset(mrioc->pel_abort_cmd.reply, 0,  in mpi3mr_memset_buffers()
4228 		    sizeof(*mrioc->pel_abort_cmd.reply));  in mpi3mr_memset_buffers()
4229 		memset(mrioc->transport_cmds.reply, 0,  in mpi3mr_memset_buffers()
4230 		    sizeof(*mrioc->transport_cmds.reply));  in mpi3mr_memset_buffers()
4232 			memset(mrioc->dev_rmhs_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4233 			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));  in mpi3mr_memset_buffers()
4235 			memset(mrioc->evtack_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4236 			    sizeof(*mrioc->evtack_cmds[i].reply));  in mpi3mr_memset_buffers()
4237 		bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_memset_buffers()
4238 			     mrioc->dev_handle_bitmap_bits);  in mpi3mr_memset_buffers()
4239 		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_memset_buffers()
4240 		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,  in mpi3mr_memset_buffers()
4244 	for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_memset_buffers()
4245 		mrioc->op_reply_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4246 		mrioc->op_reply_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4247 		mrioc->op_reply_qinfo[i].num_replies = 0;  in mpi3mr_memset_buffers()
4248 		mrioc->op_reply_qinfo[i].ephase = 0;  in mpi3mr_memset_buffers()
4249 		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);  in mpi3mr_memset_buffers()
4250 		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);  in mpi3mr_memset_buffers()
4253 		mrioc->req_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4254 		mrioc->req_qinfo[i].pi = 0;  in mpi3mr_memset_buffers()
4255 		mrioc->req_qinfo[i].num_requests = 0;  in mpi3mr_memset_buffers()
4256 		mrioc->req_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4257 		mrioc->req_qinfo[i].reply_qid = 0;  in mpi3mr_memset_buffers()
4258 		spin_lock_init(&mrioc->req_qinfo[i].q_lock);  in mpi3mr_memset_buffers()
4262 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4263 	if (mrioc->throttle_groups) {  in mpi3mr_memset_buffers()
4264 		tg = mrioc->throttle_groups;  in mpi3mr_memset_buffers()
4265 		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
4266 			tg->id = 0;  in mpi3mr_memset_buffers()
4267 			tg->fw_qd = 0;  in mpi3mr_memset_buffers()
4268 			tg->modified_qd = 0;  in mpi3mr_memset_buffers()
4269 			tg->io_divert = 0;  in mpi3mr_memset_buffers()
4270 			tg->need_qd_reduction = 0;  in mpi3mr_memset_buffers()
4271 			tg->high = 0;  in mpi3mr_memset_buffers()
4272 			tg->low = 0;  in mpi3mr_memset_buffers()
4273 			tg->qd_reduction = 0;  in mpi3mr_memset_buffers()
4274 			atomic_set(&tg->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4280  * mpi3mr_free_mem - Free memory allocated for a controller
4294 	if (mrioc->sense_buf_pool) {  in mpi3mr_free_mem()
4295 		if (mrioc->sense_buf)  in mpi3mr_free_mem()
4296 			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,  in mpi3mr_free_mem()
4297 			    mrioc->sense_buf_dma);  in mpi3mr_free_mem()
4298 		dma_pool_destroy(mrioc->sense_buf_pool);  in mpi3mr_free_mem()
4299 		mrioc->sense_buf = NULL;  in mpi3mr_free_mem()
4300 		mrioc->sense_buf_pool = NULL;  in mpi3mr_free_mem()
4302 	if (mrioc->sense_buf_q_pool) {  in mpi3mr_free_mem()
4303 		if (mrioc->sense_buf_q)  in mpi3mr_free_mem()
4304 			dma_pool_free(mrioc->sense_buf_q_pool,  in mpi3mr_free_mem()
4305 			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);  in mpi3mr_free_mem()
4306 		dma_pool_destroy(mrioc->sense_buf_q_pool);  in mpi3mr_free_mem()
4307 		mrioc->sense_buf_q = NULL;  in mpi3mr_free_mem()
4308 		mrioc->sense_buf_q_pool = NULL;  in mpi3mr_free_mem()
4311 	if (mrioc->reply_buf_pool) {  in mpi3mr_free_mem()
4312 		if (mrioc->reply_buf)  in mpi3mr_free_mem()
4313 			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,  in mpi3mr_free_mem()
4314 			    mrioc->reply_buf_dma);  in mpi3mr_free_mem()
4315 		dma_pool_destroy(mrioc->reply_buf_pool);  in mpi3mr_free_mem()
4316 		mrioc->reply_buf = NULL;  in mpi3mr_free_mem()
4317 		mrioc->reply_buf_pool = NULL;  in mpi3mr_free_mem()
4319 	if (mrioc->reply_free_q_pool) {  in mpi3mr_free_mem()
4320 		if (mrioc->reply_free_q)  in mpi3mr_free_mem()
4321 			dma_pool_free(mrioc->reply_free_q_pool,  in mpi3mr_free_mem()
4322 			    mrioc->reply_free_q, mrioc->reply_free_q_dma);  in mpi3mr_free_mem()
4323 		dma_pool_destroy(mrioc->reply_free_q_pool);  in mpi3mr_free_mem()
4324 		mrioc->reply_free_q = NULL;  in mpi3mr_free_mem()
4325 		mrioc->reply_free_q_pool = NULL;  in mpi3mr_free_mem()
4328 	for (i = 0; i < mrioc->num_op_req_q; i++)  in mpi3mr_free_mem()
4331 	for (i = 0; i < mrioc->num_op_reply_q; i++)  in mpi3mr_free_mem()
4334 	for (i = 0; i < mrioc->intr_info_count; i++) {  in mpi3mr_free_mem()
4335 		intr_info = mrioc->intr_info + i;  in mpi3mr_free_mem()
4336 		intr_info->op_reply_q = NULL;  in mpi3mr_free_mem()
4339 	kfree(mrioc->req_qinfo);  in mpi3mr_free_mem()
4340 	mrioc->req_qinfo = NULL;  in mpi3mr_free_mem()
4341 	mrioc->num_op_req_q = 0;  in mpi3mr_free_mem()
4343 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_free_mem()
4344 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_free_mem()
4345 	mrioc->num_op_reply_q = 0;  in mpi3mr_free_mem()
4347 	kfree(mrioc->init_cmds.reply);  in mpi3mr_free_mem()
4348 	mrioc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
4350 	kfree(mrioc->bsg_cmds.reply);  in mpi3mr_free_mem()
4351 	mrioc->bsg_cmds.reply = NULL;  in mpi3mr_free_mem()
4353 	kfree(mrioc->host_tm_cmds.reply);  in mpi3mr_free_mem()
4354 	mrioc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
4356 	kfree(mrioc->pel_cmds.reply);  in mpi3mr_free_mem()
4357 	mrioc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
4359 	kfree(mrioc->pel_abort_cmd.reply);  in mpi3mr_free_mem()
4360 	mrioc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
4363 		kfree(mrioc->evtack_cmds[i].reply);  in mpi3mr_free_mem()
4364 		mrioc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4367 	bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_free_mem()
4368 	mrioc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
4370 	bitmap_free(mrioc->devrem_bitmap);  in mpi3mr_free_mem()
4371 	mrioc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
4373 	bitmap_free(mrioc->evtack_cmds_bitmap);  in mpi3mr_free_mem()
4374 	mrioc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
4376 	bitmap_free(mrioc->chain_bitmap);  in mpi3mr_free_mem()
4377 	mrioc->chain_bitmap = NULL;  in mpi3mr_free_mem()
4379 	kfree(mrioc->transport_cmds.reply);  in mpi3mr_free_mem()
4380 	mrioc->transport_cmds.reply = NULL;  in mpi3mr_free_mem()
4383 		kfree(mrioc->dev_rmhs_cmds[i].reply);  in mpi3mr_free_mem()
4384 		mrioc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4387 	if (mrioc->chain_buf_pool) {  in mpi3mr_free_mem()
4388 		for (i = 0; i < mrioc->chain_buf_count; i++) {  in mpi3mr_free_mem()
4389 			if (mrioc->chain_sgl_list[i].addr) {  in mpi3mr_free_mem()
4390 				dma_pool_free(mrioc->chain_buf_pool,  in mpi3mr_free_mem()
4391 				    mrioc->chain_sgl_list[i].addr,  in mpi3mr_free_mem()
4392 				    mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_free_mem()
4393 				mrioc->chain_sgl_list[i].addr = NULL;  in mpi3mr_free_mem()
4396 		dma_pool_destroy(mrioc->chain_buf_pool);  in mpi3mr_free_mem()
4397 		mrioc->chain_buf_pool = NULL;  in mpi3mr_free_mem()
4400 	kfree(mrioc->chain_sgl_list);  in mpi3mr_free_mem()
4401 	mrioc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
4403 	if (mrioc->admin_reply_base) {  in mpi3mr_free_mem()
4404 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_free_mem()
4405 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_free_mem()
4406 		mrioc->admin_reply_base = NULL;  in mpi3mr_free_mem()
4408 	if (mrioc->admin_req_base) {  in mpi3mr_free_mem()
4409 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_free_mem()
4410 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_free_mem()
4411 		mrioc->admin_req_base = NULL;  in mpi3mr_free_mem()
4413 	if (mrioc->cfg_page) {  in mpi3mr_free_mem()
4414 		dma_free_coherent(&mrioc->pdev->dev, mrioc->cfg_page_sz,  in mpi3mr_free_mem()
4415 		    mrioc->cfg_page, mrioc->cfg_page_dma);  in mpi3mr_free_mem()
4416 		mrioc->cfg_page = NULL;  in mpi3mr_free_mem()
4418 	if (mrioc->pel_seqnum_virt) {  in mpi3mr_free_mem()
4419 		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,  in mpi3mr_free_mem()
4420 		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);  in mpi3mr_free_mem()
4421 		mrioc->pel_seqnum_virt = NULL;  in mpi3mr_free_mem()
4424 	kfree(mrioc->throttle_groups);  in mpi3mr_free_mem()
4425 	mrioc->throttle_groups = NULL;  in mpi3mr_free_mem()
4427 	kfree(mrioc->logdata_buf);  in mpi3mr_free_mem()
4428 	mrioc->logdata_buf = NULL;  in mpi3mr_free_mem()
4433  * mpi3mr_issue_ioc_shutdown - shutdown controller
4448 	if (mrioc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
4453 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4460 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4464 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4466 	if (mrioc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
4467 		timeout = mrioc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
4470 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4477 	} while (--timeout);  in mpi3mr_issue_ioc_shutdown()
4479 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4480 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4496  * mpi3mr_cleanup_ioc - Cleanup controller
4513 	if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&  in mpi3mr_cleanup_ioc()
4526  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4538 	if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_drv_cmd_comp_reset()
4539 		cmdptr->state |= MPI3MR_CMD_RESET;  in mpi3mr_drv_cmd_comp_reset()
4540 		cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_drv_cmd_comp_reset()
4541 		if (cmdptr->is_waiting) {  in mpi3mr_drv_cmd_comp_reset()
4542 			complete(&cmdptr->done);  in mpi3mr_drv_cmd_comp_reset()
4543 			cmdptr->is_waiting = 0;  in mpi3mr_drv_cmd_comp_reset()
4544 		} else if (cmdptr->callback)  in mpi3mr_drv_cmd_comp_reset()
4545 			cmdptr->callback(mrioc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
4550  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4562 	cmdptr = &mrioc->init_cmds;  in mpi3mr_flush_drv_cmds()
4565 	cmdptr = &mrioc->cfg_cmds;  in mpi3mr_flush_drv_cmds()
4568 	cmdptr = &mrioc->bsg_cmds;  in mpi3mr_flush_drv_cmds()
4570 	cmdptr = &mrioc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
4574 		cmdptr = &mrioc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
4579 		cmdptr = &mrioc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
4583 	cmdptr = &mrioc->pel_cmds;  in mpi3mr_flush_drv_cmds()
4586 	cmdptr = &mrioc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
4589 	cmdptr = &mrioc->transport_cmds;  in mpi3mr_flush_drv_cmds()
4594  * mpi3mr_pel_wait_post - Issue PEL Wait
4607 	mrioc->pel_abort_requested = false;  in mpi3mr_pel_wait_post()
4610 	drv_cmd->state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_wait_post()
4611 	drv_cmd->is_waiting = 0;  in mpi3mr_pel_wait_post()
4612 	drv_cmd->callback = mpi3mr_pel_wait_complete;  in mpi3mr_pel_wait_post()
4613 	drv_cmd->ioc_status = 0;  in mpi3mr_pel_wait_post()
4614 	drv_cmd->ioc_loginfo = 0;  in mpi3mr_pel_wait_post()
4618 	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);  in mpi3mr_pel_wait_post()
4619 	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);  in mpi3mr_pel_wait_post()
4620 	pel_wait.class = cpu_to_le16(mrioc->pel_class);  in mpi3mr_pel_wait_post()
4623 	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);  in mpi3mr_pel_wait_post()
4628 		drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_post()
4629 		drv_cmd->callback = NULL;  in mpi3mr_pel_wait_post()
4630 		drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_post()
4631 		mrioc->pel_enabled = false;  in mpi3mr_pel_wait_post()
4636  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4643  * Return: 0 on success, non-zero on failure.
4653 	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_get_seqnum_post()
4654 	mrioc->pel_cmds.is_waiting = 0;  in mpi3mr_pel_get_seqnum_post()
4655 	mrioc->pel_cmds.ioc_status = 0;  in mpi3mr_pel_get_seqnum_post()
4656 	mrioc->pel_cmds.ioc_loginfo = 0;  in mpi3mr_pel_get_seqnum_post()
4657 	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;  in mpi3mr_pel_get_seqnum_post()
4662 	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);  in mpi3mr_pel_get_seqnum_post()
4668 			drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_post()
4669 			drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_post()
4670 			drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_post()
4672 		mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_post()
4679  * mpi3mr_pel_wait_complete - PELWait Completion callback
4698 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_wait_complete()
4701 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_wait_complete()
4704 			__func__, ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
4707 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
4711 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_wait_complete()
4712 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_wait_complete()
4720 	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);  in mpi3mr_pel_wait_complete()
4732 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_wait_complete()
4733 			drv_cmd->retry_count++;  in mpi3mr_pel_wait_complete()
4735 			    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
4741 		    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
4745 	if (!mrioc->pel_abort_requested) {  in mpi3mr_pel_wait_complete()
4746 		mrioc->pel_cmds.retry_count = 0;  in mpi3mr_pel_wait_complete()
4747 		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_pel_wait_complete()
4752 	mrioc->pel_enabled = false;  in mpi3mr_pel_wait_complete()
4754 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_complete()
4755 	drv_cmd->callback = NULL;  in mpi3mr_pel_wait_complete()
4756 	drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_complete()
4760  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4778 	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;  in mpi3mr_pel_get_seqnum_complete()
4780 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_get_seqnum_complete()
4783 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_get_seqnum_complete()
4787 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_get_seqnum_complete()
4791 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_get_seqnum_complete()
4792 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_get_seqnum_complete()
4799 	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {  in mpi3mr_pel_get_seqnum_complete()
4802 		    le16_to_cpu(pel_reply->pe_log_status));  in mpi3mr_pel_get_seqnum_complete()
4807 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_get_seqnum_complete()
4808 			drv_cmd->retry_count++;  in mpi3mr_pel_get_seqnum_complete()
4811 			    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
4818 		    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
4821 	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;  in mpi3mr_pel_get_seqnum_complete()
4822 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
4827 	mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_complete()
4829 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_complete()
4830 	drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_complete()
4831 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
4835  * mpi3mr_soft_reset_handler - Reset the controller
4853  * Return: 0 on success, non-zero on failure.
4865 	    mrioc->diagsave_timeout);  in mpi3mr_soft_reset_handler()
4866 	while (mrioc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
4873 	if (!mutex_trylock(&mrioc->reset_mutex)) {  in mpi3mr_soft_reset_handler()
4879 		} while (mrioc->reset_in_progress == 1);  in mpi3mr_soft_reset_handler()
4882 		    mrioc->prev_reset_result,  in mpi3mr_soft_reset_handler()
4884 		return mrioc->prev_reset_result;  in mpi3mr_soft_reset_handler()
4889 	mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
4890 	mrioc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
4891 	mrioc->stop_bsgs = 1;  in mpi3mr_soft_reset_handler()
4892 	mrioc->prev_reset_result = -1;  in mpi3mr_soft_reset_handler()
4898 			mrioc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
4915 				    readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_soft_reset_handler()
4920 			} while (--timeout);  in mpi3mr_soft_reset_handler()
4930 	if (mrioc->num_io_throttle_group !=  in mpi3mr_soft_reset_handler()
4931 	    mrioc->facts.max_io_throttle_group) {  in mpi3mr_soft_reset_handler()
4934 		    mrioc->num_io_throttle_group,  in mpi3mr_soft_reset_handler()
4935 		    mrioc->facts.max_io_throttle_group);  in mpi3mr_soft_reset_handler()
4936 		retval = -EPERM;  in mpi3mr_soft_reset_handler()
4942 	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_soft_reset_handler()
4943 	bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_soft_reset_handler()
4944 		     mrioc->dev_handle_bitmap_bits);  in mpi3mr_soft_reset_handler()
4945 	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);  in mpi3mr_soft_reset_handler()
4951 	if (mrioc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
4952 		mrioc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
4953 		mrioc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
4959 		    mrioc->name, reset_reason);  in mpi3mr_soft_reset_handler()
4966 		mrioc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
4967 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
4968 		mrioc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
4969 		if (mrioc->pel_enabled) {  in mpi3mr_soft_reset_handler()
4970 			mrioc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
4971 			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_soft_reset_handler()
4974 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
4976 		mrioc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
4977 		spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
4978 		if (mrioc->watchdog_work_q)  in mpi3mr_soft_reset_handler()
4979 			queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_soft_reset_handler()
4980 			    &mrioc->watchdog_work,  in mpi3mr_soft_reset_handler()
4982 		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
4983 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
4984 		if (mrioc->pel_enabled)  in mpi3mr_soft_reset_handler()
4989 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
4990 		mrioc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
4991 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
4992 		retval = -1;  in mpi3mr_soft_reset_handler()
4995 	mrioc->prev_reset_result = retval;  in mpi3mr_soft_reset_handler()
4996 	mutex_unlock(&mrioc->reset_mutex);  in mpi3mr_soft_reset_handler()
5004  * mpi3mr_free_config_dma_memory - free memory for config page
5017 	if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {  in mpi3mr_free_config_dma_memory()
5018 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_config_dma_memory()
5019 		    mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_config_dma_memory()
5020 		mem_desc->addr = NULL;  in mpi3mr_free_config_dma_memory()
5025  * mpi3mr_alloc_config_dma_memory - Alloc memory for config page
5033  * Return: 0 on success, non-zero on failure.
5038 	if (mem_desc->size > mrioc->cfg_page_sz) {  in mpi3mr_alloc_config_dma_memory()
5039 		mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_config_dma_memory()
5040 		    mem_desc->size, &mem_desc->dma_addr, GFP_KERNEL);  in mpi3mr_alloc_config_dma_memory()
5041 		if (!mem_desc->addr)  in mpi3mr_alloc_config_dma_memory()
5042 			return -ENOMEM;  in mpi3mr_alloc_config_dma_memory()
5044 		mem_desc->addr = mrioc->cfg_page;  in mpi3mr_alloc_config_dma_memory()
5045 		mem_desc->dma_addr = mrioc->cfg_page_dma;  in mpi3mr_alloc_config_dma_memory()
5046 		memset(mem_desc->addr, 0, mrioc->cfg_page_sz);  in mpi3mr_alloc_config_dma_memory()
5052  * mpi3mr_post_cfg_req - Issue config requests and wait
5066  * Return: 0 on success, non-zero on failure.
5073 	mutex_lock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5074 	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_post_cfg_req()
5075 		retval = -1;  in mpi3mr_post_cfg_req()
5077 		mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5080 	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_post_cfg_req()
5081 	mrioc->cfg_cmds.is_waiting = 1;  in mpi3mr_post_cfg_req()
5082 	mrioc->cfg_cmds.callback = NULL;  in mpi3mr_post_cfg_req()
5083 	mrioc->cfg_cmds.ioc_status = 0;  in mpi3mr_post_cfg_req()
5084 	mrioc->cfg_cmds.ioc_loginfo = 0;  in mpi3mr_post_cfg_req()
5086 	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);  in mpi3mr_post_cfg_req()
5087 	cfg_req->function = MPI3_FUNCTION_CONFIG;  in mpi3mr_post_cfg_req()
5089 	init_completion(&mrioc->cfg_cmds.done);  in mpi3mr_post_cfg_req()
5091 	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_post_cfg_req()
5099 	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));  in mpi3mr_post_cfg_req()
5100 	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_post_cfg_req()
5104 		retval = -1;  in mpi3mr_post_cfg_req()
5107 	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_post_cfg_req()
5111 		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);  in mpi3mr_post_cfg_req()
5114 	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_post_cfg_req()
5115 	mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5122  * mpi3mr_process_cfg_req - config page request processor
5155  * Return: 0 on success, non-zero on failure.
5163 	int retval = -1;  in mpi3mr_process_cfg_req()
5169 	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)  in mpi3mr_process_cfg_req()
5174 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5175 			    cfg_req->page_number);  in mpi3mr_process_cfg_req()
5178 		switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {  in mpi3mr_process_cfg_req()
5180 			if (cfg_req->action  in mpi3mr_process_cfg_req()
5185 			if ((cfg_req->action ==  in mpi3mr_process_cfg_req()
5187 			    (cfg_req->action ==  in mpi3mr_process_cfg_req()
5198 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5199 			    cfg_req->page_number, cfg_hdr->page_attribute);  in mpi3mr_process_cfg_req()
5202 		mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;  in mpi3mr_process_cfg_req()
5203 		cfg_req->page_length = cfg_hdr->page_length;  in mpi3mr_process_cfg_req()
5204 		cfg_req->page_version = cfg_hdr->page_version;  in mpi3mr_process_cfg_req()
5209 	mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,  in mpi3mr_process_cfg_req()
5212 	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||  in mpi3mr_process_cfg_req()
5213 	    (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5217 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5226 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&  in mpi3mr_process_cfg_req()
5227 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5231 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5241  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5255  * Return: 0 on success, non-zero on failure.
5295 	return -1;  in mpi3mr_cfg_get_dev_pg0()
5300  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5314  * Return: 0 on success, non-zero on failure.
5355 	return -1;  in mpi3mr_cfg_get_sas_phy_pg0()
5359  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5373  * Return: 0 on success, non-zero on failure.
5414 	return -1;  in mpi3mr_cfg_get_sas_phy_pg1()
5419  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5433  * Return: 0 on success, non-zero on failure.
5475 	return -1;  in mpi3mr_cfg_get_sas_exp_pg0()
5479  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5493  * Return: 0 on success, non-zero on failure.
5535 	return -1;  in mpi3mr_cfg_get_sas_exp_pg1()
5539  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5553  * Return: 0 on success, non-zero on failure.
5594 	return -1;  in mpi3mr_cfg_get_enclosure_pg0()
5599  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5608  * Return: 0 on success, non-zero on failure.
5651 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg0()
5655  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
5664  * Return: 0 on success, non-zero on failure.
5707 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg1()
5711  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
5721  * Return: 0 on success, non-zero on failure.
5776 	return -1;  in mpi3mr_cfg_set_sas_io_unit_pg1()
5780  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
5789  * Return: 0 on success, non-zero on failure.
5832 	return -1;  in mpi3mr_cfg_get_driver_pg1()