45 	struct nvme_ctrl	ctrl;  member
51 static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)  in to_loop_ctrl()  argument
53 	return container_of(ctrl, struct nvme_loop_ctrl, ctrl);  in to_loop_ctrl()
63 	struct nvme_loop_ctrl	*ctrl;  member
74 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
80 	return queue - queue->ctrl->queues;  in nvme_loop_queue_idx()
97 		return queue->ctrl->admin_tag_set.tags[queue_idx];  in nvme_loop_tagset()
98 	return queue->ctrl->tag_set.tags[queue_idx - 1];  in nvme_loop_tagset()
115 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,  in nvme_loop_queue_response()
122 			dev_err(queue->ctrl->ctrl.device,  in nvme_loop_queue_response()
146 	nvme_reset_ctrl(&iod->queue->ctrl->ctrl);  in nvme_loop_timeout()
164 	if (!nvmf_check_ready(&queue->ctrl->ctrl, req, queue_ready))  in nvme_loop_queue_rq()
165 		return nvmf_fail_nonready_command(&queue->ctrl->ctrl, req);  in nvme_loop_queue_rq()
173 	iod->req.port = queue->ctrl->port;  in nvme_loop_queue_rq()
196 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);  in nvme_loop_submit_async_event()  local
197 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_submit_async_event()
198 	struct nvme_loop_iod *iod = &ctrl->async_event_iod;  in nvme_loop_submit_async_event()
207 		dev_err(ctrl->ctrl.device, "failed async event work\n");  in nvme_loop_submit_async_event()
214 static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,  in nvme_loop_init_iod()  argument
219 	iod->queue = &ctrl->queues[queue_idx];  in nvme_loop_init_iod()
228 	struct nvme_loop_ctrl *ctrl = set->driver_data;  in nvme_loop_init_request()  local
230 	nvme_req(req)->ctrl = &ctrl->ctrl;  in nvme_loop_init_request()
231 	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),  in nvme_loop_init_request()
232 			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);  in nvme_loop_init_request()
238 	struct nvme_loop_ctrl *ctrl = data;  in nvme_loop_init_hctx()  local
239 	struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];  in nvme_loop_init_hctx()
241 	BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);  in nvme_loop_init_hctx()
250 	struct nvme_loop_ctrl *ctrl = data;  in nvme_loop_init_admin_hctx()  local
251 	struct nvme_loop_queue *queue = &ctrl->queues[0];  in nvme_loop_init_admin_hctx()
275 static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_admin_queue()  argument
277 	clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_destroy_admin_queue()
278 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_destroy_admin_queue()
279 	blk_cleanup_queue(ctrl->ctrl.admin_q);  in nvme_loop_destroy_admin_queue()
280 	blk_mq_free_tag_set(&ctrl->admin_tag_set);  in nvme_loop_destroy_admin_queue()
285 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);  in nvme_loop_free_ctrl()  local
287 	if (list_empty(&ctrl->list))  in nvme_loop_free_ctrl()
291 	list_del(&ctrl->list);  in nvme_loop_free_ctrl()
295 		blk_cleanup_queue(ctrl->ctrl.connect_q);  in nvme_loop_free_ctrl()
296 		blk_mq_free_tag_set(&ctrl->tag_set);  in nvme_loop_free_ctrl()
298 	kfree(ctrl->queues);  in nvme_loop_free_ctrl()
301 	kfree(ctrl);  in nvme_loop_free_ctrl()
304 static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_destroy_io_queues()  argument
308 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_destroy_io_queues()
309 		clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_destroy_io_queues()
310 		nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);  in nvme_loop_destroy_io_queues()
314 static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_init_io_queues()  argument
316 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;  in nvme_loop_init_io_queues()
321 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);  in nvme_loop_init_io_queues()
325 	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);  in nvme_loop_init_io_queues()
328 		ctrl->queues[i].ctrl = ctrl;  in nvme_loop_init_io_queues()
329 		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);  in nvme_loop_init_io_queues()
333 		ctrl->ctrl.queue_count++;  in nvme_loop_init_io_queues()
339 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_init_io_queues()
343 static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_connect_io_queues()  argument
347 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {  in nvme_loop_connect_io_queues()
348 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);  in nvme_loop_connect_io_queues()
351 		set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);  in nvme_loop_connect_io_queues()
357 static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)  in nvme_loop_configure_admin_queue()  argument
361 	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));  in nvme_loop_configure_admin_queue()
362 	ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;  in nvme_loop_configure_admin_queue()
363 	ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;  in nvme_loop_configure_admin_queue()
364 	ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */  in nvme_loop_configure_admin_queue()
365 	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;  in nvme_loop_configure_admin_queue()
366 	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +  in nvme_loop_configure_admin_queue()
368 	ctrl->admin_tag_set.driver_data = ctrl;  in nvme_loop_configure_admin_queue()
369 	ctrl->admin_tag_set.nr_hw_queues = 1;  in nvme_loop_configure_admin_queue()
370 	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;  in nvme_loop_configure_admin_queue()
371 	ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;  in nvme_loop_configure_admin_queue()
373 	ctrl->queues[0].ctrl = ctrl;  in nvme_loop_configure_admin_queue()
374 	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
377 	ctrl->ctrl.queue_count = 1;  in nvme_loop_configure_admin_queue()
379 	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
382 	ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;  in nvme_loop_configure_admin_queue()
384 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
385 	if (IS_ERR(ctrl->ctrl.admin_q)) {  in nvme_loop_configure_admin_queue()
386 		error = PTR_ERR(ctrl->ctrl.admin_q);  in nvme_loop_configure_admin_queue()
390 	error = nvmf_connect_admin_queue(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
394 	set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);  in nvme_loop_configure_admin_queue()
396 	error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap);  in nvme_loop_configure_admin_queue()
398 		dev_err(ctrl->ctrl.device,  in nvme_loop_configure_admin_queue()
403 	ctrl->ctrl.sqsize =  in nvme_loop_configure_admin_queue()
404 		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);  in nvme_loop_configure_admin_queue()
406 	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);  in nvme_loop_configure_admin_queue()
410 	ctrl->ctrl.max_hw_sectors =  in nvme_loop_configure_admin_queue()
413 	error = nvme_init_identify(&ctrl->ctrl);  in nvme_loop_configure_admin_queue()
420 	blk_cleanup_queue(ctrl->ctrl.admin_q);  in nvme_loop_configure_admin_queue()
422 	blk_mq_free_tag_set(&ctrl->admin_tag_set);  in nvme_loop_configure_admin_queue()
424 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);  in nvme_loop_configure_admin_queue()
428 static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)  in nvme_loop_shutdown_ctrl()  argument
430 	if (ctrl->ctrl.queue_count > 1) {  in nvme_loop_shutdown_ctrl()
431 		nvme_stop_queues(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
432 		blk_mq_tagset_busy_iter(&ctrl->tag_set,  in nvme_loop_shutdown_ctrl()
433 					nvme_cancel_request, &ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
434 		nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_shutdown_ctrl()
437 	if (ctrl->ctrl.state == NVME_CTRL_LIVE)  in nvme_loop_shutdown_ctrl()
438 		nvme_shutdown_ctrl(&ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
440 	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);  in nvme_loop_shutdown_ctrl()
441 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,  in nvme_loop_shutdown_ctrl()
442 				nvme_cancel_request, &ctrl->ctrl);  in nvme_loop_shutdown_ctrl()
443 	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);  in nvme_loop_shutdown_ctrl()
444 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_shutdown_ctrl()
447 static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)  in nvme_loop_delete_ctrl_host()  argument
449 	nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));  in nvme_loop_delete_ctrl_host()
454 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_delete_ctrl()  local
457 	list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {  in nvme_loop_delete_ctrl()
458 		if (ctrl->ctrl.cntlid == nctrl->cntlid)  in nvme_loop_delete_ctrl()
459 			nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_delete_ctrl()
466 	struct nvme_loop_ctrl *ctrl =  in nvme_loop_reset_ctrl_work()  local
467 		container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);  in nvme_loop_reset_ctrl_work()
471 	nvme_stop_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
472 	nvme_loop_shutdown_ctrl(ctrl);  in nvme_loop_reset_ctrl_work()
474 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {  in nvme_loop_reset_ctrl_work()
480 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
484 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
488 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
492 	blk_mq_update_nr_hw_queues(&ctrl->tag_set,  in nvme_loop_reset_ctrl_work()
493 			ctrl->ctrl.queue_count - 1);  in nvme_loop_reset_ctrl_work()
495 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);  in nvme_loop_reset_ctrl_work()
498 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
503 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_reset_ctrl_work()
505 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_reset_ctrl_work()
507 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");  in nvme_loop_reset_ctrl_work()
508 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
509 	nvme_put_ctrl(&ctrl->ctrl);  in nvme_loop_reset_ctrl_work()
525 static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)  in nvme_loop_create_io_queues()  argument
529 	ret = nvme_loop_init_io_queues(ctrl);  in nvme_loop_create_io_queues()
533 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));  in nvme_loop_create_io_queues()
534 	ctrl->tag_set.ops = &nvme_loop_mq_ops;  in nvme_loop_create_io_queues()
535 	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;  in nvme_loop_create_io_queues()
536 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */  in nvme_loop_create_io_queues()
537 	ctrl->tag_set.numa_node = NUMA_NO_NODE;  in nvme_loop_create_io_queues()
538 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in nvme_loop_create_io_queues()
539 	ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +  in nvme_loop_create_io_queues()
541 	ctrl->tag_set.driver_data = ctrl;  in nvme_loop_create_io_queues()
542 	ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;  in nvme_loop_create_io_queues()
543 	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;  in nvme_loop_create_io_queues()
544 	ctrl->ctrl.tagset = &ctrl->tag_set;  in nvme_loop_create_io_queues()
546 	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);  in nvme_loop_create_io_queues()
550 	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);  in nvme_loop_create_io_queues()
551 	if (IS_ERR(ctrl->ctrl.connect_q)) {  in nvme_loop_create_io_queues()
552 		ret = PTR_ERR(ctrl->ctrl.connect_q);  in nvme_loop_create_io_queues()
556 	ret = nvme_loop_connect_io_queues(ctrl);  in nvme_loop_create_io_queues()
563 	blk_cleanup_queue(ctrl->ctrl.connect_q);  in nvme_loop_create_io_queues()
565 	blk_mq_free_tag_set(&ctrl->tag_set);  in nvme_loop_create_io_queues()
567 	nvme_loop_destroy_io_queues(ctrl);  in nvme_loop_create_io_queues()
571 static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)  in nvme_loop_find_port()  argument
578 		if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&  in nvme_loop_find_port()
579 		    strcmp(ctrl->opts->traddr, p->disc_addr.traddr))  in nvme_loop_find_port()
591 	struct nvme_loop_ctrl *ctrl;  in nvme_loop_create_ctrl()  local
595 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in nvme_loop_create_ctrl()
596 	if (!ctrl)  in nvme_loop_create_ctrl()
598 	ctrl->ctrl.opts = opts;  in nvme_loop_create_ctrl()
599 	INIT_LIST_HEAD(&ctrl->list);  in nvme_loop_create_ctrl()
601 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);  in nvme_loop_create_ctrl()
603 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,  in nvme_loop_create_ctrl()
610 	ctrl->ctrl.sqsize = opts->queue_size - 1;  in nvme_loop_create_ctrl()
611 	ctrl->ctrl.kato = opts->kato;  in nvme_loop_create_ctrl()
612 	ctrl->port = nvme_loop_find_port(&ctrl->ctrl);  in nvme_loop_create_ctrl()
614 	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),  in nvme_loop_create_ctrl()
616 	if (!ctrl->queues)  in nvme_loop_create_ctrl()
619 	ret = nvme_loop_configure_admin_queue(ctrl);  in nvme_loop_create_ctrl()
623 	if (opts->queue_size > ctrl->ctrl.maxcmd) {  in nvme_loop_create_ctrl()
625 		dev_warn(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
626 			"queue_size %zu > ctrl maxcmd %u, clamping down\n",  in nvme_loop_create_ctrl()
627 			opts->queue_size, ctrl->ctrl.maxcmd);  in nvme_loop_create_ctrl()
628 		opts->queue_size = ctrl->ctrl.maxcmd;  in nvme_loop_create_ctrl()
632 		ret = nvme_loop_create_io_queues(ctrl);  in nvme_loop_create_ctrl()
637 	nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);  in nvme_loop_create_ctrl()
639 	dev_info(ctrl->ctrl.device,  in nvme_loop_create_ctrl()
640 		 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);  in nvme_loop_create_ctrl()
642 	nvme_get_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
644 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);  in nvme_loop_create_ctrl()
648 	list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);  in nvme_loop_create_ctrl()
651 	nvme_start_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
653 	return &ctrl->ctrl;  in nvme_loop_create_ctrl()
656 	nvme_loop_destroy_admin_queue(ctrl);  in nvme_loop_create_ctrl()
658 	kfree(ctrl->queues);  in nvme_loop_create_ctrl()
660 	nvme_uninit_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
662 	nvme_put_ctrl(&ctrl->ctrl);  in nvme_loop_create_ctrl()
724 	struct nvme_loop_ctrl *ctrl, *next;  in nvme_loop_cleanup_module()  local
730 	list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)  in nvme_loop_cleanup_module()
731 		nvme_delete_ctrl(&ctrl->ctrl);  in nvme_loop_cleanup_module()