Lines Matching +full:combined +full:- +full:power +full:- +full:req
1 // SPDX-License-Identifier: GPL-2.0
7 * Copyright (c) 2011-2014, Intel Corporation.
9 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
14 #include <linux/blk-mq.h>
16 #include <linux/dma-mapping.h>
19 #include <linux/io-64-nonatomic-lo-hi.h>
69 * cannot be higher than 0x40 which effectively limits the combined
76 #define APPLE_NVME_AQ_MQ_TAG_DEPTH (APPLE_NVME_AQ_DEPTH - 1)
91 * An exception to this are Apple's vendor-specific commands (opcode 0xD8 on the
159 struct nvme_request req; member
210 if (q->is_adminq) in queue_to_apple_nvme()
218 if (q->is_adminq) in apple_nvme_queue_depth()
228 dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); in apple_nvme_rtkit_crashed()
229 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_rtkit_crashed()
238 if (bfr->iova) in apple_nvme_sart_dma_setup()
239 return -EINVAL; in apple_nvme_sart_dma_setup()
240 if (!bfr->size) in apple_nvme_sart_dma_setup()
241 return -EINVAL; in apple_nvme_sart_dma_setup()
243 bfr->buffer = in apple_nvme_sart_dma_setup()
244 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); in apple_nvme_sart_dma_setup()
245 if (!bfr->buffer) in apple_nvme_sart_dma_setup()
246 return -ENOMEM; in apple_nvme_sart_dma_setup()
248 ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_setup()
250 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_setup()
251 bfr->buffer = NULL; in apple_nvme_sart_dma_setup()
252 return -ENOMEM; in apple_nvme_sart_dma_setup()
263 apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_destroy()
264 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_destroy()
277 writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL); in apple_nvmmu_inval()
278 if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT)) in apple_nvmmu_inval()
279 dev_warn_ratelimited(anv->dev, in apple_nvmmu_inval()
287 u32 tag = nvme_tag_from_cid(cmd->common.command_id); in apple_nvme_submit_cmd()
288 struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; in apple_nvme_submit_cmd()
290 tcb->opcode = cmd->common.opcode; in apple_nvme_submit_cmd()
291 tcb->prp1 = cmd->common.dptr.prp1; in apple_nvme_submit_cmd()
292 tcb->prp2 = cmd->common.dptr.prp2; in apple_nvme_submit_cmd()
293 tcb->length = cmd->rw.length; in apple_nvme_submit_cmd()
294 tcb->command_id = tag; in apple_nvme_submit_cmd()
297 tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; in apple_nvme_submit_cmd()
299 tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; in apple_nvme_submit_cmd()
301 memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); in apple_nvme_submit_cmd()
313 spin_lock_irq(&anv->lock); in apple_nvme_submit_cmd()
314 writel(tag, q->sq_db); in apple_nvme_submit_cmd()
315 spin_unlock_irq(&anv->lock); in apple_nvme_submit_cmd()
328 const int npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); in apple_nvme_iod_alloc_size()
335 static void **apple_nvme_iod_list(struct request *req) in apple_nvme_iod_list() argument
337 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_iod_list()
339 return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); in apple_nvme_iod_list()
342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req) in apple_nvme_free_prps() argument
344 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; in apple_nvme_free_prps()
345 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_free_prps()
346 dma_addr_t dma_addr = iod->first_dma; in apple_nvme_free_prps()
349 for (i = 0; i < iod->npages; i++) { in apple_nvme_free_prps()
350 __le64 *prp_list = apple_nvme_iod_list(req)[i]; in apple_nvme_free_prps()
353 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr); in apple_nvme_free_prps()
358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req) in apple_nvme_unmap_data() argument
360 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_unmap_data()
362 if (iod->dma_len) { in apple_nvme_unmap_data()
363 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len, in apple_nvme_unmap_data()
364 rq_dma_dir(req)); in apple_nvme_unmap_data()
368 WARN_ON_ONCE(!iod->nents); in apple_nvme_unmap_data()
370 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_unmap_data()
371 if (iod->npages == 0) in apple_nvme_unmap_data()
372 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0], in apple_nvme_unmap_data()
373 iod->first_dma); in apple_nvme_unmap_data()
375 apple_nvme_free_prps(anv, req); in apple_nvme_unmap_data()
376 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_unmap_data()
388 i, &phys, sg->offset, sg->length, &sg_dma_address(sg), in apple_nvme_print_sgl()
394 struct request *req, in apple_nvme_setup_prps() argument
397 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_setup_prps()
399 int length = blk_rq_payload_bytes(req); in apple_nvme_setup_prps()
400 struct scatterlist *sg = iod->sg; in apple_nvme_setup_prps()
403 int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1); in apple_nvme_setup_prps()
405 void **list = apple_nvme_iod_list(req); in apple_nvme_setup_prps()
409 length -= (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
411 iod->first_dma = 0; in apple_nvme_setup_prps()
415 dma_len -= (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
417 dma_addr += (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
425 iod->first_dma = dma_addr; in apple_nvme_setup_prps()
431 pool = anv->prp_small_pool; in apple_nvme_setup_prps()
432 iod->npages = 0; in apple_nvme_setup_prps()
434 pool = anv->prp_page_pool; in apple_nvme_setup_prps()
435 iod->npages = 1; in apple_nvme_setup_prps()
440 iod->first_dma = dma_addr; in apple_nvme_setup_prps()
441 iod->npages = -1; in apple_nvme_setup_prps()
445 iod->first_dma = prp_dma; in apple_nvme_setup_prps()
454 list[iod->npages++] = prp_list; in apple_nvme_setup_prps()
455 prp_list[0] = old_prp_list[i - 1]; in apple_nvme_setup_prps()
456 old_prp_list[i - 1] = cpu_to_le64(prp_dma); in apple_nvme_setup_prps()
460 dma_len -= NVME_CTRL_PAGE_SIZE; in apple_nvme_setup_prps()
462 length -= NVME_CTRL_PAGE_SIZE; in apple_nvme_setup_prps()
474 cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); in apple_nvme_setup_prps()
475 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma); in apple_nvme_setup_prps()
478 apple_nvme_free_prps(anv, req); in apple_nvme_setup_prps()
481 WARN(DO_ONCE(apple_nvme_print_sgl, iod->sg, iod->nents), in apple_nvme_setup_prps()
482 "Invalid SGL for payload:%d nents:%d\n", blk_rq_payload_bytes(req), in apple_nvme_setup_prps()
483 iod->nents); in apple_nvme_setup_prps()
488 struct request *req, in apple_nvme_setup_prp_simple() argument
492 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_setup_prp_simple()
493 unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1); in apple_nvme_setup_prp_simple()
494 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset; in apple_nvme_setup_prp_simple()
496 iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0); in apple_nvme_setup_prp_simple()
497 if (dma_mapping_error(anv->dev, iod->first_dma)) in apple_nvme_setup_prp_simple()
499 iod->dma_len = bv->bv_len; in apple_nvme_setup_prp_simple()
501 cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); in apple_nvme_setup_prp_simple()
502 if (bv->bv_len > first_prp_len) in apple_nvme_setup_prp_simple()
503 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); in apple_nvme_setup_prp_simple()
508 struct request *req, in apple_nvme_map_data() argument
511 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_map_data()
515 if (blk_rq_nr_phys_segments(req) == 1) { in apple_nvme_map_data()
516 struct bio_vec bv = req_bvec(req); in apple_nvme_map_data()
519 return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw, in apple_nvme_map_data()
523 iod->dma_len = 0; in apple_nvme_map_data()
524 iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC); in apple_nvme_map_data()
525 if (!iod->sg) in apple_nvme_map_data()
527 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); in apple_nvme_map_data()
528 iod->nents = blk_rq_map_sg(req->q, req, iod->sg); in apple_nvme_map_data()
529 if (!iod->nents) in apple_nvme_map_data()
532 nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents, in apple_nvme_map_data()
533 rq_dma_dir(req), DMA_ATTR_NO_WARN); in apple_nvme_map_data()
537 ret = apple_nvme_setup_prps(anv, req, &cmnd->rw); in apple_nvme_map_data()
543 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_map_data()
545 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_map_data()
549 static __always_inline void apple_nvme_unmap_rq(struct request *req) in apple_nvme_unmap_rq() argument
551 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_unmap_rq()
552 struct apple_nvme *anv = queue_to_apple_nvme(iod->q); in apple_nvme_unmap_rq()
554 if (blk_rq_nr_phys_segments(req)) in apple_nvme_unmap_rq()
555 apple_nvme_unmap_data(anv, req); in apple_nvme_unmap_rq()
558 static void apple_nvme_complete_rq(struct request *req) in apple_nvme_complete_rq() argument
560 apple_nvme_unmap_rq(req); in apple_nvme_complete_rq()
561 nvme_complete_rq(req); in apple_nvme_complete_rq()
571 struct nvme_completion *hcqe = &q->cqes[q->cq_head]; in apple_nvme_cqe_pending()
573 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == q->cq_phase; in apple_nvme_cqe_pending()
579 if (q->is_adminq) in apple_nvme_queue_tagset()
580 return anv->admin_tagset.tags[0]; in apple_nvme_queue_tagset()
582 return anv->tagset.tags[0]; in apple_nvme_queue_tagset()
589 struct nvme_completion *cqe = &q->cqes[idx]; in apple_nvme_handle_cqe()
590 __u16 command_id = READ_ONCE(cqe->command_id); in apple_nvme_handle_cqe()
591 struct request *req; in apple_nvme_handle_cqe() local
595 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); in apple_nvme_handle_cqe()
596 if (unlikely(!req)) { in apple_nvme_handle_cqe()
597 dev_warn(anv->dev, "invalid id %d completed", command_id); in apple_nvme_handle_cqe()
601 if (!nvme_try_complete_req(req, cqe->status, cqe->result) && in apple_nvme_handle_cqe()
602 !blk_mq_add_to_batch(req, iob, nvme_req(req)->status, in apple_nvme_handle_cqe()
604 apple_nvme_complete_rq(req); in apple_nvme_handle_cqe()
609 u32 tmp = q->cq_head + 1; in apple_nvme_update_cq_head()
612 q->cq_head = 0; in apple_nvme_update_cq_head()
613 q->cq_phase ^= 1; in apple_nvme_update_cq_head()
615 q->cq_head = tmp; in apple_nvme_update_cq_head()
628 * load-load control dependency between phase and the rest of in apple_nvme_poll_cq()
632 apple_nvme_handle_cqe(q, iob, q->cq_head); in apple_nvme_poll_cq()
637 writel(q->cq_head, q->cq_db); in apple_nvme_poll_cq()
647 if (!READ_ONCE(q->enabled) && !force) in apple_nvme_handle_cq()
664 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_irq()
665 if (apple_nvme_handle_cq(&anv->ioq, false)) in apple_nvme_irq()
667 if (apple_nvme_handle_cq(&anv->adminq, false)) in apple_nvme_irq()
669 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_irq()
685 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr); in apple_nvme_create_cq()
687 c.create_cq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); in apple_nvme_create_cq()
691 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_cq()
701 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_cq()
713 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr); in apple_nvme_create_sq()
715 c.create_sq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); in apple_nvme_create_sq()
719 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_sq()
729 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_sq()
735 struct nvme_ns *ns = hctx->queue->queuedata; in apple_nvme_queue_rq()
736 struct apple_nvme_queue *q = hctx->driver_data; in apple_nvme_queue_rq()
738 struct request *req = bd->rq; in apple_nvme_queue_rq() local
739 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_queue_rq()
740 struct nvme_command *cmnd = &iod->cmd; in apple_nvme_queue_rq()
743 iod->npages = -1; in apple_nvme_queue_rq()
744 iod->nents = 0; in apple_nvme_queue_rq()
750 if (unlikely(!READ_ONCE(q->enabled))) in apple_nvme_queue_rq()
753 if (!nvme_check_ready(&anv->ctrl, req, true)) in apple_nvme_queue_rq()
754 return nvme_fail_nonready_command(&anv->ctrl, req); in apple_nvme_queue_rq()
756 ret = nvme_setup_cmd(ns, req); in apple_nvme_queue_rq()
760 if (blk_rq_nr_phys_segments(req)) { in apple_nvme_queue_rq()
761 ret = apple_nvme_map_data(anv, req, cmnd); in apple_nvme_queue_rq()
766 nvme_start_request(req); in apple_nvme_queue_rq()
771 nvme_cleanup_cmd(req); in apple_nvme_queue_rq()
778 hctx->driver_data = data; in apple_nvme_init_hctx()
783 struct request *req, unsigned int hctx_idx, in apple_nvme_init_request() argument
786 struct apple_nvme_queue *q = set->driver_data; in apple_nvme_init_request()
788 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_init_request()
789 struct nvme_request *nreq = nvme_req(req); in apple_nvme_init_request()
791 iod->q = q; in apple_nvme_init_request()
792 nreq->ctrl = &anv->ctrl; in apple_nvme_init_request()
793 nreq->cmd = &iod->cmd; in apple_nvme_init_request()
800 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_disable()
804 if (apple_rtkit_is_crashed(anv->rtk)) in apple_nvme_disable()
811 if (anv->ctrl.state == NVME_CTRL_LIVE || in apple_nvme_disable()
812 anv->ctrl.state == NVME_CTRL_RESETTING) { in apple_nvme_disable()
814 nvme_start_freeze(&anv->ctrl); in apple_nvme_disable()
822 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT); in apple_nvme_disable()
824 nvme_quiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
827 if (READ_ONCE(anv->ioq.enabled)) { in apple_nvme_disable()
842 * Both U-Boot and m1n1 also use this convention (i.e. an ANS in apple_nvme_disable()
847 nvme_disable_ctrl(&anv->ctrl, shutdown); in apple_nvme_disable()
848 nvme_disable_ctrl(&anv->ctrl, false); in apple_nvme_disable()
851 WRITE_ONCE(anv->ioq.enabled, false); in apple_nvme_disable()
852 WRITE_ONCE(anv->adminq.enabled, false); in apple_nvme_disable()
854 nvme_quiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
857 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_disable()
858 apple_nvme_handle_cq(&anv->ioq, true); in apple_nvme_disable()
859 apple_nvme_handle_cq(&anv->adminq, true); in apple_nvme_disable()
860 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_disable()
862 nvme_cancel_tagset(&anv->ctrl); in apple_nvme_disable()
863 nvme_cancel_admin_tagset(&anv->ctrl); in apple_nvme_disable()
868 * deadlocking blk-mq hot-cpu notifier. in apple_nvme_disable()
871 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
872 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
876 static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) in apple_nvme_timeout() argument
878 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_timeout()
879 struct apple_nvme_queue *q = iod->q; in apple_nvme_timeout()
882 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_timeout()
884 if (anv->ctrl.state != NVME_CTRL_LIVE) { in apple_nvme_timeout()
890 * - ctrl disable/shutdown fabrics requests in apple_nvme_timeout()
891 * - connect requests in apple_nvme_timeout()
892 * - initialization admin requests in apple_nvme_timeout()
893 * - I/O requests that entered after unquiescing and in apple_nvme_timeout()
899 dev_warn(anv->dev, in apple_nvme_timeout()
901 req->tag, q->is_adminq); in apple_nvme_timeout()
902 if (blk_mq_request_started(req) && in apple_nvme_timeout()
903 !blk_mq_request_completed(req)) { in apple_nvme_timeout()
904 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; in apple_nvme_timeout()
905 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in apple_nvme_timeout()
906 blk_mq_complete_request(req); in apple_nvme_timeout()
912 if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { in apple_nvme_timeout()
913 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_timeout()
915 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_timeout()
916 if (blk_mq_request_completed(req)) { in apple_nvme_timeout()
917 dev_warn(anv->dev, in apple_nvme_timeout()
919 req->tag, q->is_adminq); in apple_nvme_timeout()
928 dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n", in apple_nvme_timeout()
929 req->tag, q->is_adminq); in apple_nvme_timeout()
930 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in apple_nvme_timeout()
932 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_timeout()
939 struct apple_nvme_queue *q = hctx->driver_data; in apple_nvme_poll()
944 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_poll()
946 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_poll()
972 q->cq_head = 0; in apple_nvme_init_queue()
973 q->cq_phase = 1; in apple_nvme_init_queue()
974 memset(q->tcbs, 0, in apple_nvme_init_queue()
976 memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); in apple_nvme_init_queue()
977 WRITE_ONCE(q->enabled, true); in apple_nvme_init_queue()
989 if (anv->ctrl.state != NVME_CTRL_RESETTING) { in apple_nvme_reset_work()
990 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", in apple_nvme_reset_work()
991 anv->ctrl.state); in apple_nvme_reset_work()
992 ret = -ENODEV; in apple_nvme_reset_work()
997 if (apple_rtkit_is_crashed(anv->rtk)) { in apple_nvme_reset_work()
998 dev_err(anv->dev, in apple_nvme_reset_work()
1000 ret = -EIO; in apple_nvme_reset_work()
1004 /* RTKit must be shut down cleanly for the (soft)-reset to work */ in apple_nvme_reset_work()
1005 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_reset_work()
1007 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) in apple_nvme_reset_work()
1009 dev_dbg(anv->dev, "Trying to shut down RTKit before reset."); in apple_nvme_reset_work()
1010 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_reset_work()
1015 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1017 ret = reset_control_assert(anv->reset); in apple_nvme_reset_work()
1021 ret = apple_rtkit_reinit(anv->rtk); in apple_nvme_reset_work()
1025 ret = reset_control_deassert(anv->reset); in apple_nvme_reset_work()
1030 anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1031 ret = apple_rtkit_boot(anv->rtk); in apple_nvme_reset_work()
1033 dev_err(anv->dev, "ANS did not boot"); in apple_nvme_reset_work()
1037 ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, in apple_nvme_reset_work()
1042 dev_err(anv->dev, "ANS did not initialize"); in apple_nvme_reset_work()
1046 dev_dbg(anv->dev, "ANS booted successfully."); in apple_nvme_reset_work()
1049 * Limit the max command size to prevent iod->sg allocations going in apple_nvme_reset_work()
1052 anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1, in apple_nvme_reset_work()
1053 dma_max_mapping_size(anv->dev) >> 9); in apple_nvme_reset_work()
1054 anv->ctrl.max_segments = NVME_MAX_SEGS; in apple_nvme_reset_work()
1056 dma_set_max_seg_size(anv->dev, 0xffffffff); in apple_nvme_reset_work()
1066 anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); in apple_nvme_reset_work()
1070 anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); in apple_nvme_reset_work()
1073 writel(APPLE_ANS_MAX_QUEUE_DEPTH - 1, in apple_nvme_reset_work()
1074 anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); in apple_nvme_reset_work()
1079 * the co-processor complains about "completed with err BAD_CMD-" or in apple_nvme_reset_work()
1082 writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & in apple_nvme_reset_work()
1084 anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); in apple_nvme_reset_work()
1087 aqa = APPLE_NVME_AQ_DEPTH - 1; in apple_nvme_reset_work()
1089 writel(aqa, anv->mmio_nvme + NVME_REG_AQA); in apple_nvme_reset_work()
1090 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); in apple_nvme_reset_work()
1091 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); in apple_nvme_reset_work()
1094 writeq(anv->adminq.tcb_dma_addr, in apple_nvme_reset_work()
1095 anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); in apple_nvme_reset_work()
1096 writeq(anv->ioq.tcb_dma_addr, in apple_nvme_reset_work()
1097 anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); in apple_nvme_reset_work()
1099 anv->ctrl.sqsize = in apple_nvme_reset_work()
1100 APPLE_ANS_MAX_QUEUE_DEPTH - 1; /* 0's based queue depth */ in apple_nvme_reset_work()
1101 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); in apple_nvme_reset_work()
1103 dev_dbg(anv->dev, "Enabling controller now"); in apple_nvme_reset_work()
1104 ret = nvme_enable_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1108 dev_dbg(anv->dev, "Starting admin queue"); in apple_nvme_reset_work()
1109 apple_nvme_init_queue(&anv->adminq); in apple_nvme_reset_work()
1110 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_reset_work()
1112 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) { in apple_nvme_reset_work()
1113 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1115 ret = -ENODEV; in apple_nvme_reset_work()
1119 ret = nvme_init_ctrl_finish(&anv->ctrl, false); in apple_nvme_reset_work()
1123 dev_dbg(anv->dev, "Creating IOCQ"); in apple_nvme_reset_work()
1127 dev_dbg(anv->dev, "Creating IOSQ"); in apple_nvme_reset_work()
1132 apple_nvme_init_queue(&anv->ioq); in apple_nvme_reset_work()
1134 ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); in apple_nvme_reset_work()
1138 ret = -ENXIO; in apple_nvme_reset_work()
1142 anv->ctrl.queue_count = nr_io_queues + 1; in apple_nvme_reset_work()
1144 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_reset_work()
1145 nvme_wait_freeze(&anv->ctrl); in apple_nvme_reset_work()
1146 blk_mq_update_nr_hw_queues(&anv->tagset, 1); in apple_nvme_reset_work()
1147 nvme_unfreeze(&anv->ctrl); in apple_nvme_reset_work()
1149 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) { in apple_nvme_reset_work()
1150 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1152 ret = -ENODEV; in apple_nvme_reset_work()
1156 nvme_start_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1158 dev_dbg(anv->dev, "ANS boot and NVMe init completed."); in apple_nvme_reset_work()
1166 dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); in apple_nvme_reset_work()
1167 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_reset_work()
1168 nvme_get_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1170 nvme_mark_namespaces_dead(&anv->ctrl); in apple_nvme_reset_work()
1171 if (!queue_work(nvme_wq, &anv->remove_work)) in apple_nvme_reset_work()
1172 nvme_put_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1180 nvme_put_ctrl(&anv->ctrl); in apple_nvme_remove_dead_ctrl_work()
1181 device_release_driver(anv->dev); in apple_nvme_remove_dead_ctrl_work()
1186 *val = readl(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_read32()
1192 writel(val, ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_write32()
1198 *val = readq(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_read64()
1204 struct device *dev = ctrl_to_apple_nvme(ctrl)->dev; in apple_nvme_get_address()
1213 if (anv->ctrl.admin_q) in apple_nvme_free_ctrl()
1214 blk_put_queue(anv->ctrl.admin_q); in apple_nvme_free_ctrl()
1215 put_device(anv->dev); in apple_nvme_free_ctrl()
1219 .name = "apple-nvme",
1233 flush_work(&anv->ctrl.reset_work); in apple_nvme_async_probe()
1234 flush_work(&anv->ctrl.scan_work); in apple_nvme_async_probe()
1235 nvme_put_ctrl(&anv->ctrl); in apple_nvme_async_probe()
1247 anv->admin_tagset.ops = &apple_nvme_mq_admin_ops; in apple_nvme_alloc_tagsets()
1248 anv->admin_tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1249 anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; in apple_nvme_alloc_tagsets()
1250 anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT; in apple_nvme_alloc_tagsets()
1251 anv->admin_tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1252 anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1253 anv->admin_tagset.flags = BLK_MQ_F_NO_SCHED; in apple_nvme_alloc_tagsets()
1254 anv->admin_tagset.driver_data = &anv->adminq; in apple_nvme_alloc_tagsets()
1256 ret = blk_mq_alloc_tag_set(&anv->admin_tagset); in apple_nvme_alloc_tagsets()
1259 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1260 &anv->admin_tagset); in apple_nvme_alloc_tagsets()
1264 anv->tagset.ops = &apple_nvme_mq_ops; in apple_nvme_alloc_tagsets()
1265 anv->tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1266 anv->tagset.nr_maps = 1; in apple_nvme_alloc_tagsets()
1272 anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; in apple_nvme_alloc_tagsets()
1273 anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1; in apple_nvme_alloc_tagsets()
1274 anv->tagset.timeout = NVME_IO_TIMEOUT; in apple_nvme_alloc_tagsets()
1275 anv->tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1276 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1277 anv->tagset.flags = BLK_MQ_F_SHOULD_MERGE; in apple_nvme_alloc_tagsets()
1278 anv->tagset.driver_data = &anv->ioq; in apple_nvme_alloc_tagsets()
1280 ret = blk_mq_alloc_tag_set(&anv->tagset); in apple_nvme_alloc_tagsets()
1283 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1284 &anv->tagset); in apple_nvme_alloc_tagsets()
1288 anv->ctrl.admin_tagset = &anv->admin_tagset; in apple_nvme_alloc_tagsets()
1289 anv->ctrl.tagset = &anv->tagset; in apple_nvme_alloc_tagsets()
1299 q->cqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1301 &q->cq_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1302 if (!q->cqes) in apple_nvme_queue_alloc()
1303 return -ENOMEM; in apple_nvme_queue_alloc()
1305 q->sqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1307 &q->sq_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1308 if (!q->sqes) in apple_nvme_queue_alloc()
1309 return -ENOMEM; in apple_nvme_queue_alloc()
1315 q->tcbs = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1318 &q->tcb_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1319 if (!q->tcbs) in apple_nvme_queue_alloc()
1320 return -ENOMEM; in apple_nvme_queue_alloc()
1326 q->cq_phase = 1; in apple_nvme_queue_alloc()
1334 if (anv->pd_count <= 1) in apple_nvme_detach_genpd()
1337 for (i = anv->pd_count - 1; i >= 0; i--) { in apple_nvme_detach_genpd()
1338 if (anv->pd_link[i]) in apple_nvme_detach_genpd()
1339 device_link_del(anv->pd_link[i]); in apple_nvme_detach_genpd()
1340 if (!IS_ERR_OR_NULL(anv->pd_dev[i])) in apple_nvme_detach_genpd()
1341 dev_pm_domain_detach(anv->pd_dev[i], true); in apple_nvme_detach_genpd()
1347 struct device *dev = anv->dev; in apple_nvme_attach_genpd()
1350 anv->pd_count = of_count_phandle_with_args( in apple_nvme_attach_genpd()
1351 dev->of_node, "power-domains", "#power-domain-cells"); in apple_nvme_attach_genpd()
1352 if (anv->pd_count <= 1) in apple_nvme_attach_genpd()
1355 anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev), in apple_nvme_attach_genpd()
1357 if (!anv->pd_dev) in apple_nvme_attach_genpd()
1358 return -ENOMEM; in apple_nvme_attach_genpd()
1360 anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link), in apple_nvme_attach_genpd()
1362 if (!anv->pd_link) in apple_nvme_attach_genpd()
1363 return -ENOMEM; in apple_nvme_attach_genpd()
1365 for (i = 0; i < anv->pd_count; i++) { in apple_nvme_attach_genpd()
1366 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); in apple_nvme_attach_genpd()
1367 if (IS_ERR(anv->pd_dev[i])) { in apple_nvme_attach_genpd()
1369 return PTR_ERR(anv->pd_dev[i]); in apple_nvme_attach_genpd()
1372 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i], in apple_nvme_attach_genpd()
1376 if (!anv->pd_link[i]) { in apple_nvme_attach_genpd()
1378 return -EINVAL; in apple_nvme_attach_genpd()
1392 struct device *dev = &pdev->dev; in apple_nvme_alloc()
1398 return ERR_PTR(-ENOMEM); in apple_nvme_alloc()
1400 anv->dev = get_device(dev); in apple_nvme_alloc()
1401 anv->adminq.is_adminq = true; in apple_nvme_alloc()
1406 dev_err_probe(dev, ret, "Failed to attach power domains"); in apple_nvme_alloc()
1410 ret = -ENXIO; in apple_nvme_alloc()
1414 anv->irq = platform_get_irq(pdev, 0); in apple_nvme_alloc()
1415 if (anv->irq < 0) { in apple_nvme_alloc()
1416 ret = anv->irq; in apple_nvme_alloc()
1419 if (!anv->irq) { in apple_nvme_alloc()
1420 ret = -ENXIO; in apple_nvme_alloc()
1424 anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans"); in apple_nvme_alloc()
1425 if (IS_ERR(anv->mmio_coproc)) { in apple_nvme_alloc()
1426 ret = PTR_ERR(anv->mmio_coproc); in apple_nvme_alloc()
1429 anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme"); in apple_nvme_alloc()
1430 if (IS_ERR(anv->mmio_nvme)) { in apple_nvme_alloc()
1431 ret = PTR_ERR(anv->mmio_nvme); in apple_nvme_alloc()
1435 anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; in apple_nvme_alloc()
1436 anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; in apple_nvme_alloc()
1437 anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; in apple_nvme_alloc()
1438 anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; in apple_nvme_alloc()
1440 anv->sart = devm_apple_sart_get(dev); in apple_nvme_alloc()
1441 if (IS_ERR(anv->sart)) { in apple_nvme_alloc()
1442 ret = dev_err_probe(dev, PTR_ERR(anv->sart), in apple_nvme_alloc()
1447 anv->reset = devm_reset_control_array_get_exclusive(anv->dev); in apple_nvme_alloc()
1448 if (IS_ERR(anv->reset)) { in apple_nvme_alloc()
1449 ret = dev_err_probe(dev, PTR_ERR(anv->reset), in apple_nvme_alloc()
1454 INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work); in apple_nvme_alloc()
1455 INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work); in apple_nvme_alloc()
1456 spin_lock_init(&anv->lock); in apple_nvme_alloc()
1458 ret = apple_nvme_queue_alloc(anv, &anv->adminq); in apple_nvme_alloc()
1461 ret = apple_nvme_queue_alloc(anv, &anv->ioq); in apple_nvme_alloc()
1465 anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev, in apple_nvme_alloc()
1468 if (!anv->prp_page_pool) { in apple_nvme_alloc()
1469 ret = -ENOMEM; in apple_nvme_alloc()
1473 anv->prp_small_pool = in apple_nvme_alloc()
1474 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0); in apple_nvme_alloc()
1475 if (!anv->prp_small_pool) { in apple_nvme_alloc()
1476 ret = -ENOMEM; in apple_nvme_alloc()
1481 anv->iod_mempool = in apple_nvme_alloc()
1483 if (!anv->iod_mempool) { in apple_nvme_alloc()
1484 ret = -ENOMEM; in apple_nvme_alloc()
1487 ret = devm_add_action_or_reset(anv->dev, in apple_nvme_alloc()
1488 devm_apple_nvme_mempool_destroy, anv->iod_mempool); in apple_nvme_alloc()
1496 ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, in apple_nvme_alloc()
1497 "nvme-apple", anv); in apple_nvme_alloc()
1503 anv->rtk = in apple_nvme_alloc()
1505 if (IS_ERR(anv->rtk)) { in apple_nvme_alloc()
1506 ret = dev_err_probe(dev, PTR_ERR(anv->rtk), in apple_nvme_alloc()
1511 ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, in apple_nvme_alloc()
1521 put_device(anv->dev); in apple_nvme_alloc()
1534 anv->ctrl.admin_q = blk_mq_init_queue(&anv->admin_tagset); in apple_nvme_probe()
1535 if (IS_ERR(anv->ctrl.admin_q)) { in apple_nvme_probe()
1536 ret = -ENOMEM; in apple_nvme_probe()
1537 anv->ctrl.admin_q = NULL; in apple_nvme_probe()
1541 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_probe()
1547 nvme_uninit_ctrl(&anv->ctrl); in apple_nvme_probe()
1548 nvme_put_ctrl(&anv->ctrl); in apple_nvme_probe()
1557 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_remove()
1558 flush_work(&anv->ctrl.reset_work); in apple_nvme_remove()
1559 nvme_stop_ctrl(&anv->ctrl); in apple_nvme_remove()
1560 nvme_remove_namespaces(&anv->ctrl); in apple_nvme_remove()
1562 nvme_uninit_ctrl(&anv->ctrl); in apple_nvme_remove()
1564 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_remove()
1565 apple_rtkit_shutdown(anv->rtk); in apple_nvme_remove()
1577 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_shutdown()
1578 apple_rtkit_shutdown(anv->rtk); in apple_nvme_shutdown()
1585 return nvme_reset_ctrl(&anv->ctrl); in apple_nvme_resume()
1595 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_suspend()
1596 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_suspend()
1598 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_suspend()
1607 { .compatible = "apple,nvme-ans2" },
1614 .name = "nvme-apple",