Lines Matching +full:dev +full:- +full:ctrl
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2011-2014, Intel Corporation.
11 #include <linux/blk-mq.h>
12 #include <linux/blk-mq-pci.h>
23 #include <linux/t10-pi.h>
25 #include <linux/io-64-nonatomic-lo-hi.h>
26 #include <linux/io-64-nonatomic-hi-lo.h>
27 #include <linux/sed-opal.h>
28 #include <linux/pci-p2pdma.h>
33 #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes)
34 #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion))
80 return -EINVAL; in io_queue_count_set()
106 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
107 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode);
117 struct device *dev; member
136 struct nvme_ctrl ctrl; member
165 return -EINVAL; in io_queue_depth_set()
180 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) in to_nvme_dev() argument
182 return container_of(ctrl, struct nvme_dev, ctrl); in to_nvme_dev()
190 struct nvme_dev *dev; member
239 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) in nvme_dbbuf_size() argument
241 return dev->nr_allocated_queues * 8 * dev->db_stride; in nvme_dbbuf_size()
244 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) in nvme_dbbuf_dma_alloc() argument
246 unsigned int mem_size = nvme_dbbuf_size(dev); in nvme_dbbuf_dma_alloc()
248 if (dev->dbbuf_dbs) in nvme_dbbuf_dma_alloc()
251 dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size, in nvme_dbbuf_dma_alloc()
252 &dev->dbbuf_dbs_dma_addr, in nvme_dbbuf_dma_alloc()
254 if (!dev->dbbuf_dbs) in nvme_dbbuf_dma_alloc()
255 return -ENOMEM; in nvme_dbbuf_dma_alloc()
256 dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size, in nvme_dbbuf_dma_alloc()
257 &dev->dbbuf_eis_dma_addr, in nvme_dbbuf_dma_alloc()
259 if (!dev->dbbuf_eis) { in nvme_dbbuf_dma_alloc()
260 dma_free_coherent(dev->dev, mem_size, in nvme_dbbuf_dma_alloc()
261 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr); in nvme_dbbuf_dma_alloc()
262 dev->dbbuf_dbs = NULL; in nvme_dbbuf_dma_alloc()
263 return -ENOMEM; in nvme_dbbuf_dma_alloc()
269 static void nvme_dbbuf_dma_free(struct nvme_dev *dev) in nvme_dbbuf_dma_free() argument
271 unsigned int mem_size = nvme_dbbuf_size(dev); in nvme_dbbuf_dma_free()
273 if (dev->dbbuf_dbs) { in nvme_dbbuf_dma_free()
274 dma_free_coherent(dev->dev, mem_size, in nvme_dbbuf_dma_free()
275 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr); in nvme_dbbuf_dma_free()
276 dev->dbbuf_dbs = NULL; in nvme_dbbuf_dma_free()
278 if (dev->dbbuf_eis) { in nvme_dbbuf_dma_free()
279 dma_free_coherent(dev->dev, mem_size, in nvme_dbbuf_dma_free()
280 dev->dbbuf_eis, dev->dbbuf_eis_dma_addr); in nvme_dbbuf_dma_free()
281 dev->dbbuf_eis = NULL; in nvme_dbbuf_dma_free()
285 static void nvme_dbbuf_init(struct nvme_dev *dev, in nvme_dbbuf_init() argument
288 if (!dev->dbbuf_dbs || !qid) in nvme_dbbuf_init()
291 nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)]; in nvme_dbbuf_init()
292 nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)]; in nvme_dbbuf_init()
293 nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)]; in nvme_dbbuf_init()
294 nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)]; in nvme_dbbuf_init()
299 if (!nvmeq->qid) in nvme_dbbuf_free()
302 nvmeq->dbbuf_sq_db = NULL; in nvme_dbbuf_free()
303 nvmeq->dbbuf_cq_db = NULL; in nvme_dbbuf_free()
304 nvmeq->dbbuf_sq_ei = NULL; in nvme_dbbuf_free()
305 nvmeq->dbbuf_cq_ei = NULL; in nvme_dbbuf_free()
308 static void nvme_dbbuf_set(struct nvme_dev *dev) in nvme_dbbuf_set() argument
313 if (!dev->dbbuf_dbs) in nvme_dbbuf_set()
318 c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr); in nvme_dbbuf_set()
319 c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr); in nvme_dbbuf_set()
321 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) { in nvme_dbbuf_set()
322 dev_warn(dev->ctrl.device, "unable to set dbbuf\n"); in nvme_dbbuf_set()
324 nvme_dbbuf_dma_free(dev); in nvme_dbbuf_set()
326 for (i = 1; i <= dev->online_queues; i++) in nvme_dbbuf_set()
327 nvme_dbbuf_free(&dev->queues[i]); in nvme_dbbuf_set()
333 return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); in nvme_dbbuf_need_event()
377 return DIV_ROUND_UP(8 * nprps, NVME_CTRL_PAGE_SIZE - 8); in nvme_pci_npages_prp()
393 struct nvme_dev *dev = data; in nvme_admin_init_hctx() local
394 struct nvme_queue *nvmeq = &dev->queues[0]; in nvme_admin_init_hctx()
397 WARN_ON(dev->admin_tagset.tags[0] != hctx->tags); in nvme_admin_init_hctx()
399 hctx->driver_data = nvmeq; in nvme_admin_init_hctx()
406 struct nvme_dev *dev = data; in nvme_init_hctx() local
407 struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1]; in nvme_init_hctx()
409 WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags); in nvme_init_hctx()
410 hctx->driver_data = nvmeq; in nvme_init_hctx()
417 struct nvme_dev *dev = set->driver_data; in nvme_init_request() local
419 int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0; in nvme_init_request()
420 struct nvme_queue *nvmeq = &dev->queues[queue_idx]; in nvme_init_request()
423 iod->nvmeq = nvmeq; in nvme_init_request()
425 nvme_req(req)->ctrl = &dev->ctrl; in nvme_init_request()
429 static int queue_irq_offset(struct nvme_dev *dev) in queue_irq_offset() argument
432 if (dev->num_vecs > 1) in queue_irq_offset()
440 struct nvme_dev *dev = set->driver_data; in nvme_pci_map_queues() local
443 offset = queue_irq_offset(dev); in nvme_pci_map_queues()
444 for (i = 0, qoff = 0; i < set->nr_maps; i++) { in nvme_pci_map_queues()
445 struct blk_mq_queue_map *map = &set->map[i]; in nvme_pci_map_queues()
447 map->nr_queues = dev->io_queues[i]; in nvme_pci_map_queues()
448 if (!map->nr_queues) { in nvme_pci_map_queues()
455 * affinity), so use the regular blk-mq cpu mapping in nvme_pci_map_queues()
457 map->queue_offset = qoff; in nvme_pci_map_queues()
459 blk_mq_pci_map_queues(map, to_pci_dev(dev->dev), offset); in nvme_pci_map_queues()
462 qoff += map->nr_queues; in nvme_pci_map_queues()
463 offset += map->nr_queues; in nvme_pci_map_queues()
475 u16 next_tail = nvmeq->sq_tail + 1; in nvme_write_sq_db()
477 if (next_tail == nvmeq->q_depth) in nvme_write_sq_db()
479 if (next_tail != nvmeq->last_sq_tail) in nvme_write_sq_db()
483 if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail, in nvme_write_sq_db()
484 nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) in nvme_write_sq_db()
485 writel(nvmeq->sq_tail, nvmeq->q_db); in nvme_write_sq_db()
486 nvmeq->last_sq_tail = nvmeq->sq_tail; in nvme_write_sq_db()
490 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
498 spin_lock(&nvmeq->sq_lock); in nvme_submit_cmd()
499 memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), in nvme_submit_cmd()
501 if (++nvmeq->sq_tail == nvmeq->q_depth) in nvme_submit_cmd()
502 nvmeq->sq_tail = 0; in nvme_submit_cmd()
504 spin_unlock(&nvmeq->sq_lock); in nvme_submit_cmd()
509 struct nvme_queue *nvmeq = hctx->driver_data; in nvme_commit_rqs()
511 spin_lock(&nvmeq->sq_lock); in nvme_commit_rqs()
512 if (nvmeq->sq_tail != nvmeq->last_sq_tail) in nvme_commit_rqs()
514 spin_unlock(&nvmeq->sq_lock); in nvme_commit_rqs()
520 return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); in nvme_pci_iod_list()
523 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) in nvme_pci_use_sgls() argument
531 if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1)))) in nvme_pci_use_sgls()
533 if (!iod->nvmeq->qid) in nvme_pci_use_sgls()
540 static void nvme_free_prps(struct nvme_dev *dev, struct request *req) in nvme_free_prps() argument
542 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; in nvme_free_prps()
544 dma_addr_t dma_addr = iod->first_dma; in nvme_free_prps()
547 for (i = 0; i < iod->npages; i++) { in nvme_free_prps()
551 dma_pool_free(dev->prp_page_pool, prp_list, dma_addr); in nvme_free_prps()
557 static void nvme_free_sgls(struct nvme_dev *dev, struct request *req) in nvme_free_sgls() argument
559 const int last_sg = SGES_PER_PAGE - 1; in nvme_free_sgls()
561 dma_addr_t dma_addr = iod->first_dma; in nvme_free_sgls()
564 for (i = 0; i < iod->npages; i++) { in nvme_free_sgls()
568 dma_pool_free(dev->prp_page_pool, sg_list, dma_addr); in nvme_free_sgls()
574 static void nvme_unmap_sg(struct nvme_dev *dev, struct request *req) in nvme_unmap_sg() argument
578 if (is_pci_p2pdma_page(sg_page(iod->sg))) in nvme_unmap_sg()
579 pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents, in nvme_unmap_sg()
582 dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req)); in nvme_unmap_sg()
585 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req) in nvme_unmap_data() argument
589 if (iod->dma_len) { in nvme_unmap_data()
590 dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len, in nvme_unmap_data()
595 WARN_ON_ONCE(!iod->nents); in nvme_unmap_data()
597 nvme_unmap_sg(dev, req); in nvme_unmap_data()
598 if (iod->npages == 0) in nvme_unmap_data()
599 dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0], in nvme_unmap_data()
600 iod->first_dma); in nvme_unmap_data()
601 else if (iod->use_sgl) in nvme_unmap_data()
602 nvme_free_sgls(dev, req); in nvme_unmap_data()
604 nvme_free_prps(dev, req); in nvme_unmap_data()
605 mempool_free(iod->sg, dev->iod_mempool); in nvme_unmap_data()
617 i, &phys, sg->offset, sg->length, &sg_dma_address(sg), in nvme_print_sgl()
622 static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, in nvme_pci_setup_prps() argument
628 struct scatterlist *sg = iod->sg; in nvme_pci_setup_prps()
631 int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1); in nvme_pci_setup_prps()
637 length -= (NVME_CTRL_PAGE_SIZE - offset); in nvme_pci_setup_prps()
639 iod->first_dma = 0; in nvme_pci_setup_prps()
643 dma_len -= (NVME_CTRL_PAGE_SIZE - offset); in nvme_pci_setup_prps()
645 dma_addr += (NVME_CTRL_PAGE_SIZE - offset); in nvme_pci_setup_prps()
653 iod->first_dma = dma_addr; in nvme_pci_setup_prps()
659 pool = dev->prp_small_pool; in nvme_pci_setup_prps()
660 iod->npages = 0; in nvme_pci_setup_prps()
662 pool = dev->prp_page_pool; in nvme_pci_setup_prps()
663 iod->npages = 1; in nvme_pci_setup_prps()
668 iod->first_dma = dma_addr; in nvme_pci_setup_prps()
669 iod->npages = -1; in nvme_pci_setup_prps()
673 iod->first_dma = prp_dma; in nvme_pci_setup_prps()
681 list[iod->npages++] = prp_list; in nvme_pci_setup_prps()
682 prp_list[0] = old_prp_list[i - 1]; in nvme_pci_setup_prps()
683 old_prp_list[i - 1] = cpu_to_le64(prp_dma); in nvme_pci_setup_prps()
687 dma_len -= NVME_CTRL_PAGE_SIZE; in nvme_pci_setup_prps()
689 length -= NVME_CTRL_PAGE_SIZE; in nvme_pci_setup_prps()
701 cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); in nvme_pci_setup_prps()
702 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma); in nvme_pci_setup_prps()
705 nvme_free_prps(dev, req); in nvme_pci_setup_prps()
708 WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents), in nvme_pci_setup_prps()
710 blk_rq_payload_bytes(req), iod->nents); in nvme_pci_setup_prps()
717 sge->addr = cpu_to_le64(sg_dma_address(sg)); in nvme_pci_sgl_set_data()
718 sge->length = cpu_to_le32(sg_dma_len(sg)); in nvme_pci_sgl_set_data()
719 sge->type = NVME_SGL_FMT_DATA_DESC << 4; in nvme_pci_sgl_set_data()
725 sge->addr = cpu_to_le64(dma_addr); in nvme_pci_sgl_set_seg()
727 sge->length = cpu_to_le32(entries * sizeof(*sge)); in nvme_pci_sgl_set_seg()
728 sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; in nvme_pci_sgl_set_seg()
730 sge->length = cpu_to_le32(NVME_CTRL_PAGE_SIZE); in nvme_pci_sgl_set_seg()
731 sge->type = NVME_SGL_FMT_SEG_DESC << 4; in nvme_pci_sgl_set_seg()
735 static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, in nvme_pci_setup_sgls() argument
741 struct scatterlist *sg = iod->sg; in nvme_pci_setup_sgls()
746 cmd->flags = NVME_CMD_SGL_METABUF; in nvme_pci_setup_sgls()
749 nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg); in nvme_pci_setup_sgls()
754 pool = dev->prp_small_pool; in nvme_pci_setup_sgls()
755 iod->npages = 0; in nvme_pci_setup_sgls()
757 pool = dev->prp_page_pool; in nvme_pci_setup_sgls()
758 iod->npages = 1; in nvme_pci_setup_sgls()
763 iod->npages = -1; in nvme_pci_setup_sgls()
768 iod->first_dma = sgl_dma; in nvme_pci_setup_sgls()
770 nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries); in nvme_pci_setup_sgls()
775 struct nvme_sgl_desc *link = &old_sg_desc[i - 1]; in nvme_pci_setup_sgls()
782 nvme_pci_iod_list(req)[iod->npages++] = sg_list; in nvme_pci_setup_sgls()
789 } while (--entries > 0); in nvme_pci_setup_sgls()
793 nvme_free_sgls(dev, req); in nvme_pci_setup_sgls()
797 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev, in nvme_setup_prp_simple() argument
802 unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1); in nvme_setup_prp_simple()
803 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset; in nvme_setup_prp_simple()
805 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); in nvme_setup_prp_simple()
806 if (dma_mapping_error(dev->dev, iod->first_dma)) in nvme_setup_prp_simple()
808 iod->dma_len = bv->bv_len; in nvme_setup_prp_simple()
810 cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); in nvme_setup_prp_simple()
811 if (bv->bv_len > first_prp_len) in nvme_setup_prp_simple()
812 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); in nvme_setup_prp_simple()
814 cmnd->dptr.prp2 = 0; in nvme_setup_prp_simple()
818 static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev, in nvme_setup_sgl_simple() argument
824 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); in nvme_setup_sgl_simple()
825 if (dma_mapping_error(dev->dev, iod->first_dma)) in nvme_setup_sgl_simple()
827 iod->dma_len = bv->bv_len; in nvme_setup_sgl_simple()
829 cmnd->flags = NVME_CMD_SGL_METABUF; in nvme_setup_sgl_simple()
830 cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma); in nvme_setup_sgl_simple()
831 cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len); in nvme_setup_sgl_simple()
832 cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; in nvme_setup_sgl_simple()
836 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, in nvme_map_data() argument
848 return nvme_setup_prp_simple(dev, req, in nvme_map_data()
849 &cmnd->rw, &bv); in nvme_map_data()
851 if (iod->nvmeq->qid && sgl_threshold && in nvme_map_data()
852 dev->ctrl.sgls & ((1 << 0) | (1 << 1))) in nvme_map_data()
853 return nvme_setup_sgl_simple(dev, req, in nvme_map_data()
854 &cmnd->rw, &bv); in nvme_map_data()
858 iod->dma_len = 0; in nvme_map_data()
859 iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC); in nvme_map_data()
860 if (!iod->sg) in nvme_map_data()
862 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); in nvme_map_data()
863 iod->nents = blk_rq_map_sg(req->q, req, iod->sg); in nvme_map_data()
864 if (!iod->nents) in nvme_map_data()
867 if (is_pci_p2pdma_page(sg_page(iod->sg))) in nvme_map_data()
868 nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg, in nvme_map_data()
869 iod->nents, rq_dma_dir(req), DMA_ATTR_NO_WARN); in nvme_map_data()
871 nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, in nvme_map_data()
876 iod->use_sgl = nvme_pci_use_sgls(dev, req); in nvme_map_data()
877 if (iod->use_sgl) in nvme_map_data()
878 ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped); in nvme_map_data()
880 ret = nvme_pci_setup_prps(dev, req, &cmnd->rw); in nvme_map_data()
886 nvme_unmap_sg(dev, req); in nvme_map_data()
888 mempool_free(iod->sg, dev->iod_mempool); in nvme_map_data()
892 static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req, in nvme_map_metadata() argument
897 iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req), in nvme_map_metadata()
899 if (dma_mapping_error(dev->dev, iod->meta_dma)) in nvme_map_metadata()
901 cmnd->rw.metadata = cpu_to_le64(iod->meta_dma); in nvme_map_metadata()
911 struct nvme_ns *ns = hctx->queue->queuedata; in nvme_queue_rq()
912 struct nvme_queue *nvmeq = hctx->driver_data; in nvme_queue_rq()
913 struct nvme_dev *dev = nvmeq->dev; in nvme_queue_rq() local
914 struct request *req = bd->rq; in nvme_queue_rq()
916 struct nvme_command *cmnd = &iod->cmd; in nvme_queue_rq()
919 iod->aborted = 0; in nvme_queue_rq()
920 iod->npages = -1; in nvme_queue_rq()
921 iod->nents = 0; in nvme_queue_rq()
927 if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags))) in nvme_queue_rq()
935 ret = nvme_map_data(dev, req, cmnd); in nvme_queue_rq()
941 ret = nvme_map_metadata(dev, req, cmnd); in nvme_queue_rq()
947 nvme_submit_cmd(nvmeq, cmnd, bd->last); in nvme_queue_rq()
950 nvme_unmap_data(dev, req); in nvme_queue_rq()
959 struct nvme_dev *dev = iod->nvmeq->dev; in nvme_pci_complete_rq() local
962 dma_unmap_page(dev->dev, iod->meta_dma, in nvme_pci_complete_rq()
963 rq_integrity_vec(req)->bv_len, rq_dma_dir(req)); in nvme_pci_complete_rq()
966 nvme_unmap_data(dev, req); in nvme_pci_complete_rq()
973 struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head]; in nvme_cqe_pending()
975 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase; in nvme_cqe_pending()
980 u16 head = nvmeq->cq_head; in nvme_ring_cq_doorbell()
982 if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db, in nvme_ring_cq_doorbell()
983 nvmeq->dbbuf_cq_ei)) in nvme_ring_cq_doorbell()
984 writel(head, nvmeq->q_db + nvmeq->dev->db_stride); in nvme_ring_cq_doorbell()
989 if (!nvmeq->qid) in nvme_queue_tagset()
990 return nvmeq->dev->admin_tagset.tags[0]; in nvme_queue_tagset()
991 return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; in nvme_queue_tagset()
996 struct nvme_completion *cqe = &nvmeq->cqes[idx]; in nvme_handle_cqe()
997 __u16 command_id = READ_ONCE(cqe->command_id); in nvme_handle_cqe()
1006 if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) { in nvme_handle_cqe()
1007 nvme_complete_async_event(&nvmeq->dev->ctrl, in nvme_handle_cqe()
1008 cqe->status, &cqe->result); in nvme_handle_cqe()
1014 dev_warn(nvmeq->dev->ctrl.device, in nvme_handle_cqe()
1016 command_id, le16_to_cpu(cqe->sq_id)); in nvme_handle_cqe()
1020 trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); in nvme_handle_cqe()
1021 if (!nvme_try_complete_req(req, cqe->status, cqe->result)) in nvme_handle_cqe()
1027 u32 tmp = nvmeq->cq_head + 1; in nvme_update_cq_head()
1029 if (tmp == nvmeq->q_depth) { in nvme_update_cq_head()
1030 nvmeq->cq_head = 0; in nvme_update_cq_head()
1031 nvmeq->cq_phase ^= 1; in nvme_update_cq_head()
1033 nvmeq->cq_head = tmp; in nvme_update_cq_head()
1044 * load-load control dependency between phase and the rest of in nvme_process_cq()
1048 nvme_handle_cqe(nvmeq, nvmeq->cq_head); in nvme_process_cq()
1089 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); in nvme_poll_irqdisable()
1091 WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); in nvme_poll_irqdisable()
1093 disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); in nvme_poll_irqdisable()
1095 enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); in nvme_poll_irqdisable()
1100 struct nvme_queue *nvmeq = hctx->driver_data; in nvme_poll()
1106 spin_lock(&nvmeq->cq_poll_lock); in nvme_poll()
1108 spin_unlock(&nvmeq->cq_poll_lock); in nvme_poll()
1113 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl) in nvme_pci_submit_async_event() argument
1115 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_submit_async_event() local
1116 struct nvme_queue *nvmeq = &dev->queues[0]; in nvme_pci_submit_async_event()
1125 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) in adapter_delete_queue() argument
1133 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_delete_queue()
1136 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, in adapter_alloc_cq() argument
1142 if (!test_bit(NVMEQ_POLLED, &nvmeq->flags)) in adapter_alloc_cq()
1151 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); in adapter_alloc_cq()
1153 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); in adapter_alloc_cq()
1157 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_alloc_cq()
1160 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, in adapter_alloc_sq() argument
1163 struct nvme_ctrl *ctrl = &dev->ctrl; in adapter_alloc_sq() local
1168 * Some drives have a bug that auto-enables WRRU if MEDIUM isn't in adapter_alloc_sq()
1172 if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ) in adapter_alloc_sq()
1181 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); in adapter_alloc_sq()
1183 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); in adapter_alloc_sq()
1187 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in adapter_alloc_sq()
1190 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) in adapter_delete_cq() argument
1192 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); in adapter_delete_cq()
1195 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) in adapter_delete_sq() argument
1197 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); in adapter_delete_sq()
1203 struct nvme_queue *nvmeq = iod->nvmeq; in abort_endio()
1205 dev_warn(nvmeq->dev->ctrl.device, in abort_endio()
1206 "Abort status: 0x%x", nvme_req(req)->status); in abort_endio()
1207 atomic_inc(&nvmeq->dev->ctrl.abort_limit); in abort_endio()
1211 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) in nvme_should_reset() argument
1216 bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO); in nvme_should_reset()
1219 switch (dev->ctrl.state) { in nvme_should_reset()
1236 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) in nvme_warn_reset() argument
1242 result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS, in nvme_warn_reset()
1245 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1249 dev_warn(dev->ctrl.device, in nvme_warn_reset()
1257 struct nvme_queue *nvmeq = iod->nvmeq; in nvme_timeout()
1258 struct nvme_dev *dev = nvmeq->dev; in nvme_timeout() local
1261 u32 csts = readl(dev->bar + NVME_REG_CSTS); in nvme_timeout()
1267 if (pci_channel_offline(to_pci_dev(dev->dev))) in nvme_timeout()
1273 if (nvme_should_reset(dev, csts)) { in nvme_timeout()
1274 nvme_warn_reset(dev, csts); in nvme_timeout()
1275 nvme_dev_disable(dev, false); in nvme_timeout()
1276 nvme_reset_ctrl(&dev->ctrl); in nvme_timeout()
1283 if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) in nvme_timeout()
1284 nvme_poll(req->mq_hctx); in nvme_timeout()
1289 dev_warn(dev->ctrl.device, in nvme_timeout()
1291 req->tag, nvmeq->qid); in nvme_timeout()
1301 switch (dev->ctrl.state) { in nvme_timeout()
1303 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_timeout()
1306 dev_warn_ratelimited(dev->ctrl.device, in nvme_timeout()
1308 req->tag, nvmeq->qid); in nvme_timeout()
1309 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in nvme_timeout()
1310 nvme_dev_disable(dev, true); in nvme_timeout()
1323 if (!nvmeq->qid || iod->aborted) { in nvme_timeout()
1324 dev_warn(dev->ctrl.device, in nvme_timeout()
1326 req->tag, nvmeq->qid); in nvme_timeout()
1327 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in nvme_timeout()
1328 nvme_dev_disable(dev, false); in nvme_timeout()
1329 nvme_reset_ctrl(&dev->ctrl); in nvme_timeout()
1334 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) { in nvme_timeout()
1335 atomic_inc(&dev->ctrl.abort_limit); in nvme_timeout()
1338 iod->aborted = 1; in nvme_timeout()
1343 cmd.abort.sqid = cpu_to_le16(nvmeq->qid); in nvme_timeout()
1345 dev_warn(nvmeq->dev->ctrl.device, in nvme_timeout()
1347 req->tag, nvmeq->qid); in nvme_timeout()
1349 abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd, in nvme_timeout()
1352 atomic_inc(&dev->ctrl.abort_limit); in nvme_timeout()
1356 abort_req->end_io_data = NULL; in nvme_timeout()
1357 blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio); in nvme_timeout()
1369 dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq), in nvme_free_queue()
1370 (void *)nvmeq->cqes, nvmeq->cq_dma_addr); in nvme_free_queue()
1371 if (!nvmeq->sq_cmds) in nvme_free_queue()
1374 if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) { in nvme_free_queue()
1375 pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev), in nvme_free_queue()
1376 nvmeq->sq_cmds, SQ_SIZE(nvmeq)); in nvme_free_queue()
1378 dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq), in nvme_free_queue()
1379 nvmeq->sq_cmds, nvmeq->sq_dma_addr); in nvme_free_queue()
1383 static void nvme_free_queues(struct nvme_dev *dev, int lowest) in nvme_free_queues() argument
1387 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) { in nvme_free_queues()
1388 dev->ctrl.queue_count--; in nvme_free_queues()
1389 nvme_free_queue(&dev->queues[i]); in nvme_free_queues()
1394 * nvme_suspend_queue - put queue into suspended state
1399 if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags)) in nvme_suspend_queue()
1405 nvmeq->dev->online_queues--; in nvme_suspend_queue()
1406 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q) in nvme_suspend_queue()
1407 nvme_stop_admin_queue(&nvmeq->dev->ctrl); in nvme_suspend_queue()
1408 if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) in nvme_suspend_queue()
1409 pci_free_irq(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector, nvmeq); in nvme_suspend_queue()
1413 static void nvme_suspend_io_queues(struct nvme_dev *dev) in nvme_suspend_io_queues() argument
1417 for (i = dev->ctrl.queue_count - 1; i > 0; i--) in nvme_suspend_io_queues()
1418 nvme_suspend_queue(&dev->queues[i]); in nvme_suspend_io_queues()
1421 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown) in nvme_disable_admin_queue() argument
1423 struct nvme_queue *nvmeq = &dev->queues[0]; in nvme_disable_admin_queue()
1426 nvme_shutdown_ctrl(&dev->ctrl); in nvme_disable_admin_queue()
1428 nvme_disable_ctrl(&dev->ctrl); in nvme_disable_admin_queue()
1439 static void nvme_reap_pending_cqes(struct nvme_dev *dev) in nvme_reap_pending_cqes() argument
1443 for (i = dev->ctrl.queue_count - 1; i > 0; i--) { in nvme_reap_pending_cqes()
1444 spin_lock(&dev->queues[i].cq_poll_lock); in nvme_reap_pending_cqes()
1445 nvme_process_cq(&dev->queues[i]); in nvme_reap_pending_cqes()
1446 spin_unlock(&dev->queues[i].cq_poll_lock); in nvme_reap_pending_cqes()
1450 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, in nvme_cmb_qdepth() argument
1453 int q_depth = dev->q_depth; in nvme_cmb_qdepth()
1457 if (q_size_aligned * nr_io_queues > dev->cmb_size) { in nvme_cmb_qdepth()
1458 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); in nvme_cmb_qdepth()
1469 return -ENOMEM; in nvme_cmb_qdepth()
1475 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, in nvme_alloc_sq_cmds() argument
1478 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_alloc_sq_cmds()
1480 if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) { in nvme_alloc_sq_cmds()
1481 nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq)); in nvme_alloc_sq_cmds()
1482 if (nvmeq->sq_cmds) { in nvme_alloc_sq_cmds()
1483 nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev, in nvme_alloc_sq_cmds()
1484 nvmeq->sq_cmds); in nvme_alloc_sq_cmds()
1485 if (nvmeq->sq_dma_addr) { in nvme_alloc_sq_cmds()
1486 set_bit(NVMEQ_SQ_CMB, &nvmeq->flags); in nvme_alloc_sq_cmds()
1490 pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq)); in nvme_alloc_sq_cmds()
1494 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq), in nvme_alloc_sq_cmds()
1495 &nvmeq->sq_dma_addr, GFP_KERNEL); in nvme_alloc_sq_cmds()
1496 if (!nvmeq->sq_cmds) in nvme_alloc_sq_cmds()
1497 return -ENOMEM; in nvme_alloc_sq_cmds()
1501 static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) in nvme_alloc_queue() argument
1503 struct nvme_queue *nvmeq = &dev->queues[qid]; in nvme_alloc_queue()
1505 if (dev->ctrl.queue_count > qid) in nvme_alloc_queue()
1508 nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES; in nvme_alloc_queue()
1509 nvmeq->q_depth = depth; in nvme_alloc_queue()
1510 nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq), in nvme_alloc_queue()
1511 &nvmeq->cq_dma_addr, GFP_KERNEL); in nvme_alloc_queue()
1512 if (!nvmeq->cqes) in nvme_alloc_queue()
1515 if (nvme_alloc_sq_cmds(dev, nvmeq, qid)) in nvme_alloc_queue()
1518 nvmeq->dev = dev; in nvme_alloc_queue()
1519 spin_lock_init(&nvmeq->sq_lock); in nvme_alloc_queue()
1520 spin_lock_init(&nvmeq->cq_poll_lock); in nvme_alloc_queue()
1521 nvmeq->cq_head = 0; in nvme_alloc_queue()
1522 nvmeq->cq_phase = 1; in nvme_alloc_queue()
1523 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; in nvme_alloc_queue()
1524 nvmeq->qid = qid; in nvme_alloc_queue()
1525 dev->ctrl.queue_count++; in nvme_alloc_queue()
1530 dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes, in nvme_alloc_queue()
1531 nvmeq->cq_dma_addr); in nvme_alloc_queue()
1533 return -ENOMEM; in nvme_alloc_queue()
1538 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); in queue_request_irq()
1539 int nr = nvmeq->dev->ctrl.instance; in queue_request_irq()
1542 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check, in queue_request_irq()
1543 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid); in queue_request_irq()
1545 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq, in queue_request_irq()
1546 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid); in queue_request_irq()
1552 struct nvme_dev *dev = nvmeq->dev; in nvme_init_queue() local
1554 nvmeq->sq_tail = 0; in nvme_init_queue()
1555 nvmeq->last_sq_tail = 0; in nvme_init_queue()
1556 nvmeq->cq_head = 0; in nvme_init_queue()
1557 nvmeq->cq_phase = 1; in nvme_init_queue()
1558 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; in nvme_init_queue()
1559 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq)); in nvme_init_queue()
1560 nvme_dbbuf_init(dev, nvmeq, qid); in nvme_init_queue()
1561 dev->online_queues++; in nvme_init_queue()
1567 struct nvme_dev *dev = nvmeq->dev; in nvme_create_queue() local
1571 clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags); in nvme_create_queue()
1578 vector = dev->num_vecs == 1 ? 0 : qid; in nvme_create_queue()
1580 set_bit(NVMEQ_POLLED, &nvmeq->flags); in nvme_create_queue()
1582 result = adapter_alloc_cq(dev, qid, nvmeq, vector); in nvme_create_queue()
1586 result = adapter_alloc_sq(dev, qid, nvmeq); in nvme_create_queue()
1592 nvmeq->cq_vector = vector; in nvme_create_queue()
1601 set_bit(NVMEQ_ENABLED, &nvmeq->flags); in nvme_create_queue()
1605 dev->online_queues--; in nvme_create_queue()
1606 adapter_delete_sq(dev, qid); in nvme_create_queue()
1608 adapter_delete_cq(dev, qid); in nvme_create_queue()
1631 static void nvme_dev_remove_admin(struct nvme_dev *dev) in nvme_dev_remove_admin() argument
1633 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) { in nvme_dev_remove_admin()
1639 nvme_start_admin_queue(&dev->ctrl); in nvme_dev_remove_admin()
1640 blk_cleanup_queue(dev->ctrl.admin_q); in nvme_dev_remove_admin()
1641 blk_mq_free_tag_set(&dev->admin_tagset); in nvme_dev_remove_admin()
1645 static int nvme_alloc_admin_tags(struct nvme_dev *dev) in nvme_alloc_admin_tags() argument
1647 if (!dev->ctrl.admin_q) { in nvme_alloc_admin_tags()
1648 dev->admin_tagset.ops = &nvme_mq_admin_ops; in nvme_alloc_admin_tags()
1649 dev->admin_tagset.nr_hw_queues = 1; in nvme_alloc_admin_tags()
1651 dev->admin_tagset.queue_depth = NVME_AQ_MQ_TAG_DEPTH; in nvme_alloc_admin_tags()
1652 dev->admin_tagset.timeout = ADMIN_TIMEOUT; in nvme_alloc_admin_tags()
1653 dev->admin_tagset.numa_node = dev->ctrl.numa_node; in nvme_alloc_admin_tags()
1654 dev->admin_tagset.cmd_size = sizeof(struct nvme_iod); in nvme_alloc_admin_tags()
1655 dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED; in nvme_alloc_admin_tags()
1656 dev->admin_tagset.driver_data = dev; in nvme_alloc_admin_tags()
1658 if (blk_mq_alloc_tag_set(&dev->admin_tagset)) in nvme_alloc_admin_tags()
1659 return -ENOMEM; in nvme_alloc_admin_tags()
1660 dev->ctrl.admin_tagset = &dev->admin_tagset; in nvme_alloc_admin_tags()
1662 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset); in nvme_alloc_admin_tags()
1663 if (IS_ERR(dev->ctrl.admin_q)) { in nvme_alloc_admin_tags()
1664 blk_mq_free_tag_set(&dev->admin_tagset); in nvme_alloc_admin_tags()
1665 dev->ctrl.admin_q = NULL; in nvme_alloc_admin_tags()
1666 return -ENOMEM; in nvme_alloc_admin_tags()
1668 if (!blk_get_queue(dev->ctrl.admin_q)) { in nvme_alloc_admin_tags()
1669 nvme_dev_remove_admin(dev); in nvme_alloc_admin_tags()
1670 dev->ctrl.admin_q = NULL; in nvme_alloc_admin_tags()
1671 return -ENODEV; in nvme_alloc_admin_tags()
1674 nvme_start_admin_queue(&dev->ctrl); in nvme_alloc_admin_tags()
1679 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) in db_bar_size() argument
1681 return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride); in db_bar_size()
1684 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size) in nvme_remap_bar() argument
1686 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_remap_bar()
1688 if (size <= dev->bar_mapped_size) in nvme_remap_bar()
1691 return -ENOMEM; in nvme_remap_bar()
1692 if (dev->bar) in nvme_remap_bar()
1693 iounmap(dev->bar); in nvme_remap_bar()
1694 dev->bar = ioremap(pci_resource_start(pdev, 0), size); in nvme_remap_bar()
1695 if (!dev->bar) { in nvme_remap_bar()
1696 dev->bar_mapped_size = 0; in nvme_remap_bar()
1697 return -ENOMEM; in nvme_remap_bar()
1699 dev->bar_mapped_size = size; in nvme_remap_bar()
1700 dev->dbs = dev->bar + NVME_REG_DBS; in nvme_remap_bar()
1705 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) in nvme_pci_configure_admin_queue() argument
1711 result = nvme_remap_bar(dev, db_bar_size(dev, 0)); in nvme_pci_configure_admin_queue()
1715 dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ? in nvme_pci_configure_admin_queue()
1716 NVME_CAP_NSSRC(dev->ctrl.cap) : 0; in nvme_pci_configure_admin_queue()
1718 if (dev->subsystem && in nvme_pci_configure_admin_queue()
1719 (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO)) in nvme_pci_configure_admin_queue()
1720 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS); in nvme_pci_configure_admin_queue()
1722 result = nvme_disable_ctrl(&dev->ctrl); in nvme_pci_configure_admin_queue()
1726 result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH); in nvme_pci_configure_admin_queue()
1730 dev->ctrl.numa_node = dev_to_node(dev->dev); in nvme_pci_configure_admin_queue()
1732 nvmeq = &dev->queues[0]; in nvme_pci_configure_admin_queue()
1733 aqa = nvmeq->q_depth - 1; in nvme_pci_configure_admin_queue()
1736 writel(aqa, dev->bar + NVME_REG_AQA); in nvme_pci_configure_admin_queue()
1737 lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ); in nvme_pci_configure_admin_queue()
1738 lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ); in nvme_pci_configure_admin_queue()
1740 result = nvme_enable_ctrl(&dev->ctrl); in nvme_pci_configure_admin_queue()
1744 nvmeq->cq_vector = 0; in nvme_pci_configure_admin_queue()
1748 dev->online_queues--; in nvme_pci_configure_admin_queue()
1752 set_bit(NVMEQ_ENABLED, &nvmeq->flags); in nvme_pci_configure_admin_queue()
1756 static int nvme_create_io_queues(struct nvme_dev *dev) in nvme_create_io_queues() argument
1761 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) { in nvme_create_io_queues()
1762 if (nvme_alloc_queue(dev, i, dev->q_depth)) { in nvme_create_io_queues()
1763 ret = -ENOMEM; in nvme_create_io_queues()
1768 max = min(dev->max_qid, dev->ctrl.queue_count - 1); in nvme_create_io_queues()
1769 if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) { in nvme_create_io_queues()
1770 rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] + in nvme_create_io_queues()
1771 dev->io_queues[HCTX_TYPE_READ]; in nvme_create_io_queues()
1776 for (i = dev->online_queues; i <= max; i++) { in nvme_create_io_queues()
1779 ret = nvme_create_queue(&dev->queues[i], i, polled); in nvme_create_io_queues()
1793 static ssize_t nvme_cmb_show(struct device *dev, in nvme_cmb_show() argument
1797 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); in nvme_cmb_show()
1800 ndev->cmbloc, ndev->cmbsz); in nvme_cmb_show()
1804 static u64 nvme_cmb_size_unit(struct nvme_dev *dev) in nvme_cmb_size_unit() argument
1806 u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; in nvme_cmb_size_unit()
1811 static u32 nvme_cmb_size(struct nvme_dev *dev) in nvme_cmb_size() argument
1813 return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK; in nvme_cmb_size()
1816 static void nvme_map_cmb(struct nvme_dev *dev) in nvme_map_cmb() argument
1820 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_map_cmb()
1823 if (dev->cmb_size) in nvme_map_cmb()
1826 if (NVME_CAP_CMBS(dev->ctrl.cap)) in nvme_map_cmb()
1827 writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC); in nvme_map_cmb()
1829 dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); in nvme_map_cmb()
1830 if (!dev->cmbsz) in nvme_map_cmb()
1832 dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); in nvme_map_cmb()
1834 size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); in nvme_map_cmb()
1835 offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); in nvme_map_cmb()
1836 bar = NVME_CMB_BIR(dev->cmbloc); in nvme_map_cmb()
1846 if (NVME_CAP_CMBS(dev->ctrl.cap)) { in nvme_map_cmb()
1849 dev->bar + NVME_REG_CMBMSC); in nvme_map_cmb()
1857 if (size > bar_size - offset) in nvme_map_cmb()
1858 size = bar_size - offset; in nvme_map_cmb()
1861 dev_warn(dev->ctrl.device, in nvme_map_cmb()
1866 dev->cmb_size = size; in nvme_map_cmb()
1867 dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS); in nvme_map_cmb()
1869 if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) == in nvme_map_cmb()
1873 if (sysfs_add_file_to_group(&dev->ctrl.device->kobj, in nvme_map_cmb()
1875 dev_warn(dev->ctrl.device, in nvme_map_cmb()
1879 static inline void nvme_release_cmb(struct nvme_dev *dev) in nvme_release_cmb() argument
1881 if (dev->cmb_size) { in nvme_release_cmb()
1882 sysfs_remove_file_from_group(&dev->ctrl.device->kobj, in nvme_release_cmb()
1884 dev->cmb_size = 0; in nvme_release_cmb()
1888 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) in nvme_set_host_mem() argument
1890 u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT; in nvme_set_host_mem()
1891 u64 dma_addr = dev->host_mem_descs_dma; in nvme_set_host_mem()
1902 c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs); in nvme_set_host_mem()
1904 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); in nvme_set_host_mem()
1906 dev_warn(dev->ctrl.device, in nvme_set_host_mem()
1913 static void nvme_free_host_mem(struct nvme_dev *dev) in nvme_free_host_mem() argument
1917 for (i = 0; i < dev->nr_host_mem_descs; i++) { in nvme_free_host_mem()
1918 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i]; in nvme_free_host_mem()
1919 size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE; in nvme_free_host_mem()
1921 dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i], in nvme_free_host_mem()
1922 le64_to_cpu(desc->addr), in nvme_free_host_mem()
1926 kfree(dev->host_mem_desc_bufs); in nvme_free_host_mem()
1927 dev->host_mem_desc_bufs = NULL; in nvme_free_host_mem()
1928 dma_free_coherent(dev->dev, in nvme_free_host_mem()
1929 dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs), in nvme_free_host_mem()
1930 dev->host_mem_descs, dev->host_mem_descs_dma); in nvme_free_host_mem()
1931 dev->host_mem_descs = NULL; in nvme_free_host_mem()
1932 dev->nr_host_mem_descs = 0; in nvme_free_host_mem()
1935 static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred, in __nvme_alloc_host_mem() argument
1945 tmp = (preferred + chunk_size - 1); in __nvme_alloc_host_mem()
1949 if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries) in __nvme_alloc_host_mem()
1950 max_entries = dev->ctrl.hmmaxd; in __nvme_alloc_host_mem()
1952 descs = dma_alloc_coherent(dev->dev, max_entries * sizeof(*descs), in __nvme_alloc_host_mem()
1964 len = min_t(u64, chunk_size, preferred - size); in __nvme_alloc_host_mem()
1965 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL, in __nvme_alloc_host_mem()
1978 dev->nr_host_mem_descs = i; in __nvme_alloc_host_mem()
1979 dev->host_mem_size = size; in __nvme_alloc_host_mem()
1980 dev->host_mem_descs = descs; in __nvme_alloc_host_mem()
1981 dev->host_mem_descs_dma = descs_dma; in __nvme_alloc_host_mem()
1982 dev->host_mem_desc_bufs = bufs; in __nvme_alloc_host_mem()
1986 while (--i >= 0) { in __nvme_alloc_host_mem()
1989 dma_free_attrs(dev->dev, size, bufs[i], in __nvme_alloc_host_mem()
1996 dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs, in __nvme_alloc_host_mem()
1999 dev->host_mem_descs = NULL; in __nvme_alloc_host_mem()
2000 return -ENOMEM; in __nvme_alloc_host_mem()
2003 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) in nvme_alloc_host_mem() argument
2006 u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2); in nvme_alloc_host_mem()
2011 if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) { in nvme_alloc_host_mem()
2012 if (!min || dev->host_mem_size >= min) in nvme_alloc_host_mem()
2014 nvme_free_host_mem(dev); in nvme_alloc_host_mem()
2018 return -ENOMEM; in nvme_alloc_host_mem()
2021 static int nvme_setup_host_mem(struct nvme_dev *dev) in nvme_setup_host_mem() argument
2024 u64 preferred = (u64)dev->ctrl.hmpre * 4096; in nvme_setup_host_mem()
2025 u64 min = (u64)dev->ctrl.hmmin * 4096; in nvme_setup_host_mem()
2031 dev_warn(dev->ctrl.device, in nvme_setup_host_mem()
2034 nvme_free_host_mem(dev); in nvme_setup_host_mem()
2041 if (dev->host_mem_descs) { in nvme_setup_host_mem()
2042 if (dev->host_mem_size >= min) in nvme_setup_host_mem()
2045 nvme_free_host_mem(dev); in nvme_setup_host_mem()
2048 if (!dev->host_mem_descs) { in nvme_setup_host_mem()
2049 if (nvme_alloc_host_mem(dev, min, preferred)) { in nvme_setup_host_mem()
2050 dev_warn(dev->ctrl.device, in nvme_setup_host_mem()
2055 dev_info(dev->ctrl.device, in nvme_setup_host_mem()
2057 dev->host_mem_size >> ilog2(SZ_1M)); in nvme_setup_host_mem()
2060 ret = nvme_set_host_mem(dev, enable_bits); in nvme_setup_host_mem()
2062 nvme_free_host_mem(dev); in nvme_setup_host_mem()
2072 struct nvme_dev *dev = affd->priv; in nvme_calc_irq_sets() local
2073 unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; in nvme_calc_irq_sets()
2094 nr_read_queues = nrirqs - nr_write_queues; in nvme_calc_irq_sets()
2097 dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; in nvme_calc_irq_sets()
2098 affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; in nvme_calc_irq_sets()
2099 dev->io_queues[HCTX_TYPE_READ] = nr_read_queues; in nvme_calc_irq_sets()
2100 affd->set_size[HCTX_TYPE_READ] = nr_read_queues; in nvme_calc_irq_sets()
2101 affd->nr_sets = nr_read_queues ? 2 : 1; in nvme_calc_irq_sets()
2104 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues) in nvme_setup_irqs() argument
2106 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_setup_irqs()
2110 .priv = dev, in nvme_setup_irqs()
2116 * left over for non-polled I/O. in nvme_setup_irqs()
2118 poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1); in nvme_setup_irqs()
2119 dev->io_queues[HCTX_TYPE_POLL] = poll_queues; in nvme_setup_irqs()
2125 dev->io_queues[HCTX_TYPE_DEFAULT] = 1; in nvme_setup_irqs()
2126 dev->io_queues[HCTX_TYPE_READ] = 0; in nvme_setup_irqs()
2129 * We need interrupts for the admin queue and each non-polled I/O queue, in nvme_setup_irqs()
2134 if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR)) in nvme_setup_irqs()
2135 irq_queues += (nr_io_queues - poll_queues); in nvme_setup_irqs()
2140 static void nvme_disable_io_queues(struct nvme_dev *dev) in nvme_disable_io_queues() argument
2142 if (__nvme_disable_io_queues(dev, nvme_admin_delete_sq)) in nvme_disable_io_queues()
2143 __nvme_disable_io_queues(dev, nvme_admin_delete_cq); in nvme_disable_io_queues()
2146 static unsigned int nvme_max_io_queues(struct nvme_dev *dev) in nvme_max_io_queues() argument
2148 return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues; in nvme_max_io_queues()
2151 static int nvme_setup_io_queues(struct nvme_dev *dev) in nvme_setup_io_queues() argument
2153 struct nvme_queue *adminq = &dev->queues[0]; in nvme_setup_io_queues()
2154 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_setup_io_queues()
2163 dev->nr_write_queues = write_queues; in nvme_setup_io_queues()
2164 dev->nr_poll_queues = poll_queues; in nvme_setup_io_queues()
2170 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) in nvme_setup_io_queues()
2173 nr_io_queues = min(nvme_max_io_queues(dev), in nvme_setup_io_queues()
2174 dev->nr_allocated_queues - 1); in nvme_setup_io_queues()
2176 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); in nvme_setup_io_queues()
2183 clear_bit(NVMEQ_ENABLED, &adminq->flags); in nvme_setup_io_queues()
2185 if (dev->cmb_use_sqes) { in nvme_setup_io_queues()
2186 result = nvme_cmb_qdepth(dev, nr_io_queues, in nvme_setup_io_queues()
2189 dev->q_depth = result; in nvme_setup_io_queues()
2191 dev->cmb_use_sqes = false; in nvme_setup_io_queues()
2195 size = db_bar_size(dev, nr_io_queues); in nvme_setup_io_queues()
2196 result = nvme_remap_bar(dev, size); in nvme_setup_io_queues()
2199 if (!--nr_io_queues) in nvme_setup_io_queues()
2200 return -ENOMEM; in nvme_setup_io_queues()
2202 adminq->q_db = dev->dbs; in nvme_setup_io_queues()
2214 result = nvme_setup_irqs(dev, nr_io_queues); in nvme_setup_io_queues()
2216 return -EIO; in nvme_setup_io_queues()
2218 dev->num_vecs = result; in nvme_setup_io_queues()
2219 result = max(result - 1, 1); in nvme_setup_io_queues()
2220 dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL]; in nvme_setup_io_queues()
2231 set_bit(NVMEQ_ENABLED, &adminq->flags); in nvme_setup_io_queues()
2233 result = nvme_create_io_queues(dev); in nvme_setup_io_queues()
2234 if (result || dev->online_queues < 2) in nvme_setup_io_queues()
2237 if (dev->online_queues - 1 < dev->max_qid) { in nvme_setup_io_queues()
2238 nr_io_queues = dev->online_queues - 1; in nvme_setup_io_queues()
2239 nvme_disable_io_queues(dev); in nvme_setup_io_queues()
2240 nvme_suspend_io_queues(dev); in nvme_setup_io_queues()
2243 dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n", in nvme_setup_io_queues()
2244 dev->io_queues[HCTX_TYPE_DEFAULT], in nvme_setup_io_queues()
2245 dev->io_queues[HCTX_TYPE_READ], in nvme_setup_io_queues()
2246 dev->io_queues[HCTX_TYPE_POLL]); in nvme_setup_io_queues()
2252 struct nvme_queue *nvmeq = req->end_io_data; in nvme_del_queue_end()
2255 complete(&nvmeq->delete_done); in nvme_del_queue_end()
2260 struct nvme_queue *nvmeq = req->end_io_data; in nvme_del_cq_end()
2263 set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags); in nvme_del_cq_end()
2270 struct request_queue *q = nvmeq->dev->ctrl.admin_q; in nvme_delete_queue()
2276 cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid); in nvme_delete_queue()
2282 req->end_io_data = nvmeq; in nvme_delete_queue()
2284 init_completion(&nvmeq->delete_done); in nvme_delete_queue()
2291 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode) in __nvme_disable_io_queues() argument
2293 int nr_queues = dev->online_queues - 1, sent = 0; in __nvme_disable_io_queues()
2299 if (nvme_delete_queue(&dev->queues[nr_queues], opcode)) in __nvme_disable_io_queues()
2301 nr_queues--; in __nvme_disable_io_queues()
2305 struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent]; in __nvme_disable_io_queues()
2307 timeout = wait_for_completion_io_timeout(&nvmeq->delete_done, in __nvme_disable_io_queues()
2312 sent--; in __nvme_disable_io_queues()
2319 static void nvme_dev_add(struct nvme_dev *dev) in nvme_dev_add() argument
2323 if (!dev->ctrl.tagset) { in nvme_dev_add()
2324 dev->tagset.ops = &nvme_mq_ops; in nvme_dev_add()
2325 dev->tagset.nr_hw_queues = dev->online_queues - 1; in nvme_dev_add()
2326 dev->tagset.nr_maps = 2; /* default + read */ in nvme_dev_add()
2327 if (dev->io_queues[HCTX_TYPE_POLL]) in nvme_dev_add()
2328 dev->tagset.nr_maps++; in nvme_dev_add()
2329 dev->tagset.timeout = NVME_IO_TIMEOUT; in nvme_dev_add()
2330 dev->tagset.numa_node = dev->ctrl.numa_node; in nvme_dev_add()
2331 dev->tagset.queue_depth = min_t(unsigned int, dev->q_depth, in nvme_dev_add()
2332 BLK_MQ_MAX_DEPTH) - 1; in nvme_dev_add()
2333 dev->tagset.cmd_size = sizeof(struct nvme_iod); in nvme_dev_add()
2334 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE; in nvme_dev_add()
2335 dev->tagset.driver_data = dev; in nvme_dev_add()
2342 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) in nvme_dev_add()
2343 dev->tagset.reserved_tags = NVME_AQ_DEPTH; in nvme_dev_add()
2345 ret = blk_mq_alloc_tag_set(&dev->tagset); in nvme_dev_add()
2347 dev_warn(dev->ctrl.device, in nvme_dev_add()
2351 dev->ctrl.tagset = &dev->tagset; in nvme_dev_add()
2353 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1); in nvme_dev_add()
2356 nvme_free_queues(dev, dev->online_queues); in nvme_dev_add()
2359 nvme_dbbuf_set(dev); in nvme_dev_add()
2362 static int nvme_pci_enable(struct nvme_dev *dev) in nvme_pci_enable() argument
2364 int result = -ENOMEM; in nvme_pci_enable()
2365 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_pci_enable()
2372 if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64))) in nvme_pci_enable()
2375 if (readl(dev->bar + NVME_REG_CSTS) == -1) { in nvme_pci_enable()
2376 result = -ENODEV; in nvme_pci_enable()
2382 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll in nvme_pci_enable()
2389 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); in nvme_pci_enable()
2391 dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1, in nvme_pci_enable()
2393 dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */ in nvme_pci_enable()
2394 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); in nvme_pci_enable()
2395 dev->dbs = dev->bar + 4096; in nvme_pci_enable()
2398 * Some Apple controllers require a non-standard SQE size. in nvme_pci_enable()
2402 if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES) in nvme_pci_enable()
2403 dev->io_sqes = 7; in nvme_pci_enable()
2405 dev->io_sqes = NVME_NVM_IOSQES; in nvme_pci_enable()
2411 if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) { in nvme_pci_enable()
2412 dev->q_depth = 2; in nvme_pci_enable()
2413 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, " in nvme_pci_enable()
2415 dev->q_depth); in nvme_pci_enable()
2416 } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG && in nvme_pci_enable()
2417 (pdev->device == 0xa821 || pdev->device == 0xa822) && in nvme_pci_enable()
2418 NVME_CAP_MQES(dev->ctrl.cap) == 0) { in nvme_pci_enable()
2419 dev->q_depth = 64; in nvme_pci_enable()
2420 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, " in nvme_pci_enable()
2421 "set queue depth=%u\n", dev->q_depth); in nvme_pci_enable()
2428 if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) && in nvme_pci_enable()
2429 (dev->q_depth < (NVME_AQ_DEPTH + 2))) { in nvme_pci_enable()
2430 dev->q_depth = NVME_AQ_DEPTH + 2; in nvme_pci_enable()
2431 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n", in nvme_pci_enable()
2432 dev->q_depth); in nvme_pci_enable()
2436 nvme_map_cmb(dev); in nvme_pci_enable()
2447 static void nvme_dev_unmap(struct nvme_dev *dev) in nvme_dev_unmap() argument
2449 if (dev->bar) in nvme_dev_unmap()
2450 iounmap(dev->bar); in nvme_dev_unmap()
2451 pci_release_mem_regions(to_pci_dev(dev->dev)); in nvme_dev_unmap()
2454 static void nvme_pci_disable(struct nvme_dev *dev) in nvme_pci_disable() argument
2456 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_pci_disable()
2466 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) in nvme_dev_disable() argument
2469 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_dev_disable()
2471 mutex_lock(&dev->shutdown_lock); in nvme_dev_disable()
2473 u32 csts = readl(dev->bar + NVME_REG_CSTS); in nvme_dev_disable()
2475 if (dev->ctrl.state == NVME_CTRL_LIVE || in nvme_dev_disable()
2476 dev->ctrl.state == NVME_CTRL_RESETTING) { in nvme_dev_disable()
2478 nvme_start_freeze(&dev->ctrl); in nvme_dev_disable()
2481 pdev->error_state != pci_channel_io_normal); in nvme_dev_disable()
2489 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT); in nvme_dev_disable()
2491 nvme_stop_queues(&dev->ctrl); in nvme_dev_disable()
2493 if (!dead && dev->ctrl.queue_count > 0) { in nvme_dev_disable()
2494 nvme_disable_io_queues(dev); in nvme_dev_disable()
2495 nvme_disable_admin_queue(dev, shutdown); in nvme_dev_disable()
2497 nvme_suspend_io_queues(dev); in nvme_dev_disable()
2498 nvme_suspend_queue(&dev->queues[0]); in nvme_dev_disable()
2499 nvme_pci_disable(dev); in nvme_dev_disable()
2500 nvme_reap_pending_cqes(dev); in nvme_dev_disable()
2502 blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl); in nvme_dev_disable()
2503 blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl); in nvme_dev_disable()
2504 blk_mq_tagset_wait_completed_request(&dev->tagset); in nvme_dev_disable()
2505 blk_mq_tagset_wait_completed_request(&dev->admin_tagset); in nvme_dev_disable()
2510 * deadlocking blk-mq hot-cpu notifier. in nvme_dev_disable()
2513 nvme_start_queues(&dev->ctrl); in nvme_dev_disable()
2514 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) in nvme_dev_disable()
2515 nvme_start_admin_queue(&dev->ctrl); in nvme_dev_disable()
2517 mutex_unlock(&dev->shutdown_lock); in nvme_dev_disable()
2520 static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) in nvme_disable_prepare_reset() argument
2522 if (!nvme_wait_reset(&dev->ctrl)) in nvme_disable_prepare_reset()
2523 return -EBUSY; in nvme_disable_prepare_reset()
2524 nvme_dev_disable(dev, shutdown); in nvme_disable_prepare_reset()
2528 static int nvme_setup_prp_pools(struct nvme_dev *dev) in nvme_setup_prp_pools() argument
2530 dev->prp_page_pool = dma_pool_create("prp list page", dev->dev, in nvme_setup_prp_pools()
2533 if (!dev->prp_page_pool) in nvme_setup_prp_pools()
2534 return -ENOMEM; in nvme_setup_prp_pools()
2537 dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev, in nvme_setup_prp_pools()
2539 if (!dev->prp_small_pool) { in nvme_setup_prp_pools()
2540 dma_pool_destroy(dev->prp_page_pool); in nvme_setup_prp_pools()
2541 return -ENOMEM; in nvme_setup_prp_pools()
2546 static void nvme_release_prp_pools(struct nvme_dev *dev) in nvme_release_prp_pools() argument
2548 dma_pool_destroy(dev->prp_page_pool); in nvme_release_prp_pools()
2549 dma_pool_destroy(dev->prp_small_pool); in nvme_release_prp_pools()
2552 static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev) in nvme_pci_alloc_iod_mempool() argument
2559 dev->iod_mempool = mempool_create_node(1, in nvme_pci_alloc_iod_mempool()
2562 dev_to_node(dev->dev)); in nvme_pci_alloc_iod_mempool()
2563 if (!dev->iod_mempool) in nvme_pci_alloc_iod_mempool()
2564 return -ENOMEM; in nvme_pci_alloc_iod_mempool()
2568 static void nvme_free_tagset(struct nvme_dev *dev) in nvme_free_tagset() argument
2570 if (dev->tagset.tags) in nvme_free_tagset()
2571 blk_mq_free_tag_set(&dev->tagset); in nvme_free_tagset()
2572 dev->ctrl.tagset = NULL; in nvme_free_tagset()
2576 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) in nvme_pci_free_ctrl() argument
2578 struct nvme_dev *dev = to_nvme_dev(ctrl); in nvme_pci_free_ctrl() local
2580 nvme_dbbuf_dma_free(dev); in nvme_pci_free_ctrl()
2581 nvme_free_tagset(dev); in nvme_pci_free_ctrl()
2582 if (dev->ctrl.admin_q) in nvme_pci_free_ctrl()
2583 blk_put_queue(dev->ctrl.admin_q); in nvme_pci_free_ctrl()
2584 free_opal_dev(dev->ctrl.opal_dev); in nvme_pci_free_ctrl()
2585 mempool_destroy(dev->iod_mempool); in nvme_pci_free_ctrl()
2586 put_device(dev->dev); in nvme_pci_free_ctrl()
2587 kfree(dev->queues); in nvme_pci_free_ctrl()
2588 kfree(dev); in nvme_pci_free_ctrl()
2591 static void nvme_remove_dead_ctrl(struct nvme_dev *dev) in nvme_remove_dead_ctrl() argument
2597 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_remove_dead_ctrl()
2598 nvme_get_ctrl(&dev->ctrl); in nvme_remove_dead_ctrl()
2599 nvme_dev_disable(dev, false); in nvme_remove_dead_ctrl()
2600 nvme_kill_queues(&dev->ctrl); in nvme_remove_dead_ctrl()
2601 if (!queue_work(nvme_wq, &dev->remove_work)) in nvme_remove_dead_ctrl()
2602 nvme_put_ctrl(&dev->ctrl); in nvme_remove_dead_ctrl()
2607 struct nvme_dev *dev = in nvme_reset_work() local
2608 container_of(work, struct nvme_dev, ctrl.reset_work); in nvme_reset_work()
2609 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL); in nvme_reset_work()
2612 if (dev->ctrl.state != NVME_CTRL_RESETTING) { in nvme_reset_work()
2613 dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n", in nvme_reset_work()
2614 dev->ctrl.state); in nvme_reset_work()
2615 result = -ENODEV; in nvme_reset_work()
2623 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE) in nvme_reset_work()
2624 nvme_dev_disable(dev, false); in nvme_reset_work()
2625 nvme_sync_queues(&dev->ctrl); in nvme_reset_work()
2627 mutex_lock(&dev->shutdown_lock); in nvme_reset_work()
2628 result = nvme_pci_enable(dev); in nvme_reset_work()
2632 result = nvme_pci_configure_admin_queue(dev); in nvme_reset_work()
2636 result = nvme_alloc_admin_tags(dev); in nvme_reset_work()
2640 dma_set_min_align_mask(dev->dev, NVME_CTRL_PAGE_SIZE - 1); in nvme_reset_work()
2643 * Limit the max command size to prevent iod->sg allocations going in nvme_reset_work()
2646 dev->ctrl.max_hw_sectors = min_t(u32, in nvme_reset_work()
2647 NVME_MAX_KB_SZ << 1, dma_max_mapping_size(dev->dev) >> 9); in nvme_reset_work()
2648 dev->ctrl.max_segments = NVME_MAX_SEGS; in nvme_reset_work()
2653 dma_set_max_seg_size(dev->dev, 0xffffffff); in nvme_reset_work()
2655 mutex_unlock(&dev->shutdown_lock); in nvme_reset_work()
2658 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the in nvme_reset_work()
2661 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) { in nvme_reset_work()
2662 dev_warn(dev->ctrl.device, in nvme_reset_work()
2664 result = -EBUSY; in nvme_reset_work()
2672 dev->ctrl.max_integrity_segments = 1; in nvme_reset_work()
2674 result = nvme_init_identify(&dev->ctrl); in nvme_reset_work()
2678 if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) { in nvme_reset_work()
2679 if (!dev->ctrl.opal_dev) in nvme_reset_work()
2680 dev->ctrl.opal_dev = in nvme_reset_work()
2681 init_opal_dev(&dev->ctrl, &nvme_sec_submit); in nvme_reset_work()
2683 opal_unlock_from_suspend(dev->ctrl.opal_dev); in nvme_reset_work()
2685 free_opal_dev(dev->ctrl.opal_dev); in nvme_reset_work()
2686 dev->ctrl.opal_dev = NULL; in nvme_reset_work()
2689 if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) { in nvme_reset_work()
2690 result = nvme_dbbuf_dma_alloc(dev); in nvme_reset_work()
2692 dev_warn(dev->dev, in nvme_reset_work()
2696 if (dev->ctrl.hmpre) { in nvme_reset_work()
2697 result = nvme_setup_host_mem(dev); in nvme_reset_work()
2702 result = nvme_setup_io_queues(dev); in nvme_reset_work()
2710 if (dev->online_queues < 2) { in nvme_reset_work()
2711 dev_warn(dev->ctrl.device, "IO queues not created\n"); in nvme_reset_work()
2712 nvme_kill_queues(&dev->ctrl); in nvme_reset_work()
2713 nvme_remove_namespaces(&dev->ctrl); in nvme_reset_work()
2714 nvme_free_tagset(dev); in nvme_reset_work()
2716 nvme_start_queues(&dev->ctrl); in nvme_reset_work()
2717 nvme_wait_freeze(&dev->ctrl); in nvme_reset_work()
2718 nvme_dev_add(dev); in nvme_reset_work()
2719 nvme_unfreeze(&dev->ctrl); in nvme_reset_work()
2726 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { in nvme_reset_work()
2727 dev_warn(dev->ctrl.device, in nvme_reset_work()
2729 result = -ENODEV; in nvme_reset_work()
2733 nvme_start_ctrl(&dev->ctrl); in nvme_reset_work()
2737 mutex_unlock(&dev->shutdown_lock); in nvme_reset_work()
2740 dev_warn(dev->ctrl.device, in nvme_reset_work()
2742 nvme_remove_dead_ctrl(dev); in nvme_reset_work()
2747 struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work); in nvme_remove_dead_ctrl_work() local
2748 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_remove_dead_ctrl_work()
2751 device_release_driver(&pdev->dev); in nvme_remove_dead_ctrl_work()
2752 nvme_put_ctrl(&dev->ctrl); in nvme_remove_dead_ctrl_work()
2755 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) in nvme_pci_reg_read32() argument
2757 *val = readl(to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_read32()
2761 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) in nvme_pci_reg_write32() argument
2763 writel(val, to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_write32()
2767 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) in nvme_pci_reg_read64() argument
2769 *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off); in nvme_pci_reg_read64()
2773 static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size) in nvme_pci_get_address() argument
2775 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev); in nvme_pci_get_address()
2777 return snprintf(buf, size, "%s\n", dev_name(&pdev->dev)); in nvme_pci_get_address()
2793 static int nvme_dev_map(struct nvme_dev *dev) in nvme_dev_map() argument
2795 struct pci_dev *pdev = to_pci_dev(dev->dev); in nvme_dev_map()
2798 return -ENODEV; in nvme_dev_map()
2800 if (nvme_remap_bar(dev, NVME_REG_DBS + 4096)) in nvme_dev_map()
2806 return -ENODEV; in nvme_dev_map()
2811 if (pdev->vendor == 0x144d && pdev->device == 0xa802) { in check_vendor_combination_bug()
2824 } else if (pdev->vendor == 0x144d && pdev->device == 0xa804) { in check_vendor_combination_bug()
2827 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as in check_vendor_combination_bug()
2828 * within few minutes after bootup on a Coffee Lake board - in check_vendor_combination_bug()
2829 * ASUS PRIME Z370-A in check_vendor_combination_bug()
2832 (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") || in check_vendor_combination_bug()
2833 dmi_match(DMI_BOARD_NAME, "PRIME Z370-A"))) in check_vendor_combination_bug()
2835 } else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 || in check_vendor_combination_bug()
2836 pdev->device == 0xa808 || pdev->device == 0xa809)) || in check_vendor_combination_bug()
2837 (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) { in check_vendor_combination_bug()
2854 struct nvme_dev *dev = data; in nvme_async_probe() local
2856 flush_work(&dev->ctrl.reset_work); in nvme_async_probe()
2857 flush_work(&dev->ctrl.scan_work); in nvme_async_probe()
2858 nvme_put_ctrl(&dev->ctrl); in nvme_async_probe()
2864 unsigned long quirks = id->driver_data; in nvme_pci_alloc_dev()
2865 int node = dev_to_node(&pdev->dev); in nvme_pci_alloc_dev()
2866 struct nvme_dev *dev; in nvme_pci_alloc_dev() local
2867 int ret = -ENOMEM; in nvme_pci_alloc_dev()
2869 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node); in nvme_pci_alloc_dev()
2870 if (!dev) in nvme_pci_alloc_dev()
2871 return ERR_PTR(-ENOMEM); in nvme_pci_alloc_dev()
2872 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); in nvme_pci_alloc_dev()
2873 INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work); in nvme_pci_alloc_dev()
2874 mutex_init(&dev->shutdown_lock); in nvme_pci_alloc_dev()
2876 dev->nr_write_queues = write_queues; in nvme_pci_alloc_dev()
2877 dev->nr_poll_queues = poll_queues; in nvme_pci_alloc_dev()
2878 dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1; in nvme_pci_alloc_dev()
2879 dev->queues = kcalloc_node(dev->nr_allocated_queues, in nvme_pci_alloc_dev()
2881 if (!dev->queues) in nvme_pci_alloc_dev()
2884 dev->dev = get_device(&pdev->dev); in nvme_pci_alloc_dev()
2887 if (!noacpi && acpi_storage_d3(&pdev->dev)) { in nvme_pci_alloc_dev()
2892 dev_info(&pdev->dev, in nvme_pci_alloc_dev()
2896 ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops, in nvme_pci_alloc_dev()
2900 return dev; in nvme_pci_alloc_dev()
2903 put_device(dev->dev); in nvme_pci_alloc_dev()
2904 kfree(dev->queues); in nvme_pci_alloc_dev()
2906 kfree(dev); in nvme_pci_alloc_dev()
2912 struct nvme_dev *dev; in nvme_probe() local
2913 int result = -ENOMEM; in nvme_probe()
2915 dev = nvme_pci_alloc_dev(pdev, id); in nvme_probe()
2916 if (IS_ERR(dev)) in nvme_probe()
2917 return PTR_ERR(dev); in nvme_probe()
2919 result = nvme_dev_map(dev); in nvme_probe()
2923 result = nvme_setup_prp_pools(dev); in nvme_probe()
2927 result = nvme_pci_alloc_iod_mempool(dev); in nvme_probe()
2931 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev)); in nvme_probe()
2932 pci_set_drvdata(pdev, dev); in nvme_probe()
2934 nvme_reset_ctrl(&dev->ctrl); in nvme_probe()
2935 async_schedule(nvme_async_probe, dev); in nvme_probe()
2939 nvme_release_prp_pools(dev); in nvme_probe()
2941 nvme_dev_unmap(dev); in nvme_probe()
2943 nvme_uninit_ctrl(&dev->ctrl); in nvme_probe()
2949 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_reset_prepare() local
2954 * with ->remove(). in nvme_reset_prepare()
2956 nvme_disable_prepare_reset(dev, false); in nvme_reset_prepare()
2957 nvme_sync_queues(&dev->ctrl); in nvme_reset_prepare()
2962 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_reset_done() local
2964 if (!nvme_try_sched_reset(&dev->ctrl)) in nvme_reset_done()
2965 flush_work(&dev->ctrl.reset_work); in nvme_reset_done()
2970 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_shutdown() local
2972 nvme_disable_prepare_reset(dev, true); in nvme_shutdown()
2982 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_remove() local
2984 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); in nvme_remove()
2988 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); in nvme_remove()
2989 nvme_dev_disable(dev, true); in nvme_remove()
2992 flush_work(&dev->ctrl.reset_work); in nvme_remove()
2993 nvme_stop_ctrl(&dev->ctrl); in nvme_remove()
2994 nvme_remove_namespaces(&dev->ctrl); in nvme_remove()
2995 nvme_dev_disable(dev, true); in nvme_remove()
2996 nvme_release_cmb(dev); in nvme_remove()
2997 nvme_free_host_mem(dev); in nvme_remove()
2998 nvme_dev_remove_admin(dev); in nvme_remove()
2999 nvme_free_queues(dev, 0); in nvme_remove()
3000 nvme_release_prp_pools(dev); in nvme_remove()
3001 nvme_dev_unmap(dev); in nvme_remove()
3002 nvme_uninit_ctrl(&dev->ctrl); in nvme_remove()
3006 static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps) in nvme_get_power_state() argument
3008 return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps); in nvme_get_power_state()
3011 static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps) in nvme_set_power_state() argument
3013 return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL); in nvme_set_power_state()
3016 static int nvme_resume(struct device *dev) in nvme_resume() argument
3018 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); in nvme_resume()
3019 struct nvme_ctrl *ctrl = &ndev->ctrl; in nvme_resume() local
3021 if (ndev->last_ps == U32_MAX || in nvme_resume()
3022 nvme_set_power_state(ctrl, ndev->last_ps) != 0) in nvme_resume()
3023 return nvme_try_sched_reset(&ndev->ctrl); in nvme_resume()
3027 static int nvme_suspend(struct device *dev) in nvme_suspend() argument
3029 struct pci_dev *pdev = to_pci_dev(dev); in nvme_suspend()
3031 struct nvme_ctrl *ctrl = &ndev->ctrl; in nvme_suspend() local
3032 int ret = -EBUSY; in nvme_suspend()
3034 ndev->last_ps = U32_MAX; in nvme_suspend()
3041 * device does not support any non-default power states, shut down the in nvme_suspend()
3046 * down, so as to allow the platform to achieve its minimum low-power in nvme_suspend()
3054 if (pm_suspend_via_firmware() || !ctrl->npss || in nvme_suspend()
3056 ndev->nr_host_mem_descs || in nvme_suspend()
3057 (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND)) in nvme_suspend()
3060 nvme_start_freeze(ctrl); in nvme_suspend()
3061 nvme_wait_freeze(ctrl); in nvme_suspend()
3062 nvme_sync_queues(ctrl); in nvme_suspend()
3064 if (ctrl->state != NVME_CTRL_LIVE) in nvme_suspend()
3067 ret = nvme_get_power_state(ctrl, &ndev->last_ps); in nvme_suspend()
3078 ret = nvme_set_power_state(ctrl, ctrl->npss); in nvme_suspend()
3091 ctrl->npss = 0; in nvme_suspend()
3094 nvme_unfreeze(ctrl); in nvme_suspend()
3098 static int nvme_simple_suspend(struct device *dev) in nvme_simple_suspend() argument
3100 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); in nvme_simple_suspend()
3105 static int nvme_simple_resume(struct device *dev) in nvme_simple_resume() argument
3107 struct pci_dev *pdev = to_pci_dev(dev); in nvme_simple_resume()
3110 return nvme_try_sched_reset(&ndev->ctrl); in nvme_simple_resume()
3126 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_error_detected() local
3137 dev_warn(dev->ctrl.device, in nvme_error_detected()
3139 nvme_dev_disable(dev, false); in nvme_error_detected()
3142 dev_warn(dev->ctrl.device, in nvme_error_detected()
3151 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_slot_reset() local
3153 dev_info(dev->ctrl.device, "restart after slot reset\n"); in nvme_slot_reset()
3155 nvme_reset_ctrl(&dev->ctrl); in nvme_slot_reset()
3161 struct nvme_dev *dev = pci_get_drvdata(pdev); in nvme_error_resume() local
3163 flush_work(&dev->ctrl.reset_work); in nvme_error_resume()