• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVM Express device driver
4  * Copyright (c) 2011-2014, Intel Corporation.
5  */
6 
7 #include <linux/blkdev.h>
8 #include <linux/blk-mq.h>
9 #include <linux/delay.h>
10 #include <linux/errno.h>
11 #include <linux/hdreg.h>
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/backing-dev.h>
15 #include <linux/list_sort.h>
16 #include <linux/slab.h>
17 #include <linux/types.h>
18 #include <linux/pr.h>
19 #include <linux/ptrace.h>
20 #include <linux/nvme_ioctl.h>
21 #include <linux/t10-pi.h>
22 #include <linux/pm_qos.h>
23 #include <asm/unaligned.h>
24 
25 #include "nvme.h"
26 #include "fabrics.h"
27 
28 #define CREATE_TRACE_POINTS
29 #include "trace.h"
30 
31 #define NVME_MINORS		(1U << MINORBITS)
32 
33 unsigned int admin_timeout = 60;
34 module_param(admin_timeout, uint, 0644);
35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
36 EXPORT_SYMBOL_GPL(admin_timeout);
37 
38 unsigned int nvme_io_timeout = 30;
39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
42 
43 static unsigned char shutdown_timeout = 5;
44 module_param(shutdown_timeout, byte, 0644);
45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46 
47 static u8 nvme_max_retries = 5;
48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
50 
51 static unsigned long default_ps_max_latency_us = 100000;
52 module_param(default_ps_max_latency_us, ulong, 0644);
53 MODULE_PARM_DESC(default_ps_max_latency_us,
54 		 "max power saving latency for new devices; use PM QOS to change per device");
55 
56 static bool force_apst;
57 module_param(force_apst, bool, 0644);
58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59 
60 static bool streams;
61 module_param(streams, bool, 0644);
62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
63 
64 /*
65  * nvme_wq - hosts nvme related works that are not reset or delete
66  * nvme_reset_wq - hosts nvme reset works
67  * nvme_delete_wq - hosts nvme delete works
68  *
69  * nvme_wq will host works such are scan, aen handling, fw activation,
70  * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
71  * runs reset works which also flush works hosted on nvme_wq for
72  * serialization purposes. nvme_delete_wq host controller deletion
73  * works which flush reset works for serialization.
74  */
75 struct workqueue_struct *nvme_wq;
76 EXPORT_SYMBOL_GPL(nvme_wq);
77 
78 struct workqueue_struct *nvme_reset_wq;
79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
80 
81 struct workqueue_struct *nvme_delete_wq;
82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
83 
84 static LIST_HEAD(nvme_subsystems);
85 static DEFINE_MUTEX(nvme_subsystems_lock);
86 
87 static DEFINE_IDA(nvme_instance_ida);
88 static dev_t nvme_chr_devt;
89 static struct class *nvme_class;
90 static struct class *nvme_subsys_class;
91 
92 static int nvme_revalidate_disk(struct gendisk *disk);
93 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
94 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
95 					   unsigned nsid);
96 
nvme_set_queue_dying(struct nvme_ns * ns)97 static void nvme_set_queue_dying(struct nvme_ns *ns)
98 {
99 	/*
100 	 * Revalidating a dead namespace sets capacity to 0. This will end
101 	 * buffered writers dirtying pages that can't be synced.
102 	 */
103 	if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
104 		return;
105 	blk_set_queue_dying(ns->queue);
106 	/* Forcibly unquiesce queues to avoid blocking dispatch */
107 	blk_mq_unquiesce_queue(ns->queue);
108 	/*
109 	 * Revalidate after unblocking dispatchers that may be holding bd_butex
110 	 */
111 	revalidate_disk(ns->disk);
112 }
113 
nvme_queue_scan(struct nvme_ctrl * ctrl)114 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
115 {
116 	/*
117 	 * Only new queue scan work when admin and IO queues are both alive
118 	 */
119 	if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
120 		queue_work(nvme_wq, &ctrl->scan_work);
121 }
122 
123 /*
124  * Use this function to proceed with scheduling reset_work for a controller
125  * that had previously been set to the resetting state. This is intended for
126  * code paths that can't be interrupted by other reset attempts. A hot removal
127  * may prevent this from succeeding.
128  */
nvme_try_sched_reset(struct nvme_ctrl * ctrl)129 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
130 {
131 	if (ctrl->state != NVME_CTRL_RESETTING)
132 		return -EBUSY;
133 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
134 		return -EBUSY;
135 	return 0;
136 }
137 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
138 
nvme_reset_ctrl(struct nvme_ctrl * ctrl)139 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
140 {
141 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
142 		return -EBUSY;
143 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
144 		return -EBUSY;
145 	return 0;
146 }
147 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
148 
nvme_reset_ctrl_sync(struct nvme_ctrl * ctrl)149 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
150 {
151 	int ret;
152 
153 	ret = nvme_reset_ctrl(ctrl);
154 	if (!ret) {
155 		flush_work(&ctrl->reset_work);
156 		if (ctrl->state != NVME_CTRL_LIVE)
157 			ret = -ENETRESET;
158 	}
159 
160 	return ret;
161 }
162 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
163 
nvme_do_delete_ctrl(struct nvme_ctrl * ctrl)164 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
165 {
166 	dev_info(ctrl->device,
167 		 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
168 
169 	flush_work(&ctrl->reset_work);
170 	nvme_stop_ctrl(ctrl);
171 	nvme_remove_namespaces(ctrl);
172 	ctrl->ops->delete_ctrl(ctrl);
173 	nvme_uninit_ctrl(ctrl);
174 	nvme_put_ctrl(ctrl);
175 }
176 
nvme_delete_ctrl_work(struct work_struct * work)177 static void nvme_delete_ctrl_work(struct work_struct *work)
178 {
179 	struct nvme_ctrl *ctrl =
180 		container_of(work, struct nvme_ctrl, delete_work);
181 
182 	nvme_do_delete_ctrl(ctrl);
183 }
184 
nvme_delete_ctrl(struct nvme_ctrl * ctrl)185 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
186 {
187 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
188 		return -EBUSY;
189 	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
190 		return -EBUSY;
191 	return 0;
192 }
193 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
194 
nvme_delete_ctrl_sync(struct nvme_ctrl * ctrl)195 static int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
196 {
197 	int ret = 0;
198 
199 	/*
200 	 * Keep a reference until nvme_do_delete_ctrl() complete,
201 	 * since ->delete_ctrl can free the controller.
202 	 */
203 	nvme_get_ctrl(ctrl);
204 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
205 		ret = -EBUSY;
206 	if (!ret)
207 		nvme_do_delete_ctrl(ctrl);
208 	nvme_put_ctrl(ctrl);
209 	return ret;
210 }
211 
nvme_ns_has_pi(struct nvme_ns * ns)212 static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
213 {
214 	return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
215 }
216 
nvme_error_status(u16 status)217 static blk_status_t nvme_error_status(u16 status)
218 {
219 	switch (status & 0x7ff) {
220 	case NVME_SC_SUCCESS:
221 		return BLK_STS_OK;
222 	case NVME_SC_CAP_EXCEEDED:
223 		return BLK_STS_NOSPC;
224 	case NVME_SC_LBA_RANGE:
225 		return BLK_STS_TARGET;
226 	case NVME_SC_BAD_ATTRIBUTES:
227 	case NVME_SC_ONCS_NOT_SUPPORTED:
228 	case NVME_SC_INVALID_OPCODE:
229 	case NVME_SC_INVALID_FIELD:
230 	case NVME_SC_INVALID_NS:
231 		return BLK_STS_NOTSUPP;
232 	case NVME_SC_WRITE_FAULT:
233 	case NVME_SC_READ_ERROR:
234 	case NVME_SC_UNWRITTEN_BLOCK:
235 	case NVME_SC_ACCESS_DENIED:
236 	case NVME_SC_READ_ONLY:
237 	case NVME_SC_COMPARE_FAILED:
238 		return BLK_STS_MEDIUM;
239 	case NVME_SC_GUARD_CHECK:
240 	case NVME_SC_APPTAG_CHECK:
241 	case NVME_SC_REFTAG_CHECK:
242 	case NVME_SC_INVALID_PI:
243 		return BLK_STS_PROTECTION;
244 	case NVME_SC_RESERVATION_CONFLICT:
245 		return BLK_STS_NEXUS;
246 	case NVME_SC_HOST_PATH_ERROR:
247 		return BLK_STS_TRANSPORT;
248 	default:
249 		return BLK_STS_IOERR;
250 	}
251 }
252 
nvme_req_needs_retry(struct request * req)253 static inline bool nvme_req_needs_retry(struct request *req)
254 {
255 	if (blk_noretry_request(req))
256 		return false;
257 	if (nvme_req(req)->status & NVME_SC_DNR)
258 		return false;
259 	if (nvme_req(req)->retries >= nvme_max_retries)
260 		return false;
261 	return true;
262 }
263 
nvme_retry_req(struct request * req)264 static void nvme_retry_req(struct request *req)
265 {
266 	struct nvme_ns *ns = req->q->queuedata;
267 	unsigned long delay = 0;
268 	u16 crd;
269 
270 	/* The mask and shift result must be <= 3 */
271 	crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
272 	if (ns && crd)
273 		delay = ns->ctrl->crdt[crd - 1] * 100;
274 
275 	nvme_req(req)->retries++;
276 	blk_mq_requeue_request(req, false);
277 	blk_mq_delay_kick_requeue_list(req->q, delay);
278 }
279 
nvme_complete_rq(struct request * req)280 void nvme_complete_rq(struct request *req)
281 {
282 	blk_status_t status = nvme_error_status(nvme_req(req)->status);
283 
284 	trace_nvme_complete_rq(req);
285 
286 	if (nvme_req(req)->ctrl->kas)
287 		nvme_req(req)->ctrl->comp_seen = true;
288 
289 	if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
290 		if ((req->cmd_flags & REQ_NVME_MPATH) &&
291 		    blk_path_error(status)) {
292 			nvme_failover_req(req);
293 			return;
294 		}
295 
296 		if (!blk_queue_dying(req->q)) {
297 			nvme_retry_req(req);
298 			return;
299 		}
300 	}
301 
302 	nvme_trace_bio_complete(req, status);
303 	blk_mq_end_request(req, status);
304 }
305 EXPORT_SYMBOL_GPL(nvme_complete_rq);
306 
nvme_cancel_request(struct request * req,void * data,bool reserved)307 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
308 {
309 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
310 				"Cancelling I/O %d", req->tag);
311 
312 	/* don't abort one completed request */
313 	if (blk_mq_request_completed(req))
314 		return true;
315 
316 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
317 	blk_mq_complete_request(req);
318 	return true;
319 }
320 EXPORT_SYMBOL_GPL(nvme_cancel_request);
321 
nvme_change_ctrl_state(struct nvme_ctrl * ctrl,enum nvme_ctrl_state new_state)322 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
323 		enum nvme_ctrl_state new_state)
324 {
325 	enum nvme_ctrl_state old_state;
326 	unsigned long flags;
327 	bool changed = false;
328 
329 	spin_lock_irqsave(&ctrl->lock, flags);
330 
331 	old_state = ctrl->state;
332 	switch (new_state) {
333 	case NVME_CTRL_LIVE:
334 		switch (old_state) {
335 		case NVME_CTRL_NEW:
336 		case NVME_CTRL_RESETTING:
337 		case NVME_CTRL_CONNECTING:
338 			changed = true;
339 			/* FALLTHRU */
340 		default:
341 			break;
342 		}
343 		break;
344 	case NVME_CTRL_RESETTING:
345 		switch (old_state) {
346 		case NVME_CTRL_NEW:
347 		case NVME_CTRL_LIVE:
348 			changed = true;
349 			/* FALLTHRU */
350 		default:
351 			break;
352 		}
353 		break;
354 	case NVME_CTRL_CONNECTING:
355 		switch (old_state) {
356 		case NVME_CTRL_NEW:
357 		case NVME_CTRL_RESETTING:
358 			changed = true;
359 			/* FALLTHRU */
360 		default:
361 			break;
362 		}
363 		break;
364 	case NVME_CTRL_DELETING:
365 		switch (old_state) {
366 		case NVME_CTRL_LIVE:
367 		case NVME_CTRL_RESETTING:
368 		case NVME_CTRL_CONNECTING:
369 			changed = true;
370 			/* FALLTHRU */
371 		default:
372 			break;
373 		}
374 		break;
375 	case NVME_CTRL_DEAD:
376 		switch (old_state) {
377 		case NVME_CTRL_DELETING:
378 			changed = true;
379 			/* FALLTHRU */
380 		default:
381 			break;
382 		}
383 		break;
384 	default:
385 		break;
386 	}
387 
388 	if (changed) {
389 		ctrl->state = new_state;
390 		wake_up_all(&ctrl->state_wq);
391 	}
392 
393 	spin_unlock_irqrestore(&ctrl->lock, flags);
394 	if (changed && ctrl->state == NVME_CTRL_LIVE)
395 		nvme_kick_requeue_lists(ctrl);
396 	return changed;
397 }
398 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
399 
400 /*
401  * Returns true for sink states that can't ever transition back to live.
402  */
nvme_state_terminal(struct nvme_ctrl * ctrl)403 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
404 {
405 	switch (ctrl->state) {
406 	case NVME_CTRL_NEW:
407 	case NVME_CTRL_LIVE:
408 	case NVME_CTRL_RESETTING:
409 	case NVME_CTRL_CONNECTING:
410 		return false;
411 	case NVME_CTRL_DELETING:
412 	case NVME_CTRL_DEAD:
413 		return true;
414 	default:
415 		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
416 		return true;
417 	}
418 }
419 
420 /*
421  * Waits for the controller state to be resetting, or returns false if it is
422  * not possible to ever transition to that state.
423  */
nvme_wait_reset(struct nvme_ctrl * ctrl)424 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
425 {
426 	wait_event(ctrl->state_wq,
427 		   nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
428 		   nvme_state_terminal(ctrl));
429 	return ctrl->state == NVME_CTRL_RESETTING;
430 }
431 EXPORT_SYMBOL_GPL(nvme_wait_reset);
432 
nvme_free_ns_head(struct kref * ref)433 static void nvme_free_ns_head(struct kref *ref)
434 {
435 	struct nvme_ns_head *head =
436 		container_of(ref, struct nvme_ns_head, ref);
437 
438 	nvme_mpath_remove_disk(head);
439 	ida_simple_remove(&head->subsys->ns_ida, head->instance);
440 	list_del_init(&head->entry);
441 	cleanup_srcu_struct(&head->srcu);
442 	nvme_put_subsystem(head->subsys);
443 	kfree(head);
444 }
445 
nvme_put_ns_head(struct nvme_ns_head * head)446 static void nvme_put_ns_head(struct nvme_ns_head *head)
447 {
448 	kref_put(&head->ref, nvme_free_ns_head);
449 }
450 
nvme_free_ns(struct kref * kref)451 static void nvme_free_ns(struct kref *kref)
452 {
453 	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
454 
455 	if (ns->ndev)
456 		nvme_nvm_unregister(ns);
457 
458 	put_disk(ns->disk);
459 	nvme_put_ns_head(ns->head);
460 	nvme_put_ctrl(ns->ctrl);
461 	kfree(ns);
462 }
463 
nvme_put_ns(struct nvme_ns * ns)464 static void nvme_put_ns(struct nvme_ns *ns)
465 {
466 	kref_put(&ns->kref, nvme_free_ns);
467 }
468 
nvme_clear_nvme_request(struct request * req)469 static inline void nvme_clear_nvme_request(struct request *req)
470 {
471 	if (!(req->rq_flags & RQF_DONTPREP)) {
472 		nvme_req(req)->retries = 0;
473 		nvme_req(req)->flags = 0;
474 		req->rq_flags |= RQF_DONTPREP;
475 	}
476 }
477 
nvme_alloc_request(struct request_queue * q,struct nvme_command * cmd,blk_mq_req_flags_t flags,int qid)478 struct request *nvme_alloc_request(struct request_queue *q,
479 		struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
480 {
481 	unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
482 	struct request *req;
483 
484 	if (qid == NVME_QID_ANY) {
485 		req = blk_mq_alloc_request(q, op, flags);
486 	} else {
487 		req = blk_mq_alloc_request_hctx(q, op, flags,
488 				qid ? qid - 1 : 0);
489 	}
490 	if (IS_ERR(req))
491 		return req;
492 
493 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
494 	nvme_clear_nvme_request(req);
495 	nvme_req(req)->cmd = cmd;
496 
497 	return req;
498 }
499 EXPORT_SYMBOL_GPL(nvme_alloc_request);
500 
nvme_toggle_streams(struct nvme_ctrl * ctrl,bool enable)501 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
502 {
503 	struct nvme_command c;
504 
505 	memset(&c, 0, sizeof(c));
506 
507 	c.directive.opcode = nvme_admin_directive_send;
508 	c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
509 	c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
510 	c.directive.dtype = NVME_DIR_IDENTIFY;
511 	c.directive.tdtype = NVME_DIR_STREAMS;
512 	c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
513 
514 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
515 }
516 
nvme_disable_streams(struct nvme_ctrl * ctrl)517 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
518 {
519 	return nvme_toggle_streams(ctrl, false);
520 }
521 
nvme_enable_streams(struct nvme_ctrl * ctrl)522 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
523 {
524 	return nvme_toggle_streams(ctrl, true);
525 }
526 
nvme_get_stream_params(struct nvme_ctrl * ctrl,struct streams_directive_params * s,u32 nsid)527 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
528 				  struct streams_directive_params *s, u32 nsid)
529 {
530 	struct nvme_command c;
531 
532 	memset(&c, 0, sizeof(c));
533 	memset(s, 0, sizeof(*s));
534 
535 	c.directive.opcode = nvme_admin_directive_recv;
536 	c.directive.nsid = cpu_to_le32(nsid);
537 	c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
538 	c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
539 	c.directive.dtype = NVME_DIR_STREAMS;
540 
541 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
542 }
543 
nvme_configure_directives(struct nvme_ctrl * ctrl)544 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
545 {
546 	struct streams_directive_params s;
547 	int ret;
548 
549 	if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
550 		return 0;
551 	if (!streams)
552 		return 0;
553 
554 	ret = nvme_enable_streams(ctrl);
555 	if (ret)
556 		return ret;
557 
558 	ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
559 	if (ret)
560 		return ret;
561 
562 	ctrl->nssa = le16_to_cpu(s.nssa);
563 	if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
564 		dev_info(ctrl->device, "too few streams (%u) available\n",
565 					ctrl->nssa);
566 		nvme_disable_streams(ctrl);
567 		return 0;
568 	}
569 
570 	ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
571 	dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
572 	return 0;
573 }
574 
575 /*
576  * Check if 'req' has a write hint associated with it. If it does, assign
577  * a valid namespace stream to the write.
578  */
nvme_assign_write_stream(struct nvme_ctrl * ctrl,struct request * req,u16 * control,u32 * dsmgmt)579 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
580 				     struct request *req, u16 *control,
581 				     u32 *dsmgmt)
582 {
583 	enum rw_hint streamid = req->write_hint;
584 
585 	if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
586 		streamid = 0;
587 	else {
588 		streamid--;
589 		if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
590 			return;
591 
592 		*control |= NVME_RW_DTYPE_STREAMS;
593 		*dsmgmt |= streamid << 16;
594 	}
595 
596 	if (streamid < ARRAY_SIZE(req->q->write_hints))
597 		req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
598 }
599 
nvme_setup_flush(struct nvme_ns * ns,struct nvme_command * cmnd)600 static inline void nvme_setup_flush(struct nvme_ns *ns,
601 		struct nvme_command *cmnd)
602 {
603 	cmnd->common.opcode = nvme_cmd_flush;
604 	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
605 }
606 
nvme_setup_discard(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd)607 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
608 		struct nvme_command *cmnd)
609 {
610 	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
611 	struct nvme_dsm_range *range;
612 	struct bio *bio;
613 
614 	/*
615 	 * Some devices do not consider the DSM 'Number of Ranges' field when
616 	 * determining how much data to DMA. Always allocate memory for maximum
617 	 * number of segments to prevent device reading beyond end of buffer.
618 	 */
619 	static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
620 
621 	range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
622 	if (!range) {
623 		/*
624 		 * If we fail allocation our range, fallback to the controller
625 		 * discard page. If that's also busy, it's safe to return
626 		 * busy, as we know we can make progress once that's freed.
627 		 */
628 		if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
629 			return BLK_STS_RESOURCE;
630 
631 		range = page_address(ns->ctrl->discard_page);
632 	}
633 
634 	__rq_for_each_bio(bio, req) {
635 		u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
636 		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
637 
638 		if (n < segments) {
639 			range[n].cattr = cpu_to_le32(0);
640 			range[n].nlb = cpu_to_le32(nlb);
641 			range[n].slba = cpu_to_le64(slba);
642 		}
643 		n++;
644 	}
645 
646 	if (WARN_ON_ONCE(n != segments)) {
647 		if (virt_to_page(range) == ns->ctrl->discard_page)
648 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
649 		else
650 			kfree(range);
651 		return BLK_STS_IOERR;
652 	}
653 
654 	cmnd->dsm.opcode = nvme_cmd_dsm;
655 	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
656 	cmnd->dsm.nr = cpu_to_le32(segments - 1);
657 	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
658 
659 	req->special_vec.bv_page = virt_to_page(range);
660 	req->special_vec.bv_offset = offset_in_page(range);
661 	req->special_vec.bv_len = alloc_size;
662 	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
663 
664 	return BLK_STS_OK;
665 }
666 
nvme_setup_write_zeroes(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd)667 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
668 		struct request *req, struct nvme_command *cmnd)
669 {
670 	if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
671 		return nvme_setup_discard(ns, req, cmnd);
672 
673 	cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
674 	cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
675 	cmnd->write_zeroes.slba =
676 		cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
677 	cmnd->write_zeroes.length =
678 		cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
679 	cmnd->write_zeroes.control = 0;
680 	return BLK_STS_OK;
681 }
682 
nvme_setup_rw(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd)683 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
684 		struct request *req, struct nvme_command *cmnd)
685 {
686 	struct nvme_ctrl *ctrl = ns->ctrl;
687 	u16 control = 0;
688 	u32 dsmgmt = 0;
689 
690 	if (req->cmd_flags & REQ_FUA)
691 		control |= NVME_RW_FUA;
692 	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
693 		control |= NVME_RW_LR;
694 
695 	if (req->cmd_flags & REQ_RAHEAD)
696 		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
697 
698 	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
699 	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
700 	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
701 	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
702 
703 	if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
704 		nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
705 
706 	if (ns->ms) {
707 		/*
708 		 * If formated with metadata, the block layer always provides a
709 		 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
710 		 * we enable the PRACT bit for protection information or set the
711 		 * namespace capacity to zero to prevent any I/O.
712 		 */
713 		if (!blk_integrity_rq(req)) {
714 			if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
715 				return BLK_STS_NOTSUPP;
716 			control |= NVME_RW_PRINFO_PRACT;
717 		}
718 
719 		switch (ns->pi_type) {
720 		case NVME_NS_DPS_PI_TYPE3:
721 			control |= NVME_RW_PRINFO_PRCHK_GUARD;
722 			break;
723 		case NVME_NS_DPS_PI_TYPE1:
724 		case NVME_NS_DPS_PI_TYPE2:
725 			control |= NVME_RW_PRINFO_PRCHK_GUARD |
726 					NVME_RW_PRINFO_PRCHK_REF;
727 			cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
728 			break;
729 		}
730 	}
731 
732 	cmnd->rw.control = cpu_to_le16(control);
733 	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
734 	return 0;
735 }
736 
nvme_cleanup_cmd(struct request * req)737 void nvme_cleanup_cmd(struct request *req)
738 {
739 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
740 		struct nvme_ns *ns = req->rq_disk->private_data;
741 		struct page *page = req->special_vec.bv_page;
742 
743 		if (page == ns->ctrl->discard_page)
744 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
745 		else
746 			kfree(page_address(page) + req->special_vec.bv_offset);
747 	}
748 }
749 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
750 
nvme_setup_cmd(struct nvme_ns * ns,struct request * req,struct nvme_command * cmd)751 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
752 		struct nvme_command *cmd)
753 {
754 	blk_status_t ret = BLK_STS_OK;
755 
756 	nvme_clear_nvme_request(req);
757 
758 	memset(cmd, 0, sizeof(*cmd));
759 	switch (req_op(req)) {
760 	case REQ_OP_DRV_IN:
761 	case REQ_OP_DRV_OUT:
762 		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
763 		break;
764 	case REQ_OP_FLUSH:
765 		nvme_setup_flush(ns, cmd);
766 		break;
767 	case REQ_OP_WRITE_ZEROES:
768 		ret = nvme_setup_write_zeroes(ns, req, cmd);
769 		break;
770 	case REQ_OP_DISCARD:
771 		ret = nvme_setup_discard(ns, req, cmd);
772 		break;
773 	case REQ_OP_READ:
774 	case REQ_OP_WRITE:
775 		ret = nvme_setup_rw(ns, req, cmd);
776 		break;
777 	default:
778 		WARN_ON_ONCE(1);
779 		return BLK_STS_IOERR;
780 	}
781 
782 	cmd->common.command_id = req->tag;
783 	trace_nvme_setup_cmd(req, cmd);
784 	return ret;
785 }
786 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
787 
nvme_end_sync_rq(struct request * rq,blk_status_t error)788 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
789 {
790 	struct completion *waiting = rq->end_io_data;
791 
792 	rq->end_io_data = NULL;
793 	complete(waiting);
794 }
795 
nvme_execute_rq_polled(struct request_queue * q,struct gendisk * bd_disk,struct request * rq,int at_head)796 static void nvme_execute_rq_polled(struct request_queue *q,
797 		struct gendisk *bd_disk, struct request *rq, int at_head)
798 {
799 	DECLARE_COMPLETION_ONSTACK(wait);
800 
801 	WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
802 
803 	rq->cmd_flags |= REQ_HIPRI;
804 	rq->end_io_data = &wait;
805 	blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
806 
807 	while (!completion_done(&wait)) {
808 		blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
809 		cond_resched();
810 	}
811 }
812 
813 /*
814  * Returns 0 on success.  If the result is negative, it's a Linux error code;
815  * if the result is positive, it's an NVM Express status code
816  */
__nvme_submit_sync_cmd(struct request_queue * q,struct nvme_command * cmd,union nvme_result * result,void * buffer,unsigned bufflen,unsigned timeout,int qid,int at_head,blk_mq_req_flags_t flags,bool poll)817 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
818 		union nvme_result *result, void *buffer, unsigned bufflen,
819 		unsigned timeout, int qid, int at_head,
820 		blk_mq_req_flags_t flags, bool poll)
821 {
822 	struct request *req;
823 	int ret;
824 
825 	req = nvme_alloc_request(q, cmd, flags, qid);
826 	if (IS_ERR(req))
827 		return PTR_ERR(req);
828 
829 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
830 
831 	if (buffer && bufflen) {
832 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
833 		if (ret)
834 			goto out;
835 	}
836 
837 	if (poll)
838 		nvme_execute_rq_polled(req->q, NULL, req, at_head);
839 	else
840 		blk_execute_rq(req->q, NULL, req, at_head);
841 	if (result)
842 		*result = nvme_req(req)->result;
843 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
844 		ret = -EINTR;
845 	else
846 		ret = nvme_req(req)->status;
847  out:
848 	blk_mq_free_request(req);
849 	return ret;
850 }
851 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
852 
nvme_submit_sync_cmd(struct request_queue * q,struct nvme_command * cmd,void * buffer,unsigned bufflen)853 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
854 		void *buffer, unsigned bufflen)
855 {
856 	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
857 			NVME_QID_ANY, 0, 0, false);
858 }
859 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
860 
nvme_add_user_metadata(struct bio * bio,void __user * ubuf,unsigned len,u32 seed,bool write)861 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
862 		unsigned len, u32 seed, bool write)
863 {
864 	struct bio_integrity_payload *bip;
865 	int ret = -ENOMEM;
866 	void *buf;
867 
868 	buf = kmalloc(len, GFP_KERNEL);
869 	if (!buf)
870 		goto out;
871 
872 	ret = -EFAULT;
873 	if (write && copy_from_user(buf, ubuf, len))
874 		goto out_free_meta;
875 
876 	bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
877 	if (IS_ERR(bip)) {
878 		ret = PTR_ERR(bip);
879 		goto out_free_meta;
880 	}
881 
882 	bip->bip_iter.bi_size = len;
883 	bip->bip_iter.bi_sector = seed;
884 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
885 			offset_in_page(buf));
886 	if (ret == len)
887 		return buf;
888 	ret = -ENOMEM;
889 out_free_meta:
890 	kfree(buf);
891 out:
892 	return ERR_PTR(ret);
893 }
894 
nvme_submit_user_cmd(struct request_queue * q,struct nvme_command * cmd,void __user * ubuffer,unsigned bufflen,void __user * meta_buffer,unsigned meta_len,u32 meta_seed,u64 * result,unsigned timeout)895 static int nvme_submit_user_cmd(struct request_queue *q,
896 		struct nvme_command *cmd, void __user *ubuffer,
897 		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
898 		u32 meta_seed, u64 *result, unsigned timeout)
899 {
900 	bool write = nvme_is_write(cmd);
901 	struct nvme_ns *ns = q->queuedata;
902 	struct gendisk *disk = ns ? ns->disk : NULL;
903 	struct request *req;
904 	struct bio *bio = NULL;
905 	void *meta = NULL;
906 	int ret;
907 
908 	req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
909 	if (IS_ERR(req))
910 		return PTR_ERR(req);
911 
912 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
913 	nvme_req(req)->flags |= NVME_REQ_USERCMD;
914 
915 	if (ubuffer && bufflen) {
916 		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
917 				GFP_KERNEL);
918 		if (ret)
919 			goto out;
920 		bio = req->bio;
921 		bio->bi_disk = disk;
922 		if (disk && meta_buffer && meta_len) {
923 			meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
924 					meta_seed, write);
925 			if (IS_ERR(meta)) {
926 				ret = PTR_ERR(meta);
927 				goto out_unmap;
928 			}
929 			req->cmd_flags |= REQ_INTEGRITY;
930 		}
931 	}
932 
933 	blk_execute_rq(req->q, disk, req, 0);
934 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
935 		ret = -EINTR;
936 	else
937 		ret = nvme_req(req)->status;
938 	if (result)
939 		*result = le64_to_cpu(nvme_req(req)->result.u64);
940 	if (meta && !ret && !write) {
941 		if (copy_to_user(meta_buffer, meta, meta_len))
942 			ret = -EFAULT;
943 	}
944 	kfree(meta);
945  out_unmap:
946 	if (bio)
947 		blk_rq_unmap_user(bio);
948  out:
949 	blk_mq_free_request(req);
950 	return ret;
951 }
952 
nvme_keep_alive_end_io(struct request * rq,blk_status_t status)953 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
954 {
955 	struct nvme_ctrl *ctrl = rq->end_io_data;
956 	unsigned long flags;
957 	bool startka = false;
958 
959 	blk_mq_free_request(rq);
960 
961 	if (status) {
962 		dev_err(ctrl->device,
963 			"failed nvme_keep_alive_end_io error=%d\n",
964 				status);
965 		return;
966 	}
967 
968 	ctrl->comp_seen = false;
969 	spin_lock_irqsave(&ctrl->lock, flags);
970 	if (ctrl->state == NVME_CTRL_LIVE ||
971 	    ctrl->state == NVME_CTRL_CONNECTING)
972 		startka = true;
973 	spin_unlock_irqrestore(&ctrl->lock, flags);
974 	if (startka)
975 		schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
976 }
977 
nvme_keep_alive(struct nvme_ctrl * ctrl)978 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
979 {
980 	struct request *rq;
981 
982 	rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
983 			NVME_QID_ANY);
984 	if (IS_ERR(rq))
985 		return PTR_ERR(rq);
986 
987 	rq->timeout = ctrl->kato * HZ;
988 	rq->end_io_data = ctrl;
989 
990 	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
991 
992 	return 0;
993 }
994 
nvme_keep_alive_work(struct work_struct * work)995 static void nvme_keep_alive_work(struct work_struct *work)
996 {
997 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
998 			struct nvme_ctrl, ka_work);
999 	bool comp_seen = ctrl->comp_seen;
1000 
1001 	if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1002 		dev_dbg(ctrl->device,
1003 			"reschedule traffic based keep-alive timer\n");
1004 		ctrl->comp_seen = false;
1005 		schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
1006 		return;
1007 	}
1008 
1009 	if (nvme_keep_alive(ctrl)) {
1010 		/* allocation failure, reset the controller */
1011 		dev_err(ctrl->device, "keep-alive failed\n");
1012 		nvme_reset_ctrl(ctrl);
1013 		return;
1014 	}
1015 }
1016 
nvme_start_keep_alive(struct nvme_ctrl * ctrl)1017 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1018 {
1019 	if (unlikely(ctrl->kato == 0))
1020 		return;
1021 
1022 	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
1023 }
1024 
nvme_stop_keep_alive(struct nvme_ctrl * ctrl)1025 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1026 {
1027 	if (unlikely(ctrl->kato == 0))
1028 		return;
1029 
1030 	cancel_delayed_work_sync(&ctrl->ka_work);
1031 }
1032 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1033 
nvme_identify_ctrl(struct nvme_ctrl * dev,struct nvme_id_ctrl ** id)1034 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1035 {
1036 	struct nvme_command c = { };
1037 	int error;
1038 
1039 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1040 	c.identify.opcode = nvme_admin_identify;
1041 	c.identify.cns = NVME_ID_CNS_CTRL;
1042 
1043 	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1044 	if (!*id)
1045 		return -ENOMEM;
1046 
1047 	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1048 			sizeof(struct nvme_id_ctrl));
1049 	if (error)
1050 		kfree(*id);
1051 	return error;
1052 }
1053 
nvme_identify_ns_descs(struct nvme_ctrl * ctrl,unsigned nsid,struct nvme_ns_ids * ids)1054 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1055 		struct nvme_ns_ids *ids)
1056 {
1057 	struct nvme_command c = { };
1058 	int status;
1059 	void *data;
1060 	int pos;
1061 	int len;
1062 
1063 	c.identify.opcode = nvme_admin_identify;
1064 	c.identify.nsid = cpu_to_le32(nsid);
1065 	c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1066 
1067 	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1068 	if (!data)
1069 		return -ENOMEM;
1070 
1071 	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1072 				      NVME_IDENTIFY_DATA_SIZE);
1073 	if (status)
1074 		goto free_data;
1075 
1076 	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1077 		struct nvme_ns_id_desc *cur = data + pos;
1078 
1079 		if (cur->nidl == 0)
1080 			break;
1081 
1082 		switch (cur->nidt) {
1083 		case NVME_NIDT_EUI64:
1084 			if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1085 				dev_warn(ctrl->device,
1086 					 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
1087 					 cur->nidl);
1088 				goto free_data;
1089 			}
1090 			len = NVME_NIDT_EUI64_LEN;
1091 			memcpy(ids->eui64, data + pos + sizeof(*cur), len);
1092 			break;
1093 		case NVME_NIDT_NGUID:
1094 			if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1095 				dev_warn(ctrl->device,
1096 					 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
1097 					 cur->nidl);
1098 				goto free_data;
1099 			}
1100 			len = NVME_NIDT_NGUID_LEN;
1101 			memcpy(ids->nguid, data + pos + sizeof(*cur), len);
1102 			break;
1103 		case NVME_NIDT_UUID:
1104 			if (cur->nidl != NVME_NIDT_UUID_LEN) {
1105 				dev_warn(ctrl->device,
1106 					 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
1107 					 cur->nidl);
1108 				goto free_data;
1109 			}
1110 			len = NVME_NIDT_UUID_LEN;
1111 			uuid_copy(&ids->uuid, data + pos + sizeof(*cur));
1112 			break;
1113 		default:
1114 			/* Skip unknown types */
1115 			len = cur->nidl;
1116 			break;
1117 		}
1118 
1119 		len += sizeof(*cur);
1120 	}
1121 free_data:
1122 	kfree(data);
1123 	return status;
1124 }
1125 
nvme_identify_ns_list(struct nvme_ctrl * dev,unsigned nsid,__le32 * ns_list)1126 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
1127 {
1128 	struct nvme_command c = { };
1129 
1130 	c.identify.opcode = nvme_admin_identify;
1131 	c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
1132 	c.identify.nsid = cpu_to_le32(nsid);
1133 	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1134 				    NVME_IDENTIFY_DATA_SIZE);
1135 }
1136 
nvme_identify_ns(struct nvme_ctrl * ctrl,unsigned nsid,struct nvme_id_ns ** id)1137 static int nvme_identify_ns(struct nvme_ctrl *ctrl,
1138 		unsigned nsid, struct nvme_id_ns **id)
1139 {
1140 	struct nvme_command c = { };
1141 	int error;
1142 
1143 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1144 	c.identify.opcode = nvme_admin_identify;
1145 	c.identify.nsid = cpu_to_le32(nsid);
1146 	c.identify.cns = NVME_ID_CNS_NS;
1147 
1148 	*id = kmalloc(sizeof(**id), GFP_KERNEL);
1149 	if (!*id)
1150 		return -ENOMEM;
1151 
1152 	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1153 	if (error) {
1154 		dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1155 		kfree(*id);
1156 	}
1157 
1158 	return error;
1159 }
1160 
nvme_features(struct nvme_ctrl * dev,u8 op,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1161 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1162 		unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1163 {
1164 	struct nvme_command c;
1165 	union nvme_result res;
1166 	int ret;
1167 
1168 	memset(&c, 0, sizeof(c));
1169 	c.features.opcode = op;
1170 	c.features.fid = cpu_to_le32(fid);
1171 	c.features.dword11 = cpu_to_le32(dword11);
1172 
1173 	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1174 			buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1175 	if (ret >= 0 && result)
1176 		*result = le32_to_cpu(res.u32);
1177 	return ret;
1178 }
1179 
nvme_set_features(struct nvme_ctrl * dev,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1180 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1181 		      unsigned int dword11, void *buffer, size_t buflen,
1182 		      u32 *result)
1183 {
1184 	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1185 			     buflen, result);
1186 }
1187 EXPORT_SYMBOL_GPL(nvme_set_features);
1188 
nvme_get_features(struct nvme_ctrl * dev,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1189 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1190 		      unsigned int dword11, void *buffer, size_t buflen,
1191 		      u32 *result)
1192 {
1193 	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1194 			     buflen, result);
1195 }
1196 EXPORT_SYMBOL_GPL(nvme_get_features);
1197 
nvme_set_queue_count(struct nvme_ctrl * ctrl,int * count)1198 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1199 {
1200 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
1201 	u32 result;
1202 	int status, nr_io_queues;
1203 
1204 	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1205 			&result);
1206 	if (status < 0)
1207 		return status;
1208 
1209 	/*
1210 	 * Degraded controllers might return an error when setting the queue
1211 	 * count.  We still want to be able to bring them online and offer
1212 	 * access to the admin queue, as that might be only way to fix them up.
1213 	 */
1214 	if (status > 0) {
1215 		dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1216 		*count = 0;
1217 	} else {
1218 		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1219 		*count = min(*count, nr_io_queues);
1220 	}
1221 
1222 	return 0;
1223 }
1224 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1225 
1226 #define NVME_AEN_SUPPORTED \
1227 	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1228 	 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1229 
nvme_enable_aen(struct nvme_ctrl * ctrl)1230 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1231 {
1232 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1233 	int status;
1234 
1235 	if (!supported_aens)
1236 		return;
1237 
1238 	status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1239 			NULL, 0, &result);
1240 	if (status)
1241 		dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1242 			 supported_aens);
1243 
1244 	queue_work(nvme_wq, &ctrl->async_event_work);
1245 }
1246 
nvme_submit_io(struct nvme_ns * ns,struct nvme_user_io __user * uio)1247 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1248 {
1249 	struct nvme_user_io io;
1250 	struct nvme_command c;
1251 	unsigned length, meta_len;
1252 	void __user *metadata;
1253 
1254 	if (copy_from_user(&io, uio, sizeof(io)))
1255 		return -EFAULT;
1256 	if (io.flags)
1257 		return -EINVAL;
1258 
1259 	switch (io.opcode) {
1260 	case nvme_cmd_write:
1261 	case nvme_cmd_read:
1262 	case nvme_cmd_compare:
1263 		break;
1264 	default:
1265 		return -EINVAL;
1266 	}
1267 
1268 	length = (io.nblocks + 1) << ns->lba_shift;
1269 	meta_len = (io.nblocks + 1) * ns->ms;
1270 	metadata = (void __user *)(uintptr_t)io.metadata;
1271 
1272 	if (ns->ext) {
1273 		length += meta_len;
1274 		meta_len = 0;
1275 	} else if (meta_len) {
1276 		if ((io.metadata & 3) || !io.metadata)
1277 			return -EINVAL;
1278 	}
1279 
1280 	memset(&c, 0, sizeof(c));
1281 	c.rw.opcode = io.opcode;
1282 	c.rw.flags = io.flags;
1283 	c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1284 	c.rw.slba = cpu_to_le64(io.slba);
1285 	c.rw.length = cpu_to_le16(io.nblocks);
1286 	c.rw.control = cpu_to_le16(io.control);
1287 	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1288 	c.rw.reftag = cpu_to_le32(io.reftag);
1289 	c.rw.apptag = cpu_to_le16(io.apptag);
1290 	c.rw.appmask = cpu_to_le16(io.appmask);
1291 
1292 	return nvme_submit_user_cmd(ns->queue, &c,
1293 			(void __user *)(uintptr_t)io.addr, length,
1294 			metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1295 }
1296 
nvme_known_admin_effects(u8 opcode)1297 static u32 nvme_known_admin_effects(u8 opcode)
1298 {
1299 	switch (opcode) {
1300 	case nvme_admin_format_nvm:
1301 		return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
1302 					NVME_CMD_EFFECTS_CSE_MASK;
1303 	case nvme_admin_sanitize_nvm:
1304 		return NVME_CMD_EFFECTS_CSE_MASK;
1305 	default:
1306 		break;
1307 	}
1308 	return 0;
1309 }
1310 
nvme_passthru_start(struct nvme_ctrl * ctrl,struct nvme_ns * ns,u8 opcode)1311 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1312 								u8 opcode)
1313 {
1314 	u32 effects = 0;
1315 
1316 	if (ns) {
1317 		if (ctrl->effects)
1318 			effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1319 		if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1320 			dev_warn(ctrl->device,
1321 				 "IO command:%02x has unhandled effects:%08x\n",
1322 				 opcode, effects);
1323 		return 0;
1324 	}
1325 
1326 	if (ctrl->effects)
1327 		effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1328 	effects |= nvme_known_admin_effects(opcode);
1329 
1330 	/*
1331 	 * For simplicity, IO to all namespaces is quiesced even if the command
1332 	 * effects say only one namespace is affected.
1333 	 */
1334 	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1335 		mutex_lock(&ctrl->scan_lock);
1336 		mutex_lock(&ctrl->subsys->lock);
1337 		nvme_mpath_start_freeze(ctrl->subsys);
1338 		nvme_mpath_wait_freeze(ctrl->subsys);
1339 		nvme_start_freeze(ctrl);
1340 		nvme_wait_freeze(ctrl);
1341 	}
1342 	return effects;
1343 }
1344 
nvme_update_formats(struct nvme_ctrl * ctrl)1345 static void nvme_update_formats(struct nvme_ctrl *ctrl)
1346 {
1347 	struct nvme_ns *ns;
1348 
1349 	down_read(&ctrl->namespaces_rwsem);
1350 	list_for_each_entry(ns, &ctrl->namespaces, list)
1351 		if (ns->disk && nvme_revalidate_disk(ns->disk))
1352 			nvme_set_queue_dying(ns);
1353 	up_read(&ctrl->namespaces_rwsem);
1354 }
1355 
nvme_passthru_end(struct nvme_ctrl * ctrl,u32 effects)1356 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1357 {
1358 	/*
1359 	 * Revalidate LBA changes prior to unfreezing. This is necessary to
1360 	 * prevent memory corruption if a logical block size was changed by
1361 	 * this command.
1362 	 */
1363 	if (effects & NVME_CMD_EFFECTS_LBCC)
1364 		nvme_update_formats(ctrl);
1365 	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1366 		nvme_unfreeze(ctrl);
1367 		nvme_mpath_unfreeze(ctrl->subsys);
1368 		mutex_unlock(&ctrl->subsys->lock);
1369 		nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1370 		mutex_unlock(&ctrl->scan_lock);
1371 	}
1372 	if (effects & NVME_CMD_EFFECTS_CCC)
1373 		nvme_init_identify(ctrl);
1374 	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
1375 		nvme_queue_scan(ctrl);
1376 }
1377 
nvme_user_cmd(struct nvme_ctrl * ctrl,struct nvme_ns * ns,struct nvme_passthru_cmd __user * ucmd)1378 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1379 			struct nvme_passthru_cmd __user *ucmd)
1380 {
1381 	struct nvme_passthru_cmd cmd;
1382 	struct nvme_command c;
1383 	unsigned timeout = 0;
1384 	u32 effects;
1385 	u64 result;
1386 	int status;
1387 
1388 	if (!capable(CAP_SYS_ADMIN))
1389 		return -EACCES;
1390 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1391 		return -EFAULT;
1392 	if (cmd.flags)
1393 		return -EINVAL;
1394 
1395 	memset(&c, 0, sizeof(c));
1396 	c.common.opcode = cmd.opcode;
1397 	c.common.flags = cmd.flags;
1398 	c.common.nsid = cpu_to_le32(cmd.nsid);
1399 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1400 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1401 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1402 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1403 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1404 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1405 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1406 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1407 
1408 	if (cmd.timeout_ms)
1409 		timeout = msecs_to_jiffies(cmd.timeout_ms);
1410 
1411 	effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1412 	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1413 			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1414 			(void __user *)(uintptr_t)cmd.metadata,
1415 			cmd.metadata_len, 0, &result, timeout);
1416 	nvme_passthru_end(ctrl, effects);
1417 
1418 	if (status >= 0) {
1419 		if (put_user(result, &ucmd->result))
1420 			return -EFAULT;
1421 	}
1422 
1423 	return status;
1424 }
1425 
nvme_user_cmd64(struct nvme_ctrl * ctrl,struct nvme_ns * ns,struct nvme_passthru_cmd64 __user * ucmd)1426 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1427 			struct nvme_passthru_cmd64 __user *ucmd)
1428 {
1429 	struct nvme_passthru_cmd64 cmd;
1430 	struct nvme_command c;
1431 	unsigned timeout = 0;
1432 	u32 effects;
1433 	int status;
1434 
1435 	if (!capable(CAP_SYS_ADMIN))
1436 		return -EACCES;
1437 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1438 		return -EFAULT;
1439 	if (cmd.flags)
1440 		return -EINVAL;
1441 
1442 	memset(&c, 0, sizeof(c));
1443 	c.common.opcode = cmd.opcode;
1444 	c.common.flags = cmd.flags;
1445 	c.common.nsid = cpu_to_le32(cmd.nsid);
1446 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1447 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1448 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1449 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1450 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1451 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1452 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1453 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1454 
1455 	if (cmd.timeout_ms)
1456 		timeout = msecs_to_jiffies(cmd.timeout_ms);
1457 
1458 	effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1459 	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1460 			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1461 			(void __user *)(uintptr_t)cmd.metadata, cmd.metadata_len,
1462 			0, &cmd.result, timeout);
1463 	nvme_passthru_end(ctrl, effects);
1464 
1465 	if (status >= 0) {
1466 		if (put_user(cmd.result, &ucmd->result))
1467 			return -EFAULT;
1468 	}
1469 
1470 	return status;
1471 }
1472 
1473 /*
1474  * Issue ioctl requests on the first available path.  Note that unlike normal
1475  * block layer requests we will not retry failed request on another controller.
1476  */
nvme_get_ns_from_disk(struct gendisk * disk,struct nvme_ns_head ** head,int * srcu_idx)1477 static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1478 		struct nvme_ns_head **head, int *srcu_idx)
1479 {
1480 #ifdef CONFIG_NVME_MULTIPATH
1481 	if (disk->fops == &nvme_ns_head_ops) {
1482 		struct nvme_ns *ns;
1483 
1484 		*head = disk->private_data;
1485 		*srcu_idx = srcu_read_lock(&(*head)->srcu);
1486 		ns = nvme_find_path(*head);
1487 		if (!ns)
1488 			srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1489 		return ns;
1490 	}
1491 #endif
1492 	*head = NULL;
1493 	*srcu_idx = -1;
1494 	return disk->private_data;
1495 }
1496 
nvme_put_ns_from_disk(struct nvme_ns_head * head,int idx)1497 static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1498 {
1499 	if (head)
1500 		srcu_read_unlock(&head->srcu, idx);
1501 }
1502 
is_ctrl_ioctl(unsigned int cmd)1503 static bool is_ctrl_ioctl(unsigned int cmd)
1504 {
1505 	if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1506 		return true;
1507 	if (is_sed_ioctl(cmd))
1508 		return true;
1509 	return false;
1510 }
1511 
nvme_handle_ctrl_ioctl(struct nvme_ns * ns,unsigned int cmd,void __user * argp,struct nvme_ns_head * head,int srcu_idx)1512 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1513 				  void __user *argp,
1514 				  struct nvme_ns_head *head,
1515 				  int srcu_idx)
1516 {
1517 	struct nvme_ctrl *ctrl = ns->ctrl;
1518 	int ret;
1519 
1520 	nvme_get_ctrl(ns->ctrl);
1521 	nvme_put_ns_from_disk(head, srcu_idx);
1522 
1523 	switch (cmd) {
1524 	case NVME_IOCTL_ADMIN_CMD:
1525 		ret = nvme_user_cmd(ctrl, NULL, argp);
1526 		break;
1527 	case NVME_IOCTL_ADMIN64_CMD:
1528 		ret = nvme_user_cmd64(ctrl, NULL, argp);
1529 		break;
1530 	default:
1531 		ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1532 		break;
1533 	}
1534 	nvme_put_ctrl(ctrl);
1535 	return ret;
1536 }
1537 
nvme_ioctl(struct block_device * bdev,fmode_t mode,unsigned int cmd,unsigned long arg)1538 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1539 		unsigned int cmd, unsigned long arg)
1540 {
1541 	struct nvme_ns_head *head = NULL;
1542 	void __user *argp = (void __user *)arg;
1543 	struct nvme_ns *ns;
1544 	int srcu_idx, ret;
1545 
1546 	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1547 	if (unlikely(!ns))
1548 		return -EWOULDBLOCK;
1549 
1550 	/*
1551 	 * Handle ioctls that apply to the controller instead of the namespace
1552 	 * seperately and drop the ns SRCU reference early.  This avoids a
1553 	 * deadlock when deleting namespaces using the passthrough interface.
1554 	 */
1555 	if (is_ctrl_ioctl(cmd))
1556 		return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1557 
1558 	switch (cmd) {
1559 	case NVME_IOCTL_ID:
1560 		force_successful_syscall_return();
1561 		ret = ns->head->ns_id;
1562 		break;
1563 	case NVME_IOCTL_IO_CMD:
1564 		ret = nvme_user_cmd(ns->ctrl, ns, argp);
1565 		break;
1566 	case NVME_IOCTL_SUBMIT_IO:
1567 		ret = nvme_submit_io(ns, argp);
1568 		break;
1569 	case NVME_IOCTL_IO64_CMD:
1570 		ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1571 		break;
1572 	default:
1573 		if (ns->ndev)
1574 			ret = nvme_nvm_ioctl(ns, cmd, arg);
1575 		else
1576 			ret = -ENOTTY;
1577 	}
1578 
1579 	nvme_put_ns_from_disk(head, srcu_idx);
1580 	return ret;
1581 }
1582 
nvme_open(struct block_device * bdev,fmode_t mode)1583 static int nvme_open(struct block_device *bdev, fmode_t mode)
1584 {
1585 	struct nvme_ns *ns = bdev->bd_disk->private_data;
1586 
1587 #ifdef CONFIG_NVME_MULTIPATH
1588 	/* should never be called due to GENHD_FL_HIDDEN */
1589 	if (WARN_ON_ONCE(ns->head->disk))
1590 		goto fail;
1591 #endif
1592 	if (!kref_get_unless_zero(&ns->kref))
1593 		goto fail;
1594 	if (!try_module_get(ns->ctrl->ops->module))
1595 		goto fail_put_ns;
1596 
1597 	return 0;
1598 
1599 fail_put_ns:
1600 	nvme_put_ns(ns);
1601 fail:
1602 	return -ENXIO;
1603 }
1604 
nvme_release(struct gendisk * disk,fmode_t mode)1605 static void nvme_release(struct gendisk *disk, fmode_t mode)
1606 {
1607 	struct nvme_ns *ns = disk->private_data;
1608 
1609 	module_put(ns->ctrl->ops->module);
1610 	nvme_put_ns(ns);
1611 }
1612 
nvme_getgeo(struct block_device * bdev,struct hd_geometry * geo)1613 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1614 {
1615 	/* some standard values */
1616 	geo->heads = 1 << 6;
1617 	geo->sectors = 1 << 5;
1618 	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1619 	return 0;
1620 }
1621 
1622 #ifdef CONFIG_BLK_DEV_INTEGRITY
nvme_init_integrity(struct gendisk * disk,u16 ms,u8 pi_type)1623 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1624 {
1625 	struct blk_integrity integrity;
1626 
1627 	memset(&integrity, 0, sizeof(integrity));
1628 	switch (pi_type) {
1629 	case NVME_NS_DPS_PI_TYPE3:
1630 		integrity.profile = &t10_pi_type3_crc;
1631 		integrity.tag_size = sizeof(u16) + sizeof(u32);
1632 		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1633 		break;
1634 	case NVME_NS_DPS_PI_TYPE1:
1635 	case NVME_NS_DPS_PI_TYPE2:
1636 		integrity.profile = &t10_pi_type1_crc;
1637 		integrity.tag_size = sizeof(u16);
1638 		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1639 		break;
1640 	default:
1641 		integrity.profile = NULL;
1642 		break;
1643 	}
1644 	integrity.tuple_size = ms;
1645 	blk_integrity_register(disk, &integrity);
1646 	blk_queue_max_integrity_segments(disk->queue, 1);
1647 }
1648 #else
nvme_init_integrity(struct gendisk * disk,u16 ms,u8 pi_type)1649 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1650 {
1651 }
1652 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1653 
nvme_set_chunk_size(struct nvme_ns * ns)1654 static void nvme_set_chunk_size(struct nvme_ns *ns)
1655 {
1656 	u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
1657 	blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
1658 }
1659 
nvme_config_discard(struct gendisk * disk,struct nvme_ns * ns)1660 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1661 {
1662 	struct nvme_ctrl *ctrl = ns->ctrl;
1663 	struct request_queue *queue = disk->queue;
1664 	u32 size = queue_logical_block_size(queue);
1665 
1666 	if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1667 		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1668 		return;
1669 	}
1670 
1671 	if (ctrl->nr_streams && ns->sws && ns->sgs)
1672 		size *= ns->sws * ns->sgs;
1673 
1674 	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1675 			NVME_DSM_MAX_RANGES);
1676 
1677 	queue->limits.discard_alignment = 0;
1678 	queue->limits.discard_granularity = size;
1679 
1680 	/* If discard is already enabled, don't reset queue limits */
1681 	if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1682 		return;
1683 
1684 	blk_queue_max_discard_sectors(queue, UINT_MAX);
1685 	blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1686 
1687 	if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1688 		blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1689 }
1690 
nvme_config_write_zeroes(struct gendisk * disk,struct nvme_ns * ns)1691 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1692 {
1693 	u32 max_sectors;
1694 	unsigned short bs = 1 << ns->lba_shift;
1695 
1696 	if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1697 	    (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1698 		return;
1699 	/*
1700 	 * Even though NVMe spec explicitly states that MDTS is not
1701 	 * applicable to the write-zeroes:- "The restriction does not apply to
1702 	 * commands that do not transfer data between the host and the
1703 	 * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1704 	 * In order to be more cautious use controller's max_hw_sectors value
1705 	 * to configure the maximum sectors for the write-zeroes which is
1706 	 * configured based on the controller's MDTS field in the
1707 	 * nvme_init_identify() if available.
1708 	 */
1709 	if (ns->ctrl->max_hw_sectors == UINT_MAX)
1710 		max_sectors = ((u32)(USHRT_MAX + 1) * bs) >> 9;
1711 	else
1712 		max_sectors = ((u32)(ns->ctrl->max_hw_sectors + 1) * bs) >> 9;
1713 
1714 	blk_queue_max_write_zeroes_sectors(disk->queue, max_sectors);
1715 }
1716 
nvme_report_ns_ids(struct nvme_ctrl * ctrl,unsigned int nsid,struct nvme_id_ns * id,struct nvme_ns_ids * ids)1717 static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1718 		struct nvme_id_ns *id, struct nvme_ns_ids *ids)
1719 {
1720 	int ret = 0;
1721 
1722 	memset(ids, 0, sizeof(*ids));
1723 
1724 	if (ctrl->vs >= NVME_VS(1, 1, 0))
1725 		memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1726 	if (ctrl->vs >= NVME_VS(1, 2, 0))
1727 		memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1728 	if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1729 		 /* Don't treat error as fatal we potentially
1730 		  * already have a NGUID or EUI-64
1731 		  */
1732 		ret = nvme_identify_ns_descs(ctrl, nsid, ids);
1733 		if (ret)
1734 			dev_warn(ctrl->device,
1735 				 "Identify Descriptors failed (%d)\n", ret);
1736 		if (ret > 0)
1737 			ret = 0;
1738 	}
1739 	return ret;
1740 }
1741 
nvme_ns_ids_valid(struct nvme_ns_ids * ids)1742 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1743 {
1744 	return !uuid_is_null(&ids->uuid) ||
1745 		memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1746 		memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1747 }
1748 
nvme_ns_ids_equal(struct nvme_ns_ids * a,struct nvme_ns_ids * b)1749 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1750 {
1751 	return uuid_equal(&a->uuid, &b->uuid) &&
1752 		memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1753 		memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0;
1754 }
1755 
nvme_update_disk_info(struct gendisk * disk,struct nvme_ns * ns,struct nvme_id_ns * id)1756 static void nvme_update_disk_info(struct gendisk *disk,
1757 		struct nvme_ns *ns, struct nvme_id_ns *id)
1758 {
1759 	sector_t capacity = le64_to_cpu(id->nsze) << (ns->lba_shift - 9);
1760 	unsigned short bs = 1 << ns->lba_shift;
1761 	u32 atomic_bs, phys_bs, io_opt;
1762 
1763 	if (ns->lba_shift > PAGE_SHIFT) {
1764 		/* unsupported block size, set capacity to 0 later */
1765 		bs = (1 << 9);
1766 	}
1767 	blk_mq_freeze_queue(disk->queue);
1768 	blk_integrity_unregister(disk);
1769 
1770 	if (id->nabo == 0) {
1771 		/*
1772 		 * Bit 1 indicates whether NAWUPF is defined for this namespace
1773 		 * and whether it should be used instead of AWUPF. If NAWUPF ==
1774 		 * 0 then AWUPF must be used instead.
1775 		 */
1776 		if (id->nsfeat & (1 << 1) && id->nawupf)
1777 			atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1778 		else
1779 			atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1780 	} else {
1781 		atomic_bs = bs;
1782 	}
1783 	phys_bs = bs;
1784 	io_opt = bs;
1785 	if (id->nsfeat & (1 << 4)) {
1786 		/* NPWG = Namespace Preferred Write Granularity */
1787 		phys_bs *= 1 + le16_to_cpu(id->npwg);
1788 		/* NOWS = Namespace Optimal Write Size */
1789 		io_opt *= 1 + le16_to_cpu(id->nows);
1790 	}
1791 
1792 	blk_queue_logical_block_size(disk->queue, bs);
1793 	/*
1794 	 * Linux filesystems assume writing a single physical block is
1795 	 * an atomic operation. Hence limit the physical block size to the
1796 	 * value of the Atomic Write Unit Power Fail parameter.
1797 	 */
1798 	blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1799 	blk_queue_io_min(disk->queue, phys_bs);
1800 	blk_queue_io_opt(disk->queue, io_opt);
1801 
1802 	if (ns->ms && !ns->ext &&
1803 	    (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1804 		nvme_init_integrity(disk, ns->ms, ns->pi_type);
1805 	if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) ||
1806 	    ns->lba_shift > PAGE_SHIFT)
1807 		capacity = 0;
1808 
1809 	set_capacity(disk, capacity);
1810 
1811 	nvme_config_discard(disk, ns);
1812 	nvme_config_write_zeroes(disk, ns);
1813 
1814 	if (id->nsattr & (1 << 0))
1815 		set_disk_ro(disk, true);
1816 	else
1817 		set_disk_ro(disk, false);
1818 
1819 	blk_mq_unfreeze_queue(disk->queue);
1820 }
1821 
__nvme_revalidate_disk(struct gendisk * disk,struct nvme_id_ns * id)1822 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
1823 {
1824 	struct nvme_ns *ns = disk->private_data;
1825 
1826 	/*
1827 	 * If identify namespace failed, use default 512 byte block size so
1828 	 * block layer can use before failing read/write for 0 capacity.
1829 	 */
1830 	ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1831 	if (ns->lba_shift == 0)
1832 		ns->lba_shift = 9;
1833 	ns->noiob = le16_to_cpu(id->noiob);
1834 	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1835 	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
1836 	/* the PI implementation requires metadata equal t10 pi tuple size */
1837 	if (ns->ms == sizeof(struct t10_pi_tuple))
1838 		ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1839 	else
1840 		ns->pi_type = 0;
1841 
1842 	if (ns->noiob)
1843 		nvme_set_chunk_size(ns);
1844 	nvme_update_disk_info(disk, ns, id);
1845 #ifdef CONFIG_NVME_MULTIPATH
1846 	if (ns->head->disk) {
1847 		nvme_update_disk_info(ns->head->disk, ns, id);
1848 		blk_queue_stack_limits(ns->head->disk->queue, ns->queue);
1849 		revalidate_disk(ns->head->disk);
1850 	}
1851 #endif
1852 }
1853 
nvme_revalidate_disk(struct gendisk * disk)1854 static int nvme_revalidate_disk(struct gendisk *disk)
1855 {
1856 	struct nvme_ns *ns = disk->private_data;
1857 	struct nvme_ctrl *ctrl = ns->ctrl;
1858 	struct nvme_id_ns *id;
1859 	struct nvme_ns_ids ids;
1860 	int ret = 0;
1861 
1862 	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
1863 		set_capacity(disk, 0);
1864 		return -ENODEV;
1865 	}
1866 
1867 	ret = nvme_identify_ns(ctrl, ns->head->ns_id, &id);
1868 	if (ret)
1869 		goto out;
1870 
1871 	if (id->ncap == 0) {
1872 		ret = -ENODEV;
1873 		goto free_id;
1874 	}
1875 
1876 	__nvme_revalidate_disk(disk, id);
1877 	ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
1878 	if (ret)
1879 		goto free_id;
1880 
1881 	if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
1882 		dev_err(ctrl->device,
1883 			"identifiers changed for nsid %d\n", ns->head->ns_id);
1884 		ret = -ENODEV;
1885 	}
1886 
1887 free_id:
1888 	kfree(id);
1889 out:
1890 	/*
1891 	 * Only fail the function if we got a fatal error back from the
1892 	 * device, otherwise ignore the error and just move on.
1893 	 */
1894 	if (ret == -ENOMEM || (ret > 0 && !(ret & NVME_SC_DNR)))
1895 		ret = 0;
1896 	else if (ret > 0)
1897 		ret = blk_status_to_errno(nvme_error_status(ret));
1898 	return ret;
1899 }
1900 
nvme_pr_type(enum pr_type type)1901 static char nvme_pr_type(enum pr_type type)
1902 {
1903 	switch (type) {
1904 	case PR_WRITE_EXCLUSIVE:
1905 		return 1;
1906 	case PR_EXCLUSIVE_ACCESS:
1907 		return 2;
1908 	case PR_WRITE_EXCLUSIVE_REG_ONLY:
1909 		return 3;
1910 	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1911 		return 4;
1912 	case PR_WRITE_EXCLUSIVE_ALL_REGS:
1913 		return 5;
1914 	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1915 		return 6;
1916 	default:
1917 		return 0;
1918 	}
1919 };
1920 
nvme_pr_command(struct block_device * bdev,u32 cdw10,u64 key,u64 sa_key,u8 op)1921 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
1922 				u64 key, u64 sa_key, u8 op)
1923 {
1924 	struct nvme_ns_head *head = NULL;
1925 	struct nvme_ns *ns;
1926 	struct nvme_command c;
1927 	int srcu_idx, ret;
1928 	u8 data[16] = { 0, };
1929 
1930 	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1931 	if (unlikely(!ns))
1932 		return -EWOULDBLOCK;
1933 
1934 	put_unaligned_le64(key, &data[0]);
1935 	put_unaligned_le64(sa_key, &data[8]);
1936 
1937 	memset(&c, 0, sizeof(c));
1938 	c.common.opcode = op;
1939 	c.common.nsid = cpu_to_le32(ns->head->ns_id);
1940 	c.common.cdw10 = cpu_to_le32(cdw10);
1941 
1942 	ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1943 	nvme_put_ns_from_disk(head, srcu_idx);
1944 	return ret;
1945 }
1946 
nvme_pr_register(struct block_device * bdev,u64 old,u64 new,unsigned flags)1947 static int nvme_pr_register(struct block_device *bdev, u64 old,
1948 		u64 new, unsigned flags)
1949 {
1950 	u32 cdw10;
1951 
1952 	if (flags & ~PR_FL_IGNORE_KEY)
1953 		return -EOPNOTSUPP;
1954 
1955 	cdw10 = old ? 2 : 0;
1956 	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
1957 	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
1958 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
1959 }
1960 
nvme_pr_reserve(struct block_device * bdev,u64 key,enum pr_type type,unsigned flags)1961 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
1962 		enum pr_type type, unsigned flags)
1963 {
1964 	u32 cdw10;
1965 
1966 	if (flags & ~PR_FL_IGNORE_KEY)
1967 		return -EOPNOTSUPP;
1968 
1969 	cdw10 = nvme_pr_type(type) << 8;
1970 	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
1971 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
1972 }
1973 
nvme_pr_preempt(struct block_device * bdev,u64 old,u64 new,enum pr_type type,bool abort)1974 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
1975 		enum pr_type type, bool abort)
1976 {
1977 	u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
1978 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1979 }
1980 
nvme_pr_clear(struct block_device * bdev,u64 key)1981 static int nvme_pr_clear(struct block_device *bdev, u64 key)
1982 {
1983 	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1984 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
1985 }
1986 
nvme_pr_release(struct block_device * bdev,u64 key,enum pr_type type)1987 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1988 {
1989 	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
1990 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1991 }
1992 
1993 static const struct pr_ops nvme_pr_ops = {
1994 	.pr_register	= nvme_pr_register,
1995 	.pr_reserve	= nvme_pr_reserve,
1996 	.pr_release	= nvme_pr_release,
1997 	.pr_preempt	= nvme_pr_preempt,
1998 	.pr_clear	= nvme_pr_clear,
1999 };
2000 
2001 #ifdef CONFIG_BLK_SED_OPAL
nvme_sec_submit(void * data,u16 spsp,u8 secp,void * buffer,size_t len,bool send)2002 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2003 		bool send)
2004 {
2005 	struct nvme_ctrl *ctrl = data;
2006 	struct nvme_command cmd;
2007 
2008 	memset(&cmd, 0, sizeof(cmd));
2009 	if (send)
2010 		cmd.common.opcode = nvme_admin_security_send;
2011 	else
2012 		cmd.common.opcode = nvme_admin_security_recv;
2013 	cmd.common.nsid = 0;
2014 	cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2015 	cmd.common.cdw11 = cpu_to_le32(len);
2016 
2017 	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2018 				      ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2019 }
2020 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2021 #endif /* CONFIG_BLK_SED_OPAL */
2022 
2023 static const struct block_device_operations nvme_fops = {
2024 	.owner		= THIS_MODULE,
2025 	.ioctl		= nvme_ioctl,
2026 	.compat_ioctl	= nvme_ioctl,
2027 	.open		= nvme_open,
2028 	.release	= nvme_release,
2029 	.getgeo		= nvme_getgeo,
2030 	.revalidate_disk= nvme_revalidate_disk,
2031 	.pr_ops		= &nvme_pr_ops,
2032 };
2033 
2034 #ifdef CONFIG_NVME_MULTIPATH
nvme_ns_head_open(struct block_device * bdev,fmode_t mode)2035 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2036 {
2037 	struct nvme_ns_head *head = bdev->bd_disk->private_data;
2038 
2039 	if (!kref_get_unless_zero(&head->ref))
2040 		return -ENXIO;
2041 	return 0;
2042 }
2043 
nvme_ns_head_release(struct gendisk * disk,fmode_t mode)2044 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2045 {
2046 	nvme_put_ns_head(disk->private_data);
2047 }
2048 
2049 const struct block_device_operations nvme_ns_head_ops = {
2050 	.owner		= THIS_MODULE,
2051 	.open		= nvme_ns_head_open,
2052 	.release	= nvme_ns_head_release,
2053 	.ioctl		= nvme_ioctl,
2054 	.compat_ioctl	= nvme_ioctl,
2055 	.getgeo		= nvme_getgeo,
2056 	.pr_ops		= &nvme_pr_ops,
2057 };
2058 #endif /* CONFIG_NVME_MULTIPATH */
2059 
nvme_wait_ready(struct nvme_ctrl * ctrl,u64 cap,bool enabled)2060 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2061 {
2062 	unsigned long timeout =
2063 		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2064 	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2065 	int ret;
2066 
2067 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2068 		if (csts == ~0)
2069 			return -ENODEV;
2070 		if ((csts & NVME_CSTS_RDY) == bit)
2071 			break;
2072 
2073 		msleep(100);
2074 		if (fatal_signal_pending(current))
2075 			return -EINTR;
2076 		if (time_after(jiffies, timeout)) {
2077 			dev_err(ctrl->device,
2078 				"Device not ready; aborting %s\n", enabled ?
2079 						"initialisation" : "reset");
2080 			return -ENODEV;
2081 		}
2082 	}
2083 
2084 	return ret;
2085 }
2086 
2087 /*
2088  * If the device has been passed off to us in an enabled state, just clear
2089  * the enabled bit.  The spec says we should set the 'shutdown notification
2090  * bits', but doing so may cause the device to complete commands to the
2091  * admin queue ... and we don't know what memory that might be pointing at!
2092  */
nvme_disable_ctrl(struct nvme_ctrl * ctrl)2093 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2094 {
2095 	int ret;
2096 
2097 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2098 	ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2099 
2100 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2101 	if (ret)
2102 		return ret;
2103 
2104 	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2105 		msleep(NVME_QUIRK_DELAY_AMOUNT);
2106 
2107 	return nvme_wait_ready(ctrl, ctrl->cap, false);
2108 }
2109 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2110 
nvme_enable_ctrl(struct nvme_ctrl * ctrl)2111 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2112 {
2113 	/*
2114 	 * Default to a 4K page size, with the intention to update this
2115 	 * path in the future to accomodate architectures with differing
2116 	 * kernel and IO page sizes.
2117 	 */
2118 	unsigned dev_page_min, page_shift = 12;
2119 	int ret;
2120 
2121 	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2122 	if (ret) {
2123 		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2124 		return ret;
2125 	}
2126 	dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2127 
2128 	if (page_shift < dev_page_min) {
2129 		dev_err(ctrl->device,
2130 			"Minimum device page size %u too large for host (%u)\n",
2131 			1 << dev_page_min, 1 << page_shift);
2132 		return -ENODEV;
2133 	}
2134 
2135 	ctrl->page_size = 1 << page_shift;
2136 
2137 	ctrl->ctrl_config = NVME_CC_CSS_NVM;
2138 	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
2139 	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2140 	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2141 	ctrl->ctrl_config |= NVME_CC_ENABLE;
2142 
2143 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2144 	if (ret)
2145 		return ret;
2146 	return nvme_wait_ready(ctrl, ctrl->cap, true);
2147 }
2148 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2149 
nvme_shutdown_ctrl(struct nvme_ctrl * ctrl)2150 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2151 {
2152 	unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2153 	u32 csts;
2154 	int ret;
2155 
2156 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2157 	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2158 
2159 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2160 	if (ret)
2161 		return ret;
2162 
2163 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2164 		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2165 			break;
2166 
2167 		msleep(100);
2168 		if (fatal_signal_pending(current))
2169 			return -EINTR;
2170 		if (time_after(jiffies, timeout)) {
2171 			dev_err(ctrl->device,
2172 				"Device shutdown incomplete; abort shutdown\n");
2173 			return -ENODEV;
2174 		}
2175 	}
2176 
2177 	return ret;
2178 }
2179 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2180 
nvme_set_queue_limits(struct nvme_ctrl * ctrl,struct request_queue * q)2181 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2182 		struct request_queue *q)
2183 {
2184 	bool vwc = false;
2185 
2186 	if (ctrl->max_hw_sectors) {
2187 		u32 max_segments =
2188 			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
2189 
2190 		max_segments = min_not_zero(max_segments, ctrl->max_segments);
2191 		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
2192 		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
2193 	}
2194 	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2195 	    is_power_of_2(ctrl->max_hw_sectors))
2196 		blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
2197 	blk_queue_virt_boundary(q, ctrl->page_size - 1);
2198 	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2199 		vwc = true;
2200 	blk_queue_write_cache(q, vwc, vwc);
2201 }
2202 
nvme_configure_timestamp(struct nvme_ctrl * ctrl)2203 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2204 {
2205 	__le64 ts;
2206 	int ret;
2207 
2208 	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2209 		return 0;
2210 
2211 	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2212 	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2213 			NULL);
2214 	if (ret)
2215 		dev_warn_once(ctrl->device,
2216 			"could not set timestamp (%d)\n", ret);
2217 	return ret;
2218 }
2219 
nvme_configure_acre(struct nvme_ctrl * ctrl)2220 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2221 {
2222 	struct nvme_feat_host_behavior *host;
2223 	int ret;
2224 
2225 	/* Don't bother enabling the feature if retry delay is not reported */
2226 	if (!ctrl->crdt[0])
2227 		return 0;
2228 
2229 	host = kzalloc(sizeof(*host), GFP_KERNEL);
2230 	if (!host)
2231 		return 0;
2232 
2233 	host->acre = NVME_ENABLE_ACRE;
2234 	ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2235 				host, sizeof(*host), NULL);
2236 	kfree(host);
2237 	return ret;
2238 }
2239 
nvme_configure_apst(struct nvme_ctrl * ctrl)2240 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2241 {
2242 	/*
2243 	 * APST (Autonomous Power State Transition) lets us program a
2244 	 * table of power state transitions that the controller will
2245 	 * perform automatically.  We configure it with a simple
2246 	 * heuristic: we are willing to spend at most 2% of the time
2247 	 * transitioning between power states.  Therefore, when running
2248 	 * in any given state, we will enter the next lower-power
2249 	 * non-operational state after waiting 50 * (enlat + exlat)
2250 	 * microseconds, as long as that state's exit latency is under
2251 	 * the requested maximum latency.
2252 	 *
2253 	 * We will not autonomously enter any non-operational state for
2254 	 * which the total latency exceeds ps_max_latency_us.  Users
2255 	 * can set ps_max_latency_us to zero to turn off APST.
2256 	 */
2257 
2258 	unsigned apste;
2259 	struct nvme_feat_auto_pst *table;
2260 	u64 max_lat_us = 0;
2261 	int max_ps = -1;
2262 	int ret;
2263 
2264 	/*
2265 	 * If APST isn't supported or if we haven't been initialized yet,
2266 	 * then don't do anything.
2267 	 */
2268 	if (!ctrl->apsta)
2269 		return 0;
2270 
2271 	if (ctrl->npss > 31) {
2272 		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2273 		return 0;
2274 	}
2275 
2276 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2277 	if (!table)
2278 		return 0;
2279 
2280 	if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2281 		/* Turn off APST. */
2282 		apste = 0;
2283 		dev_dbg(ctrl->device, "APST disabled\n");
2284 	} else {
2285 		__le64 target = cpu_to_le64(0);
2286 		int state;
2287 
2288 		/*
2289 		 * Walk through all states from lowest- to highest-power.
2290 		 * According to the spec, lower-numbered states use more
2291 		 * power.  NPSS, despite the name, is the index of the
2292 		 * lowest-power state, not the number of states.
2293 		 */
2294 		for (state = (int)ctrl->npss; state >= 0; state--) {
2295 			u64 total_latency_us, exit_latency_us, transition_ms;
2296 
2297 			if (target)
2298 				table->entries[state] = target;
2299 
2300 			/*
2301 			 * Don't allow transitions to the deepest state
2302 			 * if it's quirked off.
2303 			 */
2304 			if (state == ctrl->npss &&
2305 			    (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2306 				continue;
2307 
2308 			/*
2309 			 * Is this state a useful non-operational state for
2310 			 * higher-power states to autonomously transition to?
2311 			 */
2312 			if (!(ctrl->psd[state].flags &
2313 			      NVME_PS_FLAGS_NON_OP_STATE))
2314 				continue;
2315 
2316 			exit_latency_us =
2317 				(u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2318 			if (exit_latency_us > ctrl->ps_max_latency_us)
2319 				continue;
2320 
2321 			total_latency_us =
2322 				exit_latency_us +
2323 				le32_to_cpu(ctrl->psd[state].entry_lat);
2324 
2325 			/*
2326 			 * This state is good.  Use it as the APST idle
2327 			 * target for higher power states.
2328 			 */
2329 			transition_ms = total_latency_us + 19;
2330 			do_div(transition_ms, 20);
2331 			if (transition_ms > (1 << 24) - 1)
2332 				transition_ms = (1 << 24) - 1;
2333 
2334 			target = cpu_to_le64((state << 3) |
2335 					     (transition_ms << 8));
2336 
2337 			if (max_ps == -1)
2338 				max_ps = state;
2339 
2340 			if (total_latency_us > max_lat_us)
2341 				max_lat_us = total_latency_us;
2342 		}
2343 
2344 		apste = 1;
2345 
2346 		if (max_ps == -1) {
2347 			dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2348 		} else {
2349 			dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2350 				max_ps, max_lat_us, (int)sizeof(*table), table);
2351 		}
2352 	}
2353 
2354 	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2355 				table, sizeof(*table), NULL);
2356 	if (ret)
2357 		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2358 
2359 	kfree(table);
2360 	return ret;
2361 }
2362 
nvme_set_latency_tolerance(struct device * dev,s32 val)2363 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2364 {
2365 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2366 	u64 latency;
2367 
2368 	switch (val) {
2369 	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2370 	case PM_QOS_LATENCY_ANY:
2371 		latency = U64_MAX;
2372 		break;
2373 
2374 	default:
2375 		latency = val;
2376 	}
2377 
2378 	if (ctrl->ps_max_latency_us != latency) {
2379 		ctrl->ps_max_latency_us = latency;
2380 		nvme_configure_apst(ctrl);
2381 	}
2382 }
2383 
2384 struct nvme_core_quirk_entry {
2385 	/*
2386 	 * NVMe model and firmware strings are padded with spaces.  For
2387 	 * simplicity, strings in the quirk table are padded with NULLs
2388 	 * instead.
2389 	 */
2390 	u16 vid;
2391 	const char *mn;
2392 	const char *fr;
2393 	unsigned long quirks;
2394 };
2395 
2396 static const struct nvme_core_quirk_entry core_quirks[] = {
2397 	{
2398 		/*
2399 		 * This Toshiba device seems to die using any APST states.  See:
2400 		 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2401 		 */
2402 		.vid = 0x1179,
2403 		.mn = "THNSF5256GPUK TOSHIBA",
2404 		.quirks = NVME_QUIRK_NO_APST,
2405 	},
2406 	{
2407 		/*
2408 		 * This LiteON CL1-3D*-Q11 firmware version has a race
2409 		 * condition associated with actions related to suspend to idle
2410 		 * LiteON has resolved the problem in future firmware
2411 		 */
2412 		.vid = 0x14a4,
2413 		.fr = "22301111",
2414 		.quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2415 	}
2416 };
2417 
2418 /* match is null-terminated but idstr is space-padded. */
string_matches(const char * idstr,const char * match,size_t len)2419 static bool string_matches(const char *idstr, const char *match, size_t len)
2420 {
2421 	size_t matchlen;
2422 
2423 	if (!match)
2424 		return true;
2425 
2426 	matchlen = strlen(match);
2427 	WARN_ON_ONCE(matchlen > len);
2428 
2429 	if (memcmp(idstr, match, matchlen))
2430 		return false;
2431 
2432 	for (; matchlen < len; matchlen++)
2433 		if (idstr[matchlen] != ' ')
2434 			return false;
2435 
2436 	return true;
2437 }
2438 
quirk_matches(const struct nvme_id_ctrl * id,const struct nvme_core_quirk_entry * q)2439 static bool quirk_matches(const struct nvme_id_ctrl *id,
2440 			  const struct nvme_core_quirk_entry *q)
2441 {
2442 	return q->vid == le16_to_cpu(id->vid) &&
2443 		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2444 		string_matches(id->fr, q->fr, sizeof(id->fr));
2445 }
2446 
nvme_init_subnqn(struct nvme_subsystem * subsys,struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2447 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2448 		struct nvme_id_ctrl *id)
2449 {
2450 	size_t nqnlen;
2451 	int off;
2452 
2453 	if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2454 		nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2455 		if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2456 			strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2457 			return;
2458 		}
2459 
2460 		if (ctrl->vs >= NVME_VS(1, 2, 1))
2461 			dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2462 	}
2463 
2464 	/* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2465 	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2466 			"nqn.2014.08.org.nvmexpress:%04x%04x",
2467 			le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2468 	memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2469 	off += sizeof(id->sn);
2470 	memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2471 	off += sizeof(id->mn);
2472 	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2473 }
2474 
nvme_release_subsystem(struct device * dev)2475 static void nvme_release_subsystem(struct device *dev)
2476 {
2477 	struct nvme_subsystem *subsys =
2478 		container_of(dev, struct nvme_subsystem, dev);
2479 
2480 	if (subsys->instance >= 0)
2481 		ida_simple_remove(&nvme_instance_ida, subsys->instance);
2482 	kfree(subsys);
2483 }
2484 
nvme_destroy_subsystem(struct kref * ref)2485 static void nvme_destroy_subsystem(struct kref *ref)
2486 {
2487 	struct nvme_subsystem *subsys =
2488 			container_of(ref, struct nvme_subsystem, ref);
2489 
2490 	mutex_lock(&nvme_subsystems_lock);
2491 	list_del(&subsys->entry);
2492 	mutex_unlock(&nvme_subsystems_lock);
2493 
2494 	ida_destroy(&subsys->ns_ida);
2495 	device_del(&subsys->dev);
2496 	put_device(&subsys->dev);
2497 }
2498 
nvme_put_subsystem(struct nvme_subsystem * subsys)2499 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2500 {
2501 	kref_put(&subsys->ref, nvme_destroy_subsystem);
2502 }
2503 
__nvme_find_get_subsystem(const char * subsysnqn)2504 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2505 {
2506 	struct nvme_subsystem *subsys;
2507 
2508 	lockdep_assert_held(&nvme_subsystems_lock);
2509 
2510 	/*
2511 	 * Fail matches for discovery subsystems. This results
2512 	 * in each discovery controller bound to a unique subsystem.
2513 	 * This avoids issues with validating controller values
2514 	 * that can only be true when there is a single unique subsystem.
2515 	 * There may be multiple and completely independent entities
2516 	 * that provide discovery controllers.
2517 	 */
2518 	if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2519 		return NULL;
2520 
2521 	list_for_each_entry(subsys, &nvme_subsystems, entry) {
2522 		if (strcmp(subsys->subnqn, subsysnqn))
2523 			continue;
2524 		if (!kref_get_unless_zero(&subsys->ref))
2525 			continue;
2526 		return subsys;
2527 	}
2528 
2529 	return NULL;
2530 }
2531 
2532 #define SUBSYS_ATTR_RO(_name, _mode, _show)			\
2533 	struct device_attribute subsys_attr_##_name = \
2534 		__ATTR(_name, _mode, _show, NULL)
2535 
nvme_subsys_show_nqn(struct device * dev,struct device_attribute * attr,char * buf)2536 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2537 				    struct device_attribute *attr,
2538 				    char *buf)
2539 {
2540 	struct nvme_subsystem *subsys =
2541 		container_of(dev, struct nvme_subsystem, dev);
2542 
2543 	return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2544 }
2545 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2546 
2547 #define nvme_subsys_show_str_function(field)				\
2548 static ssize_t subsys_##field##_show(struct device *dev,		\
2549 			    struct device_attribute *attr, char *buf)	\
2550 {									\
2551 	struct nvme_subsystem *subsys =					\
2552 		container_of(dev, struct nvme_subsystem, dev);		\
2553 	return sprintf(buf, "%.*s\n",					\
2554 		       (int)sizeof(subsys->field), subsys->field);	\
2555 }									\
2556 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2557 
2558 nvme_subsys_show_str_function(model);
2559 nvme_subsys_show_str_function(serial);
2560 nvme_subsys_show_str_function(firmware_rev);
2561 
2562 static struct attribute *nvme_subsys_attrs[] = {
2563 	&subsys_attr_model.attr,
2564 	&subsys_attr_serial.attr,
2565 	&subsys_attr_firmware_rev.attr,
2566 	&subsys_attr_subsysnqn.attr,
2567 #ifdef CONFIG_NVME_MULTIPATH
2568 	&subsys_attr_iopolicy.attr,
2569 #endif
2570 	NULL,
2571 };
2572 
2573 static struct attribute_group nvme_subsys_attrs_group = {
2574 	.attrs = nvme_subsys_attrs,
2575 };
2576 
2577 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2578 	&nvme_subsys_attrs_group,
2579 	NULL,
2580 };
2581 
nvme_validate_cntlid(struct nvme_subsystem * subsys,struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2582 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2583 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2584 {
2585 	struct nvme_ctrl *tmp;
2586 
2587 	lockdep_assert_held(&nvme_subsystems_lock);
2588 
2589 	list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2590 		if (tmp->state == NVME_CTRL_DELETING ||
2591 		    tmp->state == NVME_CTRL_DEAD)
2592 			continue;
2593 
2594 		if (tmp->cntlid == ctrl->cntlid) {
2595 			dev_err(ctrl->device,
2596 				"Duplicate cntlid %u with %s, rejecting\n",
2597 				ctrl->cntlid, dev_name(tmp->device));
2598 			return false;
2599 		}
2600 
2601 		if ((id->cmic & (1 << 1)) ||
2602 		    (ctrl->opts && ctrl->opts->discovery_nqn))
2603 			continue;
2604 
2605 		dev_err(ctrl->device,
2606 			"Subsystem does not support multiple controllers\n");
2607 		return false;
2608 	}
2609 
2610 	return true;
2611 }
2612 
nvme_init_subsystem(struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2613 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2614 {
2615 	struct nvme_subsystem *subsys, *found;
2616 	int ret;
2617 
2618 	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2619 	if (!subsys)
2620 		return -ENOMEM;
2621 
2622 	subsys->instance = -1;
2623 	mutex_init(&subsys->lock);
2624 	kref_init(&subsys->ref);
2625 	INIT_LIST_HEAD(&subsys->ctrls);
2626 	INIT_LIST_HEAD(&subsys->nsheads);
2627 	nvme_init_subnqn(subsys, ctrl, id);
2628 	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2629 	memcpy(subsys->model, id->mn, sizeof(subsys->model));
2630 	memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2631 	subsys->vendor_id = le16_to_cpu(id->vid);
2632 	subsys->cmic = id->cmic;
2633 	subsys->awupf = le16_to_cpu(id->awupf);
2634 #ifdef CONFIG_NVME_MULTIPATH
2635 	subsys->iopolicy = NVME_IOPOLICY_NUMA;
2636 #endif
2637 
2638 	subsys->dev.class = nvme_subsys_class;
2639 	subsys->dev.release = nvme_release_subsystem;
2640 	subsys->dev.groups = nvme_subsys_attrs_groups;
2641 	dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2642 	device_initialize(&subsys->dev);
2643 
2644 	mutex_lock(&nvme_subsystems_lock);
2645 	found = __nvme_find_get_subsystem(subsys->subnqn);
2646 	if (found) {
2647 		put_device(&subsys->dev);
2648 		subsys = found;
2649 
2650 		if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2651 			ret = -EINVAL;
2652 			goto out_put_subsystem;
2653 		}
2654 	} else {
2655 		ret = device_add(&subsys->dev);
2656 		if (ret) {
2657 			dev_err(ctrl->device,
2658 				"failed to register subsystem device.\n");
2659 			put_device(&subsys->dev);
2660 			goto out_unlock;
2661 		}
2662 		ida_init(&subsys->ns_ida);
2663 		list_add_tail(&subsys->entry, &nvme_subsystems);
2664 	}
2665 
2666 	ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2667 				dev_name(ctrl->device));
2668 	if (ret) {
2669 		dev_err(ctrl->device,
2670 			"failed to create sysfs link from subsystem.\n");
2671 		goto out_put_subsystem;
2672 	}
2673 
2674 	if (!found)
2675 		subsys->instance = ctrl->instance;
2676 	ctrl->subsys = subsys;
2677 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2678 	mutex_unlock(&nvme_subsystems_lock);
2679 	return 0;
2680 
2681 out_put_subsystem:
2682 	nvme_put_subsystem(subsys);
2683 out_unlock:
2684 	mutex_unlock(&nvme_subsystems_lock);
2685 	return ret;
2686 }
2687 
nvme_get_log(struct nvme_ctrl * ctrl,u32 nsid,u8 log_page,u8 lsp,void * log,size_t size,u64 offset)2688 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
2689 		void *log, size_t size, u64 offset)
2690 {
2691 	struct nvme_command c = { };
2692 	unsigned long dwlen = size / 4 - 1;
2693 
2694 	c.get_log_page.opcode = nvme_admin_get_log_page;
2695 	c.get_log_page.nsid = cpu_to_le32(nsid);
2696 	c.get_log_page.lid = log_page;
2697 	c.get_log_page.lsp = lsp;
2698 	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2699 	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2700 	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2701 	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2702 
2703 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2704 }
2705 
nvme_get_effects_log(struct nvme_ctrl * ctrl)2706 static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
2707 {
2708 	int ret;
2709 
2710 	if (!ctrl->effects)
2711 		ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
2712 
2713 	if (!ctrl->effects)
2714 		return 0;
2715 
2716 	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0,
2717 			ctrl->effects, sizeof(*ctrl->effects), 0);
2718 	if (ret) {
2719 		kfree(ctrl->effects);
2720 		ctrl->effects = NULL;
2721 	}
2722 	return ret;
2723 }
2724 
2725 /*
2726  * Initialize the cached copies of the Identify data and various controller
2727  * register in our nvme_ctrl structure.  This should be called as soon as
2728  * the admin queue is fully up and running.
2729  */
nvme_init_identify(struct nvme_ctrl * ctrl)2730 int nvme_init_identify(struct nvme_ctrl *ctrl)
2731 {
2732 	struct nvme_id_ctrl *id;
2733 	int ret, page_shift;
2734 	u32 max_hw_sectors;
2735 	bool prev_apst_enabled;
2736 
2737 	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
2738 	if (ret) {
2739 		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2740 		return ret;
2741 	}
2742 	page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2743 	ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
2744 
2745 	if (ctrl->vs >= NVME_VS(1, 1, 0))
2746 		ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
2747 
2748 	ret = nvme_identify_ctrl(ctrl, &id);
2749 	if (ret) {
2750 		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2751 		return -EIO;
2752 	}
2753 
2754 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2755 		ret = nvme_get_effects_log(ctrl);
2756 		if (ret < 0)
2757 			goto out_free;
2758 	}
2759 
2760 	if (!(ctrl->ops->flags & NVME_F_FABRICS))
2761 		ctrl->cntlid = le16_to_cpu(id->cntlid);
2762 
2763 	if (!ctrl->identified) {
2764 		int i;
2765 
2766 		ret = nvme_init_subsystem(ctrl, id);
2767 		if (ret)
2768 			goto out_free;
2769 
2770 		/*
2771 		 * Check for quirks.  Quirk can depend on firmware version,
2772 		 * so, in principle, the set of quirks present can change
2773 		 * across a reset.  As a possible future enhancement, we
2774 		 * could re-scan for quirks every time we reinitialize
2775 		 * the device, but we'd have to make sure that the driver
2776 		 * behaves intelligently if the quirks change.
2777 		 */
2778 		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
2779 			if (quirk_matches(id, &core_quirks[i]))
2780 				ctrl->quirks |= core_quirks[i].quirks;
2781 		}
2782 	}
2783 
2784 	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
2785 		dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
2786 		ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
2787 	}
2788 
2789 	ctrl->crdt[0] = le16_to_cpu(id->crdt1);
2790 	ctrl->crdt[1] = le16_to_cpu(id->crdt2);
2791 	ctrl->crdt[2] = le16_to_cpu(id->crdt3);
2792 
2793 	ctrl->oacs = le16_to_cpu(id->oacs);
2794 	ctrl->oncs = le16_to_cpu(id->oncs);
2795 	ctrl->mtfa = le16_to_cpu(id->mtfa);
2796 	ctrl->oaes = le32_to_cpu(id->oaes);
2797 	atomic_set(&ctrl->abort_limit, id->acl + 1);
2798 	ctrl->vwc = id->vwc;
2799 	if (id->mdts)
2800 		max_hw_sectors = 1 << (id->mdts + page_shift - 9);
2801 	else
2802 		max_hw_sectors = UINT_MAX;
2803 	ctrl->max_hw_sectors =
2804 		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
2805 
2806 	nvme_set_queue_limits(ctrl, ctrl->admin_q);
2807 	ctrl->sgls = le32_to_cpu(id->sgls);
2808 	ctrl->kas = le16_to_cpu(id->kas);
2809 	ctrl->max_namespaces = le32_to_cpu(id->mnan);
2810 	ctrl->ctratt = le32_to_cpu(id->ctratt);
2811 
2812 	if (id->rtd3e) {
2813 		/* us -> s */
2814 		u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
2815 
2816 		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
2817 						 shutdown_timeout, 60);
2818 
2819 		if (ctrl->shutdown_timeout != shutdown_timeout)
2820 			dev_info(ctrl->device,
2821 				 "Shutdown timeout set to %u seconds\n",
2822 				 ctrl->shutdown_timeout);
2823 	} else
2824 		ctrl->shutdown_timeout = shutdown_timeout;
2825 
2826 	ctrl->npss = id->npss;
2827 	ctrl->apsta = id->apsta;
2828 	prev_apst_enabled = ctrl->apst_enabled;
2829 	if (ctrl->quirks & NVME_QUIRK_NO_APST) {
2830 		if (force_apst && id->apsta) {
2831 			dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
2832 			ctrl->apst_enabled = true;
2833 		} else {
2834 			ctrl->apst_enabled = false;
2835 		}
2836 	} else {
2837 		ctrl->apst_enabled = id->apsta;
2838 	}
2839 	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
2840 
2841 	if (ctrl->ops->flags & NVME_F_FABRICS) {
2842 		ctrl->icdoff = le16_to_cpu(id->icdoff);
2843 		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
2844 		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
2845 		ctrl->maxcmd = le16_to_cpu(id->maxcmd);
2846 
2847 		/*
2848 		 * In fabrics we need to verify the cntlid matches the
2849 		 * admin connect
2850 		 */
2851 		if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
2852 			ret = -EINVAL;
2853 			goto out_free;
2854 		}
2855 
2856 		if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
2857 			dev_err(ctrl->device,
2858 				"keep-alive support is mandatory for fabrics\n");
2859 			ret = -EINVAL;
2860 			goto out_free;
2861 		}
2862 	} else {
2863 		ctrl->hmpre = le32_to_cpu(id->hmpre);
2864 		ctrl->hmmin = le32_to_cpu(id->hmmin);
2865 		ctrl->hmminds = le32_to_cpu(id->hmminds);
2866 		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2867 	}
2868 
2869 	ret = nvme_mpath_init(ctrl, id);
2870 	kfree(id);
2871 
2872 	if (ret < 0)
2873 		return ret;
2874 
2875 	if (ctrl->apst_enabled && !prev_apst_enabled)
2876 		dev_pm_qos_expose_latency_tolerance(ctrl->device);
2877 	else if (!ctrl->apst_enabled && prev_apst_enabled)
2878 		dev_pm_qos_hide_latency_tolerance(ctrl->device);
2879 
2880 	ret = nvme_configure_apst(ctrl);
2881 	if (ret < 0)
2882 		return ret;
2883 
2884 	ret = nvme_configure_timestamp(ctrl);
2885 	if (ret < 0)
2886 		return ret;
2887 
2888 	ret = nvme_configure_directives(ctrl);
2889 	if (ret < 0)
2890 		return ret;
2891 
2892 	ret = nvme_configure_acre(ctrl);
2893 	if (ret < 0)
2894 		return ret;
2895 
2896 	ctrl->identified = true;
2897 
2898 	return 0;
2899 
2900 out_free:
2901 	kfree(id);
2902 	return ret;
2903 }
2904 EXPORT_SYMBOL_GPL(nvme_init_identify);
2905 
nvme_dev_open(struct inode * inode,struct file * file)2906 static int nvme_dev_open(struct inode *inode, struct file *file)
2907 {
2908 	struct nvme_ctrl *ctrl =
2909 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
2910 
2911 	switch (ctrl->state) {
2912 	case NVME_CTRL_LIVE:
2913 		break;
2914 	default:
2915 		return -EWOULDBLOCK;
2916 	}
2917 
2918 	file->private_data = ctrl;
2919 	return 0;
2920 }
2921 
nvme_dev_user_cmd(struct nvme_ctrl * ctrl,void __user * argp)2922 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
2923 {
2924 	struct nvme_ns *ns;
2925 	int ret;
2926 
2927 	down_read(&ctrl->namespaces_rwsem);
2928 	if (list_empty(&ctrl->namespaces)) {
2929 		ret = -ENOTTY;
2930 		goto out_unlock;
2931 	}
2932 
2933 	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
2934 	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
2935 		dev_warn(ctrl->device,
2936 			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
2937 		ret = -EINVAL;
2938 		goto out_unlock;
2939 	}
2940 
2941 	dev_warn(ctrl->device,
2942 		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
2943 	kref_get(&ns->kref);
2944 	up_read(&ctrl->namespaces_rwsem);
2945 
2946 	ret = nvme_user_cmd(ctrl, ns, argp);
2947 	nvme_put_ns(ns);
2948 	return ret;
2949 
2950 out_unlock:
2951 	up_read(&ctrl->namespaces_rwsem);
2952 	return ret;
2953 }
2954 
nvme_dev_ioctl(struct file * file,unsigned int cmd,unsigned long arg)2955 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
2956 		unsigned long arg)
2957 {
2958 	struct nvme_ctrl *ctrl = file->private_data;
2959 	void __user *argp = (void __user *)arg;
2960 
2961 	switch (cmd) {
2962 	case NVME_IOCTL_ADMIN_CMD:
2963 		return nvme_user_cmd(ctrl, NULL, argp);
2964 	case NVME_IOCTL_ADMIN64_CMD:
2965 		return nvme_user_cmd64(ctrl, NULL, argp);
2966 	case NVME_IOCTL_IO_CMD:
2967 		return nvme_dev_user_cmd(ctrl, argp);
2968 	case NVME_IOCTL_RESET:
2969 		dev_warn(ctrl->device, "resetting controller\n");
2970 		return nvme_reset_ctrl_sync(ctrl);
2971 	case NVME_IOCTL_SUBSYS_RESET:
2972 		return nvme_reset_subsystem(ctrl);
2973 	case NVME_IOCTL_RESCAN:
2974 		nvme_queue_scan(ctrl);
2975 		return 0;
2976 	default:
2977 		return -ENOTTY;
2978 	}
2979 }
2980 
2981 static const struct file_operations nvme_dev_fops = {
2982 	.owner		= THIS_MODULE,
2983 	.open		= nvme_dev_open,
2984 	.unlocked_ioctl	= nvme_dev_ioctl,
2985 	.compat_ioctl	= nvme_dev_ioctl,
2986 };
2987 
nvme_sysfs_reset(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2988 static ssize_t nvme_sysfs_reset(struct device *dev,
2989 				struct device_attribute *attr, const char *buf,
2990 				size_t count)
2991 {
2992 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2993 	int ret;
2994 
2995 	ret = nvme_reset_ctrl_sync(ctrl);
2996 	if (ret < 0)
2997 		return ret;
2998 	return count;
2999 }
3000 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3001 
nvme_sysfs_rescan(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3002 static ssize_t nvme_sysfs_rescan(struct device *dev,
3003 				struct device_attribute *attr, const char *buf,
3004 				size_t count)
3005 {
3006 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3007 
3008 	nvme_queue_scan(ctrl);
3009 	return count;
3010 }
3011 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3012 
dev_to_ns_head(struct device * dev)3013 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3014 {
3015 	struct gendisk *disk = dev_to_disk(dev);
3016 
3017 	if (disk->fops == &nvme_fops)
3018 		return nvme_get_ns_from_dev(dev)->head;
3019 	else
3020 		return disk->private_data;
3021 }
3022 
wwid_show(struct device * dev,struct device_attribute * attr,char * buf)3023 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3024 		char *buf)
3025 {
3026 	struct nvme_ns_head *head = dev_to_ns_head(dev);
3027 	struct nvme_ns_ids *ids = &head->ids;
3028 	struct nvme_subsystem *subsys = head->subsys;
3029 	int serial_len = sizeof(subsys->serial);
3030 	int model_len = sizeof(subsys->model);
3031 
3032 	if (!uuid_is_null(&ids->uuid))
3033 		return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3034 
3035 	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3036 		return sprintf(buf, "eui.%16phN\n", ids->nguid);
3037 
3038 	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3039 		return sprintf(buf, "eui.%8phN\n", ids->eui64);
3040 
3041 	while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3042 				  subsys->serial[serial_len - 1] == '\0'))
3043 		serial_len--;
3044 	while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3045 				 subsys->model[model_len - 1] == '\0'))
3046 		model_len--;
3047 
3048 	return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3049 		serial_len, subsys->serial, model_len, subsys->model,
3050 		head->ns_id);
3051 }
3052 static DEVICE_ATTR_RO(wwid);
3053 
nguid_show(struct device * dev,struct device_attribute * attr,char * buf)3054 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3055 		char *buf)
3056 {
3057 	return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3058 }
3059 static DEVICE_ATTR_RO(nguid);
3060 
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)3061 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3062 		char *buf)
3063 {
3064 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3065 
3066 	/* For backward compatibility expose the NGUID to userspace if
3067 	 * we have no UUID set
3068 	 */
3069 	if (uuid_is_null(&ids->uuid)) {
3070 		printk_ratelimited(KERN_WARNING
3071 				   "No UUID available providing old NGUID\n");
3072 		return sprintf(buf, "%pU\n", ids->nguid);
3073 	}
3074 	return sprintf(buf, "%pU\n", &ids->uuid);
3075 }
3076 static DEVICE_ATTR_RO(uuid);
3077 
eui_show(struct device * dev,struct device_attribute * attr,char * buf)3078 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3079 		char *buf)
3080 {
3081 	return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3082 }
3083 static DEVICE_ATTR_RO(eui);
3084 
nsid_show(struct device * dev,struct device_attribute * attr,char * buf)3085 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3086 		char *buf)
3087 {
3088 	return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3089 }
3090 static DEVICE_ATTR_RO(nsid);
3091 
3092 static struct attribute *nvme_ns_id_attrs[] = {
3093 	&dev_attr_wwid.attr,
3094 	&dev_attr_uuid.attr,
3095 	&dev_attr_nguid.attr,
3096 	&dev_attr_eui.attr,
3097 	&dev_attr_nsid.attr,
3098 #ifdef CONFIG_NVME_MULTIPATH
3099 	&dev_attr_ana_grpid.attr,
3100 	&dev_attr_ana_state.attr,
3101 #endif
3102 	NULL,
3103 };
3104 
nvme_ns_id_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)3105 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3106 		struct attribute *a, int n)
3107 {
3108 	struct device *dev = container_of(kobj, struct device, kobj);
3109 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3110 
3111 	if (a == &dev_attr_uuid.attr) {
3112 		if (uuid_is_null(&ids->uuid) &&
3113 		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3114 			return 0;
3115 	}
3116 	if (a == &dev_attr_nguid.attr) {
3117 		if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3118 			return 0;
3119 	}
3120 	if (a == &dev_attr_eui.attr) {
3121 		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3122 			return 0;
3123 	}
3124 #ifdef CONFIG_NVME_MULTIPATH
3125 	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3126 		if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3127 			return 0;
3128 		if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3129 			return 0;
3130 	}
3131 #endif
3132 	return a->mode;
3133 }
3134 
3135 static const struct attribute_group nvme_ns_id_attr_group = {
3136 	.attrs		= nvme_ns_id_attrs,
3137 	.is_visible	= nvme_ns_id_attrs_are_visible,
3138 };
3139 
3140 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3141 	&nvme_ns_id_attr_group,
3142 #ifdef CONFIG_NVM
3143 	&nvme_nvm_attr_group,
3144 #endif
3145 	NULL,
3146 };
3147 
3148 #define nvme_show_str_function(field)						\
3149 static ssize_t  field##_show(struct device *dev,				\
3150 			    struct device_attribute *attr, char *buf)		\
3151 {										\
3152         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3153         return sprintf(buf, "%.*s\n",						\
3154 		(int)sizeof(ctrl->subsys->field), ctrl->subsys->field);		\
3155 }										\
3156 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3157 
3158 nvme_show_str_function(model);
3159 nvme_show_str_function(serial);
3160 nvme_show_str_function(firmware_rev);
3161 
3162 #define nvme_show_int_function(field)						\
3163 static ssize_t  field##_show(struct device *dev,				\
3164 			    struct device_attribute *attr, char *buf)		\
3165 {										\
3166         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3167         return sprintf(buf, "%d\n", ctrl->field);	\
3168 }										\
3169 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3170 
3171 nvme_show_int_function(cntlid);
3172 nvme_show_int_function(numa_node);
3173 nvme_show_int_function(queue_count);
3174 nvme_show_int_function(sqsize);
3175 
nvme_sysfs_delete(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3176 static ssize_t nvme_sysfs_delete(struct device *dev,
3177 				struct device_attribute *attr, const char *buf,
3178 				size_t count)
3179 {
3180 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3181 
3182 	if (device_remove_file_self(dev, attr))
3183 		nvme_delete_ctrl_sync(ctrl);
3184 	return count;
3185 }
3186 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3187 
nvme_sysfs_show_transport(struct device * dev,struct device_attribute * attr,char * buf)3188 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3189 					 struct device_attribute *attr,
3190 					 char *buf)
3191 {
3192 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3193 
3194 	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3195 }
3196 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3197 
nvme_sysfs_show_state(struct device * dev,struct device_attribute * attr,char * buf)3198 static ssize_t nvme_sysfs_show_state(struct device *dev,
3199 				     struct device_attribute *attr,
3200 				     char *buf)
3201 {
3202 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3203 	static const char *const state_name[] = {
3204 		[NVME_CTRL_NEW]		= "new",
3205 		[NVME_CTRL_LIVE]	= "live",
3206 		[NVME_CTRL_RESETTING]	= "resetting",
3207 		[NVME_CTRL_CONNECTING]	= "connecting",
3208 		[NVME_CTRL_DELETING]	= "deleting",
3209 		[NVME_CTRL_DEAD]	= "dead",
3210 	};
3211 
3212 	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3213 	    state_name[ctrl->state])
3214 		return sprintf(buf, "%s\n", state_name[ctrl->state]);
3215 
3216 	return sprintf(buf, "unknown state\n");
3217 }
3218 
3219 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3220 
nvme_sysfs_show_subsysnqn(struct device * dev,struct device_attribute * attr,char * buf)3221 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3222 					 struct device_attribute *attr,
3223 					 char *buf)
3224 {
3225 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3226 
3227 	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3228 }
3229 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3230 
nvme_sysfs_show_address(struct device * dev,struct device_attribute * attr,char * buf)3231 static ssize_t nvme_sysfs_show_address(struct device *dev,
3232 					 struct device_attribute *attr,
3233 					 char *buf)
3234 {
3235 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3236 
3237 	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3238 }
3239 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3240 
3241 static struct attribute *nvme_dev_attrs[] = {
3242 	&dev_attr_reset_controller.attr,
3243 	&dev_attr_rescan_controller.attr,
3244 	&dev_attr_model.attr,
3245 	&dev_attr_serial.attr,
3246 	&dev_attr_firmware_rev.attr,
3247 	&dev_attr_cntlid.attr,
3248 	&dev_attr_delete_controller.attr,
3249 	&dev_attr_transport.attr,
3250 	&dev_attr_subsysnqn.attr,
3251 	&dev_attr_address.attr,
3252 	&dev_attr_state.attr,
3253 	&dev_attr_numa_node.attr,
3254 	&dev_attr_queue_count.attr,
3255 	&dev_attr_sqsize.attr,
3256 	NULL
3257 };
3258 
nvme_dev_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)3259 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3260 		struct attribute *a, int n)
3261 {
3262 	struct device *dev = container_of(kobj, struct device, kobj);
3263 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3264 
3265 	if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3266 		return 0;
3267 	if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3268 		return 0;
3269 
3270 	return a->mode;
3271 }
3272 
3273 static struct attribute_group nvme_dev_attrs_group = {
3274 	.attrs		= nvme_dev_attrs,
3275 	.is_visible	= nvme_dev_attrs_are_visible,
3276 };
3277 
3278 static const struct attribute_group *nvme_dev_attr_groups[] = {
3279 	&nvme_dev_attrs_group,
3280 	NULL,
3281 };
3282 
__nvme_find_ns_head(struct nvme_subsystem * subsys,unsigned nsid)3283 static struct nvme_ns_head *__nvme_find_ns_head(struct nvme_subsystem *subsys,
3284 		unsigned nsid)
3285 {
3286 	struct nvme_ns_head *h;
3287 
3288 	lockdep_assert_held(&subsys->lock);
3289 
3290 	list_for_each_entry(h, &subsys->nsheads, entry) {
3291 		if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3292 			return h;
3293 	}
3294 
3295 	return NULL;
3296 }
3297 
__nvme_check_ids(struct nvme_subsystem * subsys,struct nvme_ns_head * new)3298 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3299 		struct nvme_ns_head *new)
3300 {
3301 	struct nvme_ns_head *h;
3302 
3303 	lockdep_assert_held(&subsys->lock);
3304 
3305 	list_for_each_entry(h, &subsys->nsheads, entry) {
3306 		if (nvme_ns_ids_valid(&new->ids) &&
3307 		    !list_empty(&h->list) &&
3308 		    nvme_ns_ids_equal(&new->ids, &h->ids))
3309 			return -EINVAL;
3310 	}
3311 
3312 	return 0;
3313 }
3314 
nvme_alloc_ns_head(struct nvme_ctrl * ctrl,unsigned nsid,struct nvme_id_ns * id)3315 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3316 		unsigned nsid, struct nvme_id_ns *id)
3317 {
3318 	struct nvme_ns_head *head;
3319 	size_t size = sizeof(*head);
3320 	int ret = -ENOMEM;
3321 
3322 #ifdef CONFIG_NVME_MULTIPATH
3323 	size += num_possible_nodes() * sizeof(struct nvme_ns *);
3324 #endif
3325 
3326 	head = kzalloc(size, GFP_KERNEL);
3327 	if (!head)
3328 		goto out;
3329 	ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3330 	if (ret < 0)
3331 		goto out_free_head;
3332 	head->instance = ret;
3333 	INIT_LIST_HEAD(&head->list);
3334 	ret = init_srcu_struct(&head->srcu);
3335 	if (ret)
3336 		goto out_ida_remove;
3337 	head->subsys = ctrl->subsys;
3338 	head->ns_id = nsid;
3339 	kref_init(&head->ref);
3340 
3341 	ret = nvme_report_ns_ids(ctrl, nsid, id, &head->ids);
3342 	if (ret)
3343 		goto out_cleanup_srcu;
3344 
3345 	ret = __nvme_check_ids(ctrl->subsys, head);
3346 	if (ret) {
3347 		dev_err(ctrl->device,
3348 			"duplicate IDs for nsid %d\n", nsid);
3349 		goto out_cleanup_srcu;
3350 	}
3351 
3352 	ret = nvme_mpath_alloc_disk(ctrl, head);
3353 	if (ret)
3354 		goto out_cleanup_srcu;
3355 
3356 	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3357 
3358 	kref_get(&ctrl->subsys->ref);
3359 
3360 	return head;
3361 out_cleanup_srcu:
3362 	cleanup_srcu_struct(&head->srcu);
3363 out_ida_remove:
3364 	ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3365 out_free_head:
3366 	kfree(head);
3367 out:
3368 	if (ret > 0)
3369 		ret = blk_status_to_errno(nvme_error_status(ret));
3370 	return ERR_PTR(ret);
3371 }
3372 
nvme_init_ns_head(struct nvme_ns * ns,unsigned nsid,struct nvme_id_ns * id)3373 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3374 		struct nvme_id_ns *id)
3375 {
3376 	struct nvme_ctrl *ctrl = ns->ctrl;
3377 	bool is_shared = id->nmic & (1 << 0);
3378 	struct nvme_ns_head *head = NULL;
3379 	int ret = 0;
3380 
3381 	mutex_lock(&ctrl->subsys->lock);
3382 	if (is_shared)
3383 		head = __nvme_find_ns_head(ctrl->subsys, nsid);
3384 	if (!head) {
3385 		head = nvme_alloc_ns_head(ctrl, nsid, id);
3386 		if (IS_ERR(head)) {
3387 			ret = PTR_ERR(head);
3388 			goto out_unlock;
3389 		}
3390 	} else {
3391 		struct nvme_ns_ids ids;
3392 
3393 		ret = nvme_report_ns_ids(ctrl, nsid, id, &ids);
3394 		if (ret)
3395 			goto out_unlock;
3396 
3397 		if (!nvme_ns_ids_equal(&head->ids, &ids)) {
3398 			dev_err(ctrl->device,
3399 				"IDs don't match for shared namespace %d\n",
3400 					nsid);
3401 			ret = -EINVAL;
3402 			goto out_unlock;
3403 		}
3404 	}
3405 
3406 	list_add_tail(&ns->siblings, &head->list);
3407 	ns->head = head;
3408 
3409 out_unlock:
3410 	mutex_unlock(&ctrl->subsys->lock);
3411 	if (ret > 0)
3412 		ret = blk_status_to_errno(nvme_error_status(ret));
3413 	return ret;
3414 }
3415 
ns_cmp(void * priv,struct list_head * a,struct list_head * b)3416 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3417 {
3418 	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3419 	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3420 
3421 	return nsa->head->ns_id - nsb->head->ns_id;
3422 }
3423 
nvme_find_get_ns(struct nvme_ctrl * ctrl,unsigned nsid)3424 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3425 {
3426 	struct nvme_ns *ns, *ret = NULL;
3427 
3428 	down_read(&ctrl->namespaces_rwsem);
3429 	list_for_each_entry(ns, &ctrl->namespaces, list) {
3430 		if (ns->head->ns_id == nsid) {
3431 			if (!kref_get_unless_zero(&ns->kref))
3432 				continue;
3433 			ret = ns;
3434 			break;
3435 		}
3436 		if (ns->head->ns_id > nsid)
3437 			break;
3438 	}
3439 	up_read(&ctrl->namespaces_rwsem);
3440 	return ret;
3441 }
3442 
nvme_setup_streams_ns(struct nvme_ctrl * ctrl,struct nvme_ns * ns)3443 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
3444 {
3445 	struct streams_directive_params s;
3446 	int ret;
3447 
3448 	if (!ctrl->nr_streams)
3449 		return 0;
3450 
3451 	ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
3452 	if (ret)
3453 		return ret;
3454 
3455 	ns->sws = le32_to_cpu(s.sws);
3456 	ns->sgs = le16_to_cpu(s.sgs);
3457 
3458 	if (ns->sws) {
3459 		unsigned int bs = 1 << ns->lba_shift;
3460 
3461 		blk_queue_io_min(ns->queue, bs * ns->sws);
3462 		if (ns->sgs)
3463 			blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
3464 	}
3465 
3466 	return 0;
3467 }
3468 
nvme_alloc_ns(struct nvme_ctrl * ctrl,unsigned nsid)3469 static int nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3470 {
3471 	struct nvme_ns *ns;
3472 	struct gendisk *disk;
3473 	struct nvme_id_ns *id;
3474 	char disk_name[DISK_NAME_LEN];
3475 	int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3476 
3477 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3478 	if (!ns)
3479 		return -ENOMEM;
3480 
3481 	ns->queue = blk_mq_init_queue(ctrl->tagset);
3482 	if (IS_ERR(ns->queue)) {
3483 		ret = PTR_ERR(ns->queue);
3484 		goto out_free_ns;
3485 	}
3486 
3487 	if (ctrl->opts && ctrl->opts->data_digest)
3488 		ns->queue->backing_dev_info->capabilities
3489 			|= BDI_CAP_STABLE_WRITES;
3490 
3491 	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3492 	if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3493 		blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3494 
3495 	ns->queue->queuedata = ns;
3496 	ns->ctrl = ctrl;
3497 
3498 	kref_init(&ns->kref);
3499 	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
3500 
3501 	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
3502 	nvme_set_queue_limits(ctrl, ns->queue);
3503 
3504 	ret = nvme_identify_ns(ctrl, nsid, &id);
3505 	if (ret)
3506 		goto out_free_queue;
3507 
3508 	if (id->ncap == 0) {
3509 		ret = -EINVAL;
3510 		goto out_free_id;
3511 	}
3512 
3513 	ret = nvme_init_ns_head(ns, nsid, id);
3514 	if (ret)
3515 		goto out_free_id;
3516 	nvme_setup_streams_ns(ctrl, ns);
3517 	nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3518 
3519 	disk = alloc_disk_node(0, node);
3520 	if (!disk) {
3521 		ret = -ENOMEM;
3522 		goto out_unlink_ns;
3523 	}
3524 
3525 	disk->fops = &nvme_fops;
3526 	disk->private_data = ns;
3527 	disk->queue = ns->queue;
3528 	disk->flags = flags;
3529 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3530 	ns->disk = disk;
3531 
3532 	__nvme_revalidate_disk(disk, id);
3533 
3534 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3535 		ret = nvme_nvm_register(ns, disk_name, node);
3536 		if (ret) {
3537 			dev_warn(ctrl->device, "LightNVM init failure\n");
3538 			goto out_put_disk;
3539 		}
3540 	}
3541 
3542 	down_write(&ctrl->namespaces_rwsem);
3543 	list_add_tail(&ns->list, &ctrl->namespaces);
3544 	up_write(&ctrl->namespaces_rwsem);
3545 
3546 	nvme_get_ctrl(ctrl);
3547 
3548 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3549 
3550 	nvme_mpath_add_disk(ns, id);
3551 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3552 	kfree(id);
3553 
3554 	return 0;
3555  out_put_disk:
3556 	put_disk(ns->disk);
3557  out_unlink_ns:
3558 	mutex_lock(&ctrl->subsys->lock);
3559 	list_del_rcu(&ns->siblings);
3560 	mutex_unlock(&ctrl->subsys->lock);
3561 	nvme_put_ns_head(ns->head);
3562  out_free_id:
3563 	kfree(id);
3564  out_free_queue:
3565 	blk_cleanup_queue(ns->queue);
3566  out_free_ns:
3567 	kfree(ns);
3568 	if (ret > 0)
3569 		ret = blk_status_to_errno(nvme_error_status(ret));
3570 	return ret;
3571 }
3572 
nvme_ns_remove(struct nvme_ns * ns)3573 static void nvme_ns_remove(struct nvme_ns *ns)
3574 {
3575 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3576 		return;
3577 
3578 	nvme_fault_inject_fini(&ns->fault_inject);
3579 
3580 	mutex_lock(&ns->ctrl->subsys->lock);
3581 	list_del_rcu(&ns->siblings);
3582 	mutex_unlock(&ns->ctrl->subsys->lock);
3583 	synchronize_rcu(); /* guarantee not available in head->list */
3584 	nvme_mpath_clear_current_path(ns);
3585 	synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3586 
3587 	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
3588 		del_gendisk(ns->disk);
3589 		blk_cleanup_queue(ns->queue);
3590 		if (blk_get_integrity(ns->disk))
3591 			blk_integrity_unregister(ns->disk);
3592 	}
3593 
3594 	down_write(&ns->ctrl->namespaces_rwsem);
3595 	list_del_init(&ns->list);
3596 	up_write(&ns->ctrl->namespaces_rwsem);
3597 
3598 	nvme_mpath_check_last_path(ns);
3599 	nvme_put_ns(ns);
3600 }
3601 
nvme_validate_ns(struct nvme_ctrl * ctrl,unsigned nsid)3602 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3603 {
3604 	struct nvme_ns *ns;
3605 
3606 	ns = nvme_find_get_ns(ctrl, nsid);
3607 	if (ns) {
3608 		if (ns->disk && revalidate_disk(ns->disk))
3609 			nvme_ns_remove(ns);
3610 		nvme_put_ns(ns);
3611 	} else
3612 		nvme_alloc_ns(ctrl, nsid);
3613 }
3614 
nvme_remove_invalid_namespaces(struct nvme_ctrl * ctrl,unsigned nsid)3615 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
3616 					unsigned nsid)
3617 {
3618 	struct nvme_ns *ns, *next;
3619 	LIST_HEAD(rm_list);
3620 
3621 	down_write(&ctrl->namespaces_rwsem);
3622 	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
3623 		if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
3624 			list_move_tail(&ns->list, &rm_list);
3625 	}
3626 	up_write(&ctrl->namespaces_rwsem);
3627 
3628 	list_for_each_entry_safe(ns, next, &rm_list, list)
3629 		nvme_ns_remove(ns);
3630 
3631 }
3632 
nvme_scan_ns_list(struct nvme_ctrl * ctrl,unsigned nn)3633 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
3634 {
3635 	struct nvme_ns *ns;
3636 	__le32 *ns_list;
3637 	unsigned i, j, nsid, prev = 0;
3638 	unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024);
3639 	int ret = 0;
3640 
3641 	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
3642 	if (!ns_list)
3643 		return -ENOMEM;
3644 
3645 	for (i = 0; i < num_lists; i++) {
3646 		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
3647 		if (ret)
3648 			goto free;
3649 
3650 		for (j = 0; j < min(nn, 1024U); j++) {
3651 			nsid = le32_to_cpu(ns_list[j]);
3652 			if (!nsid)
3653 				goto out;
3654 
3655 			nvme_validate_ns(ctrl, nsid);
3656 
3657 			while (++prev < nsid) {
3658 				ns = nvme_find_get_ns(ctrl, prev);
3659 				if (ns) {
3660 					nvme_ns_remove(ns);
3661 					nvme_put_ns(ns);
3662 				}
3663 			}
3664 		}
3665 		nn -= j;
3666 	}
3667  out:
3668 	nvme_remove_invalid_namespaces(ctrl, prev);
3669  free:
3670 	kfree(ns_list);
3671 	return ret;
3672 }
3673 
nvme_scan_ns_sequential(struct nvme_ctrl * ctrl,unsigned nn)3674 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
3675 {
3676 	unsigned i;
3677 
3678 	for (i = 1; i <= nn; i++)
3679 		nvme_validate_ns(ctrl, i);
3680 
3681 	nvme_remove_invalid_namespaces(ctrl, nn);
3682 }
3683 
nvme_clear_changed_ns_log(struct nvme_ctrl * ctrl)3684 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
3685 {
3686 	size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
3687 	__le32 *log;
3688 	int error;
3689 
3690 	log = kzalloc(log_size, GFP_KERNEL);
3691 	if (!log)
3692 		return;
3693 
3694 	/*
3695 	 * We need to read the log to clear the AEN, but we don't want to rely
3696 	 * on it for the changed namespace information as userspace could have
3697 	 * raced with us in reading the log page, which could cause us to miss
3698 	 * updates.
3699 	 */
3700 	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log,
3701 			log_size, 0);
3702 	if (error)
3703 		dev_warn(ctrl->device,
3704 			"reading changed ns log failed: %d\n", error);
3705 
3706 	kfree(log);
3707 }
3708 
nvme_scan_work(struct work_struct * work)3709 static void nvme_scan_work(struct work_struct *work)
3710 {
3711 	struct nvme_ctrl *ctrl =
3712 		container_of(work, struct nvme_ctrl, scan_work);
3713 	struct nvme_id_ctrl *id;
3714 	unsigned nn;
3715 
3716 	/* No tagset on a live ctrl means IO queues could not created */
3717 	if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
3718 		return;
3719 
3720 	if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
3721 		dev_info(ctrl->device, "rescanning namespaces.\n");
3722 		nvme_clear_changed_ns_log(ctrl);
3723 	}
3724 
3725 	if (nvme_identify_ctrl(ctrl, &id))
3726 		return;
3727 
3728 	mutex_lock(&ctrl->scan_lock);
3729 	nn = le32_to_cpu(id->nn);
3730 	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
3731 	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
3732 		if (!nvme_scan_ns_list(ctrl, nn))
3733 			goto out_free_id;
3734 	}
3735 	nvme_scan_ns_sequential(ctrl, nn);
3736 out_free_id:
3737 	mutex_unlock(&ctrl->scan_lock);
3738 	kfree(id);
3739 	down_write(&ctrl->namespaces_rwsem);
3740 	list_sort(NULL, &ctrl->namespaces, ns_cmp);
3741 	up_write(&ctrl->namespaces_rwsem);
3742 }
3743 
3744 /*
3745  * This function iterates the namespace list unlocked to allow recovery from
3746  * controller failure. It is up to the caller to ensure the namespace list is
3747  * not modified by scan work while this function is executing.
3748  */
nvme_remove_namespaces(struct nvme_ctrl * ctrl)3749 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
3750 {
3751 	struct nvme_ns *ns, *next;
3752 	LIST_HEAD(ns_list);
3753 
3754 	/*
3755 	 * make sure to requeue I/O to all namespaces as these
3756 	 * might result from the scan itself and must complete
3757 	 * for the scan_work to make progress
3758 	 */
3759 	nvme_mpath_clear_ctrl_paths(ctrl);
3760 
3761 	/* prevent racing with ns scanning */
3762 	flush_work(&ctrl->scan_work);
3763 
3764 	/*
3765 	 * The dead states indicates the controller was not gracefully
3766 	 * disconnected. In that case, we won't be able to flush any data while
3767 	 * removing the namespaces' disks; fail all the queues now to avoid
3768 	 * potentially having to clean up the failed sync later.
3769 	 */
3770 	if (ctrl->state == NVME_CTRL_DEAD)
3771 		nvme_kill_queues(ctrl);
3772 
3773 	down_write(&ctrl->namespaces_rwsem);
3774 	list_splice_init(&ctrl->namespaces, &ns_list);
3775 	up_write(&ctrl->namespaces_rwsem);
3776 
3777 	list_for_each_entry_safe(ns, next, &ns_list, list)
3778 		nvme_ns_remove(ns);
3779 }
3780 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
3781 
nvme_class_uevent(struct device * dev,struct kobj_uevent_env * env)3782 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
3783 {
3784 	struct nvme_ctrl *ctrl =
3785 		container_of(dev, struct nvme_ctrl, ctrl_device);
3786 	struct nvmf_ctrl_options *opts = ctrl->opts;
3787 	int ret;
3788 
3789 	ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
3790 	if (ret)
3791 		return ret;
3792 
3793 	if (opts) {
3794 		ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
3795 		if (ret)
3796 			return ret;
3797 
3798 		ret = add_uevent_var(env, "NVME_TRSVCID=%s",
3799 				opts->trsvcid ?: "none");
3800 		if (ret)
3801 			return ret;
3802 
3803 		ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
3804 				opts->host_traddr ?: "none");
3805 	}
3806 	return ret;
3807 }
3808 
nvme_aen_uevent(struct nvme_ctrl * ctrl)3809 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
3810 {
3811 	char *envp[2] = { NULL, NULL };
3812 	u32 aen_result = ctrl->aen_result;
3813 
3814 	ctrl->aen_result = 0;
3815 	if (!aen_result)
3816 		return;
3817 
3818 	envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
3819 	if (!envp[0])
3820 		return;
3821 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
3822 	kfree(envp[0]);
3823 }
3824 
nvme_async_event_work(struct work_struct * work)3825 static void nvme_async_event_work(struct work_struct *work)
3826 {
3827 	struct nvme_ctrl *ctrl =
3828 		container_of(work, struct nvme_ctrl, async_event_work);
3829 
3830 	nvme_aen_uevent(ctrl);
3831 	ctrl->ops->submit_async_event(ctrl);
3832 }
3833 
nvme_ctrl_pp_status(struct nvme_ctrl * ctrl)3834 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
3835 {
3836 
3837 	u32 csts;
3838 
3839 	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
3840 		return false;
3841 
3842 	if (csts == ~0)
3843 		return false;
3844 
3845 	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
3846 }
3847 
nvme_get_fw_slot_info(struct nvme_ctrl * ctrl)3848 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
3849 {
3850 	struct nvme_fw_slot_info_log *log;
3851 
3852 	log = kmalloc(sizeof(*log), GFP_KERNEL);
3853 	if (!log)
3854 		return;
3855 
3856 	if (nvme_get_log(ctrl, NVME_NSID_ALL, 0, NVME_LOG_FW_SLOT, log,
3857 			sizeof(*log), 0))
3858 		dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
3859 	kfree(log);
3860 }
3861 
nvme_fw_act_work(struct work_struct * work)3862 static void nvme_fw_act_work(struct work_struct *work)
3863 {
3864 	struct nvme_ctrl *ctrl = container_of(work,
3865 				struct nvme_ctrl, fw_act_work);
3866 	unsigned long fw_act_timeout;
3867 
3868 	if (ctrl->mtfa)
3869 		fw_act_timeout = jiffies +
3870 				msecs_to_jiffies(ctrl->mtfa * 100);
3871 	else
3872 		fw_act_timeout = jiffies +
3873 				msecs_to_jiffies(admin_timeout * 1000);
3874 
3875 	nvme_stop_queues(ctrl);
3876 	while (nvme_ctrl_pp_status(ctrl)) {
3877 		if (time_after(jiffies, fw_act_timeout)) {
3878 			dev_warn(ctrl->device,
3879 				"Fw activation timeout, reset controller\n");
3880 			nvme_try_sched_reset(ctrl);
3881 			return;
3882 		}
3883 		msleep(100);
3884 	}
3885 
3886 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
3887 		return;
3888 
3889 	nvme_start_queues(ctrl);
3890 	/* read FW slot information to clear the AER */
3891 	nvme_get_fw_slot_info(ctrl);
3892 }
3893 
nvme_handle_aen_notice(struct nvme_ctrl * ctrl,u32 result)3894 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
3895 {
3896 	u32 aer_notice_type = (result & 0xff00) >> 8;
3897 
3898 	trace_nvme_async_event(ctrl, aer_notice_type);
3899 
3900 	switch (aer_notice_type) {
3901 	case NVME_AER_NOTICE_NS_CHANGED:
3902 		set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
3903 		nvme_queue_scan(ctrl);
3904 		break;
3905 	case NVME_AER_NOTICE_FW_ACT_STARTING:
3906 		/*
3907 		 * We are (ab)using the RESETTING state to prevent subsequent
3908 		 * recovery actions from interfering with the controller's
3909 		 * firmware activation.
3910 		 */
3911 		if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
3912 			queue_work(nvme_wq, &ctrl->fw_act_work);
3913 		break;
3914 #ifdef CONFIG_NVME_MULTIPATH
3915 	case NVME_AER_NOTICE_ANA:
3916 		if (!ctrl->ana_log_buf)
3917 			break;
3918 		queue_work(nvme_wq, &ctrl->ana_work);
3919 		break;
3920 #endif
3921 	case NVME_AER_NOTICE_DISC_CHANGED:
3922 		ctrl->aen_result = result;
3923 		break;
3924 	default:
3925 		dev_warn(ctrl->device, "async event result %08x\n", result);
3926 	}
3927 }
3928 
nvme_complete_async_event(struct nvme_ctrl * ctrl,__le16 status,volatile union nvme_result * res)3929 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
3930 		volatile union nvme_result *res)
3931 {
3932 	u32 result = le32_to_cpu(res->u32);
3933 	u32 aer_type = result & 0x07;
3934 
3935 	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
3936 		return;
3937 
3938 	switch (aer_type) {
3939 	case NVME_AER_NOTICE:
3940 		nvme_handle_aen_notice(ctrl, result);
3941 		break;
3942 	case NVME_AER_ERROR:
3943 	case NVME_AER_SMART:
3944 	case NVME_AER_CSS:
3945 	case NVME_AER_VS:
3946 		trace_nvme_async_event(ctrl, aer_type);
3947 		ctrl->aen_result = result;
3948 		break;
3949 	default:
3950 		break;
3951 	}
3952 	queue_work(nvme_wq, &ctrl->async_event_work);
3953 }
3954 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
3955 
nvme_stop_ctrl(struct nvme_ctrl * ctrl)3956 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
3957 {
3958 	nvme_mpath_stop(ctrl);
3959 	nvme_stop_keep_alive(ctrl);
3960 	flush_work(&ctrl->async_event_work);
3961 	cancel_work_sync(&ctrl->fw_act_work);
3962 }
3963 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
3964 
nvme_start_ctrl(struct nvme_ctrl * ctrl)3965 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
3966 {
3967 	if (ctrl->kato)
3968 		nvme_start_keep_alive(ctrl);
3969 
3970 	nvme_enable_aen(ctrl);
3971 
3972 	if (ctrl->queue_count > 1) {
3973 		nvme_queue_scan(ctrl);
3974 		nvme_start_queues(ctrl);
3975 	}
3976 }
3977 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
3978 
nvme_uninit_ctrl(struct nvme_ctrl * ctrl)3979 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
3980 {
3981 	nvme_fault_inject_fini(&ctrl->fault_inject);
3982 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
3983 	cdev_device_del(&ctrl->cdev, ctrl->device);
3984 }
3985 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
3986 
nvme_free_ctrl(struct device * dev)3987 static void nvme_free_ctrl(struct device *dev)
3988 {
3989 	struct nvme_ctrl *ctrl =
3990 		container_of(dev, struct nvme_ctrl, ctrl_device);
3991 	struct nvme_subsystem *subsys = ctrl->subsys;
3992 
3993 	if (subsys && ctrl->instance != subsys->instance)
3994 		ida_simple_remove(&nvme_instance_ida, ctrl->instance);
3995 
3996 	kfree(ctrl->effects);
3997 	nvme_mpath_uninit(ctrl);
3998 	__free_page(ctrl->discard_page);
3999 
4000 	if (subsys) {
4001 		mutex_lock(&nvme_subsystems_lock);
4002 		list_del(&ctrl->subsys_entry);
4003 		sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4004 		mutex_unlock(&nvme_subsystems_lock);
4005 	}
4006 
4007 	ctrl->ops->free_ctrl(ctrl);
4008 
4009 	if (subsys)
4010 		nvme_put_subsystem(subsys);
4011 }
4012 
4013 /*
4014  * Initialize a NVMe controller structures.  This needs to be called during
4015  * earliest initialization so that we have the initialized structured around
4016  * during probing.
4017  */
nvme_init_ctrl(struct nvme_ctrl * ctrl,struct device * dev,const struct nvme_ctrl_ops * ops,unsigned long quirks)4018 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4019 		const struct nvme_ctrl_ops *ops, unsigned long quirks)
4020 {
4021 	int ret;
4022 
4023 	ctrl->state = NVME_CTRL_NEW;
4024 	spin_lock_init(&ctrl->lock);
4025 	mutex_init(&ctrl->scan_lock);
4026 	INIT_LIST_HEAD(&ctrl->namespaces);
4027 	init_rwsem(&ctrl->namespaces_rwsem);
4028 	ctrl->dev = dev;
4029 	ctrl->ops = ops;
4030 	ctrl->quirks = quirks;
4031 	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4032 	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4033 	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4034 	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4035 	init_waitqueue_head(&ctrl->state_wq);
4036 
4037 	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4038 	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4039 	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4040 
4041 	BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4042 			PAGE_SIZE);
4043 	ctrl->discard_page = alloc_page(GFP_KERNEL);
4044 	if (!ctrl->discard_page) {
4045 		ret = -ENOMEM;
4046 		goto out;
4047 	}
4048 
4049 	ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4050 	if (ret < 0)
4051 		goto out;
4052 	ctrl->instance = ret;
4053 
4054 	device_initialize(&ctrl->ctrl_device);
4055 	ctrl->device = &ctrl->ctrl_device;
4056 	ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4057 	ctrl->device->class = nvme_class;
4058 	ctrl->device->parent = ctrl->dev;
4059 	ctrl->device->groups = nvme_dev_attr_groups;
4060 	ctrl->device->release = nvme_free_ctrl;
4061 	dev_set_drvdata(ctrl->device, ctrl);
4062 	ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4063 	if (ret)
4064 		goto out_release_instance;
4065 
4066 	cdev_init(&ctrl->cdev, &nvme_dev_fops);
4067 	ctrl->cdev.owner = ops->module;
4068 	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4069 	if (ret)
4070 		goto out_free_name;
4071 
4072 	/*
4073 	 * Initialize latency tolerance controls.  The sysfs files won't
4074 	 * be visible to userspace unless the device actually supports APST.
4075 	 */
4076 	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4077 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4078 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4079 
4080 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4081 
4082 	return 0;
4083 out_free_name:
4084 	kfree_const(ctrl->device->kobj.name);
4085 out_release_instance:
4086 	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4087 out:
4088 	if (ctrl->discard_page)
4089 		__free_page(ctrl->discard_page);
4090 	return ret;
4091 }
4092 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4093 
4094 /**
4095  * nvme_kill_queues(): Ends all namespace queues
4096  * @ctrl: the dead controller that needs to end
4097  *
4098  * Call this function when the driver determines it is unable to get the
4099  * controller in a state capable of servicing IO.
4100  */
nvme_kill_queues(struct nvme_ctrl * ctrl)4101 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4102 {
4103 	struct nvme_ns *ns;
4104 
4105 	down_read(&ctrl->namespaces_rwsem);
4106 
4107 	/* Forcibly unquiesce queues to avoid blocking dispatch */
4108 	if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4109 		blk_mq_unquiesce_queue(ctrl->admin_q);
4110 
4111 	list_for_each_entry(ns, &ctrl->namespaces, list)
4112 		nvme_set_queue_dying(ns);
4113 
4114 	up_read(&ctrl->namespaces_rwsem);
4115 }
4116 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4117 
nvme_unfreeze(struct nvme_ctrl * ctrl)4118 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4119 {
4120 	struct nvme_ns *ns;
4121 
4122 	down_read(&ctrl->namespaces_rwsem);
4123 	list_for_each_entry(ns, &ctrl->namespaces, list)
4124 		blk_mq_unfreeze_queue(ns->queue);
4125 	up_read(&ctrl->namespaces_rwsem);
4126 }
4127 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4128 
nvme_wait_freeze_timeout(struct nvme_ctrl * ctrl,long timeout)4129 void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4130 {
4131 	struct nvme_ns *ns;
4132 
4133 	down_read(&ctrl->namespaces_rwsem);
4134 	list_for_each_entry(ns, &ctrl->namespaces, list) {
4135 		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4136 		if (timeout <= 0)
4137 			break;
4138 	}
4139 	up_read(&ctrl->namespaces_rwsem);
4140 }
4141 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4142 
nvme_wait_freeze(struct nvme_ctrl * ctrl)4143 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4144 {
4145 	struct nvme_ns *ns;
4146 
4147 	down_read(&ctrl->namespaces_rwsem);
4148 	list_for_each_entry(ns, &ctrl->namespaces, list)
4149 		blk_mq_freeze_queue_wait(ns->queue);
4150 	up_read(&ctrl->namespaces_rwsem);
4151 }
4152 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4153 
nvme_start_freeze(struct nvme_ctrl * ctrl)4154 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4155 {
4156 	struct nvme_ns *ns;
4157 
4158 	down_read(&ctrl->namespaces_rwsem);
4159 	list_for_each_entry(ns, &ctrl->namespaces, list)
4160 		blk_freeze_queue_start(ns->queue);
4161 	up_read(&ctrl->namespaces_rwsem);
4162 }
4163 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4164 
nvme_stop_queues(struct nvme_ctrl * ctrl)4165 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4166 {
4167 	struct nvme_ns *ns;
4168 
4169 	down_read(&ctrl->namespaces_rwsem);
4170 	list_for_each_entry(ns, &ctrl->namespaces, list)
4171 		blk_mq_quiesce_queue(ns->queue);
4172 	up_read(&ctrl->namespaces_rwsem);
4173 }
4174 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4175 
nvme_start_queues(struct nvme_ctrl * ctrl)4176 void nvme_start_queues(struct nvme_ctrl *ctrl)
4177 {
4178 	struct nvme_ns *ns;
4179 
4180 	down_read(&ctrl->namespaces_rwsem);
4181 	list_for_each_entry(ns, &ctrl->namespaces, list)
4182 		blk_mq_unquiesce_queue(ns->queue);
4183 	up_read(&ctrl->namespaces_rwsem);
4184 }
4185 EXPORT_SYMBOL_GPL(nvme_start_queues);
4186 
4187 
nvme_sync_queues(struct nvme_ctrl * ctrl)4188 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4189 {
4190 	struct nvme_ns *ns;
4191 
4192 	down_read(&ctrl->namespaces_rwsem);
4193 	list_for_each_entry(ns, &ctrl->namespaces, list)
4194 		blk_sync_queue(ns->queue);
4195 	up_read(&ctrl->namespaces_rwsem);
4196 
4197 	if (ctrl->admin_q)
4198 		blk_sync_queue(ctrl->admin_q);
4199 }
4200 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4201 
4202 /*
4203  * Check we didn't inadvertently grow the command structure sizes:
4204  */
_nvme_check_size(void)4205 static inline void _nvme_check_size(void)
4206 {
4207 	BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4208 	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4209 	BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4210 	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4211 	BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4212 	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4213 	BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4214 	BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4215 	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4216 	BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4217 	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4218 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4219 	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4220 	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4221 	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4222 	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4223 	BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4224 }
4225 
4226 
nvme_core_init(void)4227 static int __init nvme_core_init(void)
4228 {
4229 	int result = -ENOMEM;
4230 
4231 	_nvme_check_size();
4232 
4233 	nvme_wq = alloc_workqueue("nvme-wq",
4234 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4235 	if (!nvme_wq)
4236 		goto out;
4237 
4238 	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4239 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4240 	if (!nvme_reset_wq)
4241 		goto destroy_wq;
4242 
4243 	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4244 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4245 	if (!nvme_delete_wq)
4246 		goto destroy_reset_wq;
4247 
4248 	result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4249 	if (result < 0)
4250 		goto destroy_delete_wq;
4251 
4252 	nvme_class = class_create(THIS_MODULE, "nvme");
4253 	if (IS_ERR(nvme_class)) {
4254 		result = PTR_ERR(nvme_class);
4255 		goto unregister_chrdev;
4256 	}
4257 	nvme_class->dev_uevent = nvme_class_uevent;
4258 
4259 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4260 	if (IS_ERR(nvme_subsys_class)) {
4261 		result = PTR_ERR(nvme_subsys_class);
4262 		goto destroy_class;
4263 	}
4264 	return 0;
4265 
4266 destroy_class:
4267 	class_destroy(nvme_class);
4268 unregister_chrdev:
4269 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4270 destroy_delete_wq:
4271 	destroy_workqueue(nvme_delete_wq);
4272 destroy_reset_wq:
4273 	destroy_workqueue(nvme_reset_wq);
4274 destroy_wq:
4275 	destroy_workqueue(nvme_wq);
4276 out:
4277 	return result;
4278 }
4279 
nvme_core_exit(void)4280 static void __exit nvme_core_exit(void)
4281 {
4282 	class_destroy(nvme_subsys_class);
4283 	class_destroy(nvme_class);
4284 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4285 	destroy_workqueue(nvme_delete_wq);
4286 	destroy_workqueue(nvme_reset_wq);
4287 	destroy_workqueue(nvme_wq);
4288 }
4289 
4290 MODULE_LICENSE("GPL");
4291 MODULE_VERSION("1.0");
4292 module_init(nvme_core_init);
4293 module_exit(nvme_core_exit);
4294