1 /** @file
2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
3 NVM Express specification.
4
5 Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
6 This program and the accompanying materials
7 are licensed and made available under the terms and conditions of the BSD License
8 which accompanies this distribution. The full text of the license may be found at
9 http://opensource.org/licenses/bsd-license.php.
10
11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13
14 **/
15
16 #include "NvmExpress.h"
17
18 /**
19 Read Nvm Express controller capability register.
20
21 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
22 @param Cap The buffer used to store capability register content.
23
24 @return EFI_SUCCESS Successfully read the controller capability register content.
25 @return EFI_DEVICE_ERROR Fail to read the controller capability register.
26
27 **/
28 EFI_STATUS
ReadNvmeControllerCapabilities(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_CAP * Cap)29 ReadNvmeControllerCapabilities (
30 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
31 IN NVME_CAP *Cap
32 )
33 {
34 EFI_PCI_IO_PROTOCOL *PciIo;
35 EFI_STATUS Status;
36 UINT64 Data;
37
38 PciIo = Private->PciIo;
39 Status = PciIo->Mem.Read (
40 PciIo,
41 EfiPciIoWidthUint32,
42 NVME_BAR,
43 NVME_CAP_OFFSET,
44 2,
45 &Data
46 );
47
48 if (EFI_ERROR(Status)) {
49 return Status;
50 }
51
52 WriteUnaligned64 ((UINT64*)Cap, Data);
53 return EFI_SUCCESS;
54 }
55
56 /**
57 Read Nvm Express controller configuration register.
58
59 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
60 @param Cc The buffer used to store configuration register content.
61
62 @return EFI_SUCCESS Successfully read the controller configuration register content.
63 @return EFI_DEVICE_ERROR Fail to read the controller configuration register.
64
65 **/
66 EFI_STATUS
ReadNvmeControllerConfiguration(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_CC * Cc)67 ReadNvmeControllerConfiguration (
68 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
69 IN NVME_CC *Cc
70 )
71 {
72 EFI_PCI_IO_PROTOCOL *PciIo;
73 EFI_STATUS Status;
74 UINT32 Data;
75
76 PciIo = Private->PciIo;
77 Status = PciIo->Mem.Read (
78 PciIo,
79 EfiPciIoWidthUint32,
80 NVME_BAR,
81 NVME_CC_OFFSET,
82 1,
83 &Data
84 );
85
86 if (EFI_ERROR(Status)) {
87 return Status;
88 }
89
90 WriteUnaligned32 ((UINT32*)Cc, Data);
91 return EFI_SUCCESS;
92 }
93
94 /**
95 Write Nvm Express controller configuration register.
96
97 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
98 @param Cc The buffer used to store the content to be written into configuration register.
99
100 @return EFI_SUCCESS Successfully write data into the controller configuration register.
101 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register.
102
103 **/
104 EFI_STATUS
WriteNvmeControllerConfiguration(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_CC * Cc)105 WriteNvmeControllerConfiguration (
106 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
107 IN NVME_CC *Cc
108 )
109 {
110 EFI_PCI_IO_PROTOCOL *PciIo;
111 EFI_STATUS Status;
112 UINT32 Data;
113
114 PciIo = Private->PciIo;
115 Data = ReadUnaligned32 ((UINT32*)Cc);
116 Status = PciIo->Mem.Write (
117 PciIo,
118 EfiPciIoWidthUint32,
119 NVME_BAR,
120 NVME_CC_OFFSET,
121 1,
122 &Data
123 );
124
125 if (EFI_ERROR(Status)) {
126 return Status;
127 }
128
129 DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En));
130 DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css));
131 DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps));
132 DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams));
133 DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn));
134 DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes));
135 DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes));
136
137 return EFI_SUCCESS;
138 }
139
140 /**
141 Read Nvm Express controller status register.
142
143 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
144 @param Csts The buffer used to store status register content.
145
146 @return EFI_SUCCESS Successfully read the controller status register content.
147 @return EFI_DEVICE_ERROR Fail to read the controller status register.
148
149 **/
150 EFI_STATUS
ReadNvmeControllerStatus(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_CSTS * Csts)151 ReadNvmeControllerStatus (
152 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
153 IN NVME_CSTS *Csts
154 )
155 {
156 EFI_PCI_IO_PROTOCOL *PciIo;
157 EFI_STATUS Status;
158 UINT32 Data;
159
160 PciIo = Private->PciIo;
161 Status = PciIo->Mem.Read (
162 PciIo,
163 EfiPciIoWidthUint32,
164 NVME_BAR,
165 NVME_CSTS_OFFSET,
166 1,
167 &Data
168 );
169
170 if (EFI_ERROR(Status)) {
171 return Status;
172 }
173
174 WriteUnaligned32 ((UINT32*)Csts, Data);
175 return EFI_SUCCESS;
176 }
177
178 /**
179 Read Nvm Express admin queue attributes register.
180
181 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
182 @param Aqa The buffer used to store admin queue attributes register content.
183
184 @return EFI_SUCCESS Successfully read the admin queue attributes register content.
185 @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register.
186
187 **/
188 EFI_STATUS
ReadNvmeAdminQueueAttributes(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_AQA * Aqa)189 ReadNvmeAdminQueueAttributes (
190 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
191 IN NVME_AQA *Aqa
192 )
193 {
194 EFI_PCI_IO_PROTOCOL *PciIo;
195 EFI_STATUS Status;
196 UINT32 Data;
197
198 PciIo = Private->PciIo;
199 Status = PciIo->Mem.Read (
200 PciIo,
201 EfiPciIoWidthUint32,
202 NVME_BAR,
203 NVME_AQA_OFFSET,
204 1,
205 &Data
206 );
207
208 if (EFI_ERROR(Status)) {
209 return Status;
210 }
211
212 WriteUnaligned32 ((UINT32*)Aqa, Data);
213 return EFI_SUCCESS;
214 }
215
216 /**
217 Write Nvm Express admin queue attributes register.
218
219 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
220 @param Aqa The buffer used to store the content to be written into admin queue attributes register.
221
222 @return EFI_SUCCESS Successfully write data into the admin queue attributes register.
223 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register.
224
225 **/
226 EFI_STATUS
WriteNvmeAdminQueueAttributes(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_AQA * Aqa)227 WriteNvmeAdminQueueAttributes (
228 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
229 IN NVME_AQA *Aqa
230 )
231 {
232 EFI_PCI_IO_PROTOCOL *PciIo;
233 EFI_STATUS Status;
234 UINT32 Data;
235
236 PciIo = Private->PciIo;
237 Data = ReadUnaligned32 ((UINT32*)Aqa);
238 Status = PciIo->Mem.Write (
239 PciIo,
240 EfiPciIoWidthUint32,
241 NVME_BAR,
242 NVME_AQA_OFFSET,
243 1,
244 &Data
245 );
246
247 if (EFI_ERROR(Status)) {
248 return Status;
249 }
250
251 DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs));
252 DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs));
253
254 return EFI_SUCCESS;
255 }
256
257 /**
258 Read Nvm Express admin submission queue base address register.
259
260 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
261 @param Asq The buffer used to store admin submission queue base address register content.
262
263 @return EFI_SUCCESS Successfully read the admin submission queue base address register content.
264 @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register.
265
266 **/
267 EFI_STATUS
ReadNvmeAdminSubmissionQueueBaseAddress(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_ASQ * Asq)268 ReadNvmeAdminSubmissionQueueBaseAddress (
269 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
270 IN NVME_ASQ *Asq
271 )
272 {
273 EFI_PCI_IO_PROTOCOL *PciIo;
274 EFI_STATUS Status;
275 UINT64 Data;
276
277 PciIo = Private->PciIo;
278 Status = PciIo->Mem.Read (
279 PciIo,
280 EfiPciIoWidthUint32,
281 NVME_BAR,
282 NVME_ASQ_OFFSET,
283 2,
284 &Data
285 );
286
287 if (EFI_ERROR(Status)) {
288 return Status;
289 }
290
291 WriteUnaligned64 ((UINT64*)Asq, Data);
292 return EFI_SUCCESS;
293 }
294
295 /**
296 Write Nvm Express admin submission queue base address register.
297
298 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
299 @param Asq The buffer used to store the content to be written into admin submission queue base address register.
300
301 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register.
302 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register.
303
304 **/
305 EFI_STATUS
WriteNvmeAdminSubmissionQueueBaseAddress(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_ASQ * Asq)306 WriteNvmeAdminSubmissionQueueBaseAddress (
307 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
308 IN NVME_ASQ *Asq
309 )
310 {
311 EFI_PCI_IO_PROTOCOL *PciIo;
312 EFI_STATUS Status;
313 UINT64 Data;
314
315 PciIo = Private->PciIo;
316 Data = ReadUnaligned64 ((UINT64*)Asq);
317
318 Status = PciIo->Mem.Write (
319 PciIo,
320 EfiPciIoWidthUint32,
321 NVME_BAR,
322 NVME_ASQ_OFFSET,
323 2,
324 &Data
325 );
326
327 if (EFI_ERROR(Status)) {
328 return Status;
329 }
330
331 DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq));
332
333 return EFI_SUCCESS;
334 }
335
336 /**
337 Read Nvm Express admin completion queue base address register.
338
339 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
340 @param Acq The buffer used to store admin completion queue base address register content.
341
342 @return EFI_SUCCESS Successfully read the admin completion queue base address register content.
343 @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register.
344
345 **/
346 EFI_STATUS
ReadNvmeAdminCompletionQueueBaseAddress(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_ACQ * Acq)347 ReadNvmeAdminCompletionQueueBaseAddress (
348 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
349 IN NVME_ACQ *Acq
350 )
351 {
352 EFI_PCI_IO_PROTOCOL *PciIo;
353 EFI_STATUS Status;
354 UINT64 Data;
355
356 PciIo = Private->PciIo;
357
358 Status = PciIo->Mem.Read (
359 PciIo,
360 EfiPciIoWidthUint32,
361 NVME_BAR,
362 NVME_ACQ_OFFSET,
363 2,
364 &Data
365 );
366
367 if (EFI_ERROR(Status)) {
368 return Status;
369 }
370
371 WriteUnaligned64 ((UINT64*)Acq, Data);
372 return EFI_SUCCESS;
373 }
374
375 /**
376 Write Nvm Express admin completion queue base address register.
377
378 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
379 @param Acq The buffer used to store the content to be written into admin completion queue base address register.
380
381 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register.
382 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register.
383
384 **/
385 EFI_STATUS
WriteNvmeAdminCompletionQueueBaseAddress(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN NVME_ACQ * Acq)386 WriteNvmeAdminCompletionQueueBaseAddress (
387 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
388 IN NVME_ACQ *Acq
389 )
390 {
391 EFI_PCI_IO_PROTOCOL *PciIo;
392 EFI_STATUS Status;
393 UINT64 Data;
394
395 PciIo = Private->PciIo;
396 Data = ReadUnaligned64 ((UINT64*)Acq);
397
398 Status = PciIo->Mem.Write (
399 PciIo,
400 EfiPciIoWidthUint32,
401 NVME_BAR,
402 NVME_ACQ_OFFSET,
403 2,
404 &Data
405 );
406
407 if (EFI_ERROR(Status)) {
408 return Status;
409 }
410
411 DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq));
412
413 return EFI_SUCCESS;
414 }
415
416 /**
417 Disable the Nvm Express controller.
418
419 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
420
421 @return EFI_SUCCESS Successfully disable the controller.
422 @return EFI_DEVICE_ERROR Fail to disable the controller.
423
424 **/
425 EFI_STATUS
NvmeDisableController(IN NVME_CONTROLLER_PRIVATE_DATA * Private)426 NvmeDisableController (
427 IN NVME_CONTROLLER_PRIVATE_DATA *Private
428 )
429 {
430 NVME_CC Cc;
431 NVME_CSTS Csts;
432 EFI_STATUS Status;
433 UINT32 Index;
434 UINT8 Timeout;
435
436 //
437 // Read Controller Configuration Register.
438 //
439 Status = ReadNvmeControllerConfiguration (Private, &Cc);
440 if (EFI_ERROR(Status)) {
441 return Status;
442 }
443
444 Cc.En = 0;
445
446 //
447 // Disable the controller.
448 //
449 Status = WriteNvmeControllerConfiguration (Private, &Cc);
450
451 if (EFI_ERROR(Status)) {
452 return Status;
453 }
454
455 //
456 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after
457 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
458 //
459 if (Private->Cap.To == 0) {
460 Timeout = 1;
461 } else {
462 Timeout = Private->Cap.To;
463 }
464
465 for(Index = (Timeout * 500); Index != 0; --Index) {
466 gBS->Stall(1000);
467
468 //
469 // Check if the controller is initialized
470 //
471 Status = ReadNvmeControllerStatus (Private, &Csts);
472
473 if (EFI_ERROR(Status)) {
474 return Status;
475 }
476
477 if (Csts.Rdy == 0) {
478 break;
479 }
480 }
481
482 if (Index == 0) {
483 Status = EFI_DEVICE_ERROR;
484 }
485
486 DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status));
487 return Status;
488 }
489
490 /**
491 Enable the Nvm Express controller.
492
493 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
494
495 @return EFI_SUCCESS Successfully enable the controller.
496 @return EFI_DEVICE_ERROR Fail to enable the controller.
497 @return EFI_TIMEOUT Fail to enable the controller in given time slot.
498
499 **/
500 EFI_STATUS
NvmeEnableController(IN NVME_CONTROLLER_PRIVATE_DATA * Private)501 NvmeEnableController (
502 IN NVME_CONTROLLER_PRIVATE_DATA *Private
503 )
504 {
505 NVME_CC Cc;
506 NVME_CSTS Csts;
507 EFI_STATUS Status;
508 UINT32 Index;
509 UINT8 Timeout;
510
511 //
512 // Enable the controller.
513 // CC.AMS, CC.MPS and CC.CSS are all set to 0.
514 //
515 ZeroMem (&Cc, sizeof (NVME_CC));
516 Cc.En = 1;
517 Cc.Iosqes = 6;
518 Cc.Iocqes = 4;
519
520 Status = WriteNvmeControllerConfiguration (Private, &Cc);
521 if (EFI_ERROR(Status)) {
522 return Status;
523 }
524
525 //
526 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after
527 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To.
528 //
529 if (Private->Cap.To == 0) {
530 Timeout = 1;
531 } else {
532 Timeout = Private->Cap.To;
533 }
534
535 for(Index = (Timeout * 500); Index != 0; --Index) {
536 gBS->Stall(1000);
537
538 //
539 // Check if the controller is initialized
540 //
541 Status = ReadNvmeControllerStatus (Private, &Csts);
542
543 if (EFI_ERROR(Status)) {
544 return Status;
545 }
546
547 if (Csts.Rdy) {
548 break;
549 }
550 }
551
552 if (Index == 0) {
553 Status = EFI_TIMEOUT;
554 }
555
556 DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status));
557 return Status;
558 }
559
560 /**
561 Get identify controller data.
562
563 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
564 @param Buffer The buffer used to store the identify controller data.
565
566 @return EFI_SUCCESS Successfully get the identify controller data.
567 @return EFI_DEVICE_ERROR Fail to get the identify controller data.
568
569 **/
570 EFI_STATUS
NvmeIdentifyController(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN VOID * Buffer)571 NvmeIdentifyController (
572 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
573 IN VOID *Buffer
574 )
575 {
576 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
577 EFI_NVM_EXPRESS_COMMAND Command;
578 EFI_NVM_EXPRESS_COMPLETION Completion;
579 EFI_STATUS Status;
580
581 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
582 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
583 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
584
585 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
586 //
587 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h.
588 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure.
589 //
590 Command.Nsid = 0;
591
592 CommandPacket.NvmeCmd = &Command;
593 CommandPacket.NvmeCompletion = &Completion;
594 CommandPacket.TransferBuffer = Buffer;
595 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA);
596 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
597 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
598 //
599 // Set bit 0 (Cns bit) to 1 to identify a controller
600 //
601 Command.Cdw10 = 1;
602 Command.Flags = CDW10_VALID;
603
604 Status = Private->Passthru.PassThru (
605 &Private->Passthru,
606 NVME_CONTROLLER_ID,
607 &CommandPacket,
608 NULL
609 );
610
611 return Status;
612 }
613
614 /**
615 Get specified identify namespace data.
616
617 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
618 @param NamespaceId The specified namespace identifier.
619 @param Buffer The buffer used to store the identify namespace data.
620
621 @return EFI_SUCCESS Successfully get the identify namespace data.
622 @return EFI_DEVICE_ERROR Fail to get the identify namespace data.
623
624 **/
625 EFI_STATUS
NvmeIdentifyNamespace(IN NVME_CONTROLLER_PRIVATE_DATA * Private,IN UINT32 NamespaceId,IN VOID * Buffer)626 NvmeIdentifyNamespace (
627 IN NVME_CONTROLLER_PRIVATE_DATA *Private,
628 IN UINT32 NamespaceId,
629 IN VOID *Buffer
630 )
631 {
632 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
633 EFI_NVM_EXPRESS_COMMAND Command;
634 EFI_NVM_EXPRESS_COMPLETION Completion;
635 EFI_STATUS Status;
636
637 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
638 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
639 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
640
641 CommandPacket.NvmeCmd = &Command;
642 CommandPacket.NvmeCompletion = &Completion;
643
644 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD;
645 Command.Nsid = NamespaceId;
646 CommandPacket.TransferBuffer = Buffer;
647 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA);
648 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
649 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
650 //
651 // Set bit 0 (Cns bit) to 1 to identify a namespace
652 //
653 CommandPacket.NvmeCmd->Cdw10 = 0;
654 CommandPacket.NvmeCmd->Flags = CDW10_VALID;
655
656 Status = Private->Passthru.PassThru (
657 &Private->Passthru,
658 NamespaceId,
659 &CommandPacket,
660 NULL
661 );
662
663 return Status;
664 }
665
666 /**
667 Create io completion queue.
668
669 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
670
671 @return EFI_SUCCESS Successfully create io completion queue.
672 @return EFI_DEVICE_ERROR Fail to create io completion queue.
673
674 **/
675 EFI_STATUS
NvmeCreateIoCompletionQueue(IN NVME_CONTROLLER_PRIVATE_DATA * Private)676 NvmeCreateIoCompletionQueue (
677 IN NVME_CONTROLLER_PRIVATE_DATA *Private
678 )
679 {
680 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
681 EFI_NVM_EXPRESS_COMMAND Command;
682 EFI_NVM_EXPRESS_COMPLETION Completion;
683 EFI_STATUS Status;
684 NVME_ADMIN_CRIOCQ CrIoCq;
685 UINT32 Index;
686 UINT16 QueueSize;
687
688 Status = EFI_SUCCESS;
689
690 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
691 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
692 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
693 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
694 ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
695
696 CommandPacket.NvmeCmd = &Command;
697 CommandPacket.NvmeCompletion = &Completion;
698
699 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
700 CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];
701 CommandPacket.TransferLength = EFI_PAGE_SIZE;
702 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
703 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
704
705 if (Index == 1) {
706 QueueSize = NVME_CCQ_SIZE;
707 } else {
708 if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
709 QueueSize = NVME_ASYNC_CCQ_SIZE;
710 } else {
711 QueueSize = Private->Cap.Mqes;
712 }
713 }
714
715 CrIoCq.Qid = Index;
716 CrIoCq.Qsize = QueueSize;
717 CrIoCq.Pc = 1;
718 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
719 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
720
721 Status = Private->Passthru.PassThru (
722 &Private->Passthru,
723 0,
724 &CommandPacket,
725 NULL
726 );
727 if (EFI_ERROR (Status)) {
728 break;
729 }
730 }
731
732 return Status;
733 }
734
735 /**
736 Create io submission queue.
737
738 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
739
740 @return EFI_SUCCESS Successfully create io submission queue.
741 @return EFI_DEVICE_ERROR Fail to create io submission queue.
742
743 **/
744 EFI_STATUS
NvmeCreateIoSubmissionQueue(IN NVME_CONTROLLER_PRIVATE_DATA * Private)745 NvmeCreateIoSubmissionQueue (
746 IN NVME_CONTROLLER_PRIVATE_DATA *Private
747 )
748 {
749 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket;
750 EFI_NVM_EXPRESS_COMMAND Command;
751 EFI_NVM_EXPRESS_COMPLETION Completion;
752 EFI_STATUS Status;
753 NVME_ADMIN_CRIOSQ CrIoSq;
754 UINT32 Index;
755 UINT16 QueueSize;
756
757 Status = EFI_SUCCESS;
758
759 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
760 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
761 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
762 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
763 ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
764
765 CommandPacket.NvmeCmd = &Command;
766 CommandPacket.NvmeCompletion = &Completion;
767
768 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
769 CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];
770 CommandPacket.TransferLength = EFI_PAGE_SIZE;
771 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
772 CommandPacket.QueueType = NVME_ADMIN_QUEUE;
773
774 if (Index == 1) {
775 QueueSize = NVME_CSQ_SIZE;
776 } else {
777 if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {
778 QueueSize = NVME_ASYNC_CSQ_SIZE;
779 } else {
780 QueueSize = Private->Cap.Mqes;
781 }
782 }
783
784 CrIoSq.Qid = Index;
785 CrIoSq.Qsize = QueueSize;
786 CrIoSq.Pc = 1;
787 CrIoSq.Cqid = Index;
788 CrIoSq.Qprio = 0;
789 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
790 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
791
792 Status = Private->Passthru.PassThru (
793 &Private->Passthru,
794 0,
795 &CommandPacket,
796 NULL
797 );
798 if (EFI_ERROR (Status)) {
799 break;
800 }
801 }
802
803 return Status;
804 }
805
806 /**
807 Initialize the Nvm Express controller.
808
809 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure.
810
811 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully.
812 @retval Others A device error occurred while initializing the controller.
813
814 **/
815 EFI_STATUS
NvmeControllerInit(IN NVME_CONTROLLER_PRIVATE_DATA * Private)816 NvmeControllerInit (
817 IN NVME_CONTROLLER_PRIVATE_DATA *Private
818 )
819 {
820 EFI_STATUS Status;
821 EFI_PCI_IO_PROTOCOL *PciIo;
822 UINT64 Supports;
823 NVME_AQA Aqa;
824 NVME_ASQ Asq;
825 NVME_ACQ Acq;
826 UINT8 Sn[21];
827 UINT8 Mn[41];
828 //
829 // Save original PCI attributes and enable this controller.
830 //
831 PciIo = Private->PciIo;
832 Status = PciIo->Attributes (
833 PciIo,
834 EfiPciIoAttributeOperationGet,
835 0,
836 &Private->PciAttributes
837 );
838
839 if (EFI_ERROR (Status)) {
840 return Status;
841 }
842
843 Status = PciIo->Attributes (
844 PciIo,
845 EfiPciIoAttributeOperationSupported,
846 0,
847 &Supports
848 );
849
850 if (!EFI_ERROR (Status)) {
851 Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE;
852 Status = PciIo->Attributes (
853 PciIo,
854 EfiPciIoAttributeOperationEnable,
855 Supports,
856 NULL
857 );
858 }
859
860 if (EFI_ERROR (Status)) {
861 DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n"));
862 return Status;
863 }
864
865 //
866 // Enable 64-bit DMA support in the PCI layer.
867 //
868 Status = PciIo->Attributes (
869 PciIo,
870 EfiPciIoAttributeOperationEnable,
871 EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE,
872 NULL
873 );
874 if (EFI_ERROR (Status)) {
875 DEBUG ((EFI_D_WARN, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status));
876 }
877
878 //
879 // Read the Controller Capabilities register and verify that the NVM command set is supported
880 //
881 Status = ReadNvmeControllerCapabilities (Private, &Private->Cap);
882 if (EFI_ERROR (Status)) {
883 return Status;
884 }
885
886 if (Private->Cap.Css != 0x01) {
887 DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n"));
888 return EFI_UNSUPPORTED;
889 }
890
891 //
892 // Currently the driver only supports 4k page size.
893 //
894 ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
895
896 Private->Cid[0] = 0;
897 Private->Cid[1] = 0;
898 Private->Cid[2] = 0;
899 Private->Pt[0] = 0;
900 Private->Pt[1] = 0;
901 Private->Pt[2] = 0;
902 Private->SqTdbl[0].Sqt = 0;
903 Private->SqTdbl[1].Sqt = 0;
904 Private->SqTdbl[2].Sqt = 0;
905 Private->CqHdbl[0].Cqh = 0;
906 Private->CqHdbl[1].Cqh = 0;
907 Private->CqHdbl[2].Cqh = 0;
908 Private->AsyncSqHead = 0;
909
910 Status = NvmeDisableController (Private);
911
912 if (EFI_ERROR(Status)) {
913 return Status;
914 }
915
916 //
917 // set number of entries admin submission & completion queues.
918 //
919 Aqa.Asqs = NVME_ASQ_SIZE;
920 Aqa.Rsvd1 = 0;
921 Aqa.Acqs = NVME_ACQ_SIZE;
922 Aqa.Rsvd2 = 0;
923
924 //
925 // Address of admin submission queue.
926 //
927 Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF;
928
929 //
930 // Address of admin completion queue.
931 //
932 Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;
933
934 //
935 // Address of I/O submission & completion queue.
936 //
937 ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
938 Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
939 Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
940 Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
941 Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
942 Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
943 Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
944 Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
945 Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
946 Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
947 Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
948 Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
949 Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
950
951 DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
952 DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
953 DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
954 DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
955 DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
956 DEBUG ((EFI_D_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
957 DEBUG ((EFI_D_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
958 DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));
959 DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));
960
961 //
962 // Program admin queue attributes.
963 //
964 Status = WriteNvmeAdminQueueAttributes (Private, &Aqa);
965
966 if (EFI_ERROR(Status)) {
967 return Status;
968 }
969
970 //
971 // Program admin submission queue address.
972 //
973 Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq);
974
975 if (EFI_ERROR(Status)) {
976 return Status;
977 }
978
979 //
980 // Program admin completion queue address.
981 //
982 Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq);
983
984 if (EFI_ERROR(Status)) {
985 return Status;
986 }
987
988 Status = NvmeEnableController (Private);
989 if (EFI_ERROR(Status)) {
990 return Status;
991 }
992
993 //
994 // Allocate buffer for Identify Controller data
995 //
996 if (Private->ControllerData == NULL) {
997 Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA));
998
999 if (Private->ControllerData == NULL) {
1000 return EFI_OUT_OF_RESOURCES;
1001 }
1002 }
1003
1004 //
1005 // Get current Identify Controller Data
1006 //
1007 Status = NvmeIdentifyController (Private, Private->ControllerData);
1008
1009 if (EFI_ERROR(Status)) {
1010 FreePool(Private->ControllerData);
1011 Private->ControllerData = NULL;
1012 return EFI_NOT_FOUND;
1013 }
1014
1015 //
1016 // Dump NvmExpress Identify Controller Data
1017 //
1018 CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn));
1019 Sn[20] = 0;
1020 CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn));
1021 Mn[40] = 0;
1022 DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n"));
1023 DEBUG ((EFI_D_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid));
1024 DEBUG ((EFI_D_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid));
1025 DEBUG ((EFI_D_INFO, " SN : %a\n", Sn));
1026 DEBUG ((EFI_D_INFO, " MN : %a\n", Mn));
1027 DEBUG ((EFI_D_INFO, " FR : 0x%x\n", *((UINT64*)Private->ControllerData->Fr)));
1028 DEBUG ((EFI_D_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab));
1029 DEBUG ((EFI_D_INFO, " IEEE : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui));
1030 DEBUG ((EFI_D_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl));
1031 DEBUG ((EFI_D_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes));
1032 DEBUG ((EFI_D_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes));
1033 DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn));
1034
1035 //
1036 // Create two I/O completion queues.
1037 // One for blocking I/O, one for non-blocking I/O.
1038 //
1039 Status = NvmeCreateIoCompletionQueue (Private);
1040 if (EFI_ERROR(Status)) {
1041 return Status;
1042 }
1043
1044 //
1045 // Create two I/O Submission queues.
1046 // One for blocking I/O, one for non-blocking I/O.
1047 //
1048 Status = NvmeCreateIoSubmissionQueue (Private);
1049 if (EFI_ERROR(Status)) {
1050 return Status;
1051 }
1052
1053 return Status;
1054 }
1055
1056