1 /*
2 * Copyright 2023 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "priv.h"
23
24 #include <core/pci.h>
25 #include <subdev/timer.h>
26 #include <subdev/vfn.h>
27 #include <engine/fifo/chan.h>
28 #include <engine/sec2.h>
29
30 #include <nvfw/fw.h>
31
32 #include <nvrm/nvtypes.h>
33 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h>
34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h>
35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h>
36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h>
37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h>
38 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
39 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
40 #include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h>
41 #include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h>
42 #include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h>
43 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h>
44 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h>
45 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h>
46 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h>
47 #include <nvrm/535.113.01/nvidia/generated/g_allclasses.h>
48 #include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h>
49 #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>
50 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h>
51 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h>
52 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
53 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h>
54 #include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h>
55
56 #include <linux/acpi.h>
57 #include <linux/ctype.h>
58 #include <linux/parser.h>
59
60 #define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE
61 #define GSP_MSG_MAX_SIZE (GSP_MSG_MIN_SIZE * 16)
62
63 struct r535_gsp_msg {
64 u8 auth_tag_buffer[16];
65 u8 aad_buffer[16];
66 u32 checksum;
67 u32 sequence;
68 u32 elem_count;
69 u32 pad;
70 u8 data[];
71 };
72
73 #define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)
74
75 static int
r535_rpc_status_to_errno(uint32_t rpc_status)76 r535_rpc_status_to_errno(uint32_t rpc_status)
77 {
78 switch (rpc_status) {
79 case 0x55: /* NV_ERR_NOT_READY */
80 case 0x66: /* NV_ERR_TIMEOUT_RETRY */
81 return -EBUSY;
82 case 0x51: /* NV_ERR_NO_MEMORY */
83 return -ENOMEM;
84 default:
85 return -EINVAL;
86 }
87 }
88
89 static void *
r535_gsp_msgq_wait(struct nvkm_gsp * gsp,u32 repc,u32 * prepc,int * ptime)90 r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime)
91 {
92 struct r535_gsp_msg *mqe;
93 u32 size, rptr = *gsp->msgq.rptr;
94 int used;
95 u8 *msg;
96 u32 len;
97
98 size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE);
99 if (WARN_ON(!size || size >= gsp->msgq.cnt))
100 return ERR_PTR(-EINVAL);
101
102 do {
103 u32 wptr = *gsp->msgq.wptr;
104
105 used = wptr + gsp->msgq.cnt - rptr;
106 if (used >= gsp->msgq.cnt)
107 used -= gsp->msgq.cnt;
108 if (used >= size)
109 break;
110
111 usleep_range(1, 2);
112 } while (--(*ptime));
113
114 if (WARN_ON(!*ptime))
115 return ERR_PTR(-ETIMEDOUT);
116
117 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000);
118
119 if (prepc) {
120 *prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe);
121 return mqe->data;
122 }
123
124 size = ALIGN(repc + GSP_MSG_HDR_SIZE, GSP_PAGE_SIZE);
125
126 msg = kvmalloc(repc, GFP_KERNEL);
127 if (!msg)
128 return ERR_PTR(-ENOMEM);
129
130 len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);
131 len = min_t(u32, repc, len);
132 memcpy(msg, mqe->data, len);
133
134 repc -= len;
135
136 if (repc) {
137 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);
138 memcpy(msg + len, mqe, repc);
139 }
140
141 rptr = (rptr + DIV_ROUND_UP(size, GSP_PAGE_SIZE)) % gsp->msgq.cnt;
142
143 mb();
144 (*gsp->msgq.rptr) = rptr;
145 return msg;
146 }
147
148 static void *
r535_gsp_msgq_recv(struct nvkm_gsp * gsp,u32 repc,int * ptime)149 r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime)
150 {
151 return r535_gsp_msgq_wait(gsp, repc, NULL, ptime);
152 }
153
154 static int
r535_gsp_cmdq_push(struct nvkm_gsp * gsp,void * argv)155 r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv)
156 {
157 struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data);
158 struct r535_gsp_msg *cqe;
159 u32 argc = cmd->checksum;
160 u64 *ptr = (void *)cmd;
161 u64 *end;
162 u64 csum = 0;
163 int free, time = 1000000;
164 u32 wptr, size, step;
165 u32 off = 0;
166
167 argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE);
168
169 end = (u64 *)((char *)ptr + argc);
170 cmd->pad = 0;
171 cmd->checksum = 0;
172 cmd->sequence = gsp->cmdq.seq++;
173 cmd->elem_count = DIV_ROUND_UP(argc, 0x1000);
174
175 while (ptr < end)
176 csum ^= *ptr++;
177
178 cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);
179
180 wptr = *gsp->cmdq.wptr;
181 do {
182 do {
183 free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;
184 if (free >= gsp->cmdq.cnt)
185 free -= gsp->cmdq.cnt;
186 if (free >= 1)
187 break;
188
189 usleep_range(1, 2);
190 } while(--time);
191
192 if (WARN_ON(!time)) {
193 kvfree(cmd);
194 return -ETIMEDOUT;
195 }
196
197 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);
198 step = min_t(u32, free, (gsp->cmdq.cnt - wptr));
199 size = min_t(u32, argc, step * GSP_PAGE_SIZE);
200
201 memcpy(cqe, (u8 *)cmd + off, size);
202
203 wptr += DIV_ROUND_UP(size, 0x1000);
204 if (wptr == gsp->cmdq.cnt)
205 wptr = 0;
206
207 off += size;
208 argc -= size;
209 } while(argc);
210
211 nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);
212 wmb();
213 (*gsp->cmdq.wptr) = wptr;
214 mb();
215
216 nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);
217
218 kvfree(cmd);
219 return 0;
220 }
221
222 static void *
r535_gsp_cmdq_get(struct nvkm_gsp * gsp,u32 argc)223 r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc)
224 {
225 struct r535_gsp_msg *cmd;
226 u32 size = GSP_MSG_HDR_SIZE + argc;
227
228 size = ALIGN(size, GSP_MSG_MIN_SIZE);
229 cmd = kvzalloc(size, GFP_KERNEL);
230 if (!cmd)
231 return ERR_PTR(-ENOMEM);
232
233 cmd->checksum = argc;
234 return cmd->data;
235 }
236
237 struct nvfw_gsp_rpc {
238 u32 header_version;
239 u32 signature;
240 u32 length;
241 u32 function;
242 u32 rpc_result;
243 u32 rpc_result_private;
244 u32 sequence;
245 union {
246 u32 spare;
247 u32 cpuRmGfid;
248 };
249 u8 data[];
250 };
251
252 static void
r535_gsp_msg_done(struct nvkm_gsp * gsp,struct nvfw_gsp_rpc * msg)253 r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)
254 {
255 kvfree(msg);
256 }
257
258 static void
r535_gsp_msg_dump(struct nvkm_gsp * gsp,struct nvfw_gsp_rpc * msg,int lvl)259 r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)
260 {
261 if (gsp->subdev.debug >= lvl) {
262 nvkm_printk__(&gsp->subdev, lvl, info,
263 "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",
264 msg->function, msg->length, msg->length - sizeof(*msg),
265 msg->rpc_result, msg->rpc_result_private);
266 print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,
267 msg->data, msg->length - sizeof(*msg), true);
268 }
269 }
270
271 static struct nvfw_gsp_rpc *
r535_gsp_msg_recv(struct nvkm_gsp * gsp,int fn,u32 repc)272 r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc)
273 {
274 struct nvkm_subdev *subdev = &gsp->subdev;
275 struct nvfw_gsp_rpc *msg;
276 int time = 4000000, i;
277 u32 size;
278
279 retry:
280 msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time);
281 if (IS_ERR_OR_NULL(msg))
282 return msg;
283
284 msg = r535_gsp_msgq_recv(gsp, msg->length, &time);
285 if (IS_ERR_OR_NULL(msg))
286 return msg;
287
288 if (msg->rpc_result) {
289 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
290 r535_gsp_msg_done(gsp, msg);
291 return ERR_PTR(-EINVAL);
292 }
293
294 r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE);
295
296 if (fn && msg->function == fn) {
297 if (repc) {
298 if (msg->length < sizeof(*msg) + repc) {
299 nvkm_error(subdev, "msg len %d < %zd\n",
300 msg->length, sizeof(*msg) + repc);
301 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
302 r535_gsp_msg_done(gsp, msg);
303 return ERR_PTR(-EIO);
304 }
305
306 return msg;
307 }
308
309 r535_gsp_msg_done(gsp, msg);
310 return NULL;
311 }
312
313 for (i = 0; i < gsp->msgq.ntfy_nr; i++) {
314 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];
315
316 if (ntfy->fn == msg->function) {
317 if (ntfy->func)
318 ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg));
319 break;
320 }
321 }
322
323 if (i == gsp->msgq.ntfy_nr)
324 r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN);
325
326 r535_gsp_msg_done(gsp, msg);
327 if (fn)
328 goto retry;
329
330 if (*gsp->msgq.rptr != *gsp->msgq.wptr)
331 goto retry;
332
333 return NULL;
334 }
335
336 static int
r535_gsp_msg_ntfy_add(struct nvkm_gsp * gsp,u32 fn,nvkm_gsp_msg_ntfy_func func,void * priv)337 r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)
338 {
339 int ret = 0;
340
341 mutex_lock(&gsp->msgq.mutex);
342 if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {
343 ret = -ENOSPC;
344 } else {
345 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;
346 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;
347 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;
348 gsp->msgq.ntfy_nr++;
349 }
350 mutex_unlock(&gsp->msgq.mutex);
351 return ret;
352 }
353
354 static int
r535_gsp_rpc_poll(struct nvkm_gsp * gsp,u32 fn)355 r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)
356 {
357 void *repv;
358
359 mutex_lock(&gsp->cmdq.mutex);
360 repv = r535_gsp_msg_recv(gsp, fn, 0);
361 mutex_unlock(&gsp->cmdq.mutex);
362 if (IS_ERR(repv))
363 return PTR_ERR(repv);
364
365 return 0;
366 }
367
368 static void *
r535_gsp_rpc_send(struct nvkm_gsp * gsp,void * argv,bool wait,u32 repc)369 r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
370 {
371 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
372 struct nvfw_gsp_rpc *msg;
373 u32 fn = rpc->function;
374 void *repv = NULL;
375 int ret;
376
377 if (gsp->subdev.debug >= NV_DBG_TRACE) {
378 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,
379 rpc->length, rpc->length - sizeof(*rpc));
380 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,
381 rpc->data, rpc->length - sizeof(*rpc), true);
382 }
383
384 ret = r535_gsp_cmdq_push(gsp, rpc);
385 if (ret)
386 return ERR_PTR(ret);
387
388 if (wait) {
389 msg = r535_gsp_msg_recv(gsp, fn, repc);
390 if (!IS_ERR_OR_NULL(msg))
391 repv = msg->data;
392 else
393 repv = msg;
394 }
395
396 return repv;
397 }
398
399 static void
r535_gsp_event_dtor(struct nvkm_gsp_event * event)400 r535_gsp_event_dtor(struct nvkm_gsp_event *event)
401 {
402 struct nvkm_gsp_device *device = event->device;
403 struct nvkm_gsp_client *client = device->object.client;
404 struct nvkm_gsp *gsp = client->gsp;
405
406 mutex_lock(&gsp->client_id.mutex);
407 if (event->func) {
408 list_del(&event->head);
409 event->func = NULL;
410 }
411 mutex_unlock(&gsp->client_id.mutex);
412
413 nvkm_gsp_rm_free(&event->object);
414 event->device = NULL;
415 }
416
417 static int
r535_gsp_device_event_get(struct nvkm_gsp_event * event)418 r535_gsp_device_event_get(struct nvkm_gsp_event *event)
419 {
420 struct nvkm_gsp_device *device = event->device;
421 NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl;
422
423 ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice,
424 NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl));
425 if (IS_ERR(ctrl))
426 return PTR_ERR(ctrl);
427
428 ctrl->event = event->id;
429 ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT;
430 return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl);
431 }
432
433 static int
r535_gsp_device_event_ctor(struct nvkm_gsp_device * device,u32 handle,u32 id,nvkm_gsp_event_func func,struct nvkm_gsp_event * event)434 r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id,
435 nvkm_gsp_event_func func, struct nvkm_gsp_event *event)
436 {
437 struct nvkm_gsp_client *client = device->object.client;
438 struct nvkm_gsp *gsp = client->gsp;
439 NV0005_ALLOC_PARAMETERS *args;
440 int ret;
441
442 args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle,
443 NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args),
444 &event->object);
445 if (IS_ERR(args))
446 return PTR_ERR(args);
447
448 args->hParentClient = client->object.handle;
449 args->hSrcResource = 0;
450 args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX;
451 args->notifyIndex = NV01_EVENT_CLIENT_RM | id;
452 args->data = NULL;
453
454 ret = nvkm_gsp_rm_alloc_wr(&event->object, args);
455 if (ret)
456 return ret;
457
458 event->device = device;
459 event->id = id;
460
461 ret = r535_gsp_device_event_get(event);
462 if (ret) {
463 nvkm_gsp_event_dtor(event);
464 return ret;
465 }
466
467 mutex_lock(&gsp->client_id.mutex);
468 event->func = func;
469 list_add(&event->head, &client->events);
470 mutex_unlock(&gsp->client_id.mutex);
471 return 0;
472 }
473
474 static void
r535_gsp_device_dtor(struct nvkm_gsp_device * device)475 r535_gsp_device_dtor(struct nvkm_gsp_device *device)
476 {
477 nvkm_gsp_rm_free(&device->subdevice);
478 nvkm_gsp_rm_free(&device->object);
479 }
480
481 static int
r535_gsp_subdevice_ctor(struct nvkm_gsp_device * device)482 r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device)
483 {
484 NV2080_ALLOC_PARAMETERS *args;
485
486 return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args),
487 &device->subdevice);
488 }
489
490 static int
r535_gsp_device_ctor(struct nvkm_gsp_client * client,struct nvkm_gsp_device * device)491 r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device)
492 {
493 NV0080_ALLOC_PARAMETERS *args;
494 int ret;
495
496 args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args),
497 &device->object);
498 if (IS_ERR(args))
499 return PTR_ERR(args);
500
501 args->hClientShare = client->object.handle;
502
503 ret = nvkm_gsp_rm_alloc_wr(&device->object, args);
504 if (ret)
505 return ret;
506
507 ret = r535_gsp_subdevice_ctor(device);
508 if (ret)
509 nvkm_gsp_rm_free(&device->object);
510
511 return ret;
512 }
513
514 static void
r535_gsp_client_dtor(struct nvkm_gsp_client * client)515 r535_gsp_client_dtor(struct nvkm_gsp_client *client)
516 {
517 struct nvkm_gsp *gsp = client->gsp;
518
519 nvkm_gsp_rm_free(&client->object);
520
521 mutex_lock(&gsp->client_id.mutex);
522 idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff);
523 mutex_unlock(&gsp->client_id.mutex);
524
525 client->gsp = NULL;
526 }
527
528 static int
r535_gsp_client_ctor(struct nvkm_gsp * gsp,struct nvkm_gsp_client * client)529 r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client)
530 {
531 NV0000_ALLOC_PARAMETERS *args;
532 int ret;
533
534 mutex_lock(&gsp->client_id.mutex);
535 ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL);
536 mutex_unlock(&gsp->client_id.mutex);
537 if (ret < 0)
538 return ret;
539
540 client->gsp = gsp;
541 client->object.client = client;
542 INIT_LIST_HEAD(&client->events);
543
544 args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args),
545 &client->object);
546 if (IS_ERR(args)) {
547 r535_gsp_client_dtor(client);
548 return ret;
549 }
550
551 args->hClient = client->object.handle;
552 args->processID = ~0;
553
554 ret = nvkm_gsp_rm_alloc_wr(&client->object, args);
555 if (ret) {
556 r535_gsp_client_dtor(client);
557 return ret;
558 }
559
560 return 0;
561 }
562
563 static int
r535_gsp_rpc_rm_free(struct nvkm_gsp_object * object)564 r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object)
565 {
566 struct nvkm_gsp_client *client = object->client;
567 struct nvkm_gsp *gsp = client->gsp;
568 rpc_free_v03_00 *rpc;
569
570 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n",
571 client->object.handle, object->handle);
572
573 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc));
574 if (WARN_ON(IS_ERR_OR_NULL(rpc)))
575 return -EIO;
576
577 rpc->params.hRoot = client->object.handle;
578 rpc->params.hObjectParent = 0;
579 rpc->params.hObjectOld = object->handle;
580 return nvkm_gsp_rpc_wr(gsp, rpc, true);
581 }
582
583 static void
r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object * object,void * repv)584 r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv)
585 {
586 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
587
588 nvkm_gsp_rpc_done(object->client->gsp, rpc);
589 }
590
591 static void *
r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object * object,void * argv,u32 repc)592 r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc)
593 {
594 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params);
595 struct nvkm_gsp *gsp = object->client->gsp;
596 void *ret;
597
598 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc);
599 if (IS_ERR_OR_NULL(rpc))
600 return rpc;
601
602 if (rpc->status) {
603 ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status));
604 if (PTR_ERR(ret) != -EAGAIN && PTR_ERR(ret) != -EBUSY)
605 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status);
606 } else {
607 ret = repc ? rpc->params : NULL;
608 }
609
610 nvkm_gsp_rpc_done(gsp, rpc);
611
612 return ret;
613 }
614
615 static void *
r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object * object,u32 oclass,u32 argc)616 r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc)
617 {
618 struct nvkm_gsp_client *client = object->client;
619 struct nvkm_gsp *gsp = client->gsp;
620 rpc_gsp_rm_alloc_v03_00 *rpc;
621
622 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n",
623 client->object.handle, object->parent->handle, object->handle, oclass, argc);
624
625 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc);
626 if (IS_ERR(rpc))
627 return rpc;
628
629 rpc->hClient = client->object.handle;
630 rpc->hParent = object->parent->handle;
631 rpc->hObject = object->handle;
632 rpc->hClass = oclass;
633 rpc->status = 0;
634 rpc->paramsSize = argc;
635 return rpc->params;
636 }
637
638 static void
r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object * object,void * repv)639 r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv)
640 {
641 rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
642
643 if (!repv)
644 return;
645 nvkm_gsp_rpc_done(object->client->gsp, rpc);
646 }
647
648 static int
r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object * object,void ** argv,u32 repc)649 r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc)
650 {
651 rpc_gsp_rm_control_v03_00 *rpc = container_of((*argv), typeof(*rpc), params);
652 struct nvkm_gsp *gsp = object->client->gsp;
653 int ret = 0;
654
655 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc);
656 if (IS_ERR_OR_NULL(rpc)) {
657 *argv = NULL;
658 return PTR_ERR(rpc);
659 }
660
661 if (rpc->status) {
662 ret = r535_rpc_status_to_errno(rpc->status);
663 if (ret != -EAGAIN && ret != -EBUSY)
664 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n",
665 object->client->object.handle, object->handle, rpc->cmd, rpc->status);
666 }
667
668 if (repc)
669 *argv = rpc->params;
670 else
671 nvkm_gsp_rpc_done(gsp, rpc);
672
673 return ret;
674 }
675
676 static void *
r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object * object,u32 cmd,u32 argc)677 r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc)
678 {
679 struct nvkm_gsp_client *client = object->client;
680 struct nvkm_gsp *gsp = client->gsp;
681 rpc_gsp_rm_control_v03_00 *rpc;
682
683 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n",
684 client->object.handle, object->handle, cmd, argc);
685
686 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc);
687 if (IS_ERR(rpc))
688 return rpc;
689
690 rpc->hClient = client->object.handle;
691 rpc->hObject = object->handle;
692 rpc->cmd = cmd;
693 rpc->status = 0;
694 rpc->paramsSize = argc;
695 return rpc->params;
696 }
697
698 static void
r535_gsp_rpc_done(struct nvkm_gsp * gsp,void * repv)699 r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)
700 {
701 struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);
702
703 r535_gsp_msg_done(gsp, rpc);
704 }
705
706 static void *
r535_gsp_rpc_get(struct nvkm_gsp * gsp,u32 fn,u32 argc)707 r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc)
708 {
709 struct nvfw_gsp_rpc *rpc;
710
711 rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64)));
712 if (IS_ERR(rpc))
713 return ERR_CAST(rpc);
714
715 rpc->header_version = 0x03000000;
716 rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';
717 rpc->function = fn;
718 rpc->rpc_result = 0xffffffff;
719 rpc->rpc_result_private = 0xffffffff;
720 rpc->length = sizeof(*rpc) + argc;
721 return rpc->data;
722 }
723
724 static void *
r535_gsp_rpc_push(struct nvkm_gsp * gsp,void * argv,bool wait,u32 repc)725 r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
726 {
727 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
728 struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data);
729 const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg);
730 const u32 max_rpc_size = max_msg_size - sizeof(*rpc);
731 u32 rpc_size = rpc->length - sizeof(*rpc);
732 void *repv;
733
734 mutex_lock(&gsp->cmdq.mutex);
735 if (rpc_size > max_rpc_size) {
736 const u32 fn = rpc->function;
737
738 /* Adjust length, and send initial RPC. */
739 rpc->length = sizeof(*rpc) + max_rpc_size;
740 cmd->checksum = rpc->length;
741
742 repv = r535_gsp_rpc_send(gsp, argv, false, 0);
743 if (IS_ERR(repv))
744 goto done;
745
746 argv += max_rpc_size;
747 rpc_size -= max_rpc_size;
748
749 /* Remaining chunks sent as CONTINUATION_RECORD RPCs. */
750 while (rpc_size) {
751 u32 size = min(rpc_size, max_rpc_size);
752 void *next;
753
754 next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);
755 if (IS_ERR(next)) {
756 repv = next;
757 goto done;
758 }
759
760 memcpy(next, argv, size);
761
762 repv = r535_gsp_rpc_send(gsp, next, false, 0);
763 if (IS_ERR(repv))
764 goto done;
765
766 argv += size;
767 rpc_size -= size;
768 }
769
770 /* Wait for reply. */
771 if (wait) {
772 rpc = r535_gsp_msg_recv(gsp, fn, repc);
773 if (!IS_ERR_OR_NULL(rpc))
774 repv = rpc->data;
775 else
776 repv = rpc;
777 } else {
778 repv = NULL;
779 }
780 } else {
781 repv = r535_gsp_rpc_send(gsp, argv, wait, repc);
782 }
783
784 done:
785 mutex_unlock(&gsp->cmdq.mutex);
786 return repv;
787 }
788
789 const struct nvkm_gsp_rm
790 r535_gsp_rm = {
791 .rpc_get = r535_gsp_rpc_get,
792 .rpc_push = r535_gsp_rpc_push,
793 .rpc_done = r535_gsp_rpc_done,
794
795 .rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get,
796 .rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push,
797 .rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done,
798
799 .rm_alloc_get = r535_gsp_rpc_rm_alloc_get,
800 .rm_alloc_push = r535_gsp_rpc_rm_alloc_push,
801 .rm_alloc_done = r535_gsp_rpc_rm_alloc_done,
802
803 .rm_free = r535_gsp_rpc_rm_free,
804
805 .client_ctor = r535_gsp_client_ctor,
806 .client_dtor = r535_gsp_client_dtor,
807
808 .device_ctor = r535_gsp_device_ctor,
809 .device_dtor = r535_gsp_device_dtor,
810
811 .event_ctor = r535_gsp_device_event_ctor,
812 .event_dtor = r535_gsp_event_dtor,
813 };
814
815 static void
r535_gsp_msgq_work(struct work_struct * work)816 r535_gsp_msgq_work(struct work_struct *work)
817 {
818 struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);
819
820 mutex_lock(&gsp->cmdq.mutex);
821 if (*gsp->msgq.rptr != *gsp->msgq.wptr)
822 r535_gsp_msg_recv(gsp, 0, 0);
823 mutex_unlock(&gsp->cmdq.mutex);
824 }
825
826 static irqreturn_t
r535_gsp_intr(struct nvkm_inth * inth)827 r535_gsp_intr(struct nvkm_inth *inth)
828 {
829 struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);
830 struct nvkm_subdev *subdev = &gsp->subdev;
831 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);
832 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +
833 gsp->falcon.func->riscv_irqmask);
834 u32 stat = intr & inte;
835
836 if (!stat) {
837 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);
838 return IRQ_NONE;
839 }
840
841 if (stat & 0x00000040) {
842 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);
843 schedule_work(&gsp->msgq.work);
844 stat &= ~0x00000040;
845 }
846
847 if (stat) {
848 nvkm_error(subdev, "intr %08x\n", stat);
849 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);
850 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);
851 }
852
853 nvkm_falcon_intr_retrigger(&gsp->falcon);
854 return IRQ_HANDLED;
855 }
856
857 static int
r535_gsp_intr_get_table(struct nvkm_gsp * gsp)858 r535_gsp_intr_get_table(struct nvkm_gsp *gsp)
859 {
860 NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;
861 int ret = 0;
862
863 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
864 NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));
865 if (IS_ERR(ctrl))
866 return PTR_ERR(ctrl);
867
868 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl));
869 if (WARN_ON(ret)) {
870 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
871 return ret;
872 }
873
874 for (unsigned i = 0; i < ctrl->tableLen; i++) {
875 enum nvkm_subdev_type type;
876 int inst;
877
878 nvkm_debug(&gsp->subdev,
879 "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,
880 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,
881 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);
882
883 switch (ctrl->table[i].engineIdx) {
884 case MC_ENGINE_IDX_GSP:
885 type = NVKM_SUBDEV_GSP;
886 inst = 0;
887 break;
888 case MC_ENGINE_IDX_DISP:
889 type = NVKM_ENGINE_DISP;
890 inst = 0;
891 break;
892 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:
893 type = NVKM_ENGINE_CE;
894 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0;
895 break;
896 case MC_ENGINE_IDX_GR0:
897 type = NVKM_ENGINE_GR;
898 inst = 0;
899 break;
900 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:
901 type = NVKM_ENGINE_NVDEC;
902 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0;
903 break;
904 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:
905 type = NVKM_ENGINE_NVENC;
906 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC;
907 break;
908 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:
909 type = NVKM_ENGINE_NVJPG;
910 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0;
911 break;
912 case MC_ENGINE_IDX_OFA0:
913 type = NVKM_ENGINE_OFA;
914 inst = 0;
915 break;
916 default:
917 continue;
918 }
919
920 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {
921 ret = -ENOSPC;
922 break;
923 }
924
925 gsp->intr[gsp->intr_nr].type = type;
926 gsp->intr[gsp->intr_nr].inst = inst;
927 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;
928 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;
929 gsp->intr_nr++;
930 }
931
932 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
933 return ret;
934 }
935
936 static int
r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp * gsp)937 r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp)
938 {
939 GspStaticConfigInfo *rpc;
940 int last_usable = -1;
941
942 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));
943 if (IS_ERR(rpc))
944 return PTR_ERR(rpc);
945
946 gsp->internal.client.object.client = &gsp->internal.client;
947 gsp->internal.client.object.parent = NULL;
948 gsp->internal.client.object.handle = rpc->hInternalClient;
949 gsp->internal.client.gsp = gsp;
950
951 gsp->internal.device.object.client = &gsp->internal.client;
952 gsp->internal.device.object.parent = &gsp->internal.client.object;
953 gsp->internal.device.object.handle = rpc->hInternalDevice;
954
955 gsp->internal.device.subdevice.client = &gsp->internal.client;
956 gsp->internal.device.subdevice.parent = &gsp->internal.device.object;
957 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;
958
959 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;
960 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;
961
962 for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) {
963 NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg =
964 &rpc->fbRegionInfoParams.fbRegion[i];
965
966 nvkm_debug(&gsp->subdev, "fb region %d: "
967 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,
968 reg->base, reg->limit, reg->reserved, reg->performance,
969 reg->supportCompressed, reg->supportISO, reg->bProtected);
970
971 if (!reg->reserved && !reg->bProtected) {
972 if (reg->supportCompressed && reg->supportISO &&
973 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {
974 const u64 size = (reg->limit + 1) - reg->base;
975
976 gsp->fb.region[gsp->fb.region_nr].addr = reg->base;
977 gsp->fb.region[gsp->fb.region_nr].size = size;
978 gsp->fb.region_nr++;
979 }
980
981 last_usable = i;
982 }
983 }
984
985 if (last_usable >= 0) {
986 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1;
987
988 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;
989 }
990
991 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {
992 if (rpc->gpcInfo.gpcMask & BIT(gpc)) {
993 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);
994 gsp->gr.gpcs++;
995 }
996 }
997
998 nvkm_gsp_rpc_done(gsp, rpc);
999 return 0;
1000 }
1001
1002 static void
nvkm_gsp_mem_dtor(struct nvkm_gsp * gsp,struct nvkm_gsp_mem * mem)1003 nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem)
1004 {
1005 if (mem->data) {
1006 /*
1007 * Poison the buffer to catch any unexpected access from
1008 * GSP-RM if the buffer was prematurely freed.
1009 */
1010 memset(mem->data, 0xFF, mem->size);
1011
1012 dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr);
1013 memset(mem, 0, sizeof(*mem));
1014 }
1015 }
1016
1017 static int
nvkm_gsp_mem_ctor(struct nvkm_gsp * gsp,size_t size,struct nvkm_gsp_mem * mem)1018 nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem)
1019 {
1020 mem->size = size;
1021 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);
1022 if (WARN_ON(!mem->data))
1023 return -ENOMEM;
1024
1025 return 0;
1026 }
1027
1028 static int
r535_gsp_postinit(struct nvkm_gsp * gsp)1029 r535_gsp_postinit(struct nvkm_gsp *gsp)
1030 {
1031 struct nvkm_device *device = gsp->subdev.device;
1032 int ret;
1033
1034 ret = r535_gsp_rpc_get_gsp_static_info(gsp);
1035 if (WARN_ON(ret))
1036 return ret;
1037
1038 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);
1039
1040 ret = r535_gsp_intr_get_table(gsp);
1041 if (WARN_ON(ret))
1042 return ret;
1043
1044 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);
1045 if (WARN_ON(ret < 0))
1046 return ret;
1047
1048 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,
1049 r535_gsp_intr, &gsp->subdev.inth);
1050 if (WARN_ON(ret))
1051 return ret;
1052
1053 nvkm_inth_allow(&gsp->subdev.inth);
1054 nvkm_wr32(device, 0x110004, 0x00000040);
1055
1056 /* Release the DMA buffers that were needed only for boot and init */
1057 nvkm_gsp_mem_dtor(gsp, &gsp->boot.fw);
1058 nvkm_gsp_mem_dtor(gsp, &gsp->libos);
1059
1060 return ret;
1061 }
1062
1063 static int
r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp * gsp,bool suspend)1064 r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)
1065 {
1066 rpc_unloading_guest_driver_v1F_07 *rpc;
1067
1068 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));
1069 if (IS_ERR(rpc))
1070 return PTR_ERR(rpc);
1071
1072 if (suspend) {
1073 rpc->bInPMTransition = 1;
1074 rpc->bGc6Entering = 0;
1075 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1076 } else {
1077 rpc->bInPMTransition = 0;
1078 rpc->bGc6Entering = 0;
1079 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;
1080 }
1081
1082 return nvkm_gsp_rpc_wr(gsp, rpc, true);
1083 }
1084
1085 enum registry_type {
1086 REGISTRY_TABLE_ENTRY_TYPE_DWORD = 1, /* 32-bit unsigned integer */
1087 REGISTRY_TABLE_ENTRY_TYPE_BINARY = 2, /* Binary blob */
1088 REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */
1089 };
1090
1091 /* An arbitrary limit to the length of a registry key */
1092 #define REGISTRY_MAX_KEY_LENGTH 64
1093
1094 /**
1095 * registry_list_entry - linked list member for a registry key/value
1096 * @head: list_head struct
1097 * @type: dword, binary, or string
1098 * @klen: the length of name of the key
1099 * @vlen: the length of the value
1100 * @key: the key name
1101 * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD
1102 * @binary: the data, if TYPE_BINARY or TYPE_STRING
1103 *
1104 * Every registry key/value is represented internally by this struct.
1105 *
1106 * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in
1107 * @dword.
1108 *
1109 * Types BINARY and STRING are variable-length binary blobs. The only real
1110 * difference between BINARY and STRING is that STRING is null-terminated and
1111 * is expected to contain only printable characters.
1112 *
1113 * Note: it is technically possible to have multiple keys with the same name
1114 * but different types, but this is not useful since GSP-RM expects keys to
1115 * have only one specific type.
1116 */
1117 struct registry_list_entry {
1118 struct list_head head;
1119 enum registry_type type;
1120 size_t klen;
1121 char key[REGISTRY_MAX_KEY_LENGTH];
1122 size_t vlen;
1123 u32 dword; /* TYPE_DWORD */
1124 u8 binary[] __counted_by(vlen); /* TYPE_BINARY or TYPE_STRING */
1125 };
1126
1127 /**
1128 * add_registry -- adds a registry entry
1129 * @gsp: gsp pointer
1130 * @key: name of the registry key
1131 * @type: type of data
1132 * @data: pointer to value
1133 * @length: size of data, in bytes
1134 *
1135 * Adds a registry key/value pair to the registry database.
1136 *
1137 * This function collects the registry information in a linked list. After
1138 * all registry keys have been added, build_registry() is used to create the
1139 * RPC data structure.
1140 *
1141 * registry_rpc_size is a running total of the size of all registry keys.
1142 * It's used to avoid an O(n) calculation of the size when the RPC is built.
1143 *
1144 * Returns 0 on success, or negative error code on error.
1145 */
add_registry(struct nvkm_gsp * gsp,const char * key,enum registry_type type,const void * data,size_t length)1146 static int add_registry(struct nvkm_gsp *gsp, const char *key,
1147 enum registry_type type, const void *data, size_t length)
1148 {
1149 struct registry_list_entry *reg;
1150 const size_t nlen = strnlen(key, REGISTRY_MAX_KEY_LENGTH) + 1;
1151 size_t alloc_size; /* extra bytes to alloc for binary or string value */
1152
1153 if (nlen > REGISTRY_MAX_KEY_LENGTH)
1154 return -EINVAL;
1155
1156 alloc_size = (type == REGISTRY_TABLE_ENTRY_TYPE_DWORD) ? 0 : length;
1157
1158 reg = kmalloc(sizeof(*reg) + alloc_size, GFP_KERNEL);
1159 if (!reg)
1160 return -ENOMEM;
1161
1162 switch (type) {
1163 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
1164 reg->dword = *(const u32 *)(data);
1165 break;
1166 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
1167 case REGISTRY_TABLE_ENTRY_TYPE_STRING:
1168 memcpy(reg->binary, data, alloc_size);
1169 break;
1170 default:
1171 nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n",
1172 type, key);
1173 kfree(reg);
1174 return -EINVAL;
1175 }
1176
1177 memcpy(reg->key, key, nlen);
1178 reg->klen = nlen;
1179 reg->vlen = length;
1180 reg->type = type;
1181
1182 list_add_tail(®->head, &gsp->registry_list);
1183 gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size;
1184
1185 return 0;
1186 }
1187
add_registry_num(struct nvkm_gsp * gsp,const char * key,u32 value)1188 static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value)
1189 {
1190 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_DWORD,
1191 &value, sizeof(u32));
1192 }
1193
add_registry_string(struct nvkm_gsp * gsp,const char * key,const char * value)1194 static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value)
1195 {
1196 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_STRING,
1197 value, strlen(value) + 1);
1198 }
1199
1200 /**
1201 * build_registry -- create the registry RPC data
1202 * @gsp: gsp pointer
1203 * @registry: pointer to the RPC payload to fill
1204 *
1205 * After all registry key/value pairs have been added, call this function to
1206 * build the RPC.
1207 *
1208 * The registry RPC looks like this:
1209 *
1210 * +-----------------+
1211 * |NvU32 size; |
1212 * |NvU32 numEntries;|
1213 * +-----------------+
1214 * +----------------------------------------+
1215 * |PACKED_REGISTRY_ENTRY |
1216 * +----------------------------------------+
1217 * |Null-terminated key (string) for entry 0|
1218 * +----------------------------------------+
1219 * |Binary/string data value for entry 0 | (only if necessary)
1220 * +----------------------------------------+
1221 *
1222 * +----------------------------------------+
1223 * |PACKED_REGISTRY_ENTRY |
1224 * +----------------------------------------+
1225 * |Null-terminated key (string) for entry 1|
1226 * +----------------------------------------+
1227 * |Binary/string data value for entry 1 | (only if necessary)
1228 * +----------------------------------------+
1229 * ... (and so on, one copy for each entry)
1230 *
1231 *
1232 * The 'data' field of an entry is either a 32-bit integer (for type DWORD)
1233 * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING).
1234 *
1235 * All memory allocated by add_registry() is released.
1236 */
build_registry(struct nvkm_gsp * gsp,PACKED_REGISTRY_TABLE * registry)1237 static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry)
1238 {
1239 struct registry_list_entry *reg, *n;
1240 size_t str_offset;
1241 unsigned int i = 0;
1242
1243 registry->numEntries = list_count_nodes(&gsp->registry_list);
1244 str_offset = struct_size(registry, entries, registry->numEntries);
1245
1246 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
1247 registry->entries[i].type = reg->type;
1248 registry->entries[i].length = reg->vlen;
1249
1250 /* Append the key name to the table */
1251 registry->entries[i].nameOffset = str_offset;
1252 memcpy((void *)registry + str_offset, reg->key, reg->klen);
1253 str_offset += reg->klen;
1254
1255 switch (reg->type) {
1256 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:
1257 registry->entries[i].data = reg->dword;
1258 break;
1259 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:
1260 case REGISTRY_TABLE_ENTRY_TYPE_STRING:
1261 /* If the type is binary or string, also append the value */
1262 memcpy((void *)registry + str_offset, reg->binary, reg->vlen);
1263 registry->entries[i].data = str_offset;
1264 str_offset += reg->vlen;
1265 break;
1266 default:
1267 break;
1268 }
1269
1270 i++;
1271 list_del(®->head);
1272 kfree(reg);
1273 }
1274
1275 /* Double-check that we calculated the sizes correctly */
1276 WARN_ON(gsp->registry_rpc_size != str_offset);
1277
1278 registry->size = gsp->registry_rpc_size;
1279 }
1280
1281 /**
1282 * clean_registry -- clean up registry memory in case of error
1283 * @gsp: gsp pointer
1284 *
1285 * Call this function to clean up all memory allocated by add_registry()
1286 * in case of error and build_registry() is not called.
1287 */
clean_registry(struct nvkm_gsp * gsp)1288 static void clean_registry(struct nvkm_gsp *gsp)
1289 {
1290 struct registry_list_entry *reg, *n;
1291
1292 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {
1293 list_del(®->head);
1294 kfree(reg);
1295 }
1296
1297 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
1298 }
1299
1300 MODULE_PARM_DESC(NVreg_RegistryDwords,
1301 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys");
1302 static char *NVreg_RegistryDwords;
1303 module_param(NVreg_RegistryDwords, charp, 0400);
1304
1305 /* dword only */
1306 struct nv_gsp_registry_entries {
1307 const char *name;
1308 u32 value;
1309 };
1310
1311 /**
1312 * r535_registry_entries - required registry entries for GSP-RM
1313 *
1314 * This array lists registry entries that are required for GSP-RM to
1315 * function correctly.
1316 *
1317 * RMSecBusResetEnable - enables PCI secondary bus reset
1318 * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration
1319 * registers on any PCI reset.
1320 */
1321 static const struct nv_gsp_registry_entries r535_registry_entries[] = {
1322 { "RMSecBusResetEnable", 1 },
1323 { "RMForcePcieConfigSave", 1 },
1324 };
1325 #define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)
1326
1327 /**
1328 * strip - strips all characters in 'reject' from 's'
1329 * @s: string to strip
1330 * @reject: string of characters to remove
1331 *
1332 * 's' is modified.
1333 *
1334 * Returns the length of the new string.
1335 */
strip(char * s,const char * reject)1336 static size_t strip(char *s, const char *reject)
1337 {
1338 char *p = s, *p2 = s;
1339 size_t length = 0;
1340 char c;
1341
1342 do {
1343 while ((c = *p2) && strchr(reject, c))
1344 p2++;
1345
1346 *p++ = c = *p2++;
1347 length++;
1348 } while (c);
1349
1350 return length;
1351 }
1352
1353 /**
1354 * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM
1355 * @gsp: gsp pointer
1356 *
1357 * The GSP-RM registry is a set of key/value pairs that configure some aspects
1358 * of GSP-RM. The keys are strings, and the values are 32-bit integers.
1359 *
1360 * The registry is built from a combination of a static hard-coded list (see
1361 * above) and entries passed on the driver's command line.
1362 */
1363 static int
r535_gsp_rpc_set_registry(struct nvkm_gsp * gsp)1364 r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)
1365 {
1366 PACKED_REGISTRY_TABLE *rpc;
1367 unsigned int i;
1368 int ret;
1369
1370 INIT_LIST_HEAD(&gsp->registry_list);
1371 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);
1372
1373 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {
1374 ret = add_registry_num(gsp, r535_registry_entries[i].name,
1375 r535_registry_entries[i].value);
1376 if (ret)
1377 goto fail;
1378 }
1379
1380 /*
1381 * The NVreg_RegistryDwords parameter is a string of key=value
1382 * pairs separated by semicolons. We need to extract and trim each
1383 * substring, and then parse the substring to extract the key and
1384 * value.
1385 */
1386 if (NVreg_RegistryDwords) {
1387 char *p = kstrdup(NVreg_RegistryDwords, GFP_KERNEL);
1388 char *start, *next = p, *equal;
1389
1390 if (!p) {
1391 ret = -ENOMEM;
1392 goto fail;
1393 }
1394
1395 /* Remove any whitespace from the parameter string */
1396 strip(p, " \t\n");
1397
1398 while ((start = strsep(&next, ";"))) {
1399 long value;
1400
1401 equal = strchr(start, '=');
1402 if (!equal || equal == start || equal[1] == 0) {
1403 nvkm_error(&gsp->subdev,
1404 "ignoring invalid registry string '%s'\n",
1405 start);
1406 continue;
1407 }
1408
1409 /* Truncate the key=value string to just key */
1410 *equal = 0;
1411
1412 ret = kstrtol(equal + 1, 0, &value);
1413 if (!ret) {
1414 ret = add_registry_num(gsp, start, value);
1415 } else {
1416 /* Not a number, so treat it as a string */
1417 ret = add_registry_string(gsp, start, equal + 1);
1418 }
1419
1420 if (ret) {
1421 nvkm_error(&gsp->subdev,
1422 "ignoring invalid registry key/value '%s=%s'\n",
1423 start, equal + 1);
1424 continue;
1425 }
1426 }
1427
1428 kfree(p);
1429 }
1430
1431 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size);
1432 if (IS_ERR(rpc)) {
1433 ret = PTR_ERR(rpc);
1434 goto fail;
1435 }
1436
1437 build_registry(gsp, rpc);
1438
1439 return nvkm_gsp_rpc_wr(gsp, rpc, false);
1440
1441 fail:
1442 clean_registry(gsp);
1443 return ret;
1444 }
1445
1446 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1447 static void
r535_gsp_acpi_caps(acpi_handle handle,CAPS_METHOD_DATA * caps)1448 r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)
1449 {
1450 const guid_t NVOP_DSM_GUID =
1451 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,
1452 0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);
1453 u64 NVOP_DSM_REV = 0x00000100;
1454 union acpi_object argv4 = {
1455 .buffer.type = ACPI_TYPE_BUFFER,
1456 .buffer.length = 4,
1457 }, *obj;
1458
1459 caps->status = 0xffff;
1460
1461 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))
1462 return;
1463
1464 argv4.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL);
1465 if (!argv4.buffer.pointer)
1466 return;
1467
1468 obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);
1469 if (!obj)
1470 goto done;
1471
1472 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1473 WARN_ON(obj->buffer.length != 4))
1474 goto done;
1475
1476 caps->status = 0;
1477 caps->optimusCaps = *(u32 *)obj->buffer.pointer;
1478
1479 done:
1480 ACPI_FREE(obj);
1481
1482 kfree(argv4.buffer.pointer);
1483 }
1484
1485 static void
r535_gsp_acpi_jt(acpi_handle handle,JT_METHOD_DATA * jt)1486 r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)
1487 {
1488 const guid_t JT_DSM_GUID =
1489 GUID_INIT(0xCBECA351L, 0x067B, 0x4924,
1490 0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);
1491 u64 JT_DSM_REV = 0x00000103;
1492 u32 caps;
1493 union acpi_object argv4 = {
1494 .buffer.type = ACPI_TYPE_BUFFER,
1495 .buffer.length = sizeof(caps),
1496 }, *obj;
1497
1498 jt->status = 0xffff;
1499
1500 argv4.buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL);
1501 if (!argv4.buffer.pointer)
1502 return;
1503
1504 obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);
1505 if (!obj)
1506 goto done;
1507
1508 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1509 WARN_ON(obj->buffer.length != 4))
1510 goto done;
1511
1512 jt->status = 0;
1513 jt->jtCaps = *(u32 *)obj->buffer.pointer;
1514 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;
1515 jt->bSBIOSCaps = 0;
1516
1517 done:
1518 ACPI_FREE(obj);
1519
1520 kfree(argv4.buffer.pointer);
1521 }
1522
1523 static void
r535_gsp_acpi_mux_id(acpi_handle handle,u32 id,MUX_METHOD_DATA_ELEMENT * mode,MUX_METHOD_DATA_ELEMENT * part)1524 r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
1525 MUX_METHOD_DATA_ELEMENT *part)
1526 {
1527 union acpi_object mux_arg = { ACPI_TYPE_INTEGER };
1528 struct acpi_object_list input = { 1, &mux_arg };
1529 acpi_handle iter = NULL, handle_mux = NULL;
1530 acpi_status status;
1531 unsigned long long value;
1532
1533 mode->status = 0xffff;
1534 part->status = 0xffff;
1535
1536 do {
1537 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);
1538 if (ACPI_FAILURE(status) || !iter)
1539 return;
1540
1541 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
1542 if (ACPI_FAILURE(status) || value != id)
1543 continue;
1544
1545 handle_mux = iter;
1546 } while (!handle_mux);
1547
1548 if (!handle_mux)
1549 return;
1550
1551 /* I -think- 0 means "acquire" according to nvidia's driver source */
1552 input.pointer->integer.type = ACPI_TYPE_INTEGER;
1553 input.pointer->integer.value = 0;
1554
1555 status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value);
1556 if (ACPI_SUCCESS(status)) {
1557 mode->acpiId = id;
1558 mode->mode = value;
1559 mode->status = 0;
1560 }
1561
1562 status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value);
1563 if (ACPI_SUCCESS(status)) {
1564 part->acpiId = id;
1565 part->mode = value;
1566 part->status = 0;
1567 }
1568 }
1569
1570 static void
r535_gsp_acpi_mux(acpi_handle handle,DOD_METHOD_DATA * dod,MUX_METHOD_DATA * mux)1571 r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)
1572 {
1573 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);
1574
1575 for (int i = 0; i < mux->tableLen; i++) {
1576 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],
1577 &mux->acpiIdMuxPartTable[i]);
1578 }
1579 }
1580
1581 static void
r535_gsp_acpi_dod(acpi_handle handle,DOD_METHOD_DATA * dod)1582 r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)
1583 {
1584 acpi_status status;
1585 struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
1586 union acpi_object *_DOD;
1587
1588 dod->status = 0xffff;
1589
1590 status = acpi_evaluate_object(handle, "_DOD", NULL, &output);
1591 if (ACPI_FAILURE(status))
1592 return;
1593
1594 _DOD = output.pointer;
1595
1596 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||
1597 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))
1598 return;
1599
1600 for (int i = 0; i < _DOD->package.count; i++) {
1601 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))
1602 return;
1603
1604 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;
1605 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);
1606 }
1607
1608 dod->status = 0;
1609 kfree(output.pointer);
1610 }
1611 #endif
1612
1613 static void
r535_gsp_acpi_info(struct nvkm_gsp * gsp,ACPI_METHOD_DATA * acpi)1614 r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)
1615 {
1616 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1617 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);
1618
1619 if (!handle)
1620 return;
1621
1622 acpi->bValid = 1;
1623
1624 r535_gsp_acpi_dod(handle, &acpi->dodMethodData);
1625 if (acpi->dodMethodData.status == 0)
1626 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);
1627
1628 r535_gsp_acpi_jt(handle, &acpi->jtMethodData);
1629 r535_gsp_acpi_caps(handle, &acpi->capsMethodData);
1630 #endif
1631 }
1632
1633 static int
r535_gsp_rpc_set_system_info(struct nvkm_gsp * gsp)1634 r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
1635 {
1636 struct nvkm_device *device = gsp->subdev.device;
1637 struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);
1638 GspSystemInfo *info;
1639
1640 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))
1641 return -ENOSYS;
1642
1643 info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));
1644 if (IS_ERR(info))
1645 return PTR_ERR(info);
1646
1647 info->gpuPhysAddr = device->func->resource_addr(device, 0);
1648 info->gpuPhysFbAddr = device->func->resource_addr(device, 1);
1649 info->gpuPhysInstAddr = device->func->resource_addr(device, 3);
1650 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);
1651 info->maxUserVa = TASK_SIZE;
1652 info->pciConfigMirrorBase = 0x088000;
1653 info->pciConfigMirrorSize = 0x001000;
1654 r535_gsp_acpi_info(gsp, &info->acpiMethodData);
1655
1656 return nvkm_gsp_rpc_wr(gsp, info, false);
1657 }
1658
1659 static int
r535_gsp_msg_os_error_log(void * priv,u32 fn,void * repv,u32 repc)1660 r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)
1661 {
1662 struct nvkm_gsp *gsp = priv;
1663 struct nvkm_subdev *subdev = &gsp->subdev;
1664 rpc_os_error_log_v17_00 *msg = repv;
1665
1666 if (WARN_ON(repc < sizeof(*msg)))
1667 return -EINVAL;
1668
1669 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);
1670 return 0;
1671 }
1672
1673 static int
r535_gsp_msg_rc_triggered(void * priv,u32 fn,void * repv,u32 repc)1674 r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc)
1675 {
1676 rpc_rc_triggered_v17_02 *msg = repv;
1677 struct nvkm_gsp *gsp = priv;
1678 struct nvkm_subdev *subdev = &gsp->subdev;
1679 struct nvkm_chan *chan;
1680 unsigned long flags;
1681
1682 if (WARN_ON(repc < sizeof(*msg)))
1683 return -EINVAL;
1684
1685 nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n",
1686 msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope,
1687 msg->partitionAttributionId);
1688
1689 chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags);
1690 if (!chan) {
1691 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid);
1692 return 0;
1693 }
1694
1695 nvkm_chan_error(chan, false);
1696 nvkm_chan_put(&chan, flags);
1697 return 0;
1698 }
1699
1700 static int
r535_gsp_msg_mmu_fault_queued(void * priv,u32 fn,void * repv,u32 repc)1701 r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)
1702 {
1703 struct nvkm_gsp *gsp = priv;
1704 struct nvkm_subdev *subdev = &gsp->subdev;
1705
1706 WARN_ON(repc != 0);
1707
1708 nvkm_error(subdev, "mmu fault queued\n");
1709 return 0;
1710 }
1711
1712 static int
r535_gsp_msg_post_event(void * priv,u32 fn,void * repv,u32 repc)1713 r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)
1714 {
1715 struct nvkm_gsp *gsp = priv;
1716 struct nvkm_gsp_client *client;
1717 struct nvkm_subdev *subdev = &gsp->subdev;
1718 rpc_post_event_v17_00 *msg = repv;
1719
1720 if (WARN_ON(repc < sizeof(*msg)))
1721 return -EINVAL;
1722 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))
1723 return -EINVAL;
1724
1725 nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",
1726 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,
1727 msg->status, msg->eventDataSize, msg->bNotifyList);
1728
1729 mutex_lock(&gsp->client_id.mutex);
1730 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);
1731 if (client) {
1732 struct nvkm_gsp_event *event;
1733 bool handled = false;
1734
1735 list_for_each_entry(event, &client->events, head) {
1736 if (event->object.handle == msg->hEvent) {
1737 event->func(event, msg->eventData, msg->eventDataSize);
1738 handled = true;
1739 }
1740 }
1741
1742 if (!handled) {
1743 nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",
1744 msg->hClient, msg->hEvent);
1745 }
1746 } else {
1747 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);
1748 }
1749 mutex_unlock(&gsp->client_id.mutex);
1750 return 0;
1751 }
1752
1753 /**
1754 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
1755 * @priv: gsp pointer
1756 * @fn: function number (ignored)
1757 * @repv: pointer to libos print RPC
1758 * @repc: message size
1759 *
1760 * The GSP sequencer is a list of I/O commands that the GSP can send to
1761 * the driver to perform for various purposes. The most common usage is to
1762 * perform a special mid-initialization reset.
1763 */
1764 static int
r535_gsp_msg_run_cpu_sequencer(void * priv,u32 fn,void * repv,u32 repc)1765 r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)
1766 {
1767 struct nvkm_gsp *gsp = priv;
1768 struct nvkm_subdev *subdev = &gsp->subdev;
1769 struct nvkm_device *device = subdev->device;
1770 rpc_run_cpu_sequencer_v17_00 *seq = repv;
1771 int ptr = 0, ret;
1772
1773 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);
1774
1775 while (ptr < seq->cmdIndex) {
1776 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];
1777
1778 ptr += 1;
1779 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);
1780
1781 switch (cmd->opCode) {
1782 case GSP_SEQ_BUF_OPCODE_REG_WRITE: {
1783 u32 addr = cmd->payload.regWrite.addr;
1784 u32 data = cmd->payload.regWrite.val;
1785
1786 nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);
1787 nvkm_wr32(device, addr, data);
1788 }
1789 break;
1790 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {
1791 u32 addr = cmd->payload.regModify.addr;
1792 u32 mask = cmd->payload.regModify.mask;
1793 u32 data = cmd->payload.regModify.val;
1794
1795 nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);
1796 nvkm_mask(device, addr, mask, data);
1797 }
1798 break;
1799 case GSP_SEQ_BUF_OPCODE_REG_POLL: {
1800 u32 addr = cmd->payload.regPoll.addr;
1801 u32 mask = cmd->payload.regPoll.mask;
1802 u32 data = cmd->payload.regPoll.val;
1803 u32 usec = cmd->payload.regPoll.timeout ?: 4000000;
1804 //u32 error = cmd->payload.regPoll.error;
1805
1806 nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);
1807 nvkm_rd32(device, addr);
1808 nvkm_usec(device, usec,
1809 if ((nvkm_rd32(device, addr) & mask) == data)
1810 break;
1811 );
1812 }
1813 break;
1814 case GSP_SEQ_BUF_OPCODE_DELAY_US: {
1815 u32 usec = cmd->payload.delayUs.val;
1816
1817 nvkm_trace(subdev, "seq usec %d\n", usec);
1818 udelay(usec);
1819 }
1820 break;
1821 case GSP_SEQ_BUF_OPCODE_REG_STORE: {
1822 u32 addr = cmd->payload.regStore.addr;
1823 u32 slot = cmd->payload.regStore.index;
1824
1825 seq->regSaveArea[slot] = nvkm_rd32(device, addr);
1826 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,
1827 seq->regSaveArea[slot]);
1828 }
1829 break;
1830 case GSP_SEQ_BUF_OPCODE_CORE_RESET:
1831 nvkm_trace(subdev, "seq core reset\n");
1832 nvkm_falcon_reset(&gsp->falcon);
1833 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);
1834 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);
1835 break;
1836 case GSP_SEQ_BUF_OPCODE_CORE_START:
1837 nvkm_trace(subdev, "seq core start\n");
1838 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)
1839 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);
1840 else
1841 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);
1842 break;
1843 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:
1844 nvkm_trace(subdev, "seq core wait halt\n");
1845 nvkm_msec(device, 2000,
1846 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)
1847 break;
1848 );
1849 break;
1850 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {
1851 struct nvkm_sec2 *sec2 = device->sec2;
1852 u32 mbox0;
1853
1854 nvkm_trace(subdev, "seq core resume\n");
1855
1856 ret = gsp->func->reset(gsp);
1857 if (WARN_ON(ret))
1858 return ret;
1859
1860 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
1861 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
1862
1863 nvkm_falcon_start(&sec2->falcon);
1864
1865 if (nvkm_msec(device, 2000,
1866 if (nvkm_rd32(device, 0x1180f8) & 0x04000000)
1867 break;
1868 ) < 0)
1869 return -ETIMEDOUT;
1870
1871 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);
1872 if (WARN_ON(mbox0)) {
1873 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);
1874 return -EIO;
1875 }
1876
1877 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1878
1879 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1880 return -EIO;
1881 }
1882 break;
1883 default:
1884 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);
1885 return -EINVAL;
1886 }
1887 }
1888
1889 return 0;
1890 }
1891
1892 static int
r535_gsp_booter_unload(struct nvkm_gsp * gsp,u32 mbox0,u32 mbox1)1893 r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1894 {
1895 struct nvkm_subdev *subdev = &gsp->subdev;
1896 struct nvkm_device *device = subdev->device;
1897 u32 wpr2_hi;
1898 int ret;
1899
1900 wpr2_hi = nvkm_rd32(device, 0x1fa828);
1901 if (!wpr2_hi) {
1902 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n");
1903 return 0;
1904 }
1905
1906 ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1907 if (WARN_ON(ret))
1908 return ret;
1909
1910 wpr2_hi = nvkm_rd32(device, 0x1fa828);
1911 if (WARN_ON(wpr2_hi))
1912 return -EIO;
1913
1914 return 0;
1915 }
1916
1917 static int
r535_gsp_booter_load(struct nvkm_gsp * gsp,u32 mbox0,u32 mbox1)1918 r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1919 {
1920 int ret;
1921
1922 ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1923 if (ret)
1924 return ret;
1925
1926 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1927
1928 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1929 return -EIO;
1930
1931 return 0;
1932 }
1933
1934 static int
r535_gsp_wpr_meta_init(struct nvkm_gsp * gsp)1935 r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)
1936 {
1937 GspFwWprMeta *meta;
1938 int ret;
1939
1940 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta);
1941 if (ret)
1942 return ret;
1943
1944 meta = gsp->wpr_meta.data;
1945
1946 meta->magic = GSP_FW_WPR_META_MAGIC;
1947 meta->revision = GSP_FW_WPR_META_REVISION;
1948
1949 meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr;
1950 meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;
1951
1952 meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;
1953 meta->sizeOfBootloader = gsp->boot.fw.size;
1954 meta->bootloaderCodeOffset = gsp->boot.code_offset;
1955 meta->bootloaderDataOffset = gsp->boot.data_offset;
1956 meta->bootloaderManifestOffset = gsp->boot.manifest_offset;
1957
1958 meta->sysmemAddrOfSignature = gsp->sig.addr;
1959 meta->sizeOfSignature = gsp->sig.size;
1960
1961 meta->gspFwRsvdStart = gsp->fb.heap.addr;
1962 meta->nonWprHeapOffset = gsp->fb.heap.addr;
1963 meta->nonWprHeapSize = gsp->fb.heap.size;
1964 meta->gspFwWprStart = gsp->fb.wpr2.addr;
1965 meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr;
1966 meta->gspFwHeapSize = gsp->fb.wpr2.heap.size;
1967 meta->gspFwOffset = gsp->fb.wpr2.elf.addr;
1968 meta->bootBinOffset = gsp->fb.wpr2.boot.addr;
1969 meta->frtsOffset = gsp->fb.wpr2.frts.addr;
1970 meta->frtsSize = gsp->fb.wpr2.frts.size;
1971 meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000);
1972 meta->fbSize = gsp->fb.size;
1973 meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr;
1974 meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size;
1975 meta->bootCount = 0;
1976 meta->partitionRpcAddr = 0;
1977 meta->partitionRpcRequestOffset = 0;
1978 meta->partitionRpcReplyOffset = 0;
1979 meta->verified = 0;
1980 return 0;
1981 }
1982
1983 static int
r535_gsp_shared_init(struct nvkm_gsp * gsp)1984 r535_gsp_shared_init(struct nvkm_gsp *gsp)
1985 {
1986 struct {
1987 msgqTxHeader tx;
1988 msgqRxHeader rx;
1989 } *cmdq, *msgq;
1990 int ret, i;
1991
1992 gsp->shm.cmdq.size = 0x40000;
1993 gsp->shm.msgq.size = 0x40000;
1994
1995 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;
1996 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1997 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1998
1999 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +
2000 gsp->shm.cmdq.size +
2001 gsp->shm.msgq.size,
2002 &gsp->shm.mem);
2003 if (ret)
2004 return ret;
2005
2006 gsp->shm.ptes.ptr = gsp->shm.mem.data;
2007 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;
2008 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;
2009
2010 for (i = 0; i < gsp->shm.ptes.nr; i++)
2011 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);
2012
2013 cmdq = gsp->shm.cmdq.ptr;
2014 cmdq->tx.version = 0;
2015 cmdq->tx.size = gsp->shm.cmdq.size;
2016 cmdq->tx.entryOff = GSP_PAGE_SIZE;
2017 cmdq->tx.msgSize = GSP_PAGE_SIZE;
2018 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;
2019 cmdq->tx.writePtr = 0;
2020 cmdq->tx.flags = 1;
2021 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);
2022
2023 msgq = gsp->shm.msgq.ptr;
2024
2025 gsp->cmdq.cnt = cmdq->tx.msgCount;
2026 gsp->cmdq.wptr = &cmdq->tx.writePtr;
2027 gsp->cmdq.rptr = &msgq->rx.readPtr;
2028 gsp->msgq.cnt = cmdq->tx.msgCount;
2029 gsp->msgq.wptr = &msgq->tx.writePtr;
2030 gsp->msgq.rptr = &cmdq->rx.readPtr;
2031 return 0;
2032 }
2033
2034 static int
r535_gsp_rmargs_init(struct nvkm_gsp * gsp,bool resume)2035 r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)
2036 {
2037 GSP_ARGUMENTS_CACHED *args;
2038 int ret;
2039
2040 if (!resume) {
2041 ret = r535_gsp_shared_init(gsp);
2042 if (ret)
2043 return ret;
2044
2045 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);
2046 if (ret)
2047 return ret;
2048 }
2049
2050 args = gsp->rmargs.data;
2051 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;
2052 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;
2053 args->messageQueueInitArguments.cmdQueueOffset =
2054 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;
2055 args->messageQueueInitArguments.statQueueOffset =
2056 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;
2057
2058 if (!resume) {
2059 args->srInitArguments.oldLevel = 0;
2060 args->srInitArguments.flags = 0;
2061 args->srInitArguments.bInPMTransition = 0;
2062 } else {
2063 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
2064 args->srInitArguments.flags = 0;
2065 args->srInitArguments.bInPMTransition = 1;
2066 }
2067
2068 return 0;
2069 }
2070
2071 static inline u64
r535_gsp_libos_id8(const char * name)2072 r535_gsp_libos_id8(const char *name)
2073 {
2074 u64 id = 0;
2075
2076 for (int i = 0; i < sizeof(id) && *name; i++, name++)
2077 id = (id << 8) | *name;
2078
2079 return id;
2080 }
2081
2082 /**
2083 * create_pte_array() - creates a PTE array of a physically contiguous buffer
2084 * @ptes: pointer to the array
2085 * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)
2086 * @size: size of the buffer
2087 *
2088 * GSP-RM sometimes expects physically-contiguous buffers to have an array of
2089 * "PTEs" for each page in that buffer. Although in theory that allows for
2090 * the buffer to be physically discontiguous, GSP-RM does not currently
2091 * support that.
2092 *
2093 * In this case, the PTEs are DMA addresses of each page of the buffer. Since
2094 * the buffer is physically contiguous, calculating all the PTEs is simple
2095 * math.
2096 *
2097 * See memdescGetPhysAddrsForGpu()
2098 */
create_pte_array(u64 * ptes,dma_addr_t addr,size_t size)2099 static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)
2100 {
2101 unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);
2102 unsigned int i;
2103
2104 for (i = 0; i < num_pages; i++)
2105 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);
2106 }
2107
2108 /**
2109 * r535_gsp_libos_init() -- create the libos arguments structure
2110 * @gsp: gsp pointer
2111 *
2112 * The logging buffers are byte queues that contain encoded printf-like
2113 * messages from GSP-RM. They need to be decoded by a special application
2114 * that can parse the buffers.
2115 *
2116 * The 'loginit' buffer contains logs from early GSP-RM init and
2117 * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are
2118 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
2119 *
2120 * The physical address map for the log buffer is stored in the buffer
2121 * itself, starting with offset 1. Offset 0 contains the "put" pointer.
2122 *
2123 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
2124 * configured for a larger page size (e.g. 64K pages), we need to give
2125 * the GSP an array of 4K pages. Fortunately, since the buffer is
2126 * physically contiguous, it's simple math to calculate the addresses.
2127 *
2128 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently
2129 * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the
2130 * buffers to be physically contiguous anyway.
2131 *
2132 * The memory allocated for the arguments must remain until the GSP sends the
2133 * init_done RPC.
2134 *
2135 * See _kgspInitLibosLoggingStructures (allocates memory for buffers)
2136 * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
2137 */
2138 static int
r535_gsp_libos_init(struct nvkm_gsp * gsp)2139 r535_gsp_libos_init(struct nvkm_gsp *gsp)
2140 {
2141 LibosMemoryRegionInitArgument *args;
2142 int ret;
2143
2144 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);
2145 if (ret)
2146 return ret;
2147
2148 args = gsp->libos.data;
2149
2150 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);
2151 if (ret)
2152 return ret;
2153
2154 args[0].id8 = r535_gsp_libos_id8("LOGINIT");
2155 args[0].pa = gsp->loginit.addr;
2156 args[0].size = gsp->loginit.size;
2157 args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2158 args[0].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2159 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);
2160
2161 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);
2162 if (ret)
2163 return ret;
2164
2165 args[1].id8 = r535_gsp_libos_id8("LOGINTR");
2166 args[1].pa = gsp->logintr.addr;
2167 args[1].size = gsp->logintr.size;
2168 args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2169 args[1].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2170 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);
2171
2172 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);
2173 if (ret)
2174 return ret;
2175
2176 args[2].id8 = r535_gsp_libos_id8("LOGRM");
2177 args[2].pa = gsp->logrm.addr;
2178 args[2].size = gsp->logrm.size;
2179 args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2180 args[2].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2181 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);
2182
2183 ret = r535_gsp_rmargs_init(gsp, false);
2184 if (ret)
2185 return ret;
2186
2187 args[3].id8 = r535_gsp_libos_id8("RMARGS");
2188 args[3].pa = gsp->rmargs.addr;
2189 args[3].size = gsp->rmargs.size;
2190 args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
2191 args[3].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
2192 return 0;
2193 }
2194
2195 void
nvkm_gsp_sg_free(struct nvkm_device * device,struct sg_table * sgt)2196 nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)
2197 {
2198 struct scatterlist *sgl;
2199 int i;
2200
2201 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
2202
2203 for_each_sgtable_sg(sgt, sgl, i) {
2204 struct page *page = sg_page(sgl);
2205
2206 __free_page(page);
2207 }
2208
2209 sg_free_table(sgt);
2210 }
2211
2212 int
nvkm_gsp_sg(struct nvkm_device * device,u64 size,struct sg_table * sgt)2213 nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
2214 {
2215 const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);
2216 struct scatterlist *sgl;
2217 int ret, i;
2218
2219 ret = sg_alloc_table(sgt, pages, GFP_KERNEL);
2220 if (ret)
2221 return ret;
2222
2223 for_each_sgtable_sg(sgt, sgl, i) {
2224 struct page *page = alloc_page(GFP_KERNEL);
2225
2226 if (!page) {
2227 nvkm_gsp_sg_free(device, sgt);
2228 return -ENOMEM;
2229 }
2230
2231 sg_set_page(sgl, page, PAGE_SIZE, 0);
2232 }
2233
2234 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
2235 if (ret)
2236 nvkm_gsp_sg_free(device, sgt);
2237
2238 return ret;
2239 }
2240
2241 static void
nvkm_gsp_radix3_dtor(struct nvkm_gsp * gsp,struct nvkm_gsp_radix3 * rx3)2242 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
2243 {
2244 nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2);
2245 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
2246 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
2247 }
2248
2249 /**
2250 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
2251 * @gsp: gsp pointer
2252 * @sgt: S/G list to traverse
2253 * @size: size of the image, in bytes
2254 * @rx3: radix3 array to update
2255 *
2256 * The GSP uses a three-level page table, called radix3, to map the firmware.
2257 * Each 64-bit "pointer" in the table is either the bus address of an entry in
2258 * the next table (for levels 0 and 1) or the bus address of the next page in
2259 * the GSP firmware image itself.
2260 *
2261 * Level 0 contains a single entry in one page that points to the first page
2262 * of level 1.
2263 *
2264 * Level 1, since it's also only one page in size, contains up to 512 entries,
2265 * one for each page in Level 2.
2266 *
2267 * Level 2 can be up to 512 pages in size, and each of those entries points to
2268 * the next page of the firmware image. Since there can be up to 512*512
2269 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.
2270 *
2271 * Internally, the GSP has its window into system memory, but the base
2272 * physical address of the aperture is not 0. In fact, it varies depending on
2273 * the GPU architecture. Since the GPU is a PCI device, this window is
2274 * accessed via DMA and is therefore bound by IOMMU translation. The end
2275 * result is that GSP-RM must translate the bus addresses in the table to GSP
2276 * physical addresses. All this should happen transparently.
2277 *
2278 * Returns 0 on success, or negative error code
2279 *
2280 * See kgspCreateRadix3_IMPL
2281 */
2282 static int
nvkm_gsp_radix3_sg(struct nvkm_gsp * gsp,struct sg_table * sgt,u64 size,struct nvkm_gsp_radix3 * rx3)2283 nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,
2284 struct nvkm_gsp_radix3 *rx3)
2285 {
2286 struct sg_dma_page_iter sg_dma_iter;
2287 struct scatterlist *sg;
2288 size_t bufsize;
2289 u64 *pte;
2290 int ret, i, page_idx = 0;
2291
2292 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0);
2293 if (ret)
2294 return ret;
2295
2296 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1);
2297 if (ret)
2298 goto lvl1_fail;
2299
2300 // Allocate level 2
2301 bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
2302 ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2);
2303 if (ret)
2304 goto lvl2_fail;
2305
2306 // Write the bus address of level 1 to level 0
2307 pte = rx3->lvl0.data;
2308 *pte = rx3->lvl1.addr;
2309
2310 // Write the bus address of each page in level 2 to level 1
2311 pte = rx3->lvl1.data;
2312 for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0)
2313 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);
2314
2315 // Finally, write the bus address of each page in sgt to level 2
2316 for_each_sgtable_sg(&rx3->lvl2, sg, i) {
2317 void *sgl_end;
2318
2319 pte = sg_virt(sg);
2320 sgl_end = (void *)pte + sg->length;
2321
2322 for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) {
2323 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);
2324 page_idx++;
2325
2326 // Go to the next scatterlist for level 2 if we've reached the end
2327 if ((void *)pte >= sgl_end)
2328 break;
2329 }
2330 }
2331
2332 if (ret) {
2333 lvl2_fail:
2334 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
2335 lvl1_fail:
2336 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
2337 }
2338
2339 return ret;
2340 }
2341
2342 int
r535_gsp_fini(struct nvkm_gsp * gsp,bool suspend)2343 r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
2344 {
2345 u32 mbox0 = 0xff, mbox1 = 0xff;
2346 int ret;
2347
2348 if (!gsp->running)
2349 return 0;
2350
2351 if (suspend) {
2352 GspFwWprMeta *meta = gsp->wpr_meta.data;
2353 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart;
2354 GspFwSRMeta *sr;
2355
2356 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);
2357 if (ret)
2358 return ret;
2359
2360 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3);
2361 if (ret)
2362 return ret;
2363
2364 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);
2365 if (ret)
2366 return ret;
2367
2368 sr = gsp->sr.meta.data;
2369 sr->magic = GSP_FW_SR_META_MAGIC;
2370 sr->revision = GSP_FW_SR_META_REVISION;
2371 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr;
2372 sr->sizeOfSuspendResumeData = len;
2373
2374 mbox0 = lower_32_bits(gsp->sr.meta.addr);
2375 mbox1 = upper_32_bits(gsp->sr.meta.addr);
2376 }
2377
2378 ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);
2379 if (WARN_ON(ret))
2380 return ret;
2381
2382 nvkm_msec(gsp->subdev.device, 2000,
2383 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000)
2384 break;
2385 );
2386
2387 nvkm_falcon_reset(&gsp->falcon);
2388
2389 ret = nvkm_gsp_fwsec_sb(gsp);
2390 WARN_ON(ret);
2391
2392 ret = r535_gsp_booter_unload(gsp, mbox0, mbox1);
2393 WARN_ON(ret);
2394
2395 gsp->running = false;
2396 return 0;
2397 }
2398
2399 int
r535_gsp_init(struct nvkm_gsp * gsp)2400 r535_gsp_init(struct nvkm_gsp *gsp)
2401 {
2402 u32 mbox0, mbox1;
2403 int ret;
2404
2405 if (!gsp->sr.meta.data) {
2406 mbox0 = lower_32_bits(gsp->wpr_meta.addr);
2407 mbox1 = upper_32_bits(gsp->wpr_meta.addr);
2408 } else {
2409 r535_gsp_rmargs_init(gsp, true);
2410
2411 mbox0 = lower_32_bits(gsp->sr.meta.addr);
2412 mbox1 = upper_32_bits(gsp->sr.meta.addr);
2413 }
2414
2415 /* Execute booter to handle (eventually...) booting GSP-RM. */
2416 ret = r535_gsp_booter_load(gsp, mbox0, mbox1);
2417 if (WARN_ON(ret))
2418 goto done;
2419
2420 ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);
2421 if (ret)
2422 goto done;
2423
2424 gsp->running = true;
2425
2426 done:
2427 if (gsp->sr.meta.data) {
2428 nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta);
2429 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
2430 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
2431 return ret;
2432 }
2433
2434 if (ret == 0)
2435 ret = r535_gsp_postinit(gsp);
2436
2437 return ret;
2438 }
2439
2440 static int
r535_gsp_rm_boot_ctor(struct nvkm_gsp * gsp)2441 r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)
2442 {
2443 const struct firmware *fw = gsp->fws.bl;
2444 const struct nvfw_bin_hdr *hdr;
2445 RM_RISCV_UCODE_DESC *desc;
2446 int ret;
2447
2448 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);
2449 desc = (void *)fw->data + hdr->header_offset;
2450
2451 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);
2452 if (ret)
2453 return ret;
2454
2455 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);
2456
2457 gsp->boot.code_offset = desc->monitorCodeOffset;
2458 gsp->boot.data_offset = desc->monitorDataOffset;
2459 gsp->boot.manifest_offset = desc->manifestOffset;
2460 gsp->boot.app_version = desc->appVersion;
2461 return 0;
2462 }
2463
2464 static const struct nvkm_firmware_func
2465 r535_gsp_fw = {
2466 .type = NVKM_FIRMWARE_IMG_SGT,
2467 };
2468
2469 static int
r535_gsp_elf_section(struct nvkm_gsp * gsp,const char * name,const u8 ** pdata,u64 * psize)2470 r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)
2471 {
2472 const u8 *img = gsp->fws.rm->data;
2473 const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;
2474 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];
2475 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];
2476
2477 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {
2478 if (!strcmp(&names[shdr->sh_name], name)) {
2479 *pdata = &img[shdr->sh_offset];
2480 *psize = shdr->sh_size;
2481 return 0;
2482 }
2483 }
2484
2485 nvkm_error(&gsp->subdev, "section '%s' not found\n", name);
2486 return -ENOENT;
2487 }
2488
2489 static void
r535_gsp_dtor_fws(struct nvkm_gsp * gsp)2490 r535_gsp_dtor_fws(struct nvkm_gsp *gsp)
2491 {
2492 nvkm_firmware_put(gsp->fws.bl);
2493 gsp->fws.bl = NULL;
2494 nvkm_firmware_put(gsp->fws.booter.unload);
2495 gsp->fws.booter.unload = NULL;
2496 nvkm_firmware_put(gsp->fws.booter.load);
2497 gsp->fws.booter.load = NULL;
2498 nvkm_firmware_put(gsp->fws.rm);
2499 gsp->fws.rm = NULL;
2500 }
2501
2502 void
r535_gsp_dtor(struct nvkm_gsp * gsp)2503 r535_gsp_dtor(struct nvkm_gsp *gsp)
2504 {
2505 idr_destroy(&gsp->client_id.idr);
2506 mutex_destroy(&gsp->client_id.mutex);
2507
2508 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);
2509 nvkm_gsp_mem_dtor(gsp, &gsp->sig);
2510 nvkm_firmware_dtor(&gsp->fw);
2511
2512 nvkm_falcon_fw_dtor(&gsp->booter.unload);
2513 nvkm_falcon_fw_dtor(&gsp->booter.load);
2514
2515 mutex_destroy(&gsp->msgq.mutex);
2516 mutex_destroy(&gsp->cmdq.mutex);
2517
2518 r535_gsp_dtor_fws(gsp);
2519
2520 nvkm_gsp_mem_dtor(gsp, &gsp->rmargs);
2521 nvkm_gsp_mem_dtor(gsp, &gsp->wpr_meta);
2522 nvkm_gsp_mem_dtor(gsp, &gsp->shm.mem);
2523 nvkm_gsp_mem_dtor(gsp, &gsp->loginit);
2524 nvkm_gsp_mem_dtor(gsp, &gsp->logintr);
2525 nvkm_gsp_mem_dtor(gsp, &gsp->logrm);
2526 }
2527
2528 int
r535_gsp_oneinit(struct nvkm_gsp * gsp)2529 r535_gsp_oneinit(struct nvkm_gsp *gsp)
2530 {
2531 struct nvkm_device *device = gsp->subdev.device;
2532 const u8 *data;
2533 u64 size;
2534 int ret;
2535
2536 mutex_init(&gsp->cmdq.mutex);
2537 mutex_init(&gsp->msgq.mutex);
2538
2539 ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load,
2540 &device->sec2->falcon, &gsp->booter.load);
2541 if (ret)
2542 return ret;
2543
2544 ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload,
2545 &device->sec2->falcon, &gsp->booter.unload);
2546 if (ret)
2547 return ret;
2548
2549 /* Load GSP firmware from ELF image into DMA-accessible memory. */
2550 ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);
2551 if (ret)
2552 return ret;
2553
2554 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);
2555 if (ret)
2556 return ret;
2557
2558 /* Load relevant signature from ELF image. */
2559 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);
2560 if (ret)
2561 return ret;
2562
2563 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);
2564 if (ret)
2565 return ret;
2566
2567 memcpy(gsp->sig.data, data, size);
2568
2569 /* Build radix3 page table for ELF image. */
2570 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);
2571 if (ret)
2572 return ret;
2573
2574 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
2575 r535_gsp_msg_run_cpu_sequencer, gsp);
2576 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);
2577 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED,
2578 r535_gsp_msg_rc_triggered, gsp);
2579 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
2580 r535_gsp_msg_mmu_fault_queued, gsp);
2581 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);
2582 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);
2583 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);
2584 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);
2585 ret = r535_gsp_rm_boot_ctor(gsp);
2586 if (ret)
2587 return ret;
2588
2589 /* Release FW images - we've copied them to DMA buffers now. */
2590 r535_gsp_dtor_fws(gsp);
2591
2592 /* Calculate FB layout. */
2593 gsp->fb.wpr2.frts.size = 0x100000;
2594 gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
2595
2596 gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
2597 gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000);
2598
2599 gsp->fb.wpr2.elf.size = gsp->fw.len;
2600 gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000);
2601
2602 {
2603 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30);
2604
2605 gsp->fb.wpr2.heap.size =
2606 gsp->func->wpr_heap.os_carveout_size +
2607 gsp->func->wpr_heap.base_size +
2608 ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) +
2609 ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20);
2610
2611 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size);
2612 }
2613
2614 gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000);
2615 gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000);
2616
2617 gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000);
2618 gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr;
2619
2620 gsp->fb.heap.size = 0x100000;
2621 gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size;
2622
2623 ret = nvkm_gsp_fwsec_frts(gsp);
2624 if (WARN_ON(ret))
2625 return ret;
2626
2627 ret = r535_gsp_libos_init(gsp);
2628 if (WARN_ON(ret))
2629 return ret;
2630
2631 ret = r535_gsp_wpr_meta_init(gsp);
2632 if (WARN_ON(ret))
2633 return ret;
2634
2635 ret = r535_gsp_rpc_set_system_info(gsp);
2636 if (WARN_ON(ret))
2637 return ret;
2638
2639 ret = r535_gsp_rpc_set_registry(gsp);
2640 if (WARN_ON(ret))
2641 return ret;
2642
2643 /* Reset GSP into RISC-V mode. */
2644 ret = gsp->func->reset(gsp);
2645 if (WARN_ON(ret))
2646 return ret;
2647
2648 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
2649 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
2650
2651 mutex_init(&gsp->client_id.mutex);
2652 idr_init(&gsp->client_id.idr);
2653 return 0;
2654 }
2655
2656 static int
r535_gsp_load_fw(struct nvkm_gsp * gsp,const char * name,const char * ver,const struct firmware ** pfw)2657 r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver,
2658 const struct firmware **pfw)
2659 {
2660 char fwname[64];
2661
2662 snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver);
2663 return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw);
2664 }
2665
2666 int
r535_gsp_load(struct nvkm_gsp * gsp,int ver,const struct nvkm_gsp_fwif * fwif)2667 r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif)
2668 {
2669 struct nvkm_subdev *subdev = &gsp->subdev;
2670 int ret;
2671 bool enable_gsp = fwif->enable;
2672
2673 #if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT)
2674 enable_gsp = true;
2675 #endif
2676 if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp))
2677 return -EINVAL;
2678
2679 if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) ||
2680 (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) ||
2681 (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) ||
2682 (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) {
2683 r535_gsp_dtor_fws(gsp);
2684 return ret;
2685 }
2686
2687 return 0;
2688 }
2689
2690 #define NVKM_GSP_FIRMWARE(chip) \
2691 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin"); \
2692 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \
2693 MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin"); \
2694 MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")
2695
2696 NVKM_GSP_FIRMWARE(tu102);
2697 NVKM_GSP_FIRMWARE(tu104);
2698 NVKM_GSP_FIRMWARE(tu106);
2699
2700 NVKM_GSP_FIRMWARE(tu116);
2701 NVKM_GSP_FIRMWARE(tu117);
2702
2703 NVKM_GSP_FIRMWARE(ga100);
2704
2705 NVKM_GSP_FIRMWARE(ga102);
2706 NVKM_GSP_FIRMWARE(ga103);
2707 NVKM_GSP_FIRMWARE(ga104);
2708 NVKM_GSP_FIRMWARE(ga106);
2709 NVKM_GSP_FIRMWARE(ga107);
2710
2711 NVKM_GSP_FIRMWARE(ad102);
2712 NVKM_GSP_FIRMWARE(ad103);
2713 NVKM_GSP_FIRMWARE(ad104);
2714 NVKM_GSP_FIRMWARE(ad106);
2715 NVKM_GSP_FIRMWARE(ad107);
2716