1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <linux/mutex.h>
25 #include <linux/log2.h>
26 #include <linux/sched.h>
27 #include <linux/sched/mm.h>
28 #include <linux/sched/task.h>
29 #include <linux/mmu_context.h>
30 #include <linux/slab.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 #include "amdgpu_reset.h"
39
40 struct mm_struct;
41
42 #include "kfd_priv.h"
43 #include "kfd_device_queue_manager.h"
44 #include "kfd_svm.h"
45 #include "kfd_smi_events.h"
46 #include "kfd_debug.h"
47
48 /*
49 * List of struct kfd_process (field kfd_process).
50 * Unique/indexed by mm_struct*
51 */
52 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
53 DEFINE_MUTEX(kfd_processes_mutex);
54
55 DEFINE_SRCU(kfd_processes_srcu);
56
57 /* For process termination handling */
58 static struct workqueue_struct *kfd_process_wq;
59
60 /* Ordered, single-threaded workqueue for restoring evicted
61 * processes. Restoring multiple processes concurrently under memory
62 * pressure can lead to processes blocking each other from validating
63 * their BOs and result in a live-lock situation where processes
64 * remain evicted indefinitely.
65 */
66 static struct workqueue_struct *kfd_restore_wq;
67
68 static struct kfd_process *find_process(const struct task_struct *thread,
69 bool ref);
70 static void kfd_process_ref_release(struct kref *ref);
71 static struct kfd_process *create_process(const struct task_struct *thread);
72
73 static void evict_process_worker(struct work_struct *work);
74 static void restore_process_worker(struct work_struct *work);
75
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77
78 struct kfd_procfs_tree {
79 struct kobject *kobj;
80 };
81
82 static struct kfd_procfs_tree procfs;
83
84 /*
85 * Structure for SDMA activity tracking
86 */
87 struct kfd_sdma_activity_handler_workarea {
88 struct work_struct sdma_activity_work;
89 struct kfd_process_device *pdd;
90 uint64_t sdma_activity_counter;
91 };
92
93 struct temp_sdma_queue_list {
94 uint64_t __user *rptr;
95 uint64_t sdma_val;
96 unsigned int queue_id;
97 struct list_head list;
98 };
99
kfd_sdma_activity_worker(struct work_struct * work)100 static void kfd_sdma_activity_worker(struct work_struct *work)
101 {
102 struct kfd_sdma_activity_handler_workarea *workarea;
103 struct kfd_process_device *pdd;
104 uint64_t val;
105 struct mm_struct *mm;
106 struct queue *q;
107 struct qcm_process_device *qpd;
108 struct device_queue_manager *dqm;
109 int ret = 0;
110 struct temp_sdma_queue_list sdma_q_list;
111 struct temp_sdma_queue_list *sdma_q, *next;
112
113 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 sdma_activity_work);
115
116 pdd = workarea->pdd;
117 if (!pdd)
118 return;
119 dqm = pdd->dev->dqm;
120 qpd = &pdd->qpd;
121 if (!dqm || !qpd)
122 return;
123 /*
124 * Total SDMA activity is current SDMA activity + past SDMA activity
125 * Past SDMA count is stored in pdd.
126 * To get the current activity counters for all active SDMA queues,
127 * we loop over all SDMA queues and get their counts from user-space.
128 *
129 * We cannot call get_user() with dqm_lock held as it can cause
130 * a circular lock dependency situation. To read the SDMA stats,
131 * we need to do the following:
132 *
133 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 * with dqm_lock/dqm_unlock().
135 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 * Save the SDMA count for each node and also add the count to the total
137 * SDMA count counter.
138 * Its possible, during this step, a few SDMA queue nodes got deleted
139 * from the qpd->queues_list.
140 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 * If any node got deleted, its SDMA count would be captured in the sdma
142 * past activity counter. So subtract the SDMA counter stored in step 2
143 * for this node from the total SDMA count.
144 */
145 INIT_LIST_HEAD(&sdma_q_list.list);
146
147 /*
148 * Create the temp list of all SDMA queues
149 */
150 dqm_lock(dqm);
151
152 list_for_each_entry(q, &qpd->queues_list, list) {
153 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 continue;
156
157 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
158 if (!sdma_q) {
159 dqm_unlock(dqm);
160 goto cleanup;
161 }
162
163 INIT_LIST_HEAD(&sdma_q->list);
164 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
165 sdma_q->queue_id = q->properties.queue_id;
166 list_add_tail(&sdma_q->list, &sdma_q_list.list);
167 }
168
169 /*
170 * If the temp list is empty, then no SDMA queues nodes were found in
171 * qpd->queues_list. Return the past activity count as the total sdma
172 * count
173 */
174 if (list_empty(&sdma_q_list.list)) {
175 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
176 dqm_unlock(dqm);
177 return;
178 }
179
180 dqm_unlock(dqm);
181
182 /*
183 * Get the usage count for each SDMA queue in temp_list.
184 */
185 mm = get_task_mm(pdd->process->lead_thread);
186 if (!mm)
187 goto cleanup;
188
189 kthread_use_mm(mm);
190
191 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
192 val = 0;
193 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
194 if (ret) {
195 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
196 sdma_q->queue_id);
197 } else {
198 sdma_q->sdma_val = val;
199 workarea->sdma_activity_counter += val;
200 }
201 }
202
203 kthread_unuse_mm(mm);
204 mmput(mm);
205
206 /*
207 * Do a second iteration over qpd_queues_list to check if any SDMA
208 * nodes got deleted while fetching SDMA counter.
209 */
210 dqm_lock(dqm);
211
212 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
213
214 list_for_each_entry(q, &qpd->queues_list, list) {
215 if (list_empty(&sdma_q_list.list))
216 break;
217
218 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
219 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
220 continue;
221
222 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
223 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
224 (sdma_q->queue_id == q->properties.queue_id)) {
225 list_del(&sdma_q->list);
226 kfree(sdma_q);
227 break;
228 }
229 }
230 }
231
232 dqm_unlock(dqm);
233
234 /*
235 * If temp list is not empty, it implies some queues got deleted
236 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
237 * count for each node from the total SDMA count.
238 */
239 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
240 workarea->sdma_activity_counter -= sdma_q->sdma_val;
241 list_del(&sdma_q->list);
242 kfree(sdma_q);
243 }
244
245 return;
246
247 cleanup:
248 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
249 list_del(&sdma_q->list);
250 kfree(sdma_q);
251 }
252 }
253
254 /**
255 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
256 * by current process. Translates acquired wave count into number of compute units
257 * that are occupied.
258 *
259 * @attr: Handle of attribute that allows reporting of wave count. The attribute
260 * handle encapsulates GPU device it is associated with, thereby allowing collection
261 * of waves in flight, etc
262 * @buffer: Handle of user provided buffer updated with wave count
263 *
264 * Return: Number of bytes written to user buffer or an error value
265 */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)266 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
267 {
268 int cu_cnt;
269 int wave_cnt;
270 int max_waves_per_cu;
271 struct kfd_node *dev = NULL;
272 struct kfd_process *proc = NULL;
273 struct kfd_process_device *pdd = NULL;
274 int i;
275 struct kfd_cu_occupancy *cu_occupancy;
276 u32 queue_format;
277
278 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
279 dev = pdd->dev;
280 if (dev->kfd2kgd->get_cu_occupancy == NULL)
281 return -EINVAL;
282
283 cu_cnt = 0;
284 proc = pdd->process;
285 if (pdd->qpd.queue_count == 0) {
286 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
287 dev->id, proc->pasid);
288 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
289 }
290
291 /* Collect wave count from device if it supports */
292 wave_cnt = 0;
293 max_waves_per_cu = 0;
294
295 cu_occupancy = kcalloc(AMDGPU_MAX_QUEUES, sizeof(*cu_occupancy), GFP_KERNEL);
296 if (!cu_occupancy)
297 return -ENOMEM;
298
299 /*
300 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
301 * For AQL queues, because of cooperative dispatch we multiply the wave count
302 * by number of XCCs in the partition to get the total wave counts across all
303 * XCCs in the partition.
304 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
305 */
306 dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
307 &max_waves_per_cu, ffs(dev->xcc_mask) - 1);
308
309 for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
310 if (cu_occupancy[i].wave_cnt != 0 &&
311 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,
312 cu_occupancy[i].doorbell_off,
313 &queue_format)) {
314 if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
315 wave_cnt += cu_occupancy[i].wave_cnt;
316 else
317 wave_cnt += (NUM_XCC(dev->xcc_mask) *
318 cu_occupancy[i].wave_cnt);
319 }
320 }
321
322 /* Translate wave count to number of compute units */
323 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
324 kfree(cu_occupancy);
325 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
326 }
327
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)328 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
329 char *buffer)
330 {
331 if (strcmp(attr->name, "pasid") == 0) {
332 struct kfd_process *p = container_of(attr, struct kfd_process,
333 attr_pasid);
334
335 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
336 } else if (strncmp(attr->name, "vram_", 5) == 0) {
337 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
338 attr_vram);
339 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
340 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
341 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
342 attr_sdma);
343 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
344
345 INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
346 kfd_sdma_activity_worker);
347
348 sdma_activity_work_handler.pdd = pdd;
349 sdma_activity_work_handler.sdma_activity_counter = 0;
350
351 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
352
353 flush_work(&sdma_activity_work_handler.sdma_activity_work);
354 destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
355
356 return snprintf(buffer, PAGE_SIZE, "%llu\n",
357 (sdma_activity_work_handler.sdma_activity_counter)/
358 SDMA_ACTIVITY_DIVISOR);
359 } else {
360 pr_err("Invalid attribute");
361 return -EINVAL;
362 }
363
364 return 0;
365 }
366
kfd_procfs_kobj_release(struct kobject * kobj)367 static void kfd_procfs_kobj_release(struct kobject *kobj)
368 {
369 kfree(kobj);
370 }
371
372 static const struct sysfs_ops kfd_procfs_ops = {
373 .show = kfd_procfs_show,
374 };
375
376 static const struct kobj_type procfs_type = {
377 .release = kfd_procfs_kobj_release,
378 .sysfs_ops = &kfd_procfs_ops,
379 };
380
kfd_procfs_init(void)381 void kfd_procfs_init(void)
382 {
383 int ret = 0;
384
385 procfs.kobj = kfd_alloc_struct(procfs.kobj);
386 if (!procfs.kobj)
387 return;
388
389 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
390 &kfd_device->kobj, "proc");
391 if (ret) {
392 pr_warn("Could not create procfs proc folder");
393 /* If we fail to create the procfs, clean up */
394 kfd_procfs_shutdown();
395 }
396 }
397
kfd_procfs_shutdown(void)398 void kfd_procfs_shutdown(void)
399 {
400 if (procfs.kobj) {
401 kobject_del(procfs.kobj);
402 kobject_put(procfs.kobj);
403 procfs.kobj = NULL;
404 }
405 }
406
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)407 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
408 struct attribute *attr, char *buffer)
409 {
410 struct queue *q = container_of(kobj, struct queue, kobj);
411
412 if (!strcmp(attr->name, "size"))
413 return snprintf(buffer, PAGE_SIZE, "%llu",
414 q->properties.queue_size);
415 else if (!strcmp(attr->name, "type"))
416 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
417 else if (!strcmp(attr->name, "gpuid"))
418 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
419 else
420 pr_err("Invalid attribute");
421
422 return 0;
423 }
424
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)425 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
426 struct attribute *attr, char *buffer)
427 {
428 if (strcmp(attr->name, "evicted_ms") == 0) {
429 struct kfd_process_device *pdd = container_of(attr,
430 struct kfd_process_device,
431 attr_evict);
432 uint64_t evict_jiffies;
433
434 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
435
436 return snprintf(buffer,
437 PAGE_SIZE,
438 "%llu\n",
439 jiffies64_to_msecs(evict_jiffies));
440
441 /* Sysfs handle that gets CU occupancy is per device */
442 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
443 return kfd_get_cu_occupancy(attr, buffer);
444 } else {
445 pr_err("Invalid attribute");
446 }
447
448 return 0;
449 }
450
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)451 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
452 struct attribute *attr, char *buf)
453 {
454 struct kfd_process_device *pdd;
455
456 if (!strcmp(attr->name, "faults")) {
457 pdd = container_of(attr, struct kfd_process_device,
458 attr_faults);
459 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
460 }
461 if (!strcmp(attr->name, "page_in")) {
462 pdd = container_of(attr, struct kfd_process_device,
463 attr_page_in);
464 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
465 }
466 if (!strcmp(attr->name, "page_out")) {
467 pdd = container_of(attr, struct kfd_process_device,
468 attr_page_out);
469 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
470 }
471 return 0;
472 }
473
474 static struct attribute attr_queue_size = {
475 .name = "size",
476 .mode = KFD_SYSFS_FILE_MODE
477 };
478
479 static struct attribute attr_queue_type = {
480 .name = "type",
481 .mode = KFD_SYSFS_FILE_MODE
482 };
483
484 static struct attribute attr_queue_gpuid = {
485 .name = "gpuid",
486 .mode = KFD_SYSFS_FILE_MODE
487 };
488
489 static struct attribute *procfs_queue_attrs[] = {
490 &attr_queue_size,
491 &attr_queue_type,
492 &attr_queue_gpuid,
493 NULL
494 };
495 ATTRIBUTE_GROUPS(procfs_queue);
496
497 static const struct sysfs_ops procfs_queue_ops = {
498 .show = kfd_procfs_queue_show,
499 };
500
501 static const struct kobj_type procfs_queue_type = {
502 .sysfs_ops = &procfs_queue_ops,
503 .default_groups = procfs_queue_groups,
504 };
505
506 static const struct sysfs_ops procfs_stats_ops = {
507 .show = kfd_procfs_stats_show,
508 };
509
510 static const struct kobj_type procfs_stats_type = {
511 .sysfs_ops = &procfs_stats_ops,
512 .release = kfd_procfs_kobj_release,
513 };
514
515 static const struct sysfs_ops sysfs_counters_ops = {
516 .show = kfd_sysfs_counters_show,
517 };
518
519 static const struct kobj_type sysfs_counters_type = {
520 .sysfs_ops = &sysfs_counters_ops,
521 .release = kfd_procfs_kobj_release,
522 };
523
kfd_procfs_add_queue(struct queue * q)524 int kfd_procfs_add_queue(struct queue *q)
525 {
526 struct kfd_process *proc;
527 int ret;
528
529 if (!q || !q->process)
530 return -EINVAL;
531 proc = q->process;
532
533 /* Create proc/<pid>/queues/<queue id> folder */
534 if (!proc->kobj_queues)
535 return -EFAULT;
536 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
537 proc->kobj_queues, "%u", q->properties.queue_id);
538 if (ret < 0) {
539 pr_warn("Creating proc/<pid>/queues/%u failed",
540 q->properties.queue_id);
541 kobject_put(&q->kobj);
542 return ret;
543 }
544
545 return 0;
546 }
547
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)548 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
549 char *name)
550 {
551 int ret;
552
553 if (!kobj || !attr || !name)
554 return;
555
556 attr->name = name;
557 attr->mode = KFD_SYSFS_FILE_MODE;
558 sysfs_attr_init(attr);
559
560 ret = sysfs_create_file(kobj, attr);
561 if (ret)
562 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
563 }
564
kfd_procfs_add_sysfs_stats(struct kfd_process * p)565 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
566 {
567 int ret;
568 int i;
569 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
570
571 if (!p || !p->kobj)
572 return;
573
574 /*
575 * Create sysfs files for each GPU:
576 * - proc/<pid>/stats_<gpuid>/
577 * - proc/<pid>/stats_<gpuid>/evicted_ms
578 * - proc/<pid>/stats_<gpuid>/cu_occupancy
579 */
580 for (i = 0; i < p->n_pdds; i++) {
581 struct kfd_process_device *pdd = p->pdds[i];
582
583 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
584 "stats_%u", pdd->dev->id);
585 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
586 if (!pdd->kobj_stats)
587 return;
588
589 ret = kobject_init_and_add(pdd->kobj_stats,
590 &procfs_stats_type,
591 p->kobj,
592 stats_dir_filename);
593
594 if (ret) {
595 pr_warn("Creating KFD proc/stats_%s folder failed",
596 stats_dir_filename);
597 kobject_put(pdd->kobj_stats);
598 pdd->kobj_stats = NULL;
599 return;
600 }
601
602 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
603 "evicted_ms");
604 /* Add sysfs file to report compute unit occupancy */
605 if (pdd->dev->kfd2kgd->get_cu_occupancy)
606 kfd_sysfs_create_file(pdd->kobj_stats,
607 &pdd->attr_cu_occupancy,
608 "cu_occupancy");
609 }
610 }
611
kfd_procfs_add_sysfs_counters(struct kfd_process * p)612 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
613 {
614 int ret = 0;
615 int i;
616 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
617
618 if (!p || !p->kobj)
619 return;
620
621 /*
622 * Create sysfs files for each GPU which supports SVM
623 * - proc/<pid>/counters_<gpuid>/
624 * - proc/<pid>/counters_<gpuid>/faults
625 * - proc/<pid>/counters_<gpuid>/page_in
626 * - proc/<pid>/counters_<gpuid>/page_out
627 */
628 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
629 struct kfd_process_device *pdd = p->pdds[i];
630 struct kobject *kobj_counters;
631
632 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
633 "counters_%u", pdd->dev->id);
634 kobj_counters = kfd_alloc_struct(kobj_counters);
635 if (!kobj_counters)
636 return;
637
638 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
639 p->kobj, counters_dir_filename);
640 if (ret) {
641 pr_warn("Creating KFD proc/%s folder failed",
642 counters_dir_filename);
643 kobject_put(kobj_counters);
644 return;
645 }
646
647 pdd->kobj_counters = kobj_counters;
648 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
649 "faults");
650 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
651 "page_in");
652 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
653 "page_out");
654 }
655 }
656
kfd_procfs_add_sysfs_files(struct kfd_process * p)657 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
658 {
659 int i;
660
661 if (!p || !p->kobj)
662 return;
663
664 /*
665 * Create sysfs files for each GPU:
666 * - proc/<pid>/vram_<gpuid>
667 * - proc/<pid>/sdma_<gpuid>
668 */
669 for (i = 0; i < p->n_pdds; i++) {
670 struct kfd_process_device *pdd = p->pdds[i];
671
672 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
673 pdd->dev->id);
674 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
675 pdd->vram_filename);
676
677 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
678 pdd->dev->id);
679 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
680 pdd->sdma_filename);
681 }
682 }
683
kfd_procfs_del_queue(struct queue * q)684 void kfd_procfs_del_queue(struct queue *q)
685 {
686 if (!q)
687 return;
688
689 kobject_del(&q->kobj);
690 kobject_put(&q->kobj);
691 }
692
kfd_process_create_wq(void)693 int kfd_process_create_wq(void)
694 {
695 if (!kfd_process_wq)
696 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
697 if (!kfd_restore_wq)
698 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
699 WQ_FREEZABLE);
700
701 if (!kfd_process_wq || !kfd_restore_wq) {
702 kfd_process_destroy_wq();
703 return -ENOMEM;
704 }
705
706 return 0;
707 }
708
kfd_process_destroy_wq(void)709 void kfd_process_destroy_wq(void)
710 {
711 if (kfd_process_wq) {
712 destroy_workqueue(kfd_process_wq);
713 kfd_process_wq = NULL;
714 }
715 if (kfd_restore_wq) {
716 destroy_workqueue(kfd_restore_wq);
717 kfd_restore_wq = NULL;
718 }
719 }
720
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd,void ** kptr)721 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
722 struct kfd_process_device *pdd, void **kptr)
723 {
724 struct kfd_node *dev = pdd->dev;
725
726 if (kptr && *kptr) {
727 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
728 *kptr = NULL;
729 }
730
731 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
732 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
733 NULL);
734 }
735
736 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
737 * This function should be only called right after the process
738 * is created and when kfd_processes_mutex is still being held
739 * to avoid concurrency. Because of that exclusiveness, we do
740 * not need to take p->mutex.
741 */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,struct kgd_mem ** mem,void ** kptr)742 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
743 uint64_t gpu_va, uint32_t size,
744 uint32_t flags, struct kgd_mem **mem, void **kptr)
745 {
746 struct kfd_node *kdev = pdd->dev;
747 int err;
748
749 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
750 pdd->drm_priv, mem, NULL,
751 flags, false);
752 if (err)
753 goto err_alloc_mem;
754
755 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
756 pdd->drm_priv);
757 if (err)
758 goto err_map_mem;
759
760 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
761 if (err) {
762 pr_debug("Sync memory failed, wait interrupted by user signal\n");
763 goto sync_memory_failed;
764 }
765
766 if (kptr) {
767 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
768 (struct kgd_mem *)*mem, kptr, NULL);
769 if (err) {
770 pr_debug("Map GTT BO to kernel failed\n");
771 goto sync_memory_failed;
772 }
773 }
774
775 return err;
776
777 sync_memory_failed:
778 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
779
780 err_map_mem:
781 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
782 NULL);
783 err_alloc_mem:
784 *mem = NULL;
785 *kptr = NULL;
786 return err;
787 }
788
789 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
790 * process for IB usage The memory reserved is for KFD to submit
791 * IB to AMDGPU from kernel. If the memory is reserved
792 * successfully, ib_kaddr will have the CPU/kernel
793 * address. Check ib_kaddr before accessing the memory.
794 */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)795 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
796 {
797 struct qcm_process_device *qpd = &pdd->qpd;
798 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
799 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
800 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
801 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
802 struct kgd_mem *mem;
803 void *kaddr;
804 int ret;
805
806 if (qpd->ib_kaddr || !qpd->ib_base)
807 return 0;
808
809 /* ib_base is only set for dGPU */
810 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
811 &mem, &kaddr);
812 if (ret)
813 return ret;
814
815 qpd->ib_mem = mem;
816 qpd->ib_kaddr = kaddr;
817
818 return 0;
819 }
820
kfd_process_device_destroy_ib_mem(struct kfd_process_device * pdd)821 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
822 {
823 struct qcm_process_device *qpd = &pdd->qpd;
824
825 if (!qpd->ib_kaddr || !qpd->ib_base)
826 return;
827
828 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
829 }
830
kfd_create_process(struct task_struct * thread)831 struct kfd_process *kfd_create_process(struct task_struct *thread)
832 {
833 struct kfd_process *process;
834 int ret;
835
836 if (!(thread->mm && mmget_not_zero(thread->mm)))
837 return ERR_PTR(-EINVAL);
838
839 /* Only the pthreads threading model is supported. */
840 if (thread->group_leader->mm != thread->mm) {
841 mmput(thread->mm);
842 return ERR_PTR(-EINVAL);
843 }
844
845 /* If the process just called exec(3), it is possible that the
846 * cleanup of the kfd_process (following the release of the mm
847 * of the old process image) is still in the cleanup work queue.
848 * Make sure to drain any job before trying to recreate any
849 * resource for this process.
850 */
851 flush_workqueue(kfd_process_wq);
852
853 /*
854 * take kfd processes mutex before starting of process creation
855 * so there won't be a case where two threads of the same process
856 * create two kfd_process structures
857 */
858 mutex_lock(&kfd_processes_mutex);
859
860 if (kfd_is_locked()) {
861 pr_debug("KFD is locked! Cannot create process");
862 process = ERR_PTR(-EINVAL);
863 goto out;
864 }
865
866 /* A prior open of /dev/kfd could have already created the process. */
867 process = find_process(thread, false);
868 if (process) {
869 pr_debug("Process already found\n");
870 } else {
871 process = create_process(thread);
872 if (IS_ERR(process))
873 goto out;
874
875 if (!procfs.kobj)
876 goto out;
877
878 process->kobj = kfd_alloc_struct(process->kobj);
879 if (!process->kobj) {
880 pr_warn("Creating procfs kobject failed");
881 goto out;
882 }
883 ret = kobject_init_and_add(process->kobj, &procfs_type,
884 procfs.kobj, "%d",
885 (int)process->lead_thread->pid);
886 if (ret) {
887 pr_warn("Creating procfs pid directory failed");
888 kobject_put(process->kobj);
889 goto out;
890 }
891
892 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
893 "pasid");
894
895 process->kobj_queues = kobject_create_and_add("queues",
896 process->kobj);
897 if (!process->kobj_queues)
898 pr_warn("Creating KFD proc/queues folder failed");
899
900 kfd_procfs_add_sysfs_stats(process);
901 kfd_procfs_add_sysfs_files(process);
902 kfd_procfs_add_sysfs_counters(process);
903
904 init_waitqueue_head(&process->wait_irq_drain);
905 }
906 out:
907 if (!IS_ERR(process))
908 kref_get(&process->ref);
909 mutex_unlock(&kfd_processes_mutex);
910 mmput(thread->mm);
911
912 return process;
913 }
914
kfd_get_process(const struct task_struct * thread)915 struct kfd_process *kfd_get_process(const struct task_struct *thread)
916 {
917 struct kfd_process *process;
918
919 if (!thread->mm)
920 return ERR_PTR(-EINVAL);
921
922 /* Only the pthreads threading model is supported. */
923 if (thread->group_leader->mm != thread->mm)
924 return ERR_PTR(-EINVAL);
925
926 process = find_process(thread, false);
927 if (!process)
928 return ERR_PTR(-EINVAL);
929
930 return process;
931 }
932
find_process_by_mm(const struct mm_struct * mm)933 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
934 {
935 struct kfd_process *process;
936
937 hash_for_each_possible_rcu(kfd_processes_table, process,
938 kfd_processes, (uintptr_t)mm)
939 if (process->mm == mm)
940 return process;
941
942 return NULL;
943 }
944
find_process(const struct task_struct * thread,bool ref)945 static struct kfd_process *find_process(const struct task_struct *thread,
946 bool ref)
947 {
948 struct kfd_process *p;
949 int idx;
950
951 idx = srcu_read_lock(&kfd_processes_srcu);
952 p = find_process_by_mm(thread->mm);
953 if (p && ref)
954 kref_get(&p->ref);
955 srcu_read_unlock(&kfd_processes_srcu, idx);
956
957 return p;
958 }
959
kfd_unref_process(struct kfd_process * p)960 void kfd_unref_process(struct kfd_process *p)
961 {
962 kref_put(&p->ref, kfd_process_ref_release);
963 }
964
965 /* This increments the process->ref counter. */
kfd_lookup_process_by_pid(struct pid * pid)966 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
967 {
968 struct task_struct *task = NULL;
969 struct kfd_process *p = NULL;
970
971 if (!pid) {
972 task = current;
973 get_task_struct(task);
974 } else {
975 task = get_pid_task(pid, PIDTYPE_PID);
976 }
977
978 if (task) {
979 p = find_process(task, true);
980 put_task_struct(task);
981 }
982
983 return p;
984 }
985
kfd_process_device_free_bos(struct kfd_process_device * pdd)986 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
987 {
988 struct kfd_process *p = pdd->process;
989 void *mem;
990 int id;
991 int i;
992
993 /*
994 * Remove all handles from idr and release appropriate
995 * local memory object
996 */
997 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
998
999 for (i = 0; i < p->n_pdds; i++) {
1000 struct kfd_process_device *peer_pdd = p->pdds[i];
1001
1002 if (!peer_pdd->drm_priv)
1003 continue;
1004 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1005 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
1006 }
1007
1008 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
1009 pdd->drm_priv, NULL);
1010 kfd_process_device_remove_obj_handle(pdd, id);
1011 }
1012 }
1013
1014 /*
1015 * Just kunmap and unpin signal BO here. It will be freed in
1016 * kfd_process_free_outstanding_kfd_bos()
1017 */
kfd_process_kunmap_signal_bo(struct kfd_process * p)1018 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1019 {
1020 struct kfd_process_device *pdd;
1021 struct kfd_node *kdev;
1022 void *mem;
1023
1024 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1025 if (!kdev)
1026 return;
1027
1028 mutex_lock(&p->mutex);
1029
1030 pdd = kfd_get_process_device_data(kdev, p);
1031 if (!pdd)
1032 goto out;
1033
1034 mem = kfd_process_device_translate_handle(
1035 pdd, GET_IDR_HANDLE(p->signal_handle));
1036 if (!mem)
1037 goto out;
1038
1039 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1040
1041 out:
1042 mutex_unlock(&p->mutex);
1043 }
1044
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)1045 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1046 {
1047 int i;
1048
1049 for (i = 0; i < p->n_pdds; i++)
1050 kfd_process_device_free_bos(p->pdds[i]);
1051 }
1052
kfd_process_destroy_pdds(struct kfd_process * p)1053 static void kfd_process_destroy_pdds(struct kfd_process *p)
1054 {
1055 int i;
1056
1057 for (i = 0; i < p->n_pdds; i++) {
1058 struct kfd_process_device *pdd = p->pdds[i];
1059
1060 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
1061 pdd->dev->id, p->pasid);
1062
1063 kfd_process_device_destroy_cwsr_dgpu(pdd);
1064 kfd_process_device_destroy_ib_mem(pdd);
1065
1066 if (pdd->drm_file) {
1067 amdgpu_amdkfd_gpuvm_release_process_vm(
1068 pdd->dev->adev, pdd->drm_priv);
1069 fput(pdd->drm_file);
1070 }
1071
1072 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1073 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1074 get_order(KFD_CWSR_TBA_TMA_SIZE));
1075
1076 idr_destroy(&pdd->alloc_idr);
1077
1078 kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1079
1080 if (pdd->dev->kfd->shared_resources.enable_mes &&
1081 pdd->proc_ctx_cpu_ptr)
1082 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
1083 &pdd->proc_ctx_bo);
1084 /*
1085 * before destroying pdd, make sure to report availability
1086 * for auto suspend
1087 */
1088 if (pdd->runtime_inuse) {
1089 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);
1090 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1091 pdd->runtime_inuse = false;
1092 }
1093
1094 kfree(pdd);
1095 p->pdds[i] = NULL;
1096 }
1097 p->n_pdds = 0;
1098 }
1099
kfd_process_remove_sysfs(struct kfd_process * p)1100 static void kfd_process_remove_sysfs(struct kfd_process *p)
1101 {
1102 struct kfd_process_device *pdd;
1103 int i;
1104
1105 if (!p->kobj)
1106 return;
1107
1108 sysfs_remove_file(p->kobj, &p->attr_pasid);
1109 kobject_del(p->kobj_queues);
1110 kobject_put(p->kobj_queues);
1111 p->kobj_queues = NULL;
1112
1113 for (i = 0; i < p->n_pdds; i++) {
1114 pdd = p->pdds[i];
1115
1116 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1117 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1118
1119 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1120 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1121 sysfs_remove_file(pdd->kobj_stats,
1122 &pdd->attr_cu_occupancy);
1123 kobject_del(pdd->kobj_stats);
1124 kobject_put(pdd->kobj_stats);
1125 pdd->kobj_stats = NULL;
1126 }
1127
1128 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1129 pdd = p->pdds[i];
1130
1131 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1132 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1133 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1134 kobject_del(pdd->kobj_counters);
1135 kobject_put(pdd->kobj_counters);
1136 pdd->kobj_counters = NULL;
1137 }
1138
1139 kobject_del(p->kobj);
1140 kobject_put(p->kobj);
1141 p->kobj = NULL;
1142 }
1143
1144 /*
1145 * If any GPU is ongoing reset, wait for reset complete.
1146 */
kfd_process_wait_gpu_reset_complete(struct kfd_process * p)1147 static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1148 {
1149 int i;
1150
1151 for (i = 0; i < p->n_pdds; i++)
1152 flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1153 }
1154
1155 /* No process locking is needed in this function, because the process
1156 * is not findable any more. We must assume that no other thread is
1157 * using it any more, otherwise we couldn't safely free the process
1158 * structure in the end.
1159 */
kfd_process_wq_release(struct work_struct * work)1160 static void kfd_process_wq_release(struct work_struct *work)
1161 {
1162 struct kfd_process *p = container_of(work, struct kfd_process,
1163 release_work);
1164 struct dma_fence *ef;
1165
1166 kfd_process_dequeue_from_all_devices(p);
1167 pqm_uninit(&p->pqm);
1168
1169 /*
1170 * If GPU in reset, user queues may still running, wait for reset complete.
1171 */
1172 kfd_process_wait_gpu_reset_complete(p);
1173
1174 /* Signal the eviction fence after user mode queues are
1175 * destroyed. This allows any BOs to be freed without
1176 * triggering pointless evictions or waiting for fences.
1177 */
1178 synchronize_rcu();
1179 ef = rcu_access_pointer(p->ef);
1180 if (ef)
1181 dma_fence_signal(ef);
1182
1183 kfd_process_remove_sysfs(p);
1184
1185 kfd_process_kunmap_signal_bo(p);
1186 kfd_process_free_outstanding_kfd_bos(p);
1187 svm_range_list_fini(p);
1188
1189 kfd_process_destroy_pdds(p);
1190 dma_fence_put(ef);
1191
1192 kfd_event_free_process(p);
1193
1194 kfd_pasid_free(p->pasid);
1195 mutex_destroy(&p->mutex);
1196
1197 put_task_struct(p->lead_thread);
1198
1199 kfree(p);
1200 }
1201
kfd_process_ref_release(struct kref * ref)1202 static void kfd_process_ref_release(struct kref *ref)
1203 {
1204 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1205
1206 INIT_WORK(&p->release_work, kfd_process_wq_release);
1207 queue_work(kfd_process_wq, &p->release_work);
1208 }
1209
kfd_process_alloc_notifier(struct mm_struct * mm)1210 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1211 {
1212 int idx = srcu_read_lock(&kfd_processes_srcu);
1213 struct kfd_process *p = find_process_by_mm(mm);
1214
1215 srcu_read_unlock(&kfd_processes_srcu, idx);
1216
1217 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1218 }
1219
kfd_process_free_notifier(struct mmu_notifier * mn)1220 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1221 {
1222 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1223 }
1224
kfd_process_notifier_release_internal(struct kfd_process * p)1225 static void kfd_process_notifier_release_internal(struct kfd_process *p)
1226 {
1227 int i;
1228
1229 cancel_delayed_work_sync(&p->eviction_work);
1230 cancel_delayed_work_sync(&p->restore_work);
1231
1232 for (i = 0; i < p->n_pdds; i++) {
1233 struct kfd_process_device *pdd = p->pdds[i];
1234
1235 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1236 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1237 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1238 }
1239
1240 /* Indicate to other users that MM is no longer valid */
1241 p->mm = NULL;
1242 kfd_dbg_trap_disable(p);
1243
1244 if (atomic_read(&p->debugged_process_count) > 0) {
1245 struct kfd_process *target;
1246 unsigned int temp;
1247 int idx = srcu_read_lock(&kfd_processes_srcu);
1248
1249 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1250 if (target->debugger_process && target->debugger_process == p) {
1251 mutex_lock_nested(&target->mutex, 1);
1252 kfd_dbg_trap_disable(target);
1253 mutex_unlock(&target->mutex);
1254 if (atomic_read(&p->debugged_process_count) == 0)
1255 break;
1256 }
1257 }
1258
1259 srcu_read_unlock(&kfd_processes_srcu, idx);
1260 }
1261
1262 mmu_notifier_put(&p->mmu_notifier);
1263 }
1264
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1265 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1266 struct mm_struct *mm)
1267 {
1268 struct kfd_process *p;
1269
1270 /*
1271 * The kfd_process structure can not be free because the
1272 * mmu_notifier srcu is read locked
1273 */
1274 p = container_of(mn, struct kfd_process, mmu_notifier);
1275 if (WARN_ON(p->mm != mm))
1276 return;
1277
1278 mutex_lock(&kfd_processes_mutex);
1279 /*
1280 * Do early return if table is empty.
1281 *
1282 * This could potentially happen if this function is called concurrently
1283 * by mmu_notifier and by kfd_cleanup_pocesses.
1284 *
1285 */
1286 if (hash_empty(kfd_processes_table)) {
1287 mutex_unlock(&kfd_processes_mutex);
1288 return;
1289 }
1290 hash_del_rcu(&p->kfd_processes);
1291 mutex_unlock(&kfd_processes_mutex);
1292 synchronize_srcu(&kfd_processes_srcu);
1293
1294 kfd_process_notifier_release_internal(p);
1295 }
1296
1297 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1298 .release = kfd_process_notifier_release,
1299 .alloc_notifier = kfd_process_alloc_notifier,
1300 .free_notifier = kfd_process_free_notifier,
1301 };
1302
1303 /*
1304 * This code handles the case when driver is being unloaded before all
1305 * mm_struct are released. We need to safely free the kfd_process and
1306 * avoid race conditions with mmu_notifier that might try to free them.
1307 *
1308 */
kfd_cleanup_processes(void)1309 void kfd_cleanup_processes(void)
1310 {
1311 struct kfd_process *p;
1312 struct hlist_node *p_temp;
1313 unsigned int temp;
1314 HLIST_HEAD(cleanup_list);
1315
1316 /*
1317 * Move all remaining kfd_process from the process table to a
1318 * temp list for processing. Once done, callback from mmu_notifier
1319 * release will not see the kfd_process in the table and do early return,
1320 * avoiding double free issues.
1321 */
1322 mutex_lock(&kfd_processes_mutex);
1323 hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1324 hash_del_rcu(&p->kfd_processes);
1325 synchronize_srcu(&kfd_processes_srcu);
1326 hlist_add_head(&p->kfd_processes, &cleanup_list);
1327 }
1328 mutex_unlock(&kfd_processes_mutex);
1329
1330 hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1331 kfd_process_notifier_release_internal(p);
1332
1333 /*
1334 * Ensures that all outstanding free_notifier get called, triggering
1335 * the release of the kfd_process struct.
1336 */
1337 mmu_notifier_synchronize();
1338 }
1339
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1340 int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1341 {
1342 unsigned long offset;
1343 int i;
1344
1345 if (p->has_cwsr)
1346 return 0;
1347
1348 for (i = 0; i < p->n_pdds; i++) {
1349 struct kfd_node *dev = p->pdds[i]->dev;
1350 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1351
1352 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1353 continue;
1354
1355 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1356 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1357 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1358 MAP_SHARED, offset);
1359
1360 if (IS_ERR_VALUE(qpd->tba_addr)) {
1361 int err = qpd->tba_addr;
1362
1363 dev_err(dev->adev->dev,
1364 "Failure to set tba address. error %d.\n", err);
1365 qpd->tba_addr = 0;
1366 qpd->cwsr_kaddr = NULL;
1367 return err;
1368 }
1369
1370 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1371
1372 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
1373
1374 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1375 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1376 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1377 }
1378
1379 p->has_cwsr = true;
1380
1381 return 0;
1382 }
1383
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1384 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1385 {
1386 struct kfd_node *dev = pdd->dev;
1387 struct qcm_process_device *qpd = &pdd->qpd;
1388 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1389 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1390 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1391 struct kgd_mem *mem;
1392 void *kaddr;
1393 int ret;
1394
1395 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1396 return 0;
1397
1398 /* cwsr_base is only set for dGPU */
1399 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1400 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1401 if (ret)
1402 return ret;
1403
1404 qpd->cwsr_mem = mem;
1405 qpd->cwsr_kaddr = kaddr;
1406 qpd->tba_addr = qpd->cwsr_base;
1407
1408 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1409
1410 kfd_process_set_trap_debug_flag(&pdd->qpd,
1411 pdd->process->debug_trap_enabled);
1412
1413 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1414 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1415 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1416
1417 return 0;
1418 }
1419
kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device * pdd)1420 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1421 {
1422 struct kfd_node *dev = pdd->dev;
1423 struct qcm_process_device *qpd = &pdd->qpd;
1424
1425 if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1426 return;
1427
1428 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
1429 }
1430
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1431 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1432 uint64_t tba_addr,
1433 uint64_t tma_addr)
1434 {
1435 if (qpd->cwsr_kaddr) {
1436 /* KFD trap handler is bound, record as second-level TBA/TMA
1437 * in first-level TMA. First-level trap will jump to second.
1438 */
1439 uint64_t *tma =
1440 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1441 tma[0] = tba_addr;
1442 tma[1] = tma_addr;
1443 } else {
1444 /* No trap handler bound, bind as first-level TBA/TMA. */
1445 qpd->tba_addr = tba_addr;
1446 qpd->tma_addr = tma_addr;
1447 }
1448 }
1449
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1450 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1451 {
1452 int i;
1453
1454 /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1455 * boot time retry setting. Mixing processes with different
1456 * XNACK/retry settings can hang the GPU.
1457 *
1458 * Different GPUs can have different noretry settings depending
1459 * on HW bugs or limitations. We need to find at least one
1460 * XNACK mode for this process that's compatible with all GPUs.
1461 * Fortunately GPUs with retry enabled (noretry=0) can run code
1462 * built for XNACK-off. On GFXv9 it may perform slower.
1463 *
1464 * Therefore applications built for XNACK-off can always be
1465 * supported and will be our fallback if any GPU does not
1466 * support retry.
1467 */
1468 for (i = 0; i < p->n_pdds; i++) {
1469 struct kfd_node *dev = p->pdds[i]->dev;
1470
1471 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1472 * support the SVM APIs and don't need to be considered
1473 * for the XNACK mode selection.
1474 */
1475 if (!KFD_IS_SOC15(dev))
1476 continue;
1477 /* Aldebaran can always support XNACK because it can support
1478 * per-process XNACK mode selection. But let the dev->noretry
1479 * setting still influence the default XNACK mode.
1480 */
1481 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1482 if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1483 pr_debug("SRIOV platform xnack not supported\n");
1484 return false;
1485 }
1486 continue;
1487 }
1488
1489 /* GFXv10 and later GPUs do not support shader preemption
1490 * during page faults. This can lead to poor QoS for queue
1491 * management and memory-manager-related preemptions or
1492 * even deadlocks.
1493 */
1494 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1495 return false;
1496
1497 if (dev->kfd->noretry)
1498 return false;
1499 }
1500
1501 return true;
1502 }
1503
kfd_process_set_trap_debug_flag(struct qcm_process_device * qpd,bool enabled)1504 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1505 bool enabled)
1506 {
1507 if (qpd->cwsr_kaddr) {
1508 uint64_t *tma =
1509 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1510 tma[2] = enabled;
1511 }
1512 }
1513
1514 /*
1515 * On return the kfd_process is fully operational and will be freed when the
1516 * mm is released
1517 */
create_process(const struct task_struct * thread)1518 static struct kfd_process *create_process(const struct task_struct *thread)
1519 {
1520 struct kfd_process *process;
1521 struct mmu_notifier *mn;
1522 int err = -ENOMEM;
1523
1524 process = kzalloc(sizeof(*process), GFP_KERNEL);
1525 if (!process)
1526 goto err_alloc_process;
1527
1528 kref_init(&process->ref);
1529 mutex_init(&process->mutex);
1530 process->mm = thread->mm;
1531 process->lead_thread = thread->group_leader;
1532 process->n_pdds = 0;
1533 process->queues_paused = false;
1534 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1535 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1536 process->last_restore_timestamp = get_jiffies_64();
1537 err = kfd_event_init_process(process);
1538 if (err)
1539 goto err_event_init;
1540 process->is_32bit_user_mode = in_compat_syscall();
1541 process->debug_trap_enabled = false;
1542 process->debugger_process = NULL;
1543 process->exception_enable_mask = 0;
1544 atomic_set(&process->debugged_process_count, 0);
1545 sema_init(&process->runtime_enable_sema, 0);
1546
1547 process->pasid = kfd_pasid_alloc();
1548 if (process->pasid == 0) {
1549 err = -ENOSPC;
1550 goto err_alloc_pasid;
1551 }
1552
1553 err = pqm_init(&process->pqm, process);
1554 if (err != 0)
1555 goto err_process_pqm_init;
1556
1557 /* init process apertures*/
1558 err = kfd_init_apertures(process);
1559 if (err != 0)
1560 goto err_init_apertures;
1561
1562 /* Check XNACK support after PDDs are created in kfd_init_apertures */
1563 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1564
1565 err = svm_range_list_init(process);
1566 if (err)
1567 goto err_init_svm_range_list;
1568
1569 /* alloc_notifier needs to find the process in the hash table */
1570 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1571 (uintptr_t)process->mm);
1572
1573 /* Avoid free_notifier to start kfd_process_wq_release if
1574 * mmu_notifier_get failed because of pending signal.
1575 */
1576 kref_get(&process->ref);
1577
1578 /* MMU notifier registration must be the last call that can fail
1579 * because after this point we cannot unwind the process creation.
1580 * After this point, mmu_notifier_put will trigger the cleanup by
1581 * dropping the last process reference in the free_notifier.
1582 */
1583 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1584 if (IS_ERR(mn)) {
1585 err = PTR_ERR(mn);
1586 goto err_register_notifier;
1587 }
1588 BUG_ON(mn != &process->mmu_notifier);
1589
1590 kfd_unref_process(process);
1591 get_task_struct(process->lead_thread);
1592
1593 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1594
1595 return process;
1596
1597 err_register_notifier:
1598 hash_del_rcu(&process->kfd_processes);
1599 svm_range_list_fini(process);
1600 err_init_svm_range_list:
1601 kfd_process_free_outstanding_kfd_bos(process);
1602 kfd_process_destroy_pdds(process);
1603 err_init_apertures:
1604 pqm_uninit(&process->pqm);
1605 err_process_pqm_init:
1606 kfd_pasid_free(process->pasid);
1607 err_alloc_pasid:
1608 kfd_event_free_process(process);
1609 err_event_init:
1610 mutex_destroy(&process->mutex);
1611 kfree(process);
1612 err_alloc_process:
1613 return ERR_PTR(err);
1614 }
1615
kfd_get_process_device_data(struct kfd_node * dev,struct kfd_process * p)1616 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1617 struct kfd_process *p)
1618 {
1619 int i;
1620
1621 for (i = 0; i < p->n_pdds; i++)
1622 if (p->pdds[i]->dev == dev)
1623 return p->pdds[i];
1624
1625 return NULL;
1626 }
1627
kfd_create_process_device_data(struct kfd_node * dev,struct kfd_process * p)1628 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1629 struct kfd_process *p)
1630 {
1631 struct kfd_process_device *pdd = NULL;
1632
1633 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1634 return NULL;
1635 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1636 if (!pdd)
1637 return NULL;
1638
1639 pdd->dev = dev;
1640 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1641 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1642 pdd->qpd.dqm = dev->dqm;
1643 pdd->qpd.pqm = &p->pqm;
1644 pdd->qpd.evicted = 0;
1645 pdd->qpd.mapped_gws_queue = false;
1646 pdd->process = p;
1647 pdd->bound = PDD_UNBOUND;
1648 pdd->already_dequeued = false;
1649 pdd->runtime_inuse = false;
1650 atomic64_set(&pdd->vram_usage, 0);
1651 pdd->sdma_past_activity_counter = 0;
1652 pdd->user_gpu_id = dev->id;
1653 atomic64_set(&pdd->evict_duration_counter, 0);
1654
1655 p->pdds[p->n_pdds++] = pdd;
1656 if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1657 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1658 pdd->dev->adev,
1659 false,
1660 0);
1661
1662 /* Init idr used for memory handle translation */
1663 idr_init(&pdd->alloc_idr);
1664
1665 return pdd;
1666 }
1667
1668 /**
1669 * kfd_process_device_init_vm - Initialize a VM for a process-device
1670 *
1671 * @pdd: The process-device
1672 * @drm_file: Optional pointer to a DRM file descriptor
1673 *
1674 * If @drm_file is specified, it will be used to acquire the VM from
1675 * that file descriptor. If successful, the @pdd takes ownership of
1676 * the file descriptor.
1677 *
1678 * If @drm_file is NULL, a new VM is created.
1679 *
1680 * Returns 0 on success, -errno on failure.
1681 */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1682 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1683 struct file *drm_file)
1684 {
1685 struct amdgpu_fpriv *drv_priv;
1686 struct amdgpu_vm *avm;
1687 struct kfd_process *p;
1688 struct dma_fence *ef;
1689 struct kfd_node *dev;
1690 int ret;
1691
1692 if (!drm_file)
1693 return -EINVAL;
1694
1695 if (pdd->drm_priv)
1696 return -EBUSY;
1697
1698 ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1699 if (ret)
1700 return ret;
1701 avm = &drv_priv->vm;
1702
1703 p = pdd->process;
1704 dev = pdd->dev;
1705
1706 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1707 &p->kgd_process_info,
1708 p->ef ? NULL : &ef);
1709 if (ret) {
1710 dev_err(dev->adev->dev, "Failed to create process VM object\n");
1711 return ret;
1712 }
1713
1714 if (!p->ef)
1715 RCU_INIT_POINTER(p->ef, ef);
1716
1717 pdd->drm_priv = drm_file->private_data;
1718
1719 ret = kfd_process_device_reserve_ib_mem(pdd);
1720 if (ret)
1721 goto err_reserve_ib_mem;
1722 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1723 if (ret)
1724 goto err_init_cwsr;
1725
1726 ret = amdgpu_amdkfd_gpuvm_set_vm_pasid(dev->adev, avm, p->pasid);
1727 if (ret)
1728 goto err_set_pasid;
1729
1730 pdd->drm_file = drm_file;
1731
1732 return 0;
1733
1734 err_set_pasid:
1735 kfd_process_device_destroy_cwsr_dgpu(pdd);
1736 err_init_cwsr:
1737 kfd_process_device_destroy_ib_mem(pdd);
1738 err_reserve_ib_mem:
1739 pdd->drm_priv = NULL;
1740 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1741
1742 return ret;
1743 }
1744
1745 /*
1746 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1747 * to the device.
1748 * Unbinding occurs when the process dies or the device is removed.
1749 *
1750 * Assumes that the process lock is held.
1751 */
kfd_bind_process_to_device(struct kfd_node * dev,struct kfd_process * p)1752 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1753 struct kfd_process *p)
1754 {
1755 struct kfd_process_device *pdd;
1756 int err;
1757
1758 pdd = kfd_get_process_device_data(dev, p);
1759 if (!pdd) {
1760 dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1761 return ERR_PTR(-ENOMEM);
1762 }
1763
1764 if (!pdd->drm_priv)
1765 return ERR_PTR(-ENODEV);
1766
1767 /*
1768 * signal runtime-pm system to auto resume and prevent
1769 * further runtime suspend once device pdd is created until
1770 * pdd is destroyed.
1771 */
1772 if (!pdd->runtime_inuse) {
1773 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1774 if (err < 0) {
1775 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1776 return ERR_PTR(err);
1777 }
1778 }
1779
1780 /*
1781 * make sure that runtime_usage counter is incremented just once
1782 * per pdd
1783 */
1784 pdd->runtime_inuse = true;
1785
1786 return pdd;
1787 }
1788
1789 /* Create specific handle mapped to mem from process local memory idr
1790 * Assumes that the process lock is held.
1791 */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1792 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1793 void *mem)
1794 {
1795 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1796 }
1797
1798 /* Translate specific handle from process local memory idr
1799 * Assumes that the process lock is held.
1800 */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1801 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1802 int handle)
1803 {
1804 if (handle < 0)
1805 return NULL;
1806
1807 return idr_find(&pdd->alloc_idr, handle);
1808 }
1809
1810 /* Remove specific handle from process local memory idr
1811 * Assumes that the process lock is held.
1812 */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1813 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1814 int handle)
1815 {
1816 if (handle >= 0)
1817 idr_remove(&pdd->alloc_idr, handle);
1818 }
1819
1820 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid)1821 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1822 {
1823 struct kfd_process *p, *ret_p = NULL;
1824 unsigned int temp;
1825
1826 int idx = srcu_read_lock(&kfd_processes_srcu);
1827
1828 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1829 if (p->pasid == pasid) {
1830 kref_get(&p->ref);
1831 ret_p = p;
1832 break;
1833 }
1834 }
1835
1836 srcu_read_unlock(&kfd_processes_srcu, idx);
1837
1838 return ret_p;
1839 }
1840
1841 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1842 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1843 {
1844 struct kfd_process *p;
1845
1846 int idx = srcu_read_lock(&kfd_processes_srcu);
1847
1848 p = find_process_by_mm(mm);
1849 if (p)
1850 kref_get(&p->ref);
1851
1852 srcu_read_unlock(&kfd_processes_srcu, idx);
1853
1854 return p;
1855 }
1856
1857 /* kfd_process_evict_queues - Evict all user queues of a process
1858 *
1859 * Eviction is reference-counted per process-device. This means multiple
1860 * evictions from different sources can be nested safely.
1861 */
kfd_process_evict_queues(struct kfd_process * p,uint32_t trigger)1862 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1863 {
1864 int r = 0;
1865 int i;
1866 unsigned int n_evicted = 0;
1867
1868 for (i = 0; i < p->n_pdds; i++) {
1869 struct kfd_process_device *pdd = p->pdds[i];
1870 struct device *dev = pdd->dev->adev->dev;
1871
1872 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
1873 trigger);
1874
1875 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1876 &pdd->qpd);
1877 /* evict return -EIO if HWS is hang or asic is resetting, in this case
1878 * we would like to set all the queues to be in evicted state to prevent
1879 * them been add back since they actually not be saved right now.
1880 */
1881 if (r && r != -EIO) {
1882 dev_err(dev, "Failed to evict process queues\n");
1883 goto fail;
1884 }
1885 n_evicted++;
1886
1887 pdd->dev->dqm->is_hws_hang = false;
1888 }
1889
1890 return r;
1891
1892 fail:
1893 /* To keep state consistent, roll back partial eviction by
1894 * restoring queues
1895 */
1896 for (i = 0; i < p->n_pdds; i++) {
1897 struct kfd_process_device *pdd = p->pdds[i];
1898
1899 if (n_evicted == 0)
1900 break;
1901
1902 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1903
1904 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1905 &pdd->qpd))
1906 dev_err(pdd->dev->adev->dev,
1907 "Failed to restore queues\n");
1908
1909 n_evicted--;
1910 }
1911
1912 return r;
1913 }
1914
1915 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1916 int kfd_process_restore_queues(struct kfd_process *p)
1917 {
1918 int r, ret = 0;
1919 int i;
1920
1921 for (i = 0; i < p->n_pdds; i++) {
1922 struct kfd_process_device *pdd = p->pdds[i];
1923 struct device *dev = pdd->dev->adev->dev;
1924
1925 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1926
1927 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1928 &pdd->qpd);
1929 if (r) {
1930 dev_err(dev, "Failed to restore process queues\n");
1931 if (!ret)
1932 ret = r;
1933 }
1934 }
1935
1936 return ret;
1937 }
1938
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1939 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1940 {
1941 int i;
1942
1943 for (i = 0; i < p->n_pdds; i++)
1944 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1945 return i;
1946 return -EINVAL;
1947 }
1948
1949 int
kfd_process_gpuid_from_node(struct kfd_process * p,struct kfd_node * node,uint32_t * gpuid,uint32_t * gpuidx)1950 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1951 uint32_t *gpuid, uint32_t *gpuidx)
1952 {
1953 int i;
1954
1955 for (i = 0; i < p->n_pdds; i++)
1956 if (p->pdds[i] && p->pdds[i]->dev == node) {
1957 *gpuid = p->pdds[i]->user_gpu_id;
1958 *gpuidx = i;
1959 return 0;
1960 }
1961 return -EINVAL;
1962 }
1963
signal_eviction_fence(struct kfd_process * p)1964 static int signal_eviction_fence(struct kfd_process *p)
1965 {
1966 struct dma_fence *ef;
1967 int ret;
1968
1969 rcu_read_lock();
1970 ef = dma_fence_get_rcu_safe(&p->ef);
1971 rcu_read_unlock();
1972 if (!ef)
1973 return -EINVAL;
1974
1975 ret = dma_fence_signal(ef);
1976 dma_fence_put(ef);
1977
1978 return ret;
1979 }
1980
evict_process_worker(struct work_struct * work)1981 static void evict_process_worker(struct work_struct *work)
1982 {
1983 int ret;
1984 struct kfd_process *p;
1985 struct delayed_work *dwork;
1986
1987 dwork = to_delayed_work(work);
1988
1989 /* Process termination destroys this worker thread. So during the
1990 * lifetime of this thread, kfd_process p will be valid
1991 */
1992 p = container_of(dwork, struct kfd_process, eviction_work);
1993
1994 pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1995 ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
1996 if (!ret) {
1997 /* If another thread already signaled the eviction fence,
1998 * they are responsible stopping the queues and scheduling
1999 * the restore work.
2000 */
2001 if (signal_eviction_fence(p) ||
2002 mod_delayed_work(kfd_restore_wq, &p->restore_work,
2003 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2004 kfd_process_restore_queues(p);
2005
2006 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
2007 } else
2008 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
2009 }
2010
restore_process_helper(struct kfd_process * p)2011 static int restore_process_helper(struct kfd_process *p)
2012 {
2013 int ret = 0;
2014
2015 /* VMs may not have been acquired yet during debugging. */
2016 if (p->kgd_process_info) {
2017 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2018 p->kgd_process_info, &p->ef);
2019 if (ret)
2020 return ret;
2021 }
2022
2023 ret = kfd_process_restore_queues(p);
2024 if (!ret)
2025 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
2026 else
2027 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
2028
2029 return ret;
2030 }
2031
restore_process_worker(struct work_struct * work)2032 static void restore_process_worker(struct work_struct *work)
2033 {
2034 struct delayed_work *dwork;
2035 struct kfd_process *p;
2036 int ret = 0;
2037
2038 dwork = to_delayed_work(work);
2039
2040 /* Process termination destroys this worker thread. So during the
2041 * lifetime of this thread, kfd_process p will be valid
2042 */
2043 p = container_of(dwork, struct kfd_process, restore_work);
2044 pr_debug("Started restoring pasid 0x%x\n", p->pasid);
2045
2046 /* Setting last_restore_timestamp before successful restoration.
2047 * Otherwise this would have to be set by KGD (restore_process_bos)
2048 * before KFD BOs are unreserved. If not, the process can be evicted
2049 * again before the timestamp is set.
2050 * If restore fails, the timestamp will be set again in the next
2051 * attempt. This would mean that the minimum GPU quanta would be
2052 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2053 * functions)
2054 */
2055
2056 p->last_restore_timestamp = get_jiffies_64();
2057
2058 ret = restore_process_helper(p);
2059 if (ret) {
2060 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
2061 p->pasid, PROCESS_BACK_OFF_TIME_MS);
2062 if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
2063 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2064 kfd_process_restore_queues(p);
2065 }
2066 }
2067
kfd_suspend_all_processes(void)2068 void kfd_suspend_all_processes(void)
2069 {
2070 struct kfd_process *p;
2071 unsigned int temp;
2072 int idx = srcu_read_lock(&kfd_processes_srcu);
2073
2074 WARN(debug_evictions, "Evicting all processes");
2075 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2076 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2077 pr_err("Failed to suspend process 0x%x\n", p->pasid);
2078 signal_eviction_fence(p);
2079 }
2080 srcu_read_unlock(&kfd_processes_srcu, idx);
2081 }
2082
kfd_resume_all_processes(void)2083 int kfd_resume_all_processes(void)
2084 {
2085 struct kfd_process *p;
2086 unsigned int temp;
2087 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2088
2089 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2090 if (restore_process_helper(p)) {
2091 pr_err("Restore process %d failed during resume\n",
2092 p->pasid);
2093 ret = -EFAULT;
2094 }
2095 }
2096 srcu_read_unlock(&kfd_processes_srcu, idx);
2097 return ret;
2098 }
2099
kfd_reserved_mem_mmap(struct kfd_node * dev,struct kfd_process * process,struct vm_area_struct * vma)2100 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2101 struct vm_area_struct *vma)
2102 {
2103 struct kfd_process_device *pdd;
2104 struct qcm_process_device *qpd;
2105
2106 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2107 dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
2108 return -EINVAL;
2109 }
2110
2111 pdd = kfd_get_process_device_data(dev, process);
2112 if (!pdd)
2113 return -EINVAL;
2114 qpd = &pdd->qpd;
2115
2116 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2117 get_order(KFD_CWSR_TBA_TMA_SIZE));
2118 if (!qpd->cwsr_kaddr) {
2119 dev_err(dev->adev->dev,
2120 "Error allocating per process CWSR buffer.\n");
2121 return -ENOMEM;
2122 }
2123
2124 vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2125 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2126 /* Mapping pages to user process */
2127 return remap_pfn_range(vma, vma->vm_start,
2128 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2129 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2130 }
2131
2132 /* assumes caller holds process lock. */
kfd_process_drain_interrupts(struct kfd_process_device * pdd)2133 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2134 {
2135 uint32_t irq_drain_fence[8];
2136 uint8_t node_id = 0;
2137 int r = 0;
2138
2139 if (!KFD_IS_SOC15(pdd->dev))
2140 return 0;
2141
2142 pdd->process->irq_drain_is_open = true;
2143
2144 memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2145 irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2146 KFD_IRQ_FENCE_CLIENTID;
2147 irq_drain_fence[3] = pdd->process->pasid;
2148
2149 /*
2150 * For GFX 9.4.3, send the NodeId also in IH cookie DW[3]
2151 */
2152 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2153 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4)) {
2154 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2155 irq_drain_fence[3] |= node_id << 16;
2156 }
2157
2158 /* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2159 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2160 irq_drain_fence)) {
2161 pdd->process->irq_drain_is_open = false;
2162 return 0;
2163 }
2164
2165 r = wait_event_interruptible(pdd->process->wait_irq_drain,
2166 !READ_ONCE(pdd->process->irq_drain_is_open));
2167 if (r)
2168 pdd->process->irq_drain_is_open = false;
2169
2170 return r;
2171 }
2172
kfd_process_close_interrupt_drain(unsigned int pasid)2173 void kfd_process_close_interrupt_drain(unsigned int pasid)
2174 {
2175 struct kfd_process *p;
2176
2177 p = kfd_lookup_process_by_pasid(pasid);
2178
2179 if (!p)
2180 return;
2181
2182 WRITE_ONCE(p->irq_drain_is_open, false);
2183 wake_up_all(&p->wait_irq_drain);
2184 kfd_unref_process(p);
2185 }
2186
2187 struct send_exception_work_handler_workarea {
2188 struct work_struct work;
2189 struct kfd_process *p;
2190 unsigned int queue_id;
2191 uint64_t error_reason;
2192 };
2193
send_exception_work_handler(struct work_struct * work)2194 static void send_exception_work_handler(struct work_struct *work)
2195 {
2196 struct send_exception_work_handler_workarea *workarea;
2197 struct kfd_process *p;
2198 struct queue *q;
2199 struct mm_struct *mm;
2200 struct kfd_context_save_area_header __user *csa_header;
2201 uint64_t __user *err_payload_ptr;
2202 uint64_t cur_err;
2203 uint32_t ev_id;
2204
2205 workarea = container_of(work,
2206 struct send_exception_work_handler_workarea,
2207 work);
2208 p = workarea->p;
2209
2210 mm = get_task_mm(p->lead_thread);
2211
2212 if (!mm)
2213 return;
2214
2215 kthread_use_mm(mm);
2216
2217 q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2218
2219 if (!q)
2220 goto out;
2221
2222 csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2223
2224 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2225 get_user(cur_err, err_payload_ptr);
2226 cur_err |= workarea->error_reason;
2227 put_user(cur_err, err_payload_ptr);
2228 get_user(ev_id, &csa_header->err_event_id);
2229
2230 kfd_set_event(p, ev_id);
2231
2232 out:
2233 kthread_unuse_mm(mm);
2234 mmput(mm);
2235 }
2236
kfd_send_exception_to_runtime(struct kfd_process * p,unsigned int queue_id,uint64_t error_reason)2237 int kfd_send_exception_to_runtime(struct kfd_process *p,
2238 unsigned int queue_id,
2239 uint64_t error_reason)
2240 {
2241 struct send_exception_work_handler_workarea worker;
2242
2243 INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2244
2245 worker.p = p;
2246 worker.queue_id = queue_id;
2247 worker.error_reason = error_reason;
2248
2249 schedule_work(&worker.work);
2250 flush_work(&worker.work);
2251 destroy_work_on_stack(&worker.work);
2252
2253 return 0;
2254 }
2255
kfd_process_device_data_by_id(struct kfd_process * p,uint32_t gpu_id)2256 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2257 {
2258 int i;
2259
2260 if (gpu_id) {
2261 for (i = 0; i < p->n_pdds; i++) {
2262 struct kfd_process_device *pdd = p->pdds[i];
2263
2264 if (pdd->user_gpu_id == gpu_id)
2265 return pdd;
2266 }
2267 }
2268 return NULL;
2269 }
2270
kfd_process_get_user_gpu_id(struct kfd_process * p,uint32_t actual_gpu_id)2271 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2272 {
2273 int i;
2274
2275 if (!actual_gpu_id)
2276 return 0;
2277
2278 for (i = 0; i < p->n_pdds; i++) {
2279 struct kfd_process_device *pdd = p->pdds[i];
2280
2281 if (pdd->dev->id == actual_gpu_id)
2282 return pdd->user_gpu_id;
2283 }
2284 return -EINVAL;
2285 }
2286
2287 #if defined(CONFIG_DEBUG_FS)
2288
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)2289 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2290 {
2291 struct kfd_process *p;
2292 unsigned int temp;
2293 int r = 0;
2294
2295 int idx = srcu_read_lock(&kfd_processes_srcu);
2296
2297 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2298 seq_printf(m, "Process %d PASID 0x%x:\n",
2299 p->lead_thread->tgid, p->pasid);
2300
2301 mutex_lock(&p->mutex);
2302 r = pqm_debugfs_mqds(m, &p->pqm);
2303 mutex_unlock(&p->mutex);
2304
2305 if (r)
2306 break;
2307 }
2308
2309 srcu_read_unlock(&kfd_processes_srcu, idx);
2310
2311 return r;
2312 }
2313
2314 #endif
2315