1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <linux/mmu_context.h>
29 #include <linux/slab.h>
30 #include <linux/amd-iommu.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38
39 struct mm_struct;
40
41 #include "kfd_priv.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_dbgmgr.h"
44 #include "kfd_iommu.h"
45 #include "kfd_svm.h"
46
47 /*
48 * List of struct kfd_process (field kfd_process).
49 * Unique/indexed by mm_struct*
50 */
51 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
52 static DEFINE_MUTEX(kfd_processes_mutex);
53
54 DEFINE_SRCU(kfd_processes_srcu);
55
56 /* For process termination handling */
57 static struct workqueue_struct *kfd_process_wq;
58
59 /* Ordered, single-threaded workqueue for restoring evicted
60 * processes. Restoring multiple processes concurrently under memory
61 * pressure can lead to processes blocking each other from validating
62 * their BOs and result in a live-lock situation where processes
63 * remain evicted indefinitely.
64 */
65 static struct workqueue_struct *kfd_restore_wq;
66
67 static struct kfd_process *find_process(const struct task_struct *thread);
68 static void kfd_process_ref_release(struct kref *ref);
69 static struct kfd_process *create_process(const struct task_struct *thread);
70 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
71
72 static void evict_process_worker(struct work_struct *work);
73 static void restore_process_worker(struct work_struct *work);
74
75 struct kfd_procfs_tree {
76 struct kobject *kobj;
77 };
78
79 static struct kfd_procfs_tree procfs;
80
81 /*
82 * Structure for SDMA activity tracking
83 */
84 struct kfd_sdma_activity_handler_workarea {
85 struct work_struct sdma_activity_work;
86 struct kfd_process_device *pdd;
87 uint64_t sdma_activity_counter;
88 };
89
90 struct temp_sdma_queue_list {
91 uint64_t __user *rptr;
92 uint64_t sdma_val;
93 unsigned int queue_id;
94 struct list_head list;
95 };
96
kfd_sdma_activity_worker(struct work_struct * work)97 static void kfd_sdma_activity_worker(struct work_struct *work)
98 {
99 struct kfd_sdma_activity_handler_workarea *workarea;
100 struct kfd_process_device *pdd;
101 uint64_t val;
102 struct mm_struct *mm;
103 struct queue *q;
104 struct qcm_process_device *qpd;
105 struct device_queue_manager *dqm;
106 int ret = 0;
107 struct temp_sdma_queue_list sdma_q_list;
108 struct temp_sdma_queue_list *sdma_q, *next;
109
110 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
111 sdma_activity_work);
112
113 pdd = workarea->pdd;
114 if (!pdd)
115 return;
116 dqm = pdd->dev->dqm;
117 qpd = &pdd->qpd;
118 if (!dqm || !qpd)
119 return;
120 /*
121 * Total SDMA activity is current SDMA activity + past SDMA activity
122 * Past SDMA count is stored in pdd.
123 * To get the current activity counters for all active SDMA queues,
124 * we loop over all SDMA queues and get their counts from user-space.
125 *
126 * We cannot call get_user() with dqm_lock held as it can cause
127 * a circular lock dependency situation. To read the SDMA stats,
128 * we need to do the following:
129 *
130 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
131 * with dqm_lock/dqm_unlock().
132 * 2. Call get_user() for each node in temporary list without dqm_lock.
133 * Save the SDMA count for each node and also add the count to the total
134 * SDMA count counter.
135 * Its possible, during this step, a few SDMA queue nodes got deleted
136 * from the qpd->queues_list.
137 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
138 * If any node got deleted, its SDMA count would be captured in the sdma
139 * past activity counter. So subtract the SDMA counter stored in step 2
140 * for this node from the total SDMA count.
141 */
142 INIT_LIST_HEAD(&sdma_q_list.list);
143
144 /*
145 * Create the temp list of all SDMA queues
146 */
147 dqm_lock(dqm);
148
149 list_for_each_entry(q, &qpd->queues_list, list) {
150 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
151 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
152 continue;
153
154 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
155 if (!sdma_q) {
156 dqm_unlock(dqm);
157 goto cleanup;
158 }
159
160 INIT_LIST_HEAD(&sdma_q->list);
161 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
162 sdma_q->queue_id = q->properties.queue_id;
163 list_add_tail(&sdma_q->list, &sdma_q_list.list);
164 }
165
166 /*
167 * If the temp list is empty, then no SDMA queues nodes were found in
168 * qpd->queues_list. Return the past activity count as the total sdma
169 * count
170 */
171 if (list_empty(&sdma_q_list.list)) {
172 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
173 dqm_unlock(dqm);
174 return;
175 }
176
177 dqm_unlock(dqm);
178
179 /*
180 * Get the usage count for each SDMA queue in temp_list.
181 */
182 mm = get_task_mm(pdd->process->lead_thread);
183 if (!mm)
184 goto cleanup;
185
186 kthread_use_mm(mm);
187
188 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
189 val = 0;
190 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
191 if (ret) {
192 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
193 sdma_q->queue_id);
194 } else {
195 sdma_q->sdma_val = val;
196 workarea->sdma_activity_counter += val;
197 }
198 }
199
200 kthread_unuse_mm(mm);
201 mmput(mm);
202
203 /*
204 * Do a second iteration over qpd_queues_list to check if any SDMA
205 * nodes got deleted while fetching SDMA counter.
206 */
207 dqm_lock(dqm);
208
209 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
210
211 list_for_each_entry(q, &qpd->queues_list, list) {
212 if (list_empty(&sdma_q_list.list))
213 break;
214
215 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
216 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
217 continue;
218
219 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
220 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
221 (sdma_q->queue_id == q->properties.queue_id)) {
222 list_del(&sdma_q->list);
223 kfree(sdma_q);
224 break;
225 }
226 }
227 }
228
229 dqm_unlock(dqm);
230
231 /*
232 * If temp list is not empty, it implies some queues got deleted
233 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
234 * count for each node from the total SDMA count.
235 */
236 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
237 workarea->sdma_activity_counter -= sdma_q->sdma_val;
238 list_del(&sdma_q->list);
239 kfree(sdma_q);
240 }
241
242 return;
243
244 cleanup:
245 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
246 list_del(&sdma_q->list);
247 kfree(sdma_q);
248 }
249 }
250
251 /**
252 * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
253 * by current process. Translates acquired wave count into number of compute units
254 * that are occupied.
255 *
256 * @atr: Handle of attribute that allows reporting of wave count. The attribute
257 * handle encapsulates GPU device it is associated with, thereby allowing collection
258 * of waves in flight, etc
259 *
260 * @buffer: Handle of user provided buffer updated with wave count
261 *
262 * Return: Number of bytes written to user buffer or an error value
263 */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)264 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
265 {
266 int cu_cnt;
267 int wave_cnt;
268 int max_waves_per_cu;
269 struct kfd_dev *dev = NULL;
270 struct kfd_process *proc = NULL;
271 struct kfd_process_device *pdd = NULL;
272
273 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
274 dev = pdd->dev;
275 if (dev->kfd2kgd->get_cu_occupancy == NULL)
276 return -EINVAL;
277
278 cu_cnt = 0;
279 proc = pdd->process;
280 if (pdd->qpd.queue_count == 0) {
281 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
282 dev->id, proc->pasid);
283 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
284 }
285
286 /* Collect wave count from device if it supports */
287 wave_cnt = 0;
288 max_waves_per_cu = 0;
289 dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
290 &max_waves_per_cu);
291
292 /* Translate wave count to number of compute units */
293 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
294 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
295 }
296
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)297 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
298 char *buffer)
299 {
300 if (strcmp(attr->name, "pasid") == 0) {
301 struct kfd_process *p = container_of(attr, struct kfd_process,
302 attr_pasid);
303
304 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
305 } else if (strncmp(attr->name, "vram_", 5) == 0) {
306 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
307 attr_vram);
308 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
309 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
310 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
311 attr_sdma);
312 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
313
314 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
315 kfd_sdma_activity_worker);
316
317 sdma_activity_work_handler.pdd = pdd;
318 sdma_activity_work_handler.sdma_activity_counter = 0;
319
320 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
321
322 flush_work(&sdma_activity_work_handler.sdma_activity_work);
323
324 return snprintf(buffer, PAGE_SIZE, "%llu\n",
325 (sdma_activity_work_handler.sdma_activity_counter)/
326 SDMA_ACTIVITY_DIVISOR);
327 } else {
328 pr_err("Invalid attribute");
329 return -EINVAL;
330 }
331
332 return 0;
333 }
334
kfd_procfs_kobj_release(struct kobject * kobj)335 static void kfd_procfs_kobj_release(struct kobject *kobj)
336 {
337 kfree(kobj);
338 }
339
340 static const struct sysfs_ops kfd_procfs_ops = {
341 .show = kfd_procfs_show,
342 };
343
344 static struct kobj_type procfs_type = {
345 .release = kfd_procfs_kobj_release,
346 .sysfs_ops = &kfd_procfs_ops,
347 };
348
kfd_procfs_init(void)349 void kfd_procfs_init(void)
350 {
351 int ret = 0;
352
353 procfs.kobj = kfd_alloc_struct(procfs.kobj);
354 if (!procfs.kobj)
355 return;
356
357 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
358 &kfd_device->kobj, "proc");
359 if (ret) {
360 pr_warn("Could not create procfs proc folder");
361 /* If we fail to create the procfs, clean up */
362 kfd_procfs_shutdown();
363 }
364 }
365
kfd_procfs_shutdown(void)366 void kfd_procfs_shutdown(void)
367 {
368 if (procfs.kobj) {
369 kobject_del(procfs.kobj);
370 kobject_put(procfs.kobj);
371 procfs.kobj = NULL;
372 }
373 }
374
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)375 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
376 struct attribute *attr, char *buffer)
377 {
378 struct queue *q = container_of(kobj, struct queue, kobj);
379
380 if (!strcmp(attr->name, "size"))
381 return snprintf(buffer, PAGE_SIZE, "%llu",
382 q->properties.queue_size);
383 else if (!strcmp(attr->name, "type"))
384 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
385 else if (!strcmp(attr->name, "gpuid"))
386 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
387 else
388 pr_err("Invalid attribute");
389
390 return 0;
391 }
392
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)393 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
394 struct attribute *attr, char *buffer)
395 {
396 if (strcmp(attr->name, "evicted_ms") == 0) {
397 struct kfd_process_device *pdd = container_of(attr,
398 struct kfd_process_device,
399 attr_evict);
400 uint64_t evict_jiffies;
401
402 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
403
404 return snprintf(buffer,
405 PAGE_SIZE,
406 "%llu\n",
407 jiffies64_to_msecs(evict_jiffies));
408
409 /* Sysfs handle that gets CU occupancy is per device */
410 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
411 return kfd_get_cu_occupancy(attr, buffer);
412 } else {
413 pr_err("Invalid attribute");
414 }
415
416 return 0;
417 }
418
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)419 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
420 struct attribute *attr, char *buf)
421 {
422 struct kfd_process_device *pdd;
423
424 if (!strcmp(attr->name, "faults")) {
425 pdd = container_of(attr, struct kfd_process_device,
426 attr_faults);
427 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
428 }
429 if (!strcmp(attr->name, "page_in")) {
430 pdd = container_of(attr, struct kfd_process_device,
431 attr_page_in);
432 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
433 }
434 if (!strcmp(attr->name, "page_out")) {
435 pdd = container_of(attr, struct kfd_process_device,
436 attr_page_out);
437 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
438 }
439 return 0;
440 }
441
442 static struct attribute attr_queue_size = {
443 .name = "size",
444 .mode = KFD_SYSFS_FILE_MODE
445 };
446
447 static struct attribute attr_queue_type = {
448 .name = "type",
449 .mode = KFD_SYSFS_FILE_MODE
450 };
451
452 static struct attribute attr_queue_gpuid = {
453 .name = "gpuid",
454 .mode = KFD_SYSFS_FILE_MODE
455 };
456
457 static struct attribute *procfs_queue_attrs[] = {
458 &attr_queue_size,
459 &attr_queue_type,
460 &attr_queue_gpuid,
461 NULL
462 };
463
464 static const struct sysfs_ops procfs_queue_ops = {
465 .show = kfd_procfs_queue_show,
466 };
467
468 static struct kobj_type procfs_queue_type = {
469 .sysfs_ops = &procfs_queue_ops,
470 .default_attrs = procfs_queue_attrs,
471 };
472
473 static const struct sysfs_ops procfs_stats_ops = {
474 .show = kfd_procfs_stats_show,
475 };
476
477 static struct kobj_type procfs_stats_type = {
478 .sysfs_ops = &procfs_stats_ops,
479 .release = kfd_procfs_kobj_release,
480 };
481
482 static const struct sysfs_ops sysfs_counters_ops = {
483 .show = kfd_sysfs_counters_show,
484 };
485
486 static struct kobj_type sysfs_counters_type = {
487 .sysfs_ops = &sysfs_counters_ops,
488 .release = kfd_procfs_kobj_release,
489 };
490
kfd_procfs_add_queue(struct queue * q)491 int kfd_procfs_add_queue(struct queue *q)
492 {
493 struct kfd_process *proc;
494 int ret;
495
496 if (!q || !q->process)
497 return -EINVAL;
498 proc = q->process;
499
500 /* Create proc/<pid>/queues/<queue id> folder */
501 if (!proc->kobj_queues)
502 return -EFAULT;
503 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
504 proc->kobj_queues, "%u", q->properties.queue_id);
505 if (ret < 0) {
506 pr_warn("Creating proc/<pid>/queues/%u failed",
507 q->properties.queue_id);
508 kobject_put(&q->kobj);
509 return ret;
510 }
511
512 return 0;
513 }
514
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)515 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
516 char *name)
517 {
518 int ret;
519
520 if (!kobj || !attr || !name)
521 return;
522
523 attr->name = name;
524 attr->mode = KFD_SYSFS_FILE_MODE;
525 sysfs_attr_init(attr);
526
527 ret = sysfs_create_file(kobj, attr);
528 if (ret)
529 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
530 }
531
kfd_procfs_add_sysfs_stats(struct kfd_process * p)532 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
533 {
534 int ret;
535 int i;
536 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
537
538 if (!p || !p->kobj)
539 return;
540
541 /*
542 * Create sysfs files for each GPU:
543 * - proc/<pid>/stats_<gpuid>/
544 * - proc/<pid>/stats_<gpuid>/evicted_ms
545 * - proc/<pid>/stats_<gpuid>/cu_occupancy
546 */
547 for (i = 0; i < p->n_pdds; i++) {
548 struct kfd_process_device *pdd = p->pdds[i];
549
550 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
551 "stats_%u", pdd->dev->id);
552 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
553 if (!pdd->kobj_stats)
554 return;
555
556 ret = kobject_init_and_add(pdd->kobj_stats,
557 &procfs_stats_type,
558 p->kobj,
559 stats_dir_filename);
560
561 if (ret) {
562 pr_warn("Creating KFD proc/stats_%s folder failed",
563 stats_dir_filename);
564 kobject_put(pdd->kobj_stats);
565 pdd->kobj_stats = NULL;
566 return;
567 }
568
569 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
570 "evicted_ms");
571 /* Add sysfs file to report compute unit occupancy */
572 if (pdd->dev->kfd2kgd->get_cu_occupancy)
573 kfd_sysfs_create_file(pdd->kobj_stats,
574 &pdd->attr_cu_occupancy,
575 "cu_occupancy");
576 }
577 }
578
kfd_procfs_add_sysfs_counters(struct kfd_process * p)579 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
580 {
581 int ret = 0;
582 int i;
583 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
584
585 if (!p || !p->kobj)
586 return;
587
588 /*
589 * Create sysfs files for each GPU which supports SVM
590 * - proc/<pid>/counters_<gpuid>/
591 * - proc/<pid>/counters_<gpuid>/faults
592 * - proc/<pid>/counters_<gpuid>/page_in
593 * - proc/<pid>/counters_<gpuid>/page_out
594 */
595 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
596 struct kfd_process_device *pdd = p->pdds[i];
597 struct kobject *kobj_counters;
598
599 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
600 "counters_%u", pdd->dev->id);
601 kobj_counters = kfd_alloc_struct(kobj_counters);
602 if (!kobj_counters)
603 return;
604
605 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
606 p->kobj, counters_dir_filename);
607 if (ret) {
608 pr_warn("Creating KFD proc/%s folder failed",
609 counters_dir_filename);
610 kobject_put(kobj_counters);
611 return;
612 }
613
614 pdd->kobj_counters = kobj_counters;
615 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
616 "faults");
617 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
618 "page_in");
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
620 "page_out");
621 }
622 }
623
kfd_procfs_add_sysfs_files(struct kfd_process * p)624 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
625 {
626 int i;
627
628 if (!p || !p->kobj)
629 return;
630
631 /*
632 * Create sysfs files for each GPU:
633 * - proc/<pid>/vram_<gpuid>
634 * - proc/<pid>/sdma_<gpuid>
635 */
636 for (i = 0; i < p->n_pdds; i++) {
637 struct kfd_process_device *pdd = p->pdds[i];
638
639 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
640 pdd->dev->id);
641 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
642 pdd->vram_filename);
643
644 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
645 pdd->dev->id);
646 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
647 pdd->sdma_filename);
648 }
649 }
650
kfd_procfs_del_queue(struct queue * q)651 void kfd_procfs_del_queue(struct queue *q)
652 {
653 if (!q)
654 return;
655
656 kobject_del(&q->kobj);
657 kobject_put(&q->kobj);
658 }
659
kfd_process_create_wq(void)660 int kfd_process_create_wq(void)
661 {
662 if (!kfd_process_wq)
663 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
664 if (!kfd_restore_wq)
665 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
666
667 if (!kfd_process_wq || !kfd_restore_wq) {
668 kfd_process_destroy_wq();
669 return -ENOMEM;
670 }
671
672 return 0;
673 }
674
kfd_process_destroy_wq(void)675 void kfd_process_destroy_wq(void)
676 {
677 if (kfd_process_wq) {
678 destroy_workqueue(kfd_process_wq);
679 kfd_process_wq = NULL;
680 }
681 if (kfd_restore_wq) {
682 destroy_workqueue(kfd_restore_wq);
683 kfd_restore_wq = NULL;
684 }
685 }
686
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd)687 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
688 struct kfd_process_device *pdd)
689 {
690 struct kfd_dev *dev = pdd->dev;
691
692 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
693 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
694 NULL);
695 }
696
697 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
698 * This function should be only called right after the process
699 * is created and when kfd_processes_mutex is still being held
700 * to avoid concurrency. Because of that exclusiveness, we do
701 * not need to take p->mutex.
702 */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,void ** kptr)703 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
704 uint64_t gpu_va, uint32_t size,
705 uint32_t flags, void **kptr)
706 {
707 struct kfd_dev *kdev = pdd->dev;
708 struct kgd_mem *mem = NULL;
709 int handle;
710 int err;
711
712 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
713 pdd->drm_priv, &mem, NULL, flags);
714 if (err)
715 goto err_alloc_mem;
716
717 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem,
718 pdd->drm_priv, NULL);
719 if (err)
720 goto err_map_mem;
721
722 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
723 if (err) {
724 pr_debug("Sync memory failed, wait interrupted by user signal\n");
725 goto sync_memory_failed;
726 }
727
728 /* Create an obj handle so kfd_process_device_remove_obj_handle
729 * will take care of the bo removal when the process finishes.
730 * We do not need to take p->mutex, because the process is just
731 * created and the ioctls have not had the chance to run.
732 */
733 handle = kfd_process_device_create_obj_handle(pdd, mem);
734
735 if (handle < 0) {
736 err = handle;
737 goto free_gpuvm;
738 }
739
740 if (kptr) {
741 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
742 (struct kgd_mem *)mem, kptr, NULL);
743 if (err) {
744 pr_debug("Map GTT BO to kernel failed\n");
745 goto free_obj_handle;
746 }
747 }
748
749 return err;
750
751 free_obj_handle:
752 kfd_process_device_remove_obj_handle(pdd, handle);
753 free_gpuvm:
754 sync_memory_failed:
755 kfd_process_free_gpuvm(mem, pdd);
756 return err;
757
758 err_map_mem:
759 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,
760 NULL);
761 err_alloc_mem:
762 *kptr = NULL;
763 return err;
764 }
765
766 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
767 * process for IB usage The memory reserved is for KFD to submit
768 * IB to AMDGPU from kernel. If the memory is reserved
769 * successfully, ib_kaddr will have the CPU/kernel
770 * address. Check ib_kaddr before accessing the memory.
771 */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)772 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
773 {
774 struct qcm_process_device *qpd = &pdd->qpd;
775 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
776 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
777 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
778 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
779 void *kaddr;
780 int ret;
781
782 if (qpd->ib_kaddr || !qpd->ib_base)
783 return 0;
784
785 /* ib_base is only set for dGPU */
786 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
787 &kaddr);
788 if (ret)
789 return ret;
790
791 qpd->ib_kaddr = kaddr;
792
793 return 0;
794 }
795
kfd_create_process(struct file * filep)796 struct kfd_process *kfd_create_process(struct file *filep)
797 {
798 struct kfd_process *process;
799 struct task_struct *thread = current;
800 int ret;
801
802 if (!thread->mm)
803 return ERR_PTR(-EINVAL);
804
805 /* Only the pthreads threading model is supported. */
806 if (thread->group_leader->mm != thread->mm)
807 return ERR_PTR(-EINVAL);
808
809 /*
810 * take kfd processes mutex before starting of process creation
811 * so there won't be a case where two threads of the same process
812 * create two kfd_process structures
813 */
814 mutex_lock(&kfd_processes_mutex);
815
816 /* A prior open of /dev/kfd could have already created the process. */
817 process = find_process(thread);
818 if (process) {
819 pr_debug("Process already found\n");
820 } else {
821 process = create_process(thread);
822 if (IS_ERR(process))
823 goto out;
824
825 ret = kfd_process_init_cwsr_apu(process, filep);
826 if (ret)
827 goto out_destroy;
828
829 if (!procfs.kobj)
830 goto out;
831
832 process->kobj = kfd_alloc_struct(process->kobj);
833 if (!process->kobj) {
834 pr_warn("Creating procfs kobject failed");
835 goto out;
836 }
837 ret = kobject_init_and_add(process->kobj, &procfs_type,
838 procfs.kobj, "%d",
839 (int)process->lead_thread->pid);
840 if (ret) {
841 pr_warn("Creating procfs pid directory failed");
842 kobject_put(process->kobj);
843 goto out;
844 }
845
846 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
847 "pasid");
848
849 process->kobj_queues = kobject_create_and_add("queues",
850 process->kobj);
851 if (!process->kobj_queues)
852 pr_warn("Creating KFD proc/queues folder failed");
853
854 kfd_procfs_add_sysfs_stats(process);
855 kfd_procfs_add_sysfs_files(process);
856 kfd_procfs_add_sysfs_counters(process);
857 }
858 out:
859 if (!IS_ERR(process))
860 kref_get(&process->ref);
861 mutex_unlock(&kfd_processes_mutex);
862
863 return process;
864
865 out_destroy:
866 hash_del_rcu(&process->kfd_processes);
867 mutex_unlock(&kfd_processes_mutex);
868 synchronize_srcu(&kfd_processes_srcu);
869 /* kfd_process_free_notifier will trigger the cleanup */
870 mmu_notifier_put(&process->mmu_notifier);
871 return ERR_PTR(ret);
872 }
873
kfd_get_process(const struct task_struct * thread)874 struct kfd_process *kfd_get_process(const struct task_struct *thread)
875 {
876 struct kfd_process *process;
877
878 if (!thread->mm)
879 return ERR_PTR(-EINVAL);
880
881 /* Only the pthreads threading model is supported. */
882 if (thread->group_leader->mm != thread->mm)
883 return ERR_PTR(-EINVAL);
884
885 process = find_process(thread);
886 if (!process)
887 return ERR_PTR(-EINVAL);
888
889 return process;
890 }
891
find_process_by_mm(const struct mm_struct * mm)892 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
893 {
894 struct kfd_process *process;
895
896 hash_for_each_possible_rcu(kfd_processes_table, process,
897 kfd_processes, (uintptr_t)mm)
898 if (process->mm == mm)
899 return process;
900
901 return NULL;
902 }
903
find_process(const struct task_struct * thread)904 static struct kfd_process *find_process(const struct task_struct *thread)
905 {
906 struct kfd_process *p;
907 int idx;
908
909 idx = srcu_read_lock(&kfd_processes_srcu);
910 p = find_process_by_mm(thread->mm);
911 srcu_read_unlock(&kfd_processes_srcu, idx);
912
913 return p;
914 }
915
kfd_unref_process(struct kfd_process * p)916 void kfd_unref_process(struct kfd_process *p)
917 {
918 kref_put(&p->ref, kfd_process_ref_release);
919 }
920
921
kfd_process_device_free_bos(struct kfd_process_device * pdd)922 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
923 {
924 struct kfd_process *p = pdd->process;
925 void *mem;
926 int id;
927 int i;
928
929 /*
930 * Remove all handles from idr and release appropriate
931 * local memory object
932 */
933 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
934
935 for (i = 0; i < p->n_pdds; i++) {
936 struct kfd_process_device *peer_pdd = p->pdds[i];
937
938 if (!peer_pdd->drm_priv)
939 continue;
940 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
941 peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
942 }
943
944 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
945 pdd->drm_priv, NULL);
946 kfd_process_device_remove_obj_handle(pdd, id);
947 }
948 }
949
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)950 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
951 {
952 int i;
953
954 for (i = 0; i < p->n_pdds; i++)
955 kfd_process_device_free_bos(p->pdds[i]);
956 }
957
kfd_process_destroy_pdds(struct kfd_process * p)958 static void kfd_process_destroy_pdds(struct kfd_process *p)
959 {
960 int i;
961
962 for (i = 0; i < p->n_pdds; i++) {
963 struct kfd_process_device *pdd = p->pdds[i];
964
965 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
966 pdd->dev->id, p->pasid);
967
968 if (pdd->drm_file) {
969 amdgpu_amdkfd_gpuvm_release_process_vm(
970 pdd->dev->kgd, pdd->drm_priv);
971 fput(pdd->drm_file);
972 }
973
974 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
975 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
976 get_order(KFD_CWSR_TBA_TMA_SIZE));
977
978 kfree(pdd->qpd.doorbell_bitmap);
979 idr_destroy(&pdd->alloc_idr);
980
981 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
982
983 /*
984 * before destroying pdd, make sure to report availability
985 * for auto suspend
986 */
987 if (pdd->runtime_inuse) {
988 pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
989 pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
990 pdd->runtime_inuse = false;
991 }
992
993 kfree(pdd);
994 p->pdds[i] = NULL;
995 }
996 p->n_pdds = 0;
997 }
998
kfd_process_remove_sysfs(struct kfd_process * p)999 static void kfd_process_remove_sysfs(struct kfd_process *p)
1000 {
1001 struct kfd_process_device *pdd;
1002 int i;
1003
1004 if (!p->kobj)
1005 return;
1006
1007 sysfs_remove_file(p->kobj, &p->attr_pasid);
1008 kobject_del(p->kobj_queues);
1009 kobject_put(p->kobj_queues);
1010 p->kobj_queues = NULL;
1011
1012 for (i = 0; i < p->n_pdds; i++) {
1013 pdd = p->pdds[i];
1014
1015 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1016 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1017
1018 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1019 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1020 sysfs_remove_file(pdd->kobj_stats,
1021 &pdd->attr_cu_occupancy);
1022 kobject_del(pdd->kobj_stats);
1023 kobject_put(pdd->kobj_stats);
1024 pdd->kobj_stats = NULL;
1025 }
1026
1027 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1028 pdd = p->pdds[i];
1029
1030 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1031 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1032 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1033 kobject_del(pdd->kobj_counters);
1034 kobject_put(pdd->kobj_counters);
1035 pdd->kobj_counters = NULL;
1036 }
1037
1038 kobject_del(p->kobj);
1039 kobject_put(p->kobj);
1040 p->kobj = NULL;
1041 }
1042
1043 /* No process locking is needed in this function, because the process
1044 * is not findable any more. We must assume that no other thread is
1045 * using it any more, otherwise we couldn't safely free the process
1046 * structure in the end.
1047 */
kfd_process_wq_release(struct work_struct * work)1048 static void kfd_process_wq_release(struct work_struct *work)
1049 {
1050 struct kfd_process *p = container_of(work, struct kfd_process,
1051 release_work);
1052 kfd_process_remove_sysfs(p);
1053 kfd_iommu_unbind_process(p);
1054
1055 kfd_process_free_outstanding_kfd_bos(p);
1056 svm_range_list_fini(p);
1057
1058 kfd_process_destroy_pdds(p);
1059 dma_fence_put(p->ef);
1060
1061 kfd_event_free_process(p);
1062
1063 kfd_pasid_free(p->pasid);
1064 mutex_destroy(&p->mutex);
1065
1066 put_task_struct(p->lead_thread);
1067
1068 kfree(p);
1069 }
1070
kfd_process_ref_release(struct kref * ref)1071 static void kfd_process_ref_release(struct kref *ref)
1072 {
1073 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1074
1075 INIT_WORK(&p->release_work, kfd_process_wq_release);
1076 queue_work(kfd_process_wq, &p->release_work);
1077 }
1078
kfd_process_alloc_notifier(struct mm_struct * mm)1079 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1080 {
1081 int idx = srcu_read_lock(&kfd_processes_srcu);
1082 struct kfd_process *p = find_process_by_mm(mm);
1083
1084 srcu_read_unlock(&kfd_processes_srcu, idx);
1085
1086 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1087 }
1088
kfd_process_free_notifier(struct mmu_notifier * mn)1089 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1090 {
1091 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1092 }
1093
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1094 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1095 struct mm_struct *mm)
1096 {
1097 struct kfd_process *p;
1098 int i;
1099
1100 /*
1101 * The kfd_process structure can not be free because the
1102 * mmu_notifier srcu is read locked
1103 */
1104 p = container_of(mn, struct kfd_process, mmu_notifier);
1105 if (WARN_ON(p->mm != mm))
1106 return;
1107
1108 mutex_lock(&kfd_processes_mutex);
1109 hash_del_rcu(&p->kfd_processes);
1110 mutex_unlock(&kfd_processes_mutex);
1111 synchronize_srcu(&kfd_processes_srcu);
1112
1113 cancel_delayed_work_sync(&p->eviction_work);
1114 cancel_delayed_work_sync(&p->restore_work);
1115 cancel_delayed_work_sync(&p->svms.restore_work);
1116
1117 mutex_lock(&p->mutex);
1118
1119 /* Iterate over all process device data structures and if the
1120 * pdd is in debug mode, we should first force unregistration,
1121 * then we will be able to destroy the queues
1122 */
1123 for (i = 0; i < p->n_pdds; i++) {
1124 struct kfd_dev *dev = p->pdds[i]->dev;
1125
1126 mutex_lock(kfd_get_dbgmgr_mutex());
1127 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1128 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1129 kfd_dbgmgr_destroy(dev->dbgmgr);
1130 dev->dbgmgr = NULL;
1131 }
1132 }
1133 mutex_unlock(kfd_get_dbgmgr_mutex());
1134 }
1135
1136 kfd_process_dequeue_from_all_devices(p);
1137 pqm_uninit(&p->pqm);
1138
1139 /* Indicate to other users that MM is no longer valid */
1140 p->mm = NULL;
1141 /* Signal the eviction fence after user mode queues are
1142 * destroyed. This allows any BOs to be freed without
1143 * triggering pointless evictions or waiting for fences.
1144 */
1145 dma_fence_signal(p->ef);
1146
1147 mutex_unlock(&p->mutex);
1148
1149 mmu_notifier_put(&p->mmu_notifier);
1150 }
1151
1152 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1153 .release = kfd_process_notifier_release,
1154 .alloc_notifier = kfd_process_alloc_notifier,
1155 .free_notifier = kfd_process_free_notifier,
1156 };
1157
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1158 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1159 {
1160 unsigned long offset;
1161 int i;
1162
1163 for (i = 0; i < p->n_pdds; i++) {
1164 struct kfd_dev *dev = p->pdds[i]->dev;
1165 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1166
1167 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1168 continue;
1169
1170 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1171 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1172 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1173 MAP_SHARED, offset);
1174
1175 if (IS_ERR_VALUE(qpd->tba_addr)) {
1176 int err = qpd->tba_addr;
1177
1178 pr_err("Failure to set tba address. error %d.\n", err);
1179 qpd->tba_addr = 0;
1180 qpd->cwsr_kaddr = NULL;
1181 return err;
1182 }
1183
1184 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1185
1186 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1187 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1188 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1189 }
1190
1191 return 0;
1192 }
1193
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1194 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1195 {
1196 struct kfd_dev *dev = pdd->dev;
1197 struct qcm_process_device *qpd = &pdd->qpd;
1198 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1199 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1200 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1201 void *kaddr;
1202 int ret;
1203
1204 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1205 return 0;
1206
1207 /* cwsr_base is only set for dGPU */
1208 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1209 KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1210 if (ret)
1211 return ret;
1212
1213 qpd->cwsr_kaddr = kaddr;
1214 qpd->tba_addr = qpd->cwsr_base;
1215
1216 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1217
1218 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1219 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1220 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1221
1222 return 0;
1223 }
1224
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1225 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1226 uint64_t tba_addr,
1227 uint64_t tma_addr)
1228 {
1229 if (qpd->cwsr_kaddr) {
1230 /* KFD trap handler is bound, record as second-level TBA/TMA
1231 * in first-level TMA. First-level trap will jump to second.
1232 */
1233 uint64_t *tma =
1234 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1235 tma[0] = tba_addr;
1236 tma[1] = tma_addr;
1237 } else {
1238 /* No trap handler bound, bind as first-level TBA/TMA. */
1239 qpd->tba_addr = tba_addr;
1240 qpd->tma_addr = tma_addr;
1241 }
1242 }
1243
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1244 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1245 {
1246 int i;
1247
1248 /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1249 * boot time retry setting. Mixing processes with different
1250 * XNACK/retry settings can hang the GPU.
1251 *
1252 * Different GPUs can have different noretry settings depending
1253 * on HW bugs or limitations. We need to find at least one
1254 * XNACK mode for this process that's compatible with all GPUs.
1255 * Fortunately GPUs with retry enabled (noretry=0) can run code
1256 * built for XNACK-off. On GFXv9 it may perform slower.
1257 *
1258 * Therefore applications built for XNACK-off can always be
1259 * supported and will be our fallback if any GPU does not
1260 * support retry.
1261 */
1262 for (i = 0; i < p->n_pdds; i++) {
1263 struct kfd_dev *dev = p->pdds[i]->dev;
1264
1265 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1266 * support the SVM APIs and don't need to be considered
1267 * for the XNACK mode selection.
1268 */
1269 if (dev->device_info->asic_family < CHIP_VEGA10)
1270 continue;
1271 /* Aldebaran can always support XNACK because it can support
1272 * per-process XNACK mode selection. But let the dev->noretry
1273 * setting still influence the default XNACK mode.
1274 */
1275 if (supported &&
1276 dev->device_info->asic_family == CHIP_ALDEBARAN)
1277 continue;
1278
1279 /* GFXv10 and later GPUs do not support shader preemption
1280 * during page faults. This can lead to poor QoS for queue
1281 * management and memory-manager-related preemptions or
1282 * even deadlocks.
1283 */
1284 if (dev->device_info->asic_family >= CHIP_NAVI10)
1285 return false;
1286
1287 if (dev->noretry)
1288 return false;
1289 }
1290
1291 return true;
1292 }
1293
1294 /*
1295 * On return the kfd_process is fully operational and will be freed when the
1296 * mm is released
1297 */
create_process(const struct task_struct * thread)1298 static struct kfd_process *create_process(const struct task_struct *thread)
1299 {
1300 struct kfd_process *process;
1301 struct mmu_notifier *mn;
1302 int err = -ENOMEM;
1303
1304 process = kzalloc(sizeof(*process), GFP_KERNEL);
1305 if (!process)
1306 goto err_alloc_process;
1307
1308 kref_init(&process->ref);
1309 mutex_init(&process->mutex);
1310 process->mm = thread->mm;
1311 process->lead_thread = thread->group_leader;
1312 process->n_pdds = 0;
1313 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1314 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1315 process->last_restore_timestamp = get_jiffies_64();
1316 kfd_event_init_process(process);
1317 process->is_32bit_user_mode = in_compat_syscall();
1318
1319 process->pasid = kfd_pasid_alloc();
1320 if (process->pasid == 0)
1321 goto err_alloc_pasid;
1322
1323 err = pqm_init(&process->pqm, process);
1324 if (err != 0)
1325 goto err_process_pqm_init;
1326
1327 /* init process apertures*/
1328 err = kfd_init_apertures(process);
1329 if (err != 0)
1330 goto err_init_apertures;
1331
1332 /* Check XNACK support after PDDs are created in kfd_init_apertures */
1333 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1334
1335 err = svm_range_list_init(process);
1336 if (err)
1337 goto err_init_svm_range_list;
1338
1339 /* alloc_notifier needs to find the process in the hash table */
1340 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1341 (uintptr_t)process->mm);
1342
1343 /* MMU notifier registration must be the last call that can fail
1344 * because after this point we cannot unwind the process creation.
1345 * After this point, mmu_notifier_put will trigger the cleanup by
1346 * dropping the last process reference in the free_notifier.
1347 */
1348 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1349 if (IS_ERR(mn)) {
1350 err = PTR_ERR(mn);
1351 goto err_register_notifier;
1352 }
1353 BUG_ON(mn != &process->mmu_notifier);
1354
1355 get_task_struct(process->lead_thread);
1356
1357 return process;
1358
1359 err_register_notifier:
1360 hash_del_rcu(&process->kfd_processes);
1361 svm_range_list_fini(process);
1362 err_init_svm_range_list:
1363 kfd_process_free_outstanding_kfd_bos(process);
1364 kfd_process_destroy_pdds(process);
1365 err_init_apertures:
1366 pqm_uninit(&process->pqm);
1367 err_process_pqm_init:
1368 kfd_pasid_free(process->pasid);
1369 err_alloc_pasid:
1370 mutex_destroy(&process->mutex);
1371 kfree(process);
1372 err_alloc_process:
1373 return ERR_PTR(err);
1374 }
1375
init_doorbell_bitmap(struct qcm_process_device * qpd,struct kfd_dev * dev)1376 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1377 struct kfd_dev *dev)
1378 {
1379 unsigned int i;
1380 int range_start = dev->shared_resources.non_cp_doorbells_start;
1381 int range_end = dev->shared_resources.non_cp_doorbells_end;
1382
1383 if (!KFD_IS_SOC15(dev->device_info->asic_family))
1384 return 0;
1385
1386 qpd->doorbell_bitmap =
1387 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1388 BITS_PER_BYTE), GFP_KERNEL);
1389 if (!qpd->doorbell_bitmap)
1390 return -ENOMEM;
1391
1392 /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1393 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1394 pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1395 range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1396 range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1397
1398 for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1399 if (i >= range_start && i <= range_end) {
1400 set_bit(i, qpd->doorbell_bitmap);
1401 set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1402 qpd->doorbell_bitmap);
1403 }
1404 }
1405
1406 return 0;
1407 }
1408
kfd_get_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1409 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1410 struct kfd_process *p)
1411 {
1412 int i;
1413
1414 for (i = 0; i < p->n_pdds; i++)
1415 if (p->pdds[i]->dev == dev)
1416 return p->pdds[i];
1417
1418 return NULL;
1419 }
1420
kfd_create_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1421 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1422 struct kfd_process *p)
1423 {
1424 struct kfd_process_device *pdd = NULL;
1425
1426 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1427 return NULL;
1428 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1429 if (!pdd)
1430 return NULL;
1431
1432 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1433 pr_err("Failed to alloc doorbell for pdd\n");
1434 goto err_free_pdd;
1435 }
1436
1437 if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1438 pr_err("Failed to init doorbell for process\n");
1439 goto err_free_pdd;
1440 }
1441
1442 pdd->dev = dev;
1443 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1444 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1445 pdd->qpd.dqm = dev->dqm;
1446 pdd->qpd.pqm = &p->pqm;
1447 pdd->qpd.evicted = 0;
1448 pdd->qpd.mapped_gws_queue = false;
1449 pdd->process = p;
1450 pdd->bound = PDD_UNBOUND;
1451 pdd->already_dequeued = false;
1452 pdd->runtime_inuse = false;
1453 pdd->vram_usage = 0;
1454 pdd->sdma_past_activity_counter = 0;
1455 atomic64_set(&pdd->evict_duration_counter, 0);
1456 p->pdds[p->n_pdds++] = pdd;
1457
1458 /* Init idr used for memory handle translation */
1459 idr_init(&pdd->alloc_idr);
1460
1461 return pdd;
1462
1463 err_free_pdd:
1464 kfree(pdd);
1465 return NULL;
1466 }
1467
1468 /**
1469 * kfd_process_device_init_vm - Initialize a VM for a process-device
1470 *
1471 * @pdd: The process-device
1472 * @drm_file: Optional pointer to a DRM file descriptor
1473 *
1474 * If @drm_file is specified, it will be used to acquire the VM from
1475 * that file descriptor. If successful, the @pdd takes ownership of
1476 * the file descriptor.
1477 *
1478 * If @drm_file is NULL, a new VM is created.
1479 *
1480 * Returns 0 on success, -errno on failure.
1481 */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1482 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1483 struct file *drm_file)
1484 {
1485 struct kfd_process *p;
1486 struct kfd_dev *dev;
1487 int ret;
1488
1489 if (!drm_file)
1490 return -EINVAL;
1491
1492 if (pdd->drm_priv)
1493 return -EBUSY;
1494
1495 p = pdd->process;
1496 dev = pdd->dev;
1497
1498 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1499 dev->kgd, drm_file, p->pasid,
1500 &p->kgd_process_info, &p->ef);
1501 if (ret) {
1502 pr_err("Failed to create process VM object\n");
1503 return ret;
1504 }
1505 pdd->drm_priv = drm_file->private_data;
1506
1507 ret = kfd_process_device_reserve_ib_mem(pdd);
1508 if (ret)
1509 goto err_reserve_ib_mem;
1510 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1511 if (ret)
1512 goto err_init_cwsr;
1513
1514 pdd->drm_file = drm_file;
1515
1516 return 0;
1517
1518 err_init_cwsr:
1519 err_reserve_ib_mem:
1520 kfd_process_device_free_bos(pdd);
1521 pdd->drm_priv = NULL;
1522
1523 return ret;
1524 }
1525
1526 /*
1527 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1528 * to the device.
1529 * Unbinding occurs when the process dies or the device is removed.
1530 *
1531 * Assumes that the process lock is held.
1532 */
kfd_bind_process_to_device(struct kfd_dev * dev,struct kfd_process * p)1533 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1534 struct kfd_process *p)
1535 {
1536 struct kfd_process_device *pdd;
1537 int err;
1538
1539 pdd = kfd_get_process_device_data(dev, p);
1540 if (!pdd) {
1541 pr_err("Process device data doesn't exist\n");
1542 return ERR_PTR(-ENOMEM);
1543 }
1544
1545 if (!pdd->drm_priv)
1546 return ERR_PTR(-ENODEV);
1547
1548 /*
1549 * signal runtime-pm system to auto resume and prevent
1550 * further runtime suspend once device pdd is created until
1551 * pdd is destroyed.
1552 */
1553 if (!pdd->runtime_inuse) {
1554 err = pm_runtime_get_sync(dev->ddev->dev);
1555 if (err < 0) {
1556 pm_runtime_put_autosuspend(dev->ddev->dev);
1557 return ERR_PTR(err);
1558 }
1559 }
1560
1561 err = kfd_iommu_bind_process_to_device(pdd);
1562 if (err)
1563 goto out;
1564
1565 /*
1566 * make sure that runtime_usage counter is incremented just once
1567 * per pdd
1568 */
1569 pdd->runtime_inuse = true;
1570
1571 return pdd;
1572
1573 out:
1574 /* balance runpm reference count and exit with error */
1575 if (!pdd->runtime_inuse) {
1576 pm_runtime_mark_last_busy(dev->ddev->dev);
1577 pm_runtime_put_autosuspend(dev->ddev->dev);
1578 }
1579
1580 return ERR_PTR(err);
1581 }
1582
1583 /* Create specific handle mapped to mem from process local memory idr
1584 * Assumes that the process lock is held.
1585 */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1586 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1587 void *mem)
1588 {
1589 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1590 }
1591
1592 /* Translate specific handle from process local memory idr
1593 * Assumes that the process lock is held.
1594 */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1595 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1596 int handle)
1597 {
1598 if (handle < 0)
1599 return NULL;
1600
1601 return idr_find(&pdd->alloc_idr, handle);
1602 }
1603
1604 /* Remove specific handle from process local memory idr
1605 * Assumes that the process lock is held.
1606 */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1607 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1608 int handle)
1609 {
1610 if (handle >= 0)
1611 idr_remove(&pdd->alloc_idr, handle);
1612 }
1613
1614 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid)1615 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1616 {
1617 struct kfd_process *p, *ret_p = NULL;
1618 unsigned int temp;
1619
1620 int idx = srcu_read_lock(&kfd_processes_srcu);
1621
1622 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1623 if (p->pasid == pasid) {
1624 kref_get(&p->ref);
1625 ret_p = p;
1626 break;
1627 }
1628 }
1629
1630 srcu_read_unlock(&kfd_processes_srcu, idx);
1631
1632 return ret_p;
1633 }
1634
1635 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1636 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1637 {
1638 struct kfd_process *p;
1639
1640 int idx = srcu_read_lock(&kfd_processes_srcu);
1641
1642 p = find_process_by_mm(mm);
1643 if (p)
1644 kref_get(&p->ref);
1645
1646 srcu_read_unlock(&kfd_processes_srcu, idx);
1647
1648 return p;
1649 }
1650
1651 /* kfd_process_evict_queues - Evict all user queues of a process
1652 *
1653 * Eviction is reference-counted per process-device. This means multiple
1654 * evictions from different sources can be nested safely.
1655 */
kfd_process_evict_queues(struct kfd_process * p)1656 int kfd_process_evict_queues(struct kfd_process *p)
1657 {
1658 int r = 0;
1659 int i;
1660 unsigned int n_evicted = 0;
1661
1662 for (i = 0; i < p->n_pdds; i++) {
1663 struct kfd_process_device *pdd = p->pdds[i];
1664
1665 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1666 &pdd->qpd);
1667 if (r) {
1668 pr_err("Failed to evict process queues\n");
1669 goto fail;
1670 }
1671 n_evicted++;
1672 }
1673
1674 return r;
1675
1676 fail:
1677 /* To keep state consistent, roll back partial eviction by
1678 * restoring queues
1679 */
1680 for (i = 0; i < p->n_pdds; i++) {
1681 struct kfd_process_device *pdd = p->pdds[i];
1682
1683 if (n_evicted == 0)
1684 break;
1685 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1686 &pdd->qpd))
1687 pr_err("Failed to restore queues\n");
1688
1689 n_evicted--;
1690 }
1691
1692 return r;
1693 }
1694
1695 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1696 int kfd_process_restore_queues(struct kfd_process *p)
1697 {
1698 int r, ret = 0;
1699 int i;
1700
1701 for (i = 0; i < p->n_pdds; i++) {
1702 struct kfd_process_device *pdd = p->pdds[i];
1703
1704 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1705 &pdd->qpd);
1706 if (r) {
1707 pr_err("Failed to restore process queues\n");
1708 if (!ret)
1709 ret = r;
1710 }
1711 }
1712
1713 return ret;
1714 }
1715
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1716 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1717 {
1718 int i;
1719
1720 for (i = 0; i < p->n_pdds; i++)
1721 if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
1722 return i;
1723 return -EINVAL;
1724 }
1725
1726 int
kfd_process_gpuid_from_kgd(struct kfd_process * p,struct amdgpu_device * adev,uint32_t * gpuid,uint32_t * gpuidx)1727 kfd_process_gpuid_from_kgd(struct kfd_process *p, struct amdgpu_device *adev,
1728 uint32_t *gpuid, uint32_t *gpuidx)
1729 {
1730 struct kgd_dev *kgd = (struct kgd_dev *)adev;
1731 int i;
1732
1733 for (i = 0; i < p->n_pdds; i++)
1734 if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
1735 *gpuid = p->pdds[i]->dev->id;
1736 *gpuidx = i;
1737 return 0;
1738 }
1739 return -EINVAL;
1740 }
1741
evict_process_worker(struct work_struct * work)1742 static void evict_process_worker(struct work_struct *work)
1743 {
1744 int ret;
1745 struct kfd_process *p;
1746 struct delayed_work *dwork;
1747
1748 dwork = to_delayed_work(work);
1749
1750 /* Process termination destroys this worker thread. So during the
1751 * lifetime of this thread, kfd_process p will be valid
1752 */
1753 p = container_of(dwork, struct kfd_process, eviction_work);
1754 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1755 "Eviction fence mismatch\n");
1756
1757 /* Narrow window of overlap between restore and evict work
1758 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1759 * unreserves KFD BOs, it is possible to evicted again. But
1760 * restore has few more steps of finish. So lets wait for any
1761 * previous restore work to complete
1762 */
1763 flush_delayed_work(&p->restore_work);
1764
1765 pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1766 ret = kfd_process_evict_queues(p);
1767 if (!ret) {
1768 dma_fence_signal(p->ef);
1769 dma_fence_put(p->ef);
1770 p->ef = NULL;
1771 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1772 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1773
1774 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1775 } else
1776 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1777 }
1778
restore_process_worker(struct work_struct * work)1779 static void restore_process_worker(struct work_struct *work)
1780 {
1781 struct delayed_work *dwork;
1782 struct kfd_process *p;
1783 int ret = 0;
1784
1785 dwork = to_delayed_work(work);
1786
1787 /* Process termination destroys this worker thread. So during the
1788 * lifetime of this thread, kfd_process p will be valid
1789 */
1790 p = container_of(dwork, struct kfd_process, restore_work);
1791 pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1792
1793 /* Setting last_restore_timestamp before successful restoration.
1794 * Otherwise this would have to be set by KGD (restore_process_bos)
1795 * before KFD BOs are unreserved. If not, the process can be evicted
1796 * again before the timestamp is set.
1797 * If restore fails, the timestamp will be set again in the next
1798 * attempt. This would mean that the minimum GPU quanta would be
1799 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1800 * functions)
1801 */
1802
1803 p->last_restore_timestamp = get_jiffies_64();
1804 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1805 &p->ef);
1806 if (ret) {
1807 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1808 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1809 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1810 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1811 WARN(!ret, "reschedule restore work failed\n");
1812 return;
1813 }
1814
1815 ret = kfd_process_restore_queues(p);
1816 if (!ret)
1817 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1818 else
1819 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1820 }
1821
kfd_suspend_all_processes(void)1822 void kfd_suspend_all_processes(void)
1823 {
1824 struct kfd_process *p;
1825 unsigned int temp;
1826 int idx = srcu_read_lock(&kfd_processes_srcu);
1827
1828 WARN(debug_evictions, "Evicting all processes");
1829 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1830 cancel_delayed_work_sync(&p->eviction_work);
1831 cancel_delayed_work_sync(&p->restore_work);
1832
1833 if (kfd_process_evict_queues(p))
1834 pr_err("Failed to suspend process 0x%x\n", p->pasid);
1835 dma_fence_signal(p->ef);
1836 dma_fence_put(p->ef);
1837 p->ef = NULL;
1838 }
1839 srcu_read_unlock(&kfd_processes_srcu, idx);
1840 }
1841
kfd_resume_all_processes(void)1842 int kfd_resume_all_processes(void)
1843 {
1844 struct kfd_process *p;
1845 unsigned int temp;
1846 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1847
1848 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1849 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1850 pr_err("Restore process %d failed during resume\n",
1851 p->pasid);
1852 ret = -EFAULT;
1853 }
1854 }
1855 srcu_read_unlock(&kfd_processes_srcu, idx);
1856 return ret;
1857 }
1858
kfd_reserved_mem_mmap(struct kfd_dev * dev,struct kfd_process * process,struct vm_area_struct * vma)1859 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1860 struct vm_area_struct *vma)
1861 {
1862 struct kfd_process_device *pdd;
1863 struct qcm_process_device *qpd;
1864
1865 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1866 pr_err("Incorrect CWSR mapping size.\n");
1867 return -EINVAL;
1868 }
1869
1870 pdd = kfd_get_process_device_data(dev, process);
1871 if (!pdd)
1872 return -EINVAL;
1873 qpd = &pdd->qpd;
1874
1875 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1876 get_order(KFD_CWSR_TBA_TMA_SIZE));
1877 if (!qpd->cwsr_kaddr) {
1878 pr_err("Error allocating per process CWSR buffer.\n");
1879 return -ENOMEM;
1880 }
1881
1882 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1883 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1884 /* Mapping pages to user process */
1885 return remap_pfn_range(vma, vma->vm_start,
1886 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1887 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1888 }
1889
kfd_flush_tlb(struct kfd_process_device * pdd,enum TLB_FLUSH_TYPE type)1890 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
1891 {
1892 struct kfd_dev *dev = pdd->dev;
1893
1894 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1895 /* Nothing to flush until a VMID is assigned, which
1896 * only happens when the first queue is created.
1897 */
1898 if (pdd->qpd.vmid)
1899 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1900 pdd->qpd.vmid);
1901 } else {
1902 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1903 pdd->process->pasid, type);
1904 }
1905 }
1906
1907 #if defined(CONFIG_DEBUG_FS)
1908
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)1909 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1910 {
1911 struct kfd_process *p;
1912 unsigned int temp;
1913 int r = 0;
1914
1915 int idx = srcu_read_lock(&kfd_processes_srcu);
1916
1917 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1918 seq_printf(m, "Process %d PASID 0x%x:\n",
1919 p->lead_thread->tgid, p->pasid);
1920
1921 mutex_lock(&p->mutex);
1922 r = pqm_debugfs_mqds(m, &p->pqm);
1923 mutex_unlock(&p->mutex);
1924
1925 if (r)
1926 break;
1927 }
1928
1929 srcu_read_unlock(&kfd_processes_srcu, idx);
1930
1931 return r;
1932 }
1933
1934 #endif
1935
1936