1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <linux/mmu_context.h>
29 #include <linux/slab.h>
30 #include <linux/amd-iommu.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38
39 struct mm_struct;
40
41 #include "kfd_priv.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_dbgmgr.h"
44 #include "kfd_iommu.h"
45
46 /*
47 * List of struct kfd_process (field kfd_process).
48 * Unique/indexed by mm_struct*
49 */
50 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
51 static DEFINE_MUTEX(kfd_processes_mutex);
52
53 DEFINE_SRCU(kfd_processes_srcu);
54
55 /* For process termination handling */
56 static struct workqueue_struct *kfd_process_wq;
57
58 /* Ordered, single-threaded workqueue for restoring evicted
59 * processes. Restoring multiple processes concurrently under memory
60 * pressure can lead to processes blocking each other from validating
61 * their BOs and result in a live-lock situation where processes
62 * remain evicted indefinitely.
63 */
64 static struct workqueue_struct *kfd_restore_wq;
65
66 static struct kfd_process *find_process(const struct task_struct *thread);
67 static void kfd_process_ref_release(struct kref *ref);
68 static struct kfd_process *create_process(const struct task_struct *thread);
69 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
70
71 static void evict_process_worker(struct work_struct *work);
72 static void restore_process_worker(struct work_struct *work);
73
74 struct kfd_procfs_tree {
75 struct kobject *kobj;
76 };
77
78 static struct kfd_procfs_tree procfs;
79
80 /*
81 * Structure for SDMA activity tracking
82 */
83 struct kfd_sdma_activity_handler_workarea {
84 struct work_struct sdma_activity_work;
85 struct kfd_process_device *pdd;
86 uint64_t sdma_activity_counter;
87 };
88
89 struct temp_sdma_queue_list {
90 uint64_t __user *rptr;
91 uint64_t sdma_val;
92 unsigned int queue_id;
93 struct list_head list;
94 };
95
kfd_sdma_activity_worker(struct work_struct * work)96 static void kfd_sdma_activity_worker(struct work_struct *work)
97 {
98 struct kfd_sdma_activity_handler_workarea *workarea;
99 struct kfd_process_device *pdd;
100 uint64_t val;
101 struct mm_struct *mm;
102 struct queue *q;
103 struct qcm_process_device *qpd;
104 struct device_queue_manager *dqm;
105 int ret = 0;
106 struct temp_sdma_queue_list sdma_q_list;
107 struct temp_sdma_queue_list *sdma_q, *next;
108
109 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
110 sdma_activity_work);
111 if (!workarea)
112 return;
113
114 pdd = workarea->pdd;
115 if (!pdd)
116 return;
117 dqm = pdd->dev->dqm;
118 qpd = &pdd->qpd;
119 if (!dqm || !qpd)
120 return;
121 /*
122 * Total SDMA activity is current SDMA activity + past SDMA activity
123 * Past SDMA count is stored in pdd.
124 * To get the current activity counters for all active SDMA queues,
125 * we loop over all SDMA queues and get their counts from user-space.
126 *
127 * We cannot call get_user() with dqm_lock held as it can cause
128 * a circular lock dependency situation. To read the SDMA stats,
129 * we need to do the following:
130 *
131 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
132 * with dqm_lock/dqm_unlock().
133 * 2. Call get_user() for each node in temporary list without dqm_lock.
134 * Save the SDMA count for each node and also add the count to the total
135 * SDMA count counter.
136 * Its possible, during this step, a few SDMA queue nodes got deleted
137 * from the qpd->queues_list.
138 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
139 * If any node got deleted, its SDMA count would be captured in the sdma
140 * past activity counter. So subtract the SDMA counter stored in step 2
141 * for this node from the total SDMA count.
142 */
143 INIT_LIST_HEAD(&sdma_q_list.list);
144
145 /*
146 * Create the temp list of all SDMA queues
147 */
148 dqm_lock(dqm);
149
150 list_for_each_entry(q, &qpd->queues_list, list) {
151 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
152 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
153 continue;
154
155 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
156 if (!sdma_q) {
157 dqm_unlock(dqm);
158 goto cleanup;
159 }
160
161 INIT_LIST_HEAD(&sdma_q->list);
162 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
163 sdma_q->queue_id = q->properties.queue_id;
164 list_add_tail(&sdma_q->list, &sdma_q_list.list);
165 }
166
167 /*
168 * If the temp list is empty, then no SDMA queues nodes were found in
169 * qpd->queues_list. Return the past activity count as the total sdma
170 * count
171 */
172 if (list_empty(&sdma_q_list.list)) {
173 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
174 dqm_unlock(dqm);
175 return;
176 }
177
178 dqm_unlock(dqm);
179
180 /*
181 * Get the usage count for each SDMA queue in temp_list.
182 */
183 mm = get_task_mm(pdd->process->lead_thread);
184 if (!mm)
185 goto cleanup;
186
187 kthread_use_mm(mm);
188
189 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
190 val = 0;
191 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
192 if (ret) {
193 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
194 sdma_q->queue_id);
195 } else {
196 sdma_q->sdma_val = val;
197 workarea->sdma_activity_counter += val;
198 }
199 }
200
201 kthread_unuse_mm(mm);
202 mmput(mm);
203
204 /*
205 * Do a second iteration over qpd_queues_list to check if any SDMA
206 * nodes got deleted while fetching SDMA counter.
207 */
208 dqm_lock(dqm);
209
210 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
211
212 list_for_each_entry(q, &qpd->queues_list, list) {
213 if (list_empty(&sdma_q_list.list))
214 break;
215
216 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
217 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
218 continue;
219
220 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
221 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
222 (sdma_q->queue_id == q->properties.queue_id)) {
223 list_del(&sdma_q->list);
224 kfree(sdma_q);
225 break;
226 }
227 }
228 }
229
230 dqm_unlock(dqm);
231
232 /*
233 * If temp list is not empty, it implies some queues got deleted
234 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
235 * count for each node from the total SDMA count.
236 */
237 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
238 workarea->sdma_activity_counter -= sdma_q->sdma_val;
239 list_del(&sdma_q->list);
240 kfree(sdma_q);
241 }
242
243 return;
244
245 cleanup:
246 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
247 list_del(&sdma_q->list);
248 kfree(sdma_q);
249 }
250 }
251
252 /**
253 * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
254 * by current process. Translates acquired wave count into number of compute units
255 * that are occupied.
256 *
257 * @atr: Handle of attribute that allows reporting of wave count. The attribute
258 * handle encapsulates GPU device it is associated with, thereby allowing collection
259 * of waves in flight, etc
260 *
261 * @buffer: Handle of user provided buffer updated with wave count
262 *
263 * Return: Number of bytes written to user buffer or an error value
264 */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)265 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
266 {
267 int cu_cnt;
268 int wave_cnt;
269 int max_waves_per_cu;
270 struct kfd_dev *dev = NULL;
271 struct kfd_process *proc = NULL;
272 struct kfd_process_device *pdd = NULL;
273
274 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
275 dev = pdd->dev;
276 if (dev->kfd2kgd->get_cu_occupancy == NULL)
277 return -EINVAL;
278
279 cu_cnt = 0;
280 proc = pdd->process;
281 if (pdd->qpd.queue_count == 0) {
282 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
283 dev->id, proc->pasid);
284 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
285 }
286
287 /* Collect wave count from device if it supports */
288 wave_cnt = 0;
289 max_waves_per_cu = 0;
290 dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
291 &max_waves_per_cu);
292
293 /* Translate wave count to number of compute units */
294 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
295 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
296 }
297
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)298 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
299 char *buffer)
300 {
301 if (strcmp(attr->name, "pasid") == 0) {
302 struct kfd_process *p = container_of(attr, struct kfd_process,
303 attr_pasid);
304
305 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
306 } else if (strncmp(attr->name, "vram_", 5) == 0) {
307 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
308 attr_vram);
309 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
310 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
311 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
312 attr_sdma);
313 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
314
315 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
316 kfd_sdma_activity_worker);
317
318 sdma_activity_work_handler.pdd = pdd;
319 sdma_activity_work_handler.sdma_activity_counter = 0;
320
321 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
322
323 flush_work(&sdma_activity_work_handler.sdma_activity_work);
324
325 return snprintf(buffer, PAGE_SIZE, "%llu\n",
326 (sdma_activity_work_handler.sdma_activity_counter)/
327 SDMA_ACTIVITY_DIVISOR);
328 } else {
329 pr_err("Invalid attribute");
330 return -EINVAL;
331 }
332
333 return 0;
334 }
335
kfd_procfs_kobj_release(struct kobject * kobj)336 static void kfd_procfs_kobj_release(struct kobject *kobj)
337 {
338 kfree(kobj);
339 }
340
341 static const struct sysfs_ops kfd_procfs_ops = {
342 .show = kfd_procfs_show,
343 };
344
345 static struct kobj_type procfs_type = {
346 .release = kfd_procfs_kobj_release,
347 .sysfs_ops = &kfd_procfs_ops,
348 };
349
kfd_procfs_init(void)350 void kfd_procfs_init(void)
351 {
352 int ret = 0;
353
354 procfs.kobj = kfd_alloc_struct(procfs.kobj);
355 if (!procfs.kobj)
356 return;
357
358 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
359 &kfd_device->kobj, "proc");
360 if (ret) {
361 pr_warn("Could not create procfs proc folder");
362 /* If we fail to create the procfs, clean up */
363 kfd_procfs_shutdown();
364 }
365 }
366
kfd_procfs_shutdown(void)367 void kfd_procfs_shutdown(void)
368 {
369 if (procfs.kobj) {
370 kobject_del(procfs.kobj);
371 kobject_put(procfs.kobj);
372 procfs.kobj = NULL;
373 }
374 }
375
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)376 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
377 struct attribute *attr, char *buffer)
378 {
379 struct queue *q = container_of(kobj, struct queue, kobj);
380
381 if (!strcmp(attr->name, "size"))
382 return snprintf(buffer, PAGE_SIZE, "%llu",
383 q->properties.queue_size);
384 else if (!strcmp(attr->name, "type"))
385 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
386 else if (!strcmp(attr->name, "gpuid"))
387 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
388 else
389 pr_err("Invalid attribute");
390
391 return 0;
392 }
393
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)394 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
395 struct attribute *attr, char *buffer)
396 {
397 if (strcmp(attr->name, "evicted_ms") == 0) {
398 struct kfd_process_device *pdd = container_of(attr,
399 struct kfd_process_device,
400 attr_evict);
401 uint64_t evict_jiffies;
402
403 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
404
405 return snprintf(buffer,
406 PAGE_SIZE,
407 "%llu\n",
408 jiffies64_to_msecs(evict_jiffies));
409
410 /* Sysfs handle that gets CU occupancy is per device */
411 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
412 return kfd_get_cu_occupancy(attr, buffer);
413 } else {
414 pr_err("Invalid attribute");
415 }
416
417 return 0;
418 }
419
420 static struct attribute attr_queue_size = {
421 .name = "size",
422 .mode = KFD_SYSFS_FILE_MODE
423 };
424
425 static struct attribute attr_queue_type = {
426 .name = "type",
427 .mode = KFD_SYSFS_FILE_MODE
428 };
429
430 static struct attribute attr_queue_gpuid = {
431 .name = "gpuid",
432 .mode = KFD_SYSFS_FILE_MODE
433 };
434
435 static struct attribute *procfs_queue_attrs[] = {
436 &attr_queue_size,
437 &attr_queue_type,
438 &attr_queue_gpuid,
439 NULL
440 };
441
442 static const struct sysfs_ops procfs_queue_ops = {
443 .show = kfd_procfs_queue_show,
444 };
445
446 static struct kobj_type procfs_queue_type = {
447 .sysfs_ops = &procfs_queue_ops,
448 .default_attrs = procfs_queue_attrs,
449 };
450
451 static const struct sysfs_ops procfs_stats_ops = {
452 .show = kfd_procfs_stats_show,
453 };
454
455 static struct kobj_type procfs_stats_type = {
456 .sysfs_ops = &procfs_stats_ops,
457 .release = kfd_procfs_kobj_release,
458 };
459
kfd_procfs_add_queue(struct queue * q)460 int kfd_procfs_add_queue(struct queue *q)
461 {
462 struct kfd_process *proc;
463 int ret;
464
465 if (!q || !q->process)
466 return -EINVAL;
467 proc = q->process;
468
469 /* Create proc/<pid>/queues/<queue id> folder */
470 if (!proc->kobj_queues)
471 return -EFAULT;
472 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
473 proc->kobj_queues, "%u", q->properties.queue_id);
474 if (ret < 0) {
475 pr_warn("Creating proc/<pid>/queues/%u failed",
476 q->properties.queue_id);
477 kobject_put(&q->kobj);
478 return ret;
479 }
480
481 return 0;
482 }
483
kfd_sysfs_create_file(struct kfd_process * p,struct attribute * attr,char * name)484 static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr,
485 char *name)
486 {
487 int ret = 0;
488
489 if (!p || !attr || !name)
490 return -EINVAL;
491
492 attr->name = name;
493 attr->mode = KFD_SYSFS_FILE_MODE;
494 sysfs_attr_init(attr);
495
496 ret = sysfs_create_file(p->kobj, attr);
497
498 return ret;
499 }
500
kfd_procfs_add_sysfs_stats(struct kfd_process * p)501 static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
502 {
503 int ret = 0;
504 struct kfd_process_device *pdd;
505 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
506
507 if (!p)
508 return -EINVAL;
509
510 if (!p->kobj)
511 return -EFAULT;
512
513 /*
514 * Create sysfs files for each GPU:
515 * - proc/<pid>/stats_<gpuid>/
516 * - proc/<pid>/stats_<gpuid>/evicted_ms
517 * - proc/<pid>/stats_<gpuid>/cu_occupancy
518 */
519 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
520 struct kobject *kobj_stats;
521
522 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
523 "stats_%u", pdd->dev->id);
524 kobj_stats = kfd_alloc_struct(kobj_stats);
525 if (!kobj_stats)
526 return -ENOMEM;
527
528 ret = kobject_init_and_add(kobj_stats,
529 &procfs_stats_type,
530 p->kobj,
531 stats_dir_filename);
532
533 if (ret) {
534 pr_warn("Creating KFD proc/stats_%s folder failed",
535 stats_dir_filename);
536 kobject_put(kobj_stats);
537 goto err;
538 }
539
540 pdd->kobj_stats = kobj_stats;
541 pdd->attr_evict.name = "evicted_ms";
542 pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE;
543 sysfs_attr_init(&pdd->attr_evict);
544 ret = sysfs_create_file(kobj_stats, &pdd->attr_evict);
545 if (ret)
546 pr_warn("Creating eviction stats for gpuid %d failed",
547 (int)pdd->dev->id);
548
549 /* Add sysfs file to report compute unit occupancy */
550 if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
551 pdd->attr_cu_occupancy.name = "cu_occupancy";
552 pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
553 sysfs_attr_init(&pdd->attr_cu_occupancy);
554 ret = sysfs_create_file(kobj_stats,
555 &pdd->attr_cu_occupancy);
556 if (ret)
557 pr_warn("Creating %s failed for gpuid: %d",
558 pdd->attr_cu_occupancy.name,
559 (int)pdd->dev->id);
560 }
561 }
562 err:
563 return ret;
564 }
565
566
kfd_procfs_add_sysfs_files(struct kfd_process * p)567 static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
568 {
569 int ret = 0;
570 struct kfd_process_device *pdd;
571
572 if (!p)
573 return -EINVAL;
574
575 if (!p->kobj)
576 return -EFAULT;
577
578 /*
579 * Create sysfs files for each GPU:
580 * - proc/<pid>/vram_<gpuid>
581 * - proc/<pid>/sdma_<gpuid>
582 */
583 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
584 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
585 pdd->dev->id);
586 ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename);
587 if (ret)
588 pr_warn("Creating vram usage for gpu id %d failed",
589 (int)pdd->dev->id);
590
591 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
592 pdd->dev->id);
593 ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename);
594 if (ret)
595 pr_warn("Creating sdma usage for gpu id %d failed",
596 (int)pdd->dev->id);
597 }
598
599 return ret;
600 }
601
kfd_procfs_del_queue(struct queue * q)602 void kfd_procfs_del_queue(struct queue *q)
603 {
604 if (!q)
605 return;
606
607 kobject_del(&q->kobj);
608 kobject_put(&q->kobj);
609 }
610
kfd_process_create_wq(void)611 int kfd_process_create_wq(void)
612 {
613 if (!kfd_process_wq)
614 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
615 if (!kfd_restore_wq)
616 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
617
618 if (!kfd_process_wq || !kfd_restore_wq) {
619 kfd_process_destroy_wq();
620 return -ENOMEM;
621 }
622
623 return 0;
624 }
625
kfd_process_destroy_wq(void)626 void kfd_process_destroy_wq(void)
627 {
628 if (kfd_process_wq) {
629 destroy_workqueue(kfd_process_wq);
630 kfd_process_wq = NULL;
631 }
632 if (kfd_restore_wq) {
633 destroy_workqueue(kfd_restore_wq);
634 kfd_restore_wq = NULL;
635 }
636 }
637
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd)638 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
639 struct kfd_process_device *pdd)
640 {
641 struct kfd_dev *dev = pdd->dev;
642
643 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);
644 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);
645 }
646
647 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
648 * This function should be only called right after the process
649 * is created and when kfd_processes_mutex is still being held
650 * to avoid concurrency. Because of that exclusiveness, we do
651 * not need to take p->mutex.
652 */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,void ** kptr)653 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
654 uint64_t gpu_va, uint32_t size,
655 uint32_t flags, void **kptr)
656 {
657 struct kfd_dev *kdev = pdd->dev;
658 struct kgd_mem *mem = NULL;
659 int handle;
660 int err;
661
662 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
663 pdd->vm, &mem, NULL, flags);
664 if (err)
665 goto err_alloc_mem;
666
667 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
668 if (err)
669 goto err_map_mem;
670
671 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
672 if (err) {
673 pr_debug("Sync memory failed, wait interrupted by user signal\n");
674 goto sync_memory_failed;
675 }
676
677 /* Create an obj handle so kfd_process_device_remove_obj_handle
678 * will take care of the bo removal when the process finishes.
679 * We do not need to take p->mutex, because the process is just
680 * created and the ioctls have not had the chance to run.
681 */
682 handle = kfd_process_device_create_obj_handle(pdd, mem);
683
684 if (handle < 0) {
685 err = handle;
686 goto free_gpuvm;
687 }
688
689 if (kptr) {
690 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
691 (struct kgd_mem *)mem, kptr, NULL);
692 if (err) {
693 pr_debug("Map GTT BO to kernel failed\n");
694 goto free_obj_handle;
695 }
696 }
697
698 return err;
699
700 free_obj_handle:
701 kfd_process_device_remove_obj_handle(pdd, handle);
702 free_gpuvm:
703 sync_memory_failed:
704 kfd_process_free_gpuvm(mem, pdd);
705 return err;
706
707 err_map_mem:
708 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);
709 err_alloc_mem:
710 *kptr = NULL;
711 return err;
712 }
713
714 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
715 * process for IB usage The memory reserved is for KFD to submit
716 * IB to AMDGPU from kernel. If the memory is reserved
717 * successfully, ib_kaddr will have the CPU/kernel
718 * address. Check ib_kaddr before accessing the memory.
719 */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)720 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
721 {
722 struct qcm_process_device *qpd = &pdd->qpd;
723 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
724 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
725 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
726 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
727 void *kaddr;
728 int ret;
729
730 if (qpd->ib_kaddr || !qpd->ib_base)
731 return 0;
732
733 /* ib_base is only set for dGPU */
734 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
735 &kaddr);
736 if (ret)
737 return ret;
738
739 qpd->ib_kaddr = kaddr;
740
741 return 0;
742 }
743
kfd_create_process(struct file * filep)744 struct kfd_process *kfd_create_process(struct file *filep)
745 {
746 struct kfd_process *process;
747 struct task_struct *thread = current;
748 int ret;
749
750 if (!thread->mm)
751 return ERR_PTR(-EINVAL);
752
753 /* Only the pthreads threading model is supported. */
754 if (thread->group_leader->mm != thread->mm)
755 return ERR_PTR(-EINVAL);
756
757 /*
758 * take kfd processes mutex before starting of process creation
759 * so there won't be a case where two threads of the same process
760 * create two kfd_process structures
761 */
762 mutex_lock(&kfd_processes_mutex);
763
764 /* A prior open of /dev/kfd could have already created the process. */
765 process = find_process(thread);
766 if (process) {
767 pr_debug("Process already found\n");
768 } else {
769 process = create_process(thread);
770 if (IS_ERR(process))
771 goto out;
772
773 ret = kfd_process_init_cwsr_apu(process, filep);
774 if (ret) {
775 process = ERR_PTR(ret);
776 goto out;
777 }
778
779 if (!procfs.kobj)
780 goto out;
781
782 process->kobj = kfd_alloc_struct(process->kobj);
783 if (!process->kobj) {
784 pr_warn("Creating procfs kobject failed");
785 goto out;
786 }
787 ret = kobject_init_and_add(process->kobj, &procfs_type,
788 procfs.kobj, "%d",
789 (int)process->lead_thread->pid);
790 if (ret) {
791 pr_warn("Creating procfs pid directory failed");
792 kobject_put(process->kobj);
793 goto out;
794 }
795
796 process->attr_pasid.name = "pasid";
797 process->attr_pasid.mode = KFD_SYSFS_FILE_MODE;
798 sysfs_attr_init(&process->attr_pasid);
799 ret = sysfs_create_file(process->kobj, &process->attr_pasid);
800 if (ret)
801 pr_warn("Creating pasid for pid %d failed",
802 (int)process->lead_thread->pid);
803
804 process->kobj_queues = kobject_create_and_add("queues",
805 process->kobj);
806 if (!process->kobj_queues)
807 pr_warn("Creating KFD proc/queues folder failed");
808
809 ret = kfd_procfs_add_sysfs_stats(process);
810 if (ret)
811 pr_warn("Creating sysfs stats dir for pid %d failed",
812 (int)process->lead_thread->pid);
813
814 ret = kfd_procfs_add_sysfs_files(process);
815 if (ret)
816 pr_warn("Creating sysfs usage file for pid %d failed",
817 (int)process->lead_thread->pid);
818 }
819 out:
820 if (!IS_ERR(process))
821 kref_get(&process->ref);
822 mutex_unlock(&kfd_processes_mutex);
823
824 return process;
825 }
826
kfd_get_process(const struct task_struct * thread)827 struct kfd_process *kfd_get_process(const struct task_struct *thread)
828 {
829 struct kfd_process *process;
830
831 if (!thread->mm)
832 return ERR_PTR(-EINVAL);
833
834 /* Only the pthreads threading model is supported. */
835 if (thread->group_leader->mm != thread->mm)
836 return ERR_PTR(-EINVAL);
837
838 process = find_process(thread);
839 if (!process)
840 return ERR_PTR(-EINVAL);
841
842 return process;
843 }
844
find_process_by_mm(const struct mm_struct * mm)845 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
846 {
847 struct kfd_process *process;
848
849 hash_for_each_possible_rcu(kfd_processes_table, process,
850 kfd_processes, (uintptr_t)mm)
851 if (process->mm == mm)
852 return process;
853
854 return NULL;
855 }
856
find_process(const struct task_struct * thread)857 static struct kfd_process *find_process(const struct task_struct *thread)
858 {
859 struct kfd_process *p;
860 int idx;
861
862 idx = srcu_read_lock(&kfd_processes_srcu);
863 p = find_process_by_mm(thread->mm);
864 srcu_read_unlock(&kfd_processes_srcu, idx);
865
866 return p;
867 }
868
kfd_unref_process(struct kfd_process * p)869 void kfd_unref_process(struct kfd_process *p)
870 {
871 kref_put(&p->ref, kfd_process_ref_release);
872 }
873
kfd_process_device_free_bos(struct kfd_process_device * pdd)874 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
875 {
876 struct kfd_process *p = pdd->process;
877 void *mem;
878 int id;
879
880 /*
881 * Remove all handles from idr and release appropriate
882 * local memory object
883 */
884 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
885 struct kfd_process_device *peer_pdd;
886
887 list_for_each_entry(peer_pdd, &p->per_device_data,
888 per_device_list) {
889 if (!peer_pdd->vm)
890 continue;
891 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
892 peer_pdd->dev->kgd, mem, peer_pdd->vm);
893 }
894
895 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);
896 kfd_process_device_remove_obj_handle(pdd, id);
897 }
898 }
899
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)900 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
901 {
902 struct kfd_process_device *pdd;
903
904 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
905 kfd_process_device_free_bos(pdd);
906 }
907
kfd_process_destroy_pdds(struct kfd_process * p)908 static void kfd_process_destroy_pdds(struct kfd_process *p)
909 {
910 struct kfd_process_device *pdd, *temp;
911
912 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
913 per_device_list) {
914 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
915 pdd->dev->id, p->pasid);
916
917 if (pdd->drm_file) {
918 amdgpu_amdkfd_gpuvm_release_process_vm(
919 pdd->dev->kgd, pdd->vm);
920 fput(pdd->drm_file);
921 }
922 else if (pdd->vm)
923 amdgpu_amdkfd_gpuvm_destroy_process_vm(
924 pdd->dev->kgd, pdd->vm);
925
926 list_del(&pdd->per_device_list);
927
928 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
929 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
930 get_order(KFD_CWSR_TBA_TMA_SIZE));
931
932 kfree(pdd->qpd.doorbell_bitmap);
933 idr_destroy(&pdd->alloc_idr);
934
935 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
936
937 /*
938 * before destroying pdd, make sure to report availability
939 * for auto suspend
940 */
941 if (pdd->runtime_inuse) {
942 pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
943 pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
944 pdd->runtime_inuse = false;
945 }
946
947 kfree(pdd);
948 }
949 }
950
951 /* No process locking is needed in this function, because the process
952 * is not findable any more. We must assume that no other thread is
953 * using it any more, otherwise we couldn't safely free the process
954 * structure in the end.
955 */
kfd_process_wq_release(struct work_struct * work)956 static void kfd_process_wq_release(struct work_struct *work)
957 {
958 struct kfd_process *p = container_of(work, struct kfd_process,
959 release_work);
960 struct kfd_process_device *pdd;
961
962 /* Remove the procfs files */
963 if (p->kobj) {
964 sysfs_remove_file(p->kobj, &p->attr_pasid);
965 kobject_del(p->kobj_queues);
966 kobject_put(p->kobj_queues);
967 p->kobj_queues = NULL;
968
969 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
970 sysfs_remove_file(p->kobj, &pdd->attr_vram);
971 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
972
973 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
974 if (pdd->dev->kfd2kgd->get_cu_occupancy)
975 sysfs_remove_file(pdd->kobj_stats,
976 &pdd->attr_cu_occupancy);
977 kobject_del(pdd->kobj_stats);
978 kobject_put(pdd->kobj_stats);
979 pdd->kobj_stats = NULL;
980 }
981
982 kobject_del(p->kobj);
983 kobject_put(p->kobj);
984 p->kobj = NULL;
985 }
986
987 kfd_iommu_unbind_process(p);
988
989 kfd_process_free_outstanding_kfd_bos(p);
990
991 kfd_process_destroy_pdds(p);
992 dma_fence_put(p->ef);
993
994 kfd_event_free_process(p);
995
996 kfd_pasid_free(p->pasid);
997 mutex_destroy(&p->mutex);
998
999 put_task_struct(p->lead_thread);
1000
1001 kfree(p);
1002 }
1003
kfd_process_ref_release(struct kref * ref)1004 static void kfd_process_ref_release(struct kref *ref)
1005 {
1006 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1007
1008 INIT_WORK(&p->release_work, kfd_process_wq_release);
1009 queue_work(kfd_process_wq, &p->release_work);
1010 }
1011
kfd_process_free_notifier(struct mmu_notifier * mn)1012 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1013 {
1014 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1015 }
1016
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1017 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1018 struct mm_struct *mm)
1019 {
1020 struct kfd_process *p;
1021 struct kfd_process_device *pdd = NULL;
1022
1023 /*
1024 * The kfd_process structure can not be free because the
1025 * mmu_notifier srcu is read locked
1026 */
1027 p = container_of(mn, struct kfd_process, mmu_notifier);
1028 if (WARN_ON(p->mm != mm))
1029 return;
1030
1031 mutex_lock(&kfd_processes_mutex);
1032 hash_del_rcu(&p->kfd_processes);
1033 mutex_unlock(&kfd_processes_mutex);
1034 synchronize_srcu(&kfd_processes_srcu);
1035
1036 cancel_delayed_work_sync(&p->eviction_work);
1037 cancel_delayed_work_sync(&p->restore_work);
1038
1039 mutex_lock(&p->mutex);
1040
1041 /* Iterate over all process device data structures and if the
1042 * pdd is in debug mode, we should first force unregistration,
1043 * then we will be able to destroy the queues
1044 */
1045 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1046 struct kfd_dev *dev = pdd->dev;
1047
1048 mutex_lock(kfd_get_dbgmgr_mutex());
1049 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1050 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1051 kfd_dbgmgr_destroy(dev->dbgmgr);
1052 dev->dbgmgr = NULL;
1053 }
1054 }
1055 mutex_unlock(kfd_get_dbgmgr_mutex());
1056 }
1057
1058 kfd_process_dequeue_from_all_devices(p);
1059 pqm_uninit(&p->pqm);
1060
1061 /* Indicate to other users that MM is no longer valid */
1062 p->mm = NULL;
1063 /* Signal the eviction fence after user mode queues are
1064 * destroyed. This allows any BOs to be freed without
1065 * triggering pointless evictions or waiting for fences.
1066 */
1067 dma_fence_signal(p->ef);
1068
1069 mutex_unlock(&p->mutex);
1070
1071 mmu_notifier_put(&p->mmu_notifier);
1072 }
1073
1074 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1075 .release = kfd_process_notifier_release,
1076 .free_notifier = kfd_process_free_notifier,
1077 };
1078
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1079 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1080 {
1081 unsigned long offset;
1082 struct kfd_process_device *pdd;
1083
1084 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1085 struct kfd_dev *dev = pdd->dev;
1086 struct qcm_process_device *qpd = &pdd->qpd;
1087
1088 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1089 continue;
1090
1091 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1092 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1093 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1094 MAP_SHARED, offset);
1095
1096 if (IS_ERR_VALUE(qpd->tba_addr)) {
1097 int err = qpd->tba_addr;
1098
1099 pr_err("Failure to set tba address. error %d.\n", err);
1100 qpd->tba_addr = 0;
1101 qpd->cwsr_kaddr = NULL;
1102 return err;
1103 }
1104
1105 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1106
1107 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1108 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1109 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1110 }
1111
1112 return 0;
1113 }
1114
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1115 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1116 {
1117 struct kfd_dev *dev = pdd->dev;
1118 struct qcm_process_device *qpd = &pdd->qpd;
1119 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1120 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1121 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1122 void *kaddr;
1123 int ret;
1124
1125 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1126 return 0;
1127
1128 /* cwsr_base is only set for dGPU */
1129 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1130 KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1131 if (ret)
1132 return ret;
1133
1134 qpd->cwsr_kaddr = kaddr;
1135 qpd->tba_addr = qpd->cwsr_base;
1136
1137 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1138
1139 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1140 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1141 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1142
1143 return 0;
1144 }
1145
1146 /*
1147 * On return the kfd_process is fully operational and will be freed when the
1148 * mm is released
1149 */
create_process(const struct task_struct * thread)1150 static struct kfd_process *create_process(const struct task_struct *thread)
1151 {
1152 struct kfd_process *process;
1153 int err = -ENOMEM;
1154
1155 process = kzalloc(sizeof(*process), GFP_KERNEL);
1156 if (!process)
1157 goto err_alloc_process;
1158
1159 kref_init(&process->ref);
1160 mutex_init(&process->mutex);
1161 process->mm = thread->mm;
1162 process->lead_thread = thread->group_leader;
1163 INIT_LIST_HEAD(&process->per_device_data);
1164 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1165 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1166 process->last_restore_timestamp = get_jiffies_64();
1167 kfd_event_init_process(process);
1168 process->is_32bit_user_mode = in_compat_syscall();
1169
1170 process->pasid = kfd_pasid_alloc();
1171 if (process->pasid == 0)
1172 goto err_alloc_pasid;
1173
1174 err = pqm_init(&process->pqm, process);
1175 if (err != 0)
1176 goto err_process_pqm_init;
1177
1178 /* init process apertures*/
1179 err = kfd_init_apertures(process);
1180 if (err != 0)
1181 goto err_init_apertures;
1182
1183 /* Must be last, have to use release destruction after this */
1184 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
1185 err = mmu_notifier_register(&process->mmu_notifier, process->mm);
1186 if (err)
1187 goto err_register_notifier;
1188
1189 get_task_struct(process->lead_thread);
1190 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1191 (uintptr_t)process->mm);
1192
1193 return process;
1194
1195 err_register_notifier:
1196 kfd_process_free_outstanding_kfd_bos(process);
1197 kfd_process_destroy_pdds(process);
1198 err_init_apertures:
1199 pqm_uninit(&process->pqm);
1200 err_process_pqm_init:
1201 kfd_pasid_free(process->pasid);
1202 err_alloc_pasid:
1203 mutex_destroy(&process->mutex);
1204 kfree(process);
1205 err_alloc_process:
1206 return ERR_PTR(err);
1207 }
1208
init_doorbell_bitmap(struct qcm_process_device * qpd,struct kfd_dev * dev)1209 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1210 struct kfd_dev *dev)
1211 {
1212 unsigned int i;
1213 int range_start = dev->shared_resources.non_cp_doorbells_start;
1214 int range_end = dev->shared_resources.non_cp_doorbells_end;
1215
1216 if (!KFD_IS_SOC15(dev->device_info->asic_family))
1217 return 0;
1218
1219 qpd->doorbell_bitmap =
1220 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1221 BITS_PER_BYTE), GFP_KERNEL);
1222 if (!qpd->doorbell_bitmap)
1223 return -ENOMEM;
1224
1225 /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1226 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1227 pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1228 range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1229 range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1230
1231 for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1232 if (i >= range_start && i <= range_end) {
1233 set_bit(i, qpd->doorbell_bitmap);
1234 set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1235 qpd->doorbell_bitmap);
1236 }
1237 }
1238
1239 return 0;
1240 }
1241
kfd_get_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1242 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1243 struct kfd_process *p)
1244 {
1245 struct kfd_process_device *pdd = NULL;
1246
1247 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
1248 if (pdd->dev == dev)
1249 return pdd;
1250
1251 return NULL;
1252 }
1253
kfd_create_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1254 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1255 struct kfd_process *p)
1256 {
1257 struct kfd_process_device *pdd = NULL;
1258
1259 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1260 if (!pdd)
1261 return NULL;
1262
1263 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1264 pr_err("Failed to alloc doorbell for pdd\n");
1265 goto err_free_pdd;
1266 }
1267
1268 if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1269 pr_err("Failed to init doorbell for process\n");
1270 goto err_free_pdd;
1271 }
1272
1273 pdd->dev = dev;
1274 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1275 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1276 pdd->qpd.dqm = dev->dqm;
1277 pdd->qpd.pqm = &p->pqm;
1278 pdd->qpd.evicted = 0;
1279 pdd->qpd.mapped_gws_queue = false;
1280 pdd->process = p;
1281 pdd->bound = PDD_UNBOUND;
1282 pdd->already_dequeued = false;
1283 pdd->runtime_inuse = false;
1284 pdd->vram_usage = 0;
1285 pdd->sdma_past_activity_counter = 0;
1286 atomic64_set(&pdd->evict_duration_counter, 0);
1287 list_add(&pdd->per_device_list, &p->per_device_data);
1288
1289 /* Init idr used for memory handle translation */
1290 idr_init(&pdd->alloc_idr);
1291
1292 return pdd;
1293
1294 err_free_pdd:
1295 kfree(pdd);
1296 return NULL;
1297 }
1298
1299 /**
1300 * kfd_process_device_init_vm - Initialize a VM for a process-device
1301 *
1302 * @pdd: The process-device
1303 * @drm_file: Optional pointer to a DRM file descriptor
1304 *
1305 * If @drm_file is specified, it will be used to acquire the VM from
1306 * that file descriptor. If successful, the @pdd takes ownership of
1307 * the file descriptor.
1308 *
1309 * If @drm_file is NULL, a new VM is created.
1310 *
1311 * Returns 0 on success, -errno on failure.
1312 */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1313 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1314 struct file *drm_file)
1315 {
1316 struct kfd_process *p;
1317 struct kfd_dev *dev;
1318 int ret;
1319
1320 if (pdd->vm)
1321 return drm_file ? -EBUSY : 0;
1322
1323 p = pdd->process;
1324 dev = pdd->dev;
1325
1326 if (drm_file)
1327 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1328 dev->kgd, drm_file, p->pasid,
1329 &pdd->vm, &p->kgd_process_info, &p->ef);
1330 else
1331 ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid,
1332 &pdd->vm, &p->kgd_process_info, &p->ef);
1333 if (ret) {
1334 pr_err("Failed to create process VM object\n");
1335 return ret;
1336 }
1337
1338 amdgpu_vm_set_task_info(pdd->vm);
1339
1340 ret = kfd_process_device_reserve_ib_mem(pdd);
1341 if (ret)
1342 goto err_reserve_ib_mem;
1343 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1344 if (ret)
1345 goto err_init_cwsr;
1346
1347 pdd->drm_file = drm_file;
1348
1349 return 0;
1350
1351 err_init_cwsr:
1352 err_reserve_ib_mem:
1353 kfd_process_device_free_bos(pdd);
1354 if (!drm_file)
1355 amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm);
1356 pdd->vm = NULL;
1357
1358 return ret;
1359 }
1360
1361 /*
1362 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1363 * to the device.
1364 * Unbinding occurs when the process dies or the device is removed.
1365 *
1366 * Assumes that the process lock is held.
1367 */
kfd_bind_process_to_device(struct kfd_dev * dev,struct kfd_process * p)1368 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1369 struct kfd_process *p)
1370 {
1371 struct kfd_process_device *pdd;
1372 int err;
1373
1374 pdd = kfd_get_process_device_data(dev, p);
1375 if (!pdd) {
1376 pr_err("Process device data doesn't exist\n");
1377 return ERR_PTR(-ENOMEM);
1378 }
1379
1380 /*
1381 * signal runtime-pm system to auto resume and prevent
1382 * further runtime suspend once device pdd is created until
1383 * pdd is destroyed.
1384 */
1385 if (!pdd->runtime_inuse) {
1386 err = pm_runtime_get_sync(dev->ddev->dev);
1387 if (err < 0) {
1388 pm_runtime_put_autosuspend(dev->ddev->dev);
1389 return ERR_PTR(err);
1390 }
1391 }
1392
1393 err = kfd_iommu_bind_process_to_device(pdd);
1394 if (err)
1395 goto out;
1396
1397 err = kfd_process_device_init_vm(pdd, NULL);
1398 if (err)
1399 goto out;
1400
1401 /*
1402 * make sure that runtime_usage counter is incremented just once
1403 * per pdd
1404 */
1405 pdd->runtime_inuse = true;
1406
1407 return pdd;
1408
1409 out:
1410 /* balance runpm reference count and exit with error */
1411 if (!pdd->runtime_inuse) {
1412 pm_runtime_mark_last_busy(dev->ddev->dev);
1413 pm_runtime_put_autosuspend(dev->ddev->dev);
1414 }
1415
1416 return ERR_PTR(err);
1417 }
1418
kfd_get_first_process_device_data(struct kfd_process * p)1419 struct kfd_process_device *kfd_get_first_process_device_data(
1420 struct kfd_process *p)
1421 {
1422 return list_first_entry(&p->per_device_data,
1423 struct kfd_process_device,
1424 per_device_list);
1425 }
1426
kfd_get_next_process_device_data(struct kfd_process * p,struct kfd_process_device * pdd)1427 struct kfd_process_device *kfd_get_next_process_device_data(
1428 struct kfd_process *p,
1429 struct kfd_process_device *pdd)
1430 {
1431 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
1432 return NULL;
1433 return list_next_entry(pdd, per_device_list);
1434 }
1435
kfd_has_process_device_data(struct kfd_process * p)1436 bool kfd_has_process_device_data(struct kfd_process *p)
1437 {
1438 return !(list_empty(&p->per_device_data));
1439 }
1440
1441 /* Create specific handle mapped to mem from process local memory idr
1442 * Assumes that the process lock is held.
1443 */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1444 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1445 void *mem)
1446 {
1447 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1448 }
1449
1450 /* Translate specific handle from process local memory idr
1451 * Assumes that the process lock is held.
1452 */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1453 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1454 int handle)
1455 {
1456 if (handle < 0)
1457 return NULL;
1458
1459 return idr_find(&pdd->alloc_idr, handle);
1460 }
1461
1462 /* Remove specific handle from process local memory idr
1463 * Assumes that the process lock is held.
1464 */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1465 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1466 int handle)
1467 {
1468 if (handle >= 0)
1469 idr_remove(&pdd->alloc_idr, handle);
1470 }
1471
1472 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid)1473 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1474 {
1475 struct kfd_process *p, *ret_p = NULL;
1476 unsigned int temp;
1477
1478 int idx = srcu_read_lock(&kfd_processes_srcu);
1479
1480 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1481 if (p->pasid == pasid) {
1482 kref_get(&p->ref);
1483 ret_p = p;
1484 break;
1485 }
1486 }
1487
1488 srcu_read_unlock(&kfd_processes_srcu, idx);
1489
1490 return ret_p;
1491 }
1492
1493 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1494 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1495 {
1496 struct kfd_process *p;
1497
1498 int idx = srcu_read_lock(&kfd_processes_srcu);
1499
1500 p = find_process_by_mm(mm);
1501 if (p)
1502 kref_get(&p->ref);
1503
1504 srcu_read_unlock(&kfd_processes_srcu, idx);
1505
1506 return p;
1507 }
1508
1509 /* kfd_process_evict_queues - Evict all user queues of a process
1510 *
1511 * Eviction is reference-counted per process-device. This means multiple
1512 * evictions from different sources can be nested safely.
1513 */
kfd_process_evict_queues(struct kfd_process * p)1514 int kfd_process_evict_queues(struct kfd_process *p)
1515 {
1516 struct kfd_process_device *pdd;
1517 int r = 0;
1518 unsigned int n_evicted = 0;
1519
1520 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1521 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1522 &pdd->qpd);
1523 if (r) {
1524 pr_err("Failed to evict process queues\n");
1525 goto fail;
1526 }
1527 n_evicted++;
1528 }
1529
1530 return r;
1531
1532 fail:
1533 /* To keep state consistent, roll back partial eviction by
1534 * restoring queues
1535 */
1536 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1537 if (n_evicted == 0)
1538 break;
1539 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1540 &pdd->qpd))
1541 pr_err("Failed to restore queues\n");
1542
1543 n_evicted--;
1544 }
1545
1546 return r;
1547 }
1548
1549 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1550 int kfd_process_restore_queues(struct kfd_process *p)
1551 {
1552 struct kfd_process_device *pdd;
1553 int r, ret = 0;
1554
1555 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1556 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1557 &pdd->qpd);
1558 if (r) {
1559 pr_err("Failed to restore process queues\n");
1560 if (!ret)
1561 ret = r;
1562 }
1563 }
1564
1565 return ret;
1566 }
1567
evict_process_worker(struct work_struct * work)1568 static void evict_process_worker(struct work_struct *work)
1569 {
1570 int ret;
1571 struct kfd_process *p;
1572 struct delayed_work *dwork;
1573
1574 dwork = to_delayed_work(work);
1575
1576 /* Process termination destroys this worker thread. So during the
1577 * lifetime of this thread, kfd_process p will be valid
1578 */
1579 p = container_of(dwork, struct kfd_process, eviction_work);
1580 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1581 "Eviction fence mismatch\n");
1582
1583 /* Narrow window of overlap between restore and evict work
1584 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1585 * unreserves KFD BOs, it is possible to evicted again. But
1586 * restore has few more steps of finish. So lets wait for any
1587 * previous restore work to complete
1588 */
1589 flush_delayed_work(&p->restore_work);
1590
1591 pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1592 ret = kfd_process_evict_queues(p);
1593 if (!ret) {
1594 dma_fence_signal(p->ef);
1595 dma_fence_put(p->ef);
1596 p->ef = NULL;
1597 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1598 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1599
1600 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1601 } else
1602 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1603 }
1604
restore_process_worker(struct work_struct * work)1605 static void restore_process_worker(struct work_struct *work)
1606 {
1607 struct delayed_work *dwork;
1608 struct kfd_process *p;
1609 int ret = 0;
1610
1611 dwork = to_delayed_work(work);
1612
1613 /* Process termination destroys this worker thread. So during the
1614 * lifetime of this thread, kfd_process p will be valid
1615 */
1616 p = container_of(dwork, struct kfd_process, restore_work);
1617 pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1618
1619 /* Setting last_restore_timestamp before successful restoration.
1620 * Otherwise this would have to be set by KGD (restore_process_bos)
1621 * before KFD BOs are unreserved. If not, the process can be evicted
1622 * again before the timestamp is set.
1623 * If restore fails, the timestamp will be set again in the next
1624 * attempt. This would mean that the minimum GPU quanta would be
1625 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1626 * functions)
1627 */
1628
1629 p->last_restore_timestamp = get_jiffies_64();
1630 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1631 &p->ef);
1632 if (ret) {
1633 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1634 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1635 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1636 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1637 WARN(!ret, "reschedule restore work failed\n");
1638 return;
1639 }
1640
1641 ret = kfd_process_restore_queues(p);
1642 if (!ret)
1643 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1644 else
1645 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1646 }
1647
kfd_suspend_all_processes(void)1648 void kfd_suspend_all_processes(void)
1649 {
1650 struct kfd_process *p;
1651 unsigned int temp;
1652 int idx = srcu_read_lock(&kfd_processes_srcu);
1653
1654 WARN(debug_evictions, "Evicting all processes");
1655 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1656 cancel_delayed_work_sync(&p->eviction_work);
1657 cancel_delayed_work_sync(&p->restore_work);
1658
1659 if (kfd_process_evict_queues(p))
1660 pr_err("Failed to suspend process 0x%x\n", p->pasid);
1661 dma_fence_signal(p->ef);
1662 dma_fence_put(p->ef);
1663 p->ef = NULL;
1664 }
1665 srcu_read_unlock(&kfd_processes_srcu, idx);
1666 }
1667
kfd_resume_all_processes(void)1668 int kfd_resume_all_processes(void)
1669 {
1670 struct kfd_process *p;
1671 unsigned int temp;
1672 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1673
1674 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1675 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1676 pr_err("Restore process %d failed during resume\n",
1677 p->pasid);
1678 ret = -EFAULT;
1679 }
1680 }
1681 srcu_read_unlock(&kfd_processes_srcu, idx);
1682 return ret;
1683 }
1684
kfd_reserved_mem_mmap(struct kfd_dev * dev,struct kfd_process * process,struct vm_area_struct * vma)1685 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1686 struct vm_area_struct *vma)
1687 {
1688 struct kfd_process_device *pdd;
1689 struct qcm_process_device *qpd;
1690
1691 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1692 pr_err("Incorrect CWSR mapping size.\n");
1693 return -EINVAL;
1694 }
1695
1696 pdd = kfd_get_process_device_data(dev, process);
1697 if (!pdd)
1698 return -EINVAL;
1699 qpd = &pdd->qpd;
1700
1701 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1702 get_order(KFD_CWSR_TBA_TMA_SIZE));
1703 if (!qpd->cwsr_kaddr) {
1704 pr_err("Error allocating per process CWSR buffer.\n");
1705 return -ENOMEM;
1706 }
1707
1708 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1709 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1710 /* Mapping pages to user process */
1711 return remap_pfn_range(vma, vma->vm_start,
1712 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1713 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1714 }
1715
kfd_flush_tlb(struct kfd_process_device * pdd)1716 void kfd_flush_tlb(struct kfd_process_device *pdd)
1717 {
1718 struct kfd_dev *dev = pdd->dev;
1719
1720 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1721 /* Nothing to flush until a VMID is assigned, which
1722 * only happens when the first queue is created.
1723 */
1724 if (pdd->qpd.vmid)
1725 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1726 pdd->qpd.vmid);
1727 } else {
1728 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1729 pdd->process->pasid);
1730 }
1731 }
1732
1733 #if defined(CONFIG_DEBUG_FS)
1734
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)1735 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1736 {
1737 struct kfd_process *p;
1738 unsigned int temp;
1739 int r = 0;
1740
1741 int idx = srcu_read_lock(&kfd_processes_srcu);
1742
1743 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1744 seq_printf(m, "Process %d PASID 0x%x:\n",
1745 p->lead_thread->tgid, p->pasid);
1746
1747 mutex_lock(&p->mutex);
1748 r = pqm_debugfs_mqds(m, &p->pqm);
1749 mutex_unlock(&p->mutex);
1750
1751 if (r)
1752 break;
1753 }
1754
1755 srcu_read_unlock(&kfd_processes_srcu, idx);
1756
1757 return r;
1758 }
1759
1760 #endif
1761
1762