• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <linux/mmu_context.h>
29 #include <linux/slab.h>
30 #include <linux/amd-iommu.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 
39 struct mm_struct;
40 
41 #include "kfd_priv.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_dbgmgr.h"
44 #include "kfd_iommu.h"
45 #include "kfd_svm.h"
46 
47 /*
48  * List of struct kfd_process (field kfd_process).
49  * Unique/indexed by mm_struct*
50  */
51 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
52 static DEFINE_MUTEX(kfd_processes_mutex);
53 
54 DEFINE_SRCU(kfd_processes_srcu);
55 
56 /* For process termination handling */
57 static struct workqueue_struct *kfd_process_wq;
58 
59 /* Ordered, single-threaded workqueue for restoring evicted
60  * processes. Restoring multiple processes concurrently under memory
61  * pressure can lead to processes blocking each other from validating
62  * their BOs and result in a live-lock situation where processes
63  * remain evicted indefinitely.
64  */
65 static struct workqueue_struct *kfd_restore_wq;
66 
67 static struct kfd_process *find_process(const struct task_struct *thread);
68 static void kfd_process_ref_release(struct kref *ref);
69 static struct kfd_process *create_process(const struct task_struct *thread);
70 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
71 
72 static void evict_process_worker(struct work_struct *work);
73 static void restore_process_worker(struct work_struct *work);
74 
75 struct kfd_procfs_tree {
76 	struct kobject *kobj;
77 };
78 
79 static struct kfd_procfs_tree procfs;
80 
81 /*
82  * Structure for SDMA activity tracking
83  */
84 struct kfd_sdma_activity_handler_workarea {
85 	struct work_struct sdma_activity_work;
86 	struct kfd_process_device *pdd;
87 	uint64_t sdma_activity_counter;
88 };
89 
90 struct temp_sdma_queue_list {
91 	uint64_t __user *rptr;
92 	uint64_t sdma_val;
93 	unsigned int queue_id;
94 	struct list_head list;
95 };
96 
kfd_sdma_activity_worker(struct work_struct * work)97 static void kfd_sdma_activity_worker(struct work_struct *work)
98 {
99 	struct kfd_sdma_activity_handler_workarea *workarea;
100 	struct kfd_process_device *pdd;
101 	uint64_t val;
102 	struct mm_struct *mm;
103 	struct queue *q;
104 	struct qcm_process_device *qpd;
105 	struct device_queue_manager *dqm;
106 	int ret = 0;
107 	struct temp_sdma_queue_list sdma_q_list;
108 	struct temp_sdma_queue_list *sdma_q, *next;
109 
110 	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
111 				sdma_activity_work);
112 
113 	pdd = workarea->pdd;
114 	if (!pdd)
115 		return;
116 	dqm = pdd->dev->dqm;
117 	qpd = &pdd->qpd;
118 	if (!dqm || !qpd)
119 		return;
120 	/*
121 	 * Total SDMA activity is current SDMA activity + past SDMA activity
122 	 * Past SDMA count is stored in pdd.
123 	 * To get the current activity counters for all active SDMA queues,
124 	 * we loop over all SDMA queues and get their counts from user-space.
125 	 *
126 	 * We cannot call get_user() with dqm_lock held as it can cause
127 	 * a circular lock dependency situation. To read the SDMA stats,
128 	 * we need to do the following:
129 	 *
130 	 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
131 	 *    with dqm_lock/dqm_unlock().
132 	 * 2. Call get_user() for each node in temporary list without dqm_lock.
133 	 *    Save the SDMA count for each node and also add the count to the total
134 	 *    SDMA count counter.
135 	 *    Its possible, during this step, a few SDMA queue nodes got deleted
136 	 *    from the qpd->queues_list.
137 	 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
138 	 *    If any node got deleted, its SDMA count would be captured in the sdma
139 	 *    past activity counter. So subtract the SDMA counter stored in step 2
140 	 *    for this node from the total SDMA count.
141 	 */
142 	INIT_LIST_HEAD(&sdma_q_list.list);
143 
144 	/*
145 	 * Create the temp list of all SDMA queues
146 	 */
147 	dqm_lock(dqm);
148 
149 	list_for_each_entry(q, &qpd->queues_list, list) {
150 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
151 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
152 			continue;
153 
154 		sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
155 		if (!sdma_q) {
156 			dqm_unlock(dqm);
157 			goto cleanup;
158 		}
159 
160 		INIT_LIST_HEAD(&sdma_q->list);
161 		sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
162 		sdma_q->queue_id = q->properties.queue_id;
163 		list_add_tail(&sdma_q->list, &sdma_q_list.list);
164 	}
165 
166 	/*
167 	 * If the temp list is empty, then no SDMA queues nodes were found in
168 	 * qpd->queues_list. Return the past activity count as the total sdma
169 	 * count
170 	 */
171 	if (list_empty(&sdma_q_list.list)) {
172 		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
173 		dqm_unlock(dqm);
174 		return;
175 	}
176 
177 	dqm_unlock(dqm);
178 
179 	/*
180 	 * Get the usage count for each SDMA queue in temp_list.
181 	 */
182 	mm = get_task_mm(pdd->process->lead_thread);
183 	if (!mm)
184 		goto cleanup;
185 
186 	kthread_use_mm(mm);
187 
188 	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
189 		val = 0;
190 		ret = read_sdma_queue_counter(sdma_q->rptr, &val);
191 		if (ret) {
192 			pr_debug("Failed to read SDMA queue active counter for queue id: %d",
193 				 sdma_q->queue_id);
194 		} else {
195 			sdma_q->sdma_val = val;
196 			workarea->sdma_activity_counter += val;
197 		}
198 	}
199 
200 	kthread_unuse_mm(mm);
201 	mmput(mm);
202 
203 	/*
204 	 * Do a second iteration over qpd_queues_list to check if any SDMA
205 	 * nodes got deleted while fetching SDMA counter.
206 	 */
207 	dqm_lock(dqm);
208 
209 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
210 
211 	list_for_each_entry(q, &qpd->queues_list, list) {
212 		if (list_empty(&sdma_q_list.list))
213 			break;
214 
215 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
216 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
217 			continue;
218 
219 		list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
220 			if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
221 			     (sdma_q->queue_id == q->properties.queue_id)) {
222 				list_del(&sdma_q->list);
223 				kfree(sdma_q);
224 				break;
225 			}
226 		}
227 	}
228 
229 	dqm_unlock(dqm);
230 
231 	/*
232 	 * If temp list is not empty, it implies some queues got deleted
233 	 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
234 	 * count for each node from the total SDMA count.
235 	 */
236 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
237 		workarea->sdma_activity_counter -= sdma_q->sdma_val;
238 		list_del(&sdma_q->list);
239 		kfree(sdma_q);
240 	}
241 
242 	return;
243 
244 cleanup:
245 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
246 		list_del(&sdma_q->list);
247 		kfree(sdma_q);
248 	}
249 }
250 
251 /**
252  * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
253  * by current process. Translates acquired wave count into number of compute units
254  * that are occupied.
255  *
256  * @atr: Handle of attribute that allows reporting of wave count. The attribute
257  * handle encapsulates GPU device it is associated with, thereby allowing collection
258  * of waves in flight, etc
259  *
260  * @buffer: Handle of user provided buffer updated with wave count
261  *
262  * Return: Number of bytes written to user buffer or an error value
263  */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)264 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
265 {
266 	int cu_cnt;
267 	int wave_cnt;
268 	int max_waves_per_cu;
269 	struct kfd_dev *dev = NULL;
270 	struct kfd_process *proc = NULL;
271 	struct kfd_process_device *pdd = NULL;
272 
273 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
274 	dev = pdd->dev;
275 	if (dev->kfd2kgd->get_cu_occupancy == NULL)
276 		return -EINVAL;
277 
278 	cu_cnt = 0;
279 	proc = pdd->process;
280 	if (pdd->qpd.queue_count == 0) {
281 		pr_debug("Gpu-Id: %d has no active queues for process %d\n",
282 			 dev->id, proc->pasid);
283 		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
284 	}
285 
286 	/* Collect wave count from device if it supports */
287 	wave_cnt = 0;
288 	max_waves_per_cu = 0;
289 	dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
290 			&max_waves_per_cu);
291 
292 	/* Translate wave count to number of compute units */
293 	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
294 	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
295 }
296 
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)297 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
298 			       char *buffer)
299 {
300 	if (strcmp(attr->name, "pasid") == 0) {
301 		struct kfd_process *p = container_of(attr, struct kfd_process,
302 						     attr_pasid);
303 
304 		return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
305 	} else if (strncmp(attr->name, "vram_", 5) == 0) {
306 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
307 							      attr_vram);
308 		return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
309 	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
310 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
311 							      attr_sdma);
312 		struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
313 
314 		INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
315 					kfd_sdma_activity_worker);
316 
317 		sdma_activity_work_handler.pdd = pdd;
318 		sdma_activity_work_handler.sdma_activity_counter = 0;
319 
320 		schedule_work(&sdma_activity_work_handler.sdma_activity_work);
321 
322 		flush_work(&sdma_activity_work_handler.sdma_activity_work);
323 
324 		return snprintf(buffer, PAGE_SIZE, "%llu\n",
325 				(sdma_activity_work_handler.sdma_activity_counter)/
326 				 SDMA_ACTIVITY_DIVISOR);
327 	} else {
328 		pr_err("Invalid attribute");
329 		return -EINVAL;
330 	}
331 
332 	return 0;
333 }
334 
kfd_procfs_kobj_release(struct kobject * kobj)335 static void kfd_procfs_kobj_release(struct kobject *kobj)
336 {
337 	kfree(kobj);
338 }
339 
340 static const struct sysfs_ops kfd_procfs_ops = {
341 	.show = kfd_procfs_show,
342 };
343 
344 static struct kobj_type procfs_type = {
345 	.release = kfd_procfs_kobj_release,
346 	.sysfs_ops = &kfd_procfs_ops,
347 };
348 
kfd_procfs_init(void)349 void kfd_procfs_init(void)
350 {
351 	int ret = 0;
352 
353 	procfs.kobj = kfd_alloc_struct(procfs.kobj);
354 	if (!procfs.kobj)
355 		return;
356 
357 	ret = kobject_init_and_add(procfs.kobj, &procfs_type,
358 				   &kfd_device->kobj, "proc");
359 	if (ret) {
360 		pr_warn("Could not create procfs proc folder");
361 		/* If we fail to create the procfs, clean up */
362 		kfd_procfs_shutdown();
363 	}
364 }
365 
kfd_procfs_shutdown(void)366 void kfd_procfs_shutdown(void)
367 {
368 	if (procfs.kobj) {
369 		kobject_del(procfs.kobj);
370 		kobject_put(procfs.kobj);
371 		procfs.kobj = NULL;
372 	}
373 }
374 
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)375 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
376 				     struct attribute *attr, char *buffer)
377 {
378 	struct queue *q = container_of(kobj, struct queue, kobj);
379 
380 	if (!strcmp(attr->name, "size"))
381 		return snprintf(buffer, PAGE_SIZE, "%llu",
382 				q->properties.queue_size);
383 	else if (!strcmp(attr->name, "type"))
384 		return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
385 	else if (!strcmp(attr->name, "gpuid"))
386 		return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
387 	else
388 		pr_err("Invalid attribute");
389 
390 	return 0;
391 }
392 
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)393 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
394 				     struct attribute *attr, char *buffer)
395 {
396 	if (strcmp(attr->name, "evicted_ms") == 0) {
397 		struct kfd_process_device *pdd = container_of(attr,
398 				struct kfd_process_device,
399 				attr_evict);
400 		uint64_t evict_jiffies;
401 
402 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
403 
404 		return snprintf(buffer,
405 				PAGE_SIZE,
406 				"%llu\n",
407 				jiffies64_to_msecs(evict_jiffies));
408 
409 	/* Sysfs handle that gets CU occupancy is per device */
410 	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
411 		return kfd_get_cu_occupancy(attr, buffer);
412 	} else {
413 		pr_err("Invalid attribute");
414 	}
415 
416 	return 0;
417 }
418 
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)419 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
420 				       struct attribute *attr, char *buf)
421 {
422 	struct kfd_process_device *pdd;
423 
424 	if (!strcmp(attr->name, "faults")) {
425 		pdd = container_of(attr, struct kfd_process_device,
426 				   attr_faults);
427 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
428 	}
429 	if (!strcmp(attr->name, "page_in")) {
430 		pdd = container_of(attr, struct kfd_process_device,
431 				   attr_page_in);
432 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
433 	}
434 	if (!strcmp(attr->name, "page_out")) {
435 		pdd = container_of(attr, struct kfd_process_device,
436 				   attr_page_out);
437 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
438 	}
439 	return 0;
440 }
441 
442 static struct attribute attr_queue_size = {
443 	.name = "size",
444 	.mode = KFD_SYSFS_FILE_MODE
445 };
446 
447 static struct attribute attr_queue_type = {
448 	.name = "type",
449 	.mode = KFD_SYSFS_FILE_MODE
450 };
451 
452 static struct attribute attr_queue_gpuid = {
453 	.name = "gpuid",
454 	.mode = KFD_SYSFS_FILE_MODE
455 };
456 
457 static struct attribute *procfs_queue_attrs[] = {
458 	&attr_queue_size,
459 	&attr_queue_type,
460 	&attr_queue_gpuid,
461 	NULL
462 };
463 
464 static const struct sysfs_ops procfs_queue_ops = {
465 	.show = kfd_procfs_queue_show,
466 };
467 
468 static struct kobj_type procfs_queue_type = {
469 	.sysfs_ops = &procfs_queue_ops,
470 	.default_attrs = procfs_queue_attrs,
471 };
472 
473 static const struct sysfs_ops procfs_stats_ops = {
474 	.show = kfd_procfs_stats_show,
475 };
476 
477 static struct kobj_type procfs_stats_type = {
478 	.sysfs_ops = &procfs_stats_ops,
479 	.release = kfd_procfs_kobj_release,
480 };
481 
482 static const struct sysfs_ops sysfs_counters_ops = {
483 	.show = kfd_sysfs_counters_show,
484 };
485 
486 static struct kobj_type sysfs_counters_type = {
487 	.sysfs_ops = &sysfs_counters_ops,
488 	.release = kfd_procfs_kobj_release,
489 };
490 
kfd_procfs_add_queue(struct queue * q)491 int kfd_procfs_add_queue(struct queue *q)
492 {
493 	struct kfd_process *proc;
494 	int ret;
495 
496 	if (!q || !q->process)
497 		return -EINVAL;
498 	proc = q->process;
499 
500 	/* Create proc/<pid>/queues/<queue id> folder */
501 	if (!proc->kobj_queues)
502 		return -EFAULT;
503 	ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
504 			proc->kobj_queues, "%u", q->properties.queue_id);
505 	if (ret < 0) {
506 		pr_warn("Creating proc/<pid>/queues/%u failed",
507 			q->properties.queue_id);
508 		kobject_put(&q->kobj);
509 		return ret;
510 	}
511 
512 	return 0;
513 }
514 
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)515 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
516 				 char *name)
517 {
518 	int ret;
519 
520 	if (!kobj || !attr || !name)
521 		return;
522 
523 	attr->name = name;
524 	attr->mode = KFD_SYSFS_FILE_MODE;
525 	sysfs_attr_init(attr);
526 
527 	ret = sysfs_create_file(kobj, attr);
528 	if (ret)
529 		pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
530 }
531 
kfd_procfs_add_sysfs_stats(struct kfd_process * p)532 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
533 {
534 	int ret;
535 	int i;
536 	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
537 
538 	if (!p || !p->kobj)
539 		return;
540 
541 	/*
542 	 * Create sysfs files for each GPU:
543 	 * - proc/<pid>/stats_<gpuid>/
544 	 * - proc/<pid>/stats_<gpuid>/evicted_ms
545 	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
546 	 */
547 	for (i = 0; i < p->n_pdds; i++) {
548 		struct kfd_process_device *pdd = p->pdds[i];
549 
550 		snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
551 				"stats_%u", pdd->dev->id);
552 		pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
553 		if (!pdd->kobj_stats)
554 			return;
555 
556 		ret = kobject_init_and_add(pdd->kobj_stats,
557 					   &procfs_stats_type,
558 					   p->kobj,
559 					   stats_dir_filename);
560 
561 		if (ret) {
562 			pr_warn("Creating KFD proc/stats_%s folder failed",
563 				stats_dir_filename);
564 			kobject_put(pdd->kobj_stats);
565 			pdd->kobj_stats = NULL;
566 			return;
567 		}
568 
569 		kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
570 				      "evicted_ms");
571 		/* Add sysfs file to report compute unit occupancy */
572 		if (pdd->dev->kfd2kgd->get_cu_occupancy)
573 			kfd_sysfs_create_file(pdd->kobj_stats,
574 					      &pdd->attr_cu_occupancy,
575 					      "cu_occupancy");
576 	}
577 }
578 
kfd_procfs_add_sysfs_counters(struct kfd_process * p)579 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
580 {
581 	int ret = 0;
582 	int i;
583 	char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
584 
585 	if (!p || !p->kobj)
586 		return;
587 
588 	/*
589 	 * Create sysfs files for each GPU which supports SVM
590 	 * - proc/<pid>/counters_<gpuid>/
591 	 * - proc/<pid>/counters_<gpuid>/faults
592 	 * - proc/<pid>/counters_<gpuid>/page_in
593 	 * - proc/<pid>/counters_<gpuid>/page_out
594 	 */
595 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
596 		struct kfd_process_device *pdd = p->pdds[i];
597 		struct kobject *kobj_counters;
598 
599 		snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
600 			"counters_%u", pdd->dev->id);
601 		kobj_counters = kfd_alloc_struct(kobj_counters);
602 		if (!kobj_counters)
603 			return;
604 
605 		ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
606 					   p->kobj, counters_dir_filename);
607 		if (ret) {
608 			pr_warn("Creating KFD proc/%s folder failed",
609 				counters_dir_filename);
610 			kobject_put(kobj_counters);
611 			return;
612 		}
613 
614 		pdd->kobj_counters = kobj_counters;
615 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
616 				      "faults");
617 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
618 				      "page_in");
619 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
620 				      "page_out");
621 	}
622 }
623 
kfd_procfs_add_sysfs_files(struct kfd_process * p)624 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
625 {
626 	int i;
627 
628 	if (!p || !p->kobj)
629 		return;
630 
631 	/*
632 	 * Create sysfs files for each GPU:
633 	 * - proc/<pid>/vram_<gpuid>
634 	 * - proc/<pid>/sdma_<gpuid>
635 	 */
636 	for (i = 0; i < p->n_pdds; i++) {
637 		struct kfd_process_device *pdd = p->pdds[i];
638 
639 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
640 			 pdd->dev->id);
641 		kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
642 				      pdd->vram_filename);
643 
644 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
645 			 pdd->dev->id);
646 		kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
647 					    pdd->sdma_filename);
648 	}
649 }
650 
kfd_procfs_del_queue(struct queue * q)651 void kfd_procfs_del_queue(struct queue *q)
652 {
653 	if (!q)
654 		return;
655 
656 	kobject_del(&q->kobj);
657 	kobject_put(&q->kobj);
658 }
659 
kfd_process_create_wq(void)660 int kfd_process_create_wq(void)
661 {
662 	if (!kfd_process_wq)
663 		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
664 	if (!kfd_restore_wq)
665 		kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
666 
667 	if (!kfd_process_wq || !kfd_restore_wq) {
668 		kfd_process_destroy_wq();
669 		return -ENOMEM;
670 	}
671 
672 	return 0;
673 }
674 
kfd_process_destroy_wq(void)675 void kfd_process_destroy_wq(void)
676 {
677 	if (kfd_process_wq) {
678 		destroy_workqueue(kfd_process_wq);
679 		kfd_process_wq = NULL;
680 	}
681 	if (kfd_restore_wq) {
682 		destroy_workqueue(kfd_restore_wq);
683 		kfd_restore_wq = NULL;
684 	}
685 }
686 
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd)687 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
688 			struct kfd_process_device *pdd)
689 {
690 	struct kfd_dev *dev = pdd->dev;
691 
692 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
693 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
694 					       NULL);
695 }
696 
697 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
698  *	This function should be only called right after the process
699  *	is created and when kfd_processes_mutex is still being held
700  *	to avoid concurrency. Because of that exclusiveness, we do
701  *	not need to take p->mutex.
702  */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,void ** kptr)703 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
704 				   uint64_t gpu_va, uint32_t size,
705 				   uint32_t flags, void **kptr)
706 {
707 	struct kfd_dev *kdev = pdd->dev;
708 	struct kgd_mem *mem = NULL;
709 	int handle;
710 	int err;
711 
712 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
713 						 pdd->drm_priv, &mem, NULL, flags);
714 	if (err)
715 		goto err_alloc_mem;
716 
717 	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem,
718 			pdd->drm_priv, NULL);
719 	if (err)
720 		goto err_map_mem;
721 
722 	err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
723 	if (err) {
724 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
725 		goto sync_memory_failed;
726 	}
727 
728 	/* Create an obj handle so kfd_process_device_remove_obj_handle
729 	 * will take care of the bo removal when the process finishes.
730 	 * We do not need to take p->mutex, because the process is just
731 	 * created and the ioctls have not had the chance to run.
732 	 */
733 	handle = kfd_process_device_create_obj_handle(pdd, mem);
734 
735 	if (handle < 0) {
736 		err = handle;
737 		goto free_gpuvm;
738 	}
739 
740 	if (kptr) {
741 		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
742 				(struct kgd_mem *)mem, kptr, NULL);
743 		if (err) {
744 			pr_debug("Map GTT BO to kernel failed\n");
745 			goto free_obj_handle;
746 		}
747 	}
748 
749 	return err;
750 
751 free_obj_handle:
752 	kfd_process_device_remove_obj_handle(pdd, handle);
753 free_gpuvm:
754 sync_memory_failed:
755 	kfd_process_free_gpuvm(mem, pdd);
756 	return err;
757 
758 err_map_mem:
759 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,
760 					       NULL);
761 err_alloc_mem:
762 	*kptr = NULL;
763 	return err;
764 }
765 
766 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
767  *	process for IB usage The memory reserved is for KFD to submit
768  *	IB to AMDGPU from kernel.  If the memory is reserved
769  *	successfully, ib_kaddr will have the CPU/kernel
770  *	address. Check ib_kaddr before accessing the memory.
771  */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)772 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
773 {
774 	struct qcm_process_device *qpd = &pdd->qpd;
775 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
776 			KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
777 			KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
778 			KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
779 	void *kaddr;
780 	int ret;
781 
782 	if (qpd->ib_kaddr || !qpd->ib_base)
783 		return 0;
784 
785 	/* ib_base is only set for dGPU */
786 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
787 				      &kaddr);
788 	if (ret)
789 		return ret;
790 
791 	qpd->ib_kaddr = kaddr;
792 
793 	return 0;
794 }
795 
kfd_create_process(struct file * filep)796 struct kfd_process *kfd_create_process(struct file *filep)
797 {
798 	struct kfd_process *process;
799 	struct task_struct *thread = current;
800 	int ret;
801 
802 	if (!thread->mm)
803 		return ERR_PTR(-EINVAL);
804 
805 	/* Only the pthreads threading model is supported. */
806 	if (thread->group_leader->mm != thread->mm)
807 		return ERR_PTR(-EINVAL);
808 
809 	/*
810 	 * take kfd processes mutex before starting of process creation
811 	 * so there won't be a case where two threads of the same process
812 	 * create two kfd_process structures
813 	 */
814 	mutex_lock(&kfd_processes_mutex);
815 
816 	/* A prior open of /dev/kfd could have already created the process. */
817 	process = find_process(thread);
818 	if (process) {
819 		pr_debug("Process already found\n");
820 	} else {
821 		process = create_process(thread);
822 		if (IS_ERR(process))
823 			goto out;
824 
825 		ret = kfd_process_init_cwsr_apu(process, filep);
826 		if (ret)
827 			goto out_destroy;
828 
829 		if (!procfs.kobj)
830 			goto out;
831 
832 		process->kobj = kfd_alloc_struct(process->kobj);
833 		if (!process->kobj) {
834 			pr_warn("Creating procfs kobject failed");
835 			goto out;
836 		}
837 		ret = kobject_init_and_add(process->kobj, &procfs_type,
838 					   procfs.kobj, "%d",
839 					   (int)process->lead_thread->pid);
840 		if (ret) {
841 			pr_warn("Creating procfs pid directory failed");
842 			kobject_put(process->kobj);
843 			goto out;
844 		}
845 
846 		kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
847 				      "pasid");
848 
849 		process->kobj_queues = kobject_create_and_add("queues",
850 							process->kobj);
851 		if (!process->kobj_queues)
852 			pr_warn("Creating KFD proc/queues folder failed");
853 
854 		kfd_procfs_add_sysfs_stats(process);
855 		kfd_procfs_add_sysfs_files(process);
856 		kfd_procfs_add_sysfs_counters(process);
857 	}
858 out:
859 	if (!IS_ERR(process))
860 		kref_get(&process->ref);
861 	mutex_unlock(&kfd_processes_mutex);
862 
863 	return process;
864 
865 out_destroy:
866 	hash_del_rcu(&process->kfd_processes);
867 	mutex_unlock(&kfd_processes_mutex);
868 	synchronize_srcu(&kfd_processes_srcu);
869 	/* kfd_process_free_notifier will trigger the cleanup */
870 	mmu_notifier_put(&process->mmu_notifier);
871 	return ERR_PTR(ret);
872 }
873 
kfd_get_process(const struct task_struct * thread)874 struct kfd_process *kfd_get_process(const struct task_struct *thread)
875 {
876 	struct kfd_process *process;
877 
878 	if (!thread->mm)
879 		return ERR_PTR(-EINVAL);
880 
881 	/* Only the pthreads threading model is supported. */
882 	if (thread->group_leader->mm != thread->mm)
883 		return ERR_PTR(-EINVAL);
884 
885 	process = find_process(thread);
886 	if (!process)
887 		return ERR_PTR(-EINVAL);
888 
889 	return process;
890 }
891 
find_process_by_mm(const struct mm_struct * mm)892 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
893 {
894 	struct kfd_process *process;
895 
896 	hash_for_each_possible_rcu(kfd_processes_table, process,
897 					kfd_processes, (uintptr_t)mm)
898 		if (process->mm == mm)
899 			return process;
900 
901 	return NULL;
902 }
903 
find_process(const struct task_struct * thread)904 static struct kfd_process *find_process(const struct task_struct *thread)
905 {
906 	struct kfd_process *p;
907 	int idx;
908 
909 	idx = srcu_read_lock(&kfd_processes_srcu);
910 	p = find_process_by_mm(thread->mm);
911 	srcu_read_unlock(&kfd_processes_srcu, idx);
912 
913 	return p;
914 }
915 
kfd_unref_process(struct kfd_process * p)916 void kfd_unref_process(struct kfd_process *p)
917 {
918 	kref_put(&p->ref, kfd_process_ref_release);
919 }
920 
921 
kfd_process_device_free_bos(struct kfd_process_device * pdd)922 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
923 {
924 	struct kfd_process *p = pdd->process;
925 	void *mem;
926 	int id;
927 	int i;
928 
929 	/*
930 	 * Remove all handles from idr and release appropriate
931 	 * local memory object
932 	 */
933 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
934 
935 		for (i = 0; i < p->n_pdds; i++) {
936 			struct kfd_process_device *peer_pdd = p->pdds[i];
937 
938 			if (!peer_pdd->drm_priv)
939 				continue;
940 			amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
941 				peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
942 		}
943 
944 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
945 						       pdd->drm_priv, NULL);
946 		kfd_process_device_remove_obj_handle(pdd, id);
947 	}
948 }
949 
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)950 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
951 {
952 	int i;
953 
954 	for (i = 0; i < p->n_pdds; i++)
955 		kfd_process_device_free_bos(p->pdds[i]);
956 }
957 
kfd_process_destroy_pdds(struct kfd_process * p)958 static void kfd_process_destroy_pdds(struct kfd_process *p)
959 {
960 	int i;
961 
962 	for (i = 0; i < p->n_pdds; i++) {
963 		struct kfd_process_device *pdd = p->pdds[i];
964 
965 		pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
966 				pdd->dev->id, p->pasid);
967 
968 		if (pdd->drm_file) {
969 			amdgpu_amdkfd_gpuvm_release_process_vm(
970 					pdd->dev->kgd, pdd->drm_priv);
971 			fput(pdd->drm_file);
972 		}
973 
974 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
975 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
976 				get_order(KFD_CWSR_TBA_TMA_SIZE));
977 
978 		kfree(pdd->qpd.doorbell_bitmap);
979 		idr_destroy(&pdd->alloc_idr);
980 
981 		kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
982 
983 		/*
984 		 * before destroying pdd, make sure to report availability
985 		 * for auto suspend
986 		 */
987 		if (pdd->runtime_inuse) {
988 			pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
989 			pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
990 			pdd->runtime_inuse = false;
991 		}
992 
993 		kfree(pdd);
994 		p->pdds[i] = NULL;
995 	}
996 	p->n_pdds = 0;
997 }
998 
kfd_process_remove_sysfs(struct kfd_process * p)999 static void kfd_process_remove_sysfs(struct kfd_process *p)
1000 {
1001 	struct kfd_process_device *pdd;
1002 	int i;
1003 
1004 	if (!p->kobj)
1005 		return;
1006 
1007 	sysfs_remove_file(p->kobj, &p->attr_pasid);
1008 	kobject_del(p->kobj_queues);
1009 	kobject_put(p->kobj_queues);
1010 	p->kobj_queues = NULL;
1011 
1012 	for (i = 0; i < p->n_pdds; i++) {
1013 		pdd = p->pdds[i];
1014 
1015 		sysfs_remove_file(p->kobj, &pdd->attr_vram);
1016 		sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1017 
1018 		sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1019 		if (pdd->dev->kfd2kgd->get_cu_occupancy)
1020 			sysfs_remove_file(pdd->kobj_stats,
1021 					  &pdd->attr_cu_occupancy);
1022 		kobject_del(pdd->kobj_stats);
1023 		kobject_put(pdd->kobj_stats);
1024 		pdd->kobj_stats = NULL;
1025 	}
1026 
1027 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1028 		pdd = p->pdds[i];
1029 
1030 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1031 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1032 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1033 		kobject_del(pdd->kobj_counters);
1034 		kobject_put(pdd->kobj_counters);
1035 		pdd->kobj_counters = NULL;
1036 	}
1037 
1038 	kobject_del(p->kobj);
1039 	kobject_put(p->kobj);
1040 	p->kobj = NULL;
1041 }
1042 
1043 /* No process locking is needed in this function, because the process
1044  * is not findable any more. We must assume that no other thread is
1045  * using it any more, otherwise we couldn't safely free the process
1046  * structure in the end.
1047  */
kfd_process_wq_release(struct work_struct * work)1048 static void kfd_process_wq_release(struct work_struct *work)
1049 {
1050 	struct kfd_process *p = container_of(work, struct kfd_process,
1051 					     release_work);
1052 	kfd_process_remove_sysfs(p);
1053 	kfd_iommu_unbind_process(p);
1054 
1055 	kfd_process_free_outstanding_kfd_bos(p);
1056 	svm_range_list_fini(p);
1057 
1058 	kfd_process_destroy_pdds(p);
1059 	dma_fence_put(p->ef);
1060 
1061 	kfd_event_free_process(p);
1062 
1063 	kfd_pasid_free(p->pasid);
1064 	mutex_destroy(&p->mutex);
1065 
1066 	put_task_struct(p->lead_thread);
1067 
1068 	kfree(p);
1069 }
1070 
kfd_process_ref_release(struct kref * ref)1071 static void kfd_process_ref_release(struct kref *ref)
1072 {
1073 	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1074 
1075 	INIT_WORK(&p->release_work, kfd_process_wq_release);
1076 	queue_work(kfd_process_wq, &p->release_work);
1077 }
1078 
kfd_process_alloc_notifier(struct mm_struct * mm)1079 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1080 {
1081 	int idx = srcu_read_lock(&kfd_processes_srcu);
1082 	struct kfd_process *p = find_process_by_mm(mm);
1083 
1084 	srcu_read_unlock(&kfd_processes_srcu, idx);
1085 
1086 	return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1087 }
1088 
kfd_process_free_notifier(struct mmu_notifier * mn)1089 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1090 {
1091 	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1092 }
1093 
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1094 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1095 					struct mm_struct *mm)
1096 {
1097 	struct kfd_process *p;
1098 	int i;
1099 
1100 	/*
1101 	 * The kfd_process structure can not be free because the
1102 	 * mmu_notifier srcu is read locked
1103 	 */
1104 	p = container_of(mn, struct kfd_process, mmu_notifier);
1105 	if (WARN_ON(p->mm != mm))
1106 		return;
1107 
1108 	mutex_lock(&kfd_processes_mutex);
1109 	hash_del_rcu(&p->kfd_processes);
1110 	mutex_unlock(&kfd_processes_mutex);
1111 	synchronize_srcu(&kfd_processes_srcu);
1112 
1113 	cancel_delayed_work_sync(&p->eviction_work);
1114 	cancel_delayed_work_sync(&p->restore_work);
1115 	cancel_delayed_work_sync(&p->svms.restore_work);
1116 
1117 	mutex_lock(&p->mutex);
1118 
1119 	/* Iterate over all process device data structures and if the
1120 	 * pdd is in debug mode, we should first force unregistration,
1121 	 * then we will be able to destroy the queues
1122 	 */
1123 	for (i = 0; i < p->n_pdds; i++) {
1124 		struct kfd_dev *dev = p->pdds[i]->dev;
1125 
1126 		mutex_lock(kfd_get_dbgmgr_mutex());
1127 		if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1128 			if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1129 				kfd_dbgmgr_destroy(dev->dbgmgr);
1130 				dev->dbgmgr = NULL;
1131 			}
1132 		}
1133 		mutex_unlock(kfd_get_dbgmgr_mutex());
1134 	}
1135 
1136 	kfd_process_dequeue_from_all_devices(p);
1137 	pqm_uninit(&p->pqm);
1138 
1139 	/* Indicate to other users that MM is no longer valid */
1140 	p->mm = NULL;
1141 	/* Signal the eviction fence after user mode queues are
1142 	 * destroyed. This allows any BOs to be freed without
1143 	 * triggering pointless evictions or waiting for fences.
1144 	 */
1145 	dma_fence_signal(p->ef);
1146 
1147 	mutex_unlock(&p->mutex);
1148 
1149 	mmu_notifier_put(&p->mmu_notifier);
1150 }
1151 
1152 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1153 	.release = kfd_process_notifier_release,
1154 	.alloc_notifier = kfd_process_alloc_notifier,
1155 	.free_notifier = kfd_process_free_notifier,
1156 };
1157 
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1158 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1159 {
1160 	unsigned long  offset;
1161 	int i;
1162 
1163 	for (i = 0; i < p->n_pdds; i++) {
1164 		struct kfd_dev *dev = p->pdds[i]->dev;
1165 		struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1166 
1167 		if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1168 			continue;
1169 
1170 		offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1171 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1172 			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1173 			MAP_SHARED, offset);
1174 
1175 		if (IS_ERR_VALUE(qpd->tba_addr)) {
1176 			int err = qpd->tba_addr;
1177 
1178 			pr_err("Failure to set tba address. error %d.\n", err);
1179 			qpd->tba_addr = 0;
1180 			qpd->cwsr_kaddr = NULL;
1181 			return err;
1182 		}
1183 
1184 		memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1185 
1186 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1187 		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1188 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1189 	}
1190 
1191 	return 0;
1192 }
1193 
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1194 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1195 {
1196 	struct kfd_dev *dev = pdd->dev;
1197 	struct qcm_process_device *qpd = &pdd->qpd;
1198 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1199 			| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1200 			| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1201 	void *kaddr;
1202 	int ret;
1203 
1204 	if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1205 		return 0;
1206 
1207 	/* cwsr_base is only set for dGPU */
1208 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1209 				      KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1210 	if (ret)
1211 		return ret;
1212 
1213 	qpd->cwsr_kaddr = kaddr;
1214 	qpd->tba_addr = qpd->cwsr_base;
1215 
1216 	memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1217 
1218 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1219 	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1220 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1221 
1222 	return 0;
1223 }
1224 
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1225 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1226 				  uint64_t tba_addr,
1227 				  uint64_t tma_addr)
1228 {
1229 	if (qpd->cwsr_kaddr) {
1230 		/* KFD trap handler is bound, record as second-level TBA/TMA
1231 		 * in first-level TMA. First-level trap will jump to second.
1232 		 */
1233 		uint64_t *tma =
1234 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1235 		tma[0] = tba_addr;
1236 		tma[1] = tma_addr;
1237 	} else {
1238 		/* No trap handler bound, bind as first-level TBA/TMA. */
1239 		qpd->tba_addr = tba_addr;
1240 		qpd->tma_addr = tma_addr;
1241 	}
1242 }
1243 
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1244 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1245 {
1246 	int i;
1247 
1248 	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
1249 	 * boot time retry setting. Mixing processes with different
1250 	 * XNACK/retry settings can hang the GPU.
1251 	 *
1252 	 * Different GPUs can have different noretry settings depending
1253 	 * on HW bugs or limitations. We need to find at least one
1254 	 * XNACK mode for this process that's compatible with all GPUs.
1255 	 * Fortunately GPUs with retry enabled (noretry=0) can run code
1256 	 * built for XNACK-off. On GFXv9 it may perform slower.
1257 	 *
1258 	 * Therefore applications built for XNACK-off can always be
1259 	 * supported and will be our fallback if any GPU does not
1260 	 * support retry.
1261 	 */
1262 	for (i = 0; i < p->n_pdds; i++) {
1263 		struct kfd_dev *dev = p->pdds[i]->dev;
1264 
1265 		/* Only consider GFXv9 and higher GPUs. Older GPUs don't
1266 		 * support the SVM APIs and don't need to be considered
1267 		 * for the XNACK mode selection.
1268 		 */
1269 		if (dev->device_info->asic_family < CHIP_VEGA10)
1270 			continue;
1271 		/* Aldebaran can always support XNACK because it can support
1272 		 * per-process XNACK mode selection. But let the dev->noretry
1273 		 * setting still influence the default XNACK mode.
1274 		 */
1275 		if (supported &&
1276 		    dev->device_info->asic_family == CHIP_ALDEBARAN)
1277 			continue;
1278 
1279 		/* GFXv10 and later GPUs do not support shader preemption
1280 		 * during page faults. This can lead to poor QoS for queue
1281 		 * management and memory-manager-related preemptions or
1282 		 * even deadlocks.
1283 		 */
1284 		if (dev->device_info->asic_family >= CHIP_NAVI10)
1285 			return false;
1286 
1287 		if (dev->noretry)
1288 			return false;
1289 	}
1290 
1291 	return true;
1292 }
1293 
1294 /*
1295  * On return the kfd_process is fully operational and will be freed when the
1296  * mm is released
1297  */
create_process(const struct task_struct * thread)1298 static struct kfd_process *create_process(const struct task_struct *thread)
1299 {
1300 	struct kfd_process *process;
1301 	struct mmu_notifier *mn;
1302 	int err = -ENOMEM;
1303 
1304 	process = kzalloc(sizeof(*process), GFP_KERNEL);
1305 	if (!process)
1306 		goto err_alloc_process;
1307 
1308 	kref_init(&process->ref);
1309 	mutex_init(&process->mutex);
1310 	process->mm = thread->mm;
1311 	process->lead_thread = thread->group_leader;
1312 	process->n_pdds = 0;
1313 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1314 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1315 	process->last_restore_timestamp = get_jiffies_64();
1316 	kfd_event_init_process(process);
1317 	process->is_32bit_user_mode = in_compat_syscall();
1318 
1319 	process->pasid = kfd_pasid_alloc();
1320 	if (process->pasid == 0)
1321 		goto err_alloc_pasid;
1322 
1323 	err = pqm_init(&process->pqm, process);
1324 	if (err != 0)
1325 		goto err_process_pqm_init;
1326 
1327 	/* init process apertures*/
1328 	err = kfd_init_apertures(process);
1329 	if (err != 0)
1330 		goto err_init_apertures;
1331 
1332 	/* Check XNACK support after PDDs are created in kfd_init_apertures */
1333 	process->xnack_enabled = kfd_process_xnack_mode(process, false);
1334 
1335 	err = svm_range_list_init(process);
1336 	if (err)
1337 		goto err_init_svm_range_list;
1338 
1339 	/* alloc_notifier needs to find the process in the hash table */
1340 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1341 			(uintptr_t)process->mm);
1342 
1343 	/* MMU notifier registration must be the last call that can fail
1344 	 * because after this point we cannot unwind the process creation.
1345 	 * After this point, mmu_notifier_put will trigger the cleanup by
1346 	 * dropping the last process reference in the free_notifier.
1347 	 */
1348 	mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1349 	if (IS_ERR(mn)) {
1350 		err = PTR_ERR(mn);
1351 		goto err_register_notifier;
1352 	}
1353 	BUG_ON(mn != &process->mmu_notifier);
1354 
1355 	get_task_struct(process->lead_thread);
1356 
1357 	return process;
1358 
1359 err_register_notifier:
1360 	hash_del_rcu(&process->kfd_processes);
1361 	svm_range_list_fini(process);
1362 err_init_svm_range_list:
1363 	kfd_process_free_outstanding_kfd_bos(process);
1364 	kfd_process_destroy_pdds(process);
1365 err_init_apertures:
1366 	pqm_uninit(&process->pqm);
1367 err_process_pqm_init:
1368 	kfd_pasid_free(process->pasid);
1369 err_alloc_pasid:
1370 	mutex_destroy(&process->mutex);
1371 	kfree(process);
1372 err_alloc_process:
1373 	return ERR_PTR(err);
1374 }
1375 
init_doorbell_bitmap(struct qcm_process_device * qpd,struct kfd_dev * dev)1376 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1377 			struct kfd_dev *dev)
1378 {
1379 	unsigned int i;
1380 	int range_start = dev->shared_resources.non_cp_doorbells_start;
1381 	int range_end = dev->shared_resources.non_cp_doorbells_end;
1382 
1383 	if (!KFD_IS_SOC15(dev->device_info->asic_family))
1384 		return 0;
1385 
1386 	qpd->doorbell_bitmap =
1387 		kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1388 				     BITS_PER_BYTE), GFP_KERNEL);
1389 	if (!qpd->doorbell_bitmap)
1390 		return -ENOMEM;
1391 
1392 	/* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1393 	pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1394 	pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1395 			range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1396 			range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1397 
1398 	for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1399 		if (i >= range_start && i <= range_end) {
1400 			set_bit(i, qpd->doorbell_bitmap);
1401 			set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1402 				qpd->doorbell_bitmap);
1403 		}
1404 	}
1405 
1406 	return 0;
1407 }
1408 
kfd_get_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1409 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1410 							struct kfd_process *p)
1411 {
1412 	int i;
1413 
1414 	for (i = 0; i < p->n_pdds; i++)
1415 		if (p->pdds[i]->dev == dev)
1416 			return p->pdds[i];
1417 
1418 	return NULL;
1419 }
1420 
kfd_create_process_device_data(struct kfd_dev * dev,struct kfd_process * p)1421 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1422 							struct kfd_process *p)
1423 {
1424 	struct kfd_process_device *pdd = NULL;
1425 
1426 	if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1427 		return NULL;
1428 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1429 	if (!pdd)
1430 		return NULL;
1431 
1432 	if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1433 		pr_err("Failed to alloc doorbell for pdd\n");
1434 		goto err_free_pdd;
1435 	}
1436 
1437 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1438 		pr_err("Failed to init doorbell for process\n");
1439 		goto err_free_pdd;
1440 	}
1441 
1442 	pdd->dev = dev;
1443 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
1444 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1445 	pdd->qpd.dqm = dev->dqm;
1446 	pdd->qpd.pqm = &p->pqm;
1447 	pdd->qpd.evicted = 0;
1448 	pdd->qpd.mapped_gws_queue = false;
1449 	pdd->process = p;
1450 	pdd->bound = PDD_UNBOUND;
1451 	pdd->already_dequeued = false;
1452 	pdd->runtime_inuse = false;
1453 	pdd->vram_usage = 0;
1454 	pdd->sdma_past_activity_counter = 0;
1455 	atomic64_set(&pdd->evict_duration_counter, 0);
1456 	p->pdds[p->n_pdds++] = pdd;
1457 
1458 	/* Init idr used for memory handle translation */
1459 	idr_init(&pdd->alloc_idr);
1460 
1461 	return pdd;
1462 
1463 err_free_pdd:
1464 	kfree(pdd);
1465 	return NULL;
1466 }
1467 
1468 /**
1469  * kfd_process_device_init_vm - Initialize a VM for a process-device
1470  *
1471  * @pdd: The process-device
1472  * @drm_file: Optional pointer to a DRM file descriptor
1473  *
1474  * If @drm_file is specified, it will be used to acquire the VM from
1475  * that file descriptor. If successful, the @pdd takes ownership of
1476  * the file descriptor.
1477  *
1478  * If @drm_file is NULL, a new VM is created.
1479  *
1480  * Returns 0 on success, -errno on failure.
1481  */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1482 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1483 			       struct file *drm_file)
1484 {
1485 	struct kfd_process *p;
1486 	struct kfd_dev *dev;
1487 	int ret;
1488 
1489 	if (!drm_file)
1490 		return -EINVAL;
1491 
1492 	if (pdd->drm_priv)
1493 		return -EBUSY;
1494 
1495 	p = pdd->process;
1496 	dev = pdd->dev;
1497 
1498 	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1499 		dev->kgd, drm_file, p->pasid,
1500 		&p->kgd_process_info, &p->ef);
1501 	if (ret) {
1502 		pr_err("Failed to create process VM object\n");
1503 		return ret;
1504 	}
1505 	pdd->drm_priv = drm_file->private_data;
1506 
1507 	ret = kfd_process_device_reserve_ib_mem(pdd);
1508 	if (ret)
1509 		goto err_reserve_ib_mem;
1510 	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1511 	if (ret)
1512 		goto err_init_cwsr;
1513 
1514 	pdd->drm_file = drm_file;
1515 
1516 	return 0;
1517 
1518 err_init_cwsr:
1519 err_reserve_ib_mem:
1520 	kfd_process_device_free_bos(pdd);
1521 	pdd->drm_priv = NULL;
1522 
1523 	return ret;
1524 }
1525 
1526 /*
1527  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1528  * to the device.
1529  * Unbinding occurs when the process dies or the device is removed.
1530  *
1531  * Assumes that the process lock is held.
1532  */
kfd_bind_process_to_device(struct kfd_dev * dev,struct kfd_process * p)1533 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1534 							struct kfd_process *p)
1535 {
1536 	struct kfd_process_device *pdd;
1537 	int err;
1538 
1539 	pdd = kfd_get_process_device_data(dev, p);
1540 	if (!pdd) {
1541 		pr_err("Process device data doesn't exist\n");
1542 		return ERR_PTR(-ENOMEM);
1543 	}
1544 
1545 	if (!pdd->drm_priv)
1546 		return ERR_PTR(-ENODEV);
1547 
1548 	/*
1549 	 * signal runtime-pm system to auto resume and prevent
1550 	 * further runtime suspend once device pdd is created until
1551 	 * pdd is destroyed.
1552 	 */
1553 	if (!pdd->runtime_inuse) {
1554 		err = pm_runtime_get_sync(dev->ddev->dev);
1555 		if (err < 0) {
1556 			pm_runtime_put_autosuspend(dev->ddev->dev);
1557 			return ERR_PTR(err);
1558 		}
1559 	}
1560 
1561 	err = kfd_iommu_bind_process_to_device(pdd);
1562 	if (err)
1563 		goto out;
1564 
1565 	/*
1566 	 * make sure that runtime_usage counter is incremented just once
1567 	 * per pdd
1568 	 */
1569 	pdd->runtime_inuse = true;
1570 
1571 	return pdd;
1572 
1573 out:
1574 	/* balance runpm reference count and exit with error */
1575 	if (!pdd->runtime_inuse) {
1576 		pm_runtime_mark_last_busy(dev->ddev->dev);
1577 		pm_runtime_put_autosuspend(dev->ddev->dev);
1578 	}
1579 
1580 	return ERR_PTR(err);
1581 }
1582 
1583 /* Create specific handle mapped to mem from process local memory idr
1584  * Assumes that the process lock is held.
1585  */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1586 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1587 					void *mem)
1588 {
1589 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1590 }
1591 
1592 /* Translate specific handle from process local memory idr
1593  * Assumes that the process lock is held.
1594  */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1595 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1596 					int handle)
1597 {
1598 	if (handle < 0)
1599 		return NULL;
1600 
1601 	return idr_find(&pdd->alloc_idr, handle);
1602 }
1603 
1604 /* Remove specific handle from process local memory idr
1605  * Assumes that the process lock is held.
1606  */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1607 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1608 					int handle)
1609 {
1610 	if (handle >= 0)
1611 		idr_remove(&pdd->alloc_idr, handle);
1612 }
1613 
1614 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid)1615 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1616 {
1617 	struct kfd_process *p, *ret_p = NULL;
1618 	unsigned int temp;
1619 
1620 	int idx = srcu_read_lock(&kfd_processes_srcu);
1621 
1622 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1623 		if (p->pasid == pasid) {
1624 			kref_get(&p->ref);
1625 			ret_p = p;
1626 			break;
1627 		}
1628 	}
1629 
1630 	srcu_read_unlock(&kfd_processes_srcu, idx);
1631 
1632 	return ret_p;
1633 }
1634 
1635 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1636 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1637 {
1638 	struct kfd_process *p;
1639 
1640 	int idx = srcu_read_lock(&kfd_processes_srcu);
1641 
1642 	p = find_process_by_mm(mm);
1643 	if (p)
1644 		kref_get(&p->ref);
1645 
1646 	srcu_read_unlock(&kfd_processes_srcu, idx);
1647 
1648 	return p;
1649 }
1650 
1651 /* kfd_process_evict_queues - Evict all user queues of a process
1652  *
1653  * Eviction is reference-counted per process-device. This means multiple
1654  * evictions from different sources can be nested safely.
1655  */
kfd_process_evict_queues(struct kfd_process * p)1656 int kfd_process_evict_queues(struct kfd_process *p)
1657 {
1658 	int r = 0;
1659 	int i;
1660 	unsigned int n_evicted = 0;
1661 
1662 	for (i = 0; i < p->n_pdds; i++) {
1663 		struct kfd_process_device *pdd = p->pdds[i];
1664 
1665 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1666 							    &pdd->qpd);
1667 		if (r) {
1668 			pr_err("Failed to evict process queues\n");
1669 			goto fail;
1670 		}
1671 		n_evicted++;
1672 	}
1673 
1674 	return r;
1675 
1676 fail:
1677 	/* To keep state consistent, roll back partial eviction by
1678 	 * restoring queues
1679 	 */
1680 	for (i = 0; i < p->n_pdds; i++) {
1681 		struct kfd_process_device *pdd = p->pdds[i];
1682 
1683 		if (n_evicted == 0)
1684 			break;
1685 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1686 							      &pdd->qpd))
1687 			pr_err("Failed to restore queues\n");
1688 
1689 		n_evicted--;
1690 	}
1691 
1692 	return r;
1693 }
1694 
1695 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1696 int kfd_process_restore_queues(struct kfd_process *p)
1697 {
1698 	int r, ret = 0;
1699 	int i;
1700 
1701 	for (i = 0; i < p->n_pdds; i++) {
1702 		struct kfd_process_device *pdd = p->pdds[i];
1703 
1704 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1705 							      &pdd->qpd);
1706 		if (r) {
1707 			pr_err("Failed to restore process queues\n");
1708 			if (!ret)
1709 				ret = r;
1710 		}
1711 	}
1712 
1713 	return ret;
1714 }
1715 
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1716 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1717 {
1718 	int i;
1719 
1720 	for (i = 0; i < p->n_pdds; i++)
1721 		if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
1722 			return i;
1723 	return -EINVAL;
1724 }
1725 
1726 int
kfd_process_gpuid_from_kgd(struct kfd_process * p,struct amdgpu_device * adev,uint32_t * gpuid,uint32_t * gpuidx)1727 kfd_process_gpuid_from_kgd(struct kfd_process *p, struct amdgpu_device *adev,
1728 			   uint32_t *gpuid, uint32_t *gpuidx)
1729 {
1730 	struct kgd_dev *kgd = (struct kgd_dev *)adev;
1731 	int i;
1732 
1733 	for (i = 0; i < p->n_pdds; i++)
1734 		if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
1735 			*gpuid = p->pdds[i]->dev->id;
1736 			*gpuidx = i;
1737 			return 0;
1738 		}
1739 	return -EINVAL;
1740 }
1741 
evict_process_worker(struct work_struct * work)1742 static void evict_process_worker(struct work_struct *work)
1743 {
1744 	int ret;
1745 	struct kfd_process *p;
1746 	struct delayed_work *dwork;
1747 
1748 	dwork = to_delayed_work(work);
1749 
1750 	/* Process termination destroys this worker thread. So during the
1751 	 * lifetime of this thread, kfd_process p will be valid
1752 	 */
1753 	p = container_of(dwork, struct kfd_process, eviction_work);
1754 	WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1755 		  "Eviction fence mismatch\n");
1756 
1757 	/* Narrow window of overlap between restore and evict work
1758 	 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1759 	 * unreserves KFD BOs, it is possible to evicted again. But
1760 	 * restore has few more steps of finish. So lets wait for any
1761 	 * previous restore work to complete
1762 	 */
1763 	flush_delayed_work(&p->restore_work);
1764 
1765 	pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1766 	ret = kfd_process_evict_queues(p);
1767 	if (!ret) {
1768 		dma_fence_signal(p->ef);
1769 		dma_fence_put(p->ef);
1770 		p->ef = NULL;
1771 		queue_delayed_work(kfd_restore_wq, &p->restore_work,
1772 				msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1773 
1774 		pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1775 	} else
1776 		pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1777 }
1778 
restore_process_worker(struct work_struct * work)1779 static void restore_process_worker(struct work_struct *work)
1780 {
1781 	struct delayed_work *dwork;
1782 	struct kfd_process *p;
1783 	int ret = 0;
1784 
1785 	dwork = to_delayed_work(work);
1786 
1787 	/* Process termination destroys this worker thread. So during the
1788 	 * lifetime of this thread, kfd_process p will be valid
1789 	 */
1790 	p = container_of(dwork, struct kfd_process, restore_work);
1791 	pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1792 
1793 	/* Setting last_restore_timestamp before successful restoration.
1794 	 * Otherwise this would have to be set by KGD (restore_process_bos)
1795 	 * before KFD BOs are unreserved. If not, the process can be evicted
1796 	 * again before the timestamp is set.
1797 	 * If restore fails, the timestamp will be set again in the next
1798 	 * attempt. This would mean that the minimum GPU quanta would be
1799 	 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1800 	 * functions)
1801 	 */
1802 
1803 	p->last_restore_timestamp = get_jiffies_64();
1804 	ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1805 						     &p->ef);
1806 	if (ret) {
1807 		pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1808 			 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1809 		ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1810 				msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1811 		WARN(!ret, "reschedule restore work failed\n");
1812 		return;
1813 	}
1814 
1815 	ret = kfd_process_restore_queues(p);
1816 	if (!ret)
1817 		pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1818 	else
1819 		pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1820 }
1821 
kfd_suspend_all_processes(void)1822 void kfd_suspend_all_processes(void)
1823 {
1824 	struct kfd_process *p;
1825 	unsigned int temp;
1826 	int idx = srcu_read_lock(&kfd_processes_srcu);
1827 
1828 	WARN(debug_evictions, "Evicting all processes");
1829 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1830 		cancel_delayed_work_sync(&p->eviction_work);
1831 		cancel_delayed_work_sync(&p->restore_work);
1832 
1833 		if (kfd_process_evict_queues(p))
1834 			pr_err("Failed to suspend process 0x%x\n", p->pasid);
1835 		dma_fence_signal(p->ef);
1836 		dma_fence_put(p->ef);
1837 		p->ef = NULL;
1838 	}
1839 	srcu_read_unlock(&kfd_processes_srcu, idx);
1840 }
1841 
kfd_resume_all_processes(void)1842 int kfd_resume_all_processes(void)
1843 {
1844 	struct kfd_process *p;
1845 	unsigned int temp;
1846 	int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1847 
1848 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1849 		if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1850 			pr_err("Restore process %d failed during resume\n",
1851 			       p->pasid);
1852 			ret = -EFAULT;
1853 		}
1854 	}
1855 	srcu_read_unlock(&kfd_processes_srcu, idx);
1856 	return ret;
1857 }
1858 
kfd_reserved_mem_mmap(struct kfd_dev * dev,struct kfd_process * process,struct vm_area_struct * vma)1859 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1860 			  struct vm_area_struct *vma)
1861 {
1862 	struct kfd_process_device *pdd;
1863 	struct qcm_process_device *qpd;
1864 
1865 	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1866 		pr_err("Incorrect CWSR mapping size.\n");
1867 		return -EINVAL;
1868 	}
1869 
1870 	pdd = kfd_get_process_device_data(dev, process);
1871 	if (!pdd)
1872 		return -EINVAL;
1873 	qpd = &pdd->qpd;
1874 
1875 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1876 					get_order(KFD_CWSR_TBA_TMA_SIZE));
1877 	if (!qpd->cwsr_kaddr) {
1878 		pr_err("Error allocating per process CWSR buffer.\n");
1879 		return -ENOMEM;
1880 	}
1881 
1882 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1883 		| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1884 	/* Mapping pages to user process */
1885 	return remap_pfn_range(vma, vma->vm_start,
1886 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1887 			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1888 }
1889 
kfd_flush_tlb(struct kfd_process_device * pdd,enum TLB_FLUSH_TYPE type)1890 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
1891 {
1892 	struct kfd_dev *dev = pdd->dev;
1893 
1894 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1895 		/* Nothing to flush until a VMID is assigned, which
1896 		 * only happens when the first queue is created.
1897 		 */
1898 		if (pdd->qpd.vmid)
1899 			amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1900 							pdd->qpd.vmid);
1901 	} else {
1902 		amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1903 					pdd->process->pasid, type);
1904 	}
1905 }
1906 
1907 #if defined(CONFIG_DEBUG_FS)
1908 
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)1909 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1910 {
1911 	struct kfd_process *p;
1912 	unsigned int temp;
1913 	int r = 0;
1914 
1915 	int idx = srcu_read_lock(&kfd_processes_srcu);
1916 
1917 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1918 		seq_printf(m, "Process %d PASID 0x%x:\n",
1919 			   p->lead_thread->tgid, p->pasid);
1920 
1921 		mutex_lock(&p->mutex);
1922 		r = pqm_debugfs_mqds(m, &p->pqm);
1923 		mutex_unlock(&p->mutex);
1924 
1925 		if (r)
1926 			break;
1927 	}
1928 
1929 	srcu_read_unlock(&kfd_processes_srcu, idx);
1930 
1931 	return r;
1932 }
1933 
1934 #endif
1935 
1936