1 /*
2 *
3 * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18
19
20 /**
21 * @file mali_kbase_mmu.c
22 * Base kernel MMU management.
23 */
24
25 /* #define DEBUG 1 */
26 #include <linux/kernel.h>
27 #include <linux/dma-mapping.h>
28 #include <mali_kbase.h>
29 #include <mali_midg_regmap.h>
30 #if defined(CONFIG_MALI_GATOR_SUPPORT)
31 #include <mali_kbase_gator.h>
32 #endif
33 #include <mali_kbase_tlstream.h>
34 #include <mali_kbase_instr_defs.h>
35 #include <mali_kbase_debug.h>
36
37 #define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
38
39 #include <mali_kbase_defs.h>
40 #include <mali_kbase_hw.h>
41 #include <mali_kbase_mmu_hw.h>
42 #include <mali_kbase_hwaccess_jm.h>
43 #include <mali_kbase_time.h>
44 #include <mali_kbase_mem.h>
45
46 #define KBASE_MMU_PAGE_ENTRIES 512
47
48 /**
49 * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches.
50 * @kctx: The KBase context.
51 * @vpfn: The virtual page frame number to start the flush on.
52 * @nr: The number of pages to flush.
53 * @sync: Set if the operation should be synchronous or not.
54 *
55 * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs.
56 *
57 * If sync is not set then transactions still in flight when the flush is issued
58 * may use the old page tables and the data they write will not be written out
59 * to memory, this function returns after the flush has been issued but
60 * before all accesses which might effect the flushed region have completed.
61 *
62 * If sync is set then accesses in the flushed region will be drained
63 * before data is flush and invalidated through L1, L2 and into memory,
64 * after which point this function will return.
65 */
66 static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
67 u64 vpfn, size_t nr, bool sync);
68
69 /**
70 * kbase_mmu_sync_pgd - sync page directory to memory
71 * @kbdev: Device pointer.
72 * @handle: Address of DMA region.
73 * @size: Size of the region to sync.
74 *
75 * This should be called after each page directory update.
76 */
77
kbase_mmu_sync_pgd(struct kbase_device * kbdev,dma_addr_t handle,size_t size)78 static void kbase_mmu_sync_pgd(struct kbase_device *kbdev,
79 dma_addr_t handle, size_t size)
80 {
81 /* If page table is not coherent then ensure the gpu can read
82 * the pages from memory
83 */
84 if (kbdev->system_coherency != COHERENCY_ACE)
85 dma_sync_single_for_device(kbdev->dev, handle, size,
86 DMA_TO_DEVICE);
87 }
88
89 /*
90 * Definitions:
91 * - PGD: Page Directory.
92 * - PTE: Page Table Entry. A 64bit value pointing to the next
93 * level of translation
94 * - ATE: Address Transation Entry. A 64bit value pointing to
95 * a 4kB physical page.
96 */
97
98 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
99 struct kbase_as *as, const char *reason_str);
100
101
make_multiple(size_t minimum,size_t multiple)102 static size_t make_multiple(size_t minimum, size_t multiple)
103 {
104 size_t remainder = minimum % multiple;
105
106 if (remainder == 0)
107 return minimum;
108
109 return minimum + multiple - remainder;
110 }
111
page_fault_worker(struct work_struct * data)112 void page_fault_worker(struct work_struct *data)
113 {
114 u64 fault_pfn;
115 u32 fault_status;
116 size_t new_pages;
117 size_t fault_rel_pfn;
118 struct kbase_as *faulting_as;
119 int as_no;
120 struct kbase_context *kctx;
121 struct kbase_device *kbdev;
122 struct kbase_va_region *region;
123 int err;
124 bool grown = false;
125
126 faulting_as = container_of(data, struct kbase_as, work_pagefault);
127 fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
128 as_no = faulting_as->number;
129
130 kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
131
132 /* Grab the context that was already refcounted in kbase_mmu_interrupt().
133 * Therefore, it cannot be scheduled out of this AS until we explicitly release it
134 */
135 kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
136 if (WARN_ON(!kctx)) {
137 atomic_dec(&kbdev->faults_pending);
138 return;
139 }
140
141 KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
142
143 if (unlikely(faulting_as->protected_mode))
144 {
145 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
146 "Protected mode fault");
147 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
148 KBASE_MMU_FAULT_TYPE_PAGE);
149
150 goto fault_done;
151 }
152
153 fault_status = faulting_as->fault_status;
154 switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
155
156 case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
157 /* need to check against the region to handle this one */
158 break;
159
160 case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
161 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
162 "Permission failure");
163 goto fault_done;
164
165 case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
166 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
167 "Translation table bus fault");
168 goto fault_done;
169
170 case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
171 /* nothing to do, but we don't expect this fault currently */
172 dev_warn(kbdev->dev, "Access flag unexpectedly set");
173 goto fault_done;
174
175 case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT:
176 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
177 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
178 "Address size fault");
179 else
180 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
181 "Unknown fault code");
182 goto fault_done;
183
184 case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT:
185 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
186 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
187 "Memory attributes fault");
188 else
189 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
190 "Unknown fault code");
191 goto fault_done;
192
193 default:
194 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
195 "Unknown fault code");
196 goto fault_done;
197 }
198
199 /* so we have a translation fault, let's see if it is for growable
200 * memory */
201 kbase_gpu_vm_lock(kctx);
202
203 region = kbase_region_tracker_find_region_enclosing_address(kctx,
204 faulting_as->fault_addr);
205 if (!region || region->flags & KBASE_REG_FREE) {
206 kbase_gpu_vm_unlock(kctx);
207 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
208 "Memory is not mapped on the GPU");
209 goto fault_done;
210 }
211
212 if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) {
213 kbase_gpu_vm_unlock(kctx);
214 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
215 "DMA-BUF is not mapped on the GPU");
216 goto fault_done;
217 }
218
219 if ((region->flags & GROWABLE_FLAGS_REQUIRED)
220 != GROWABLE_FLAGS_REQUIRED) {
221 kbase_gpu_vm_unlock(kctx);
222 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
223 "Memory is not growable");
224 goto fault_done;
225 }
226
227 if ((region->flags & KBASE_REG_DONT_NEED)) {
228 kbase_gpu_vm_unlock(kctx);
229 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
230 "Don't need memory can't be grown");
231 goto fault_done;
232 }
233
234 /* find the size we need to grow it by */
235 /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address
236 * validating the fault_adress to be within a size_t from the start_pfn */
237 fault_rel_pfn = fault_pfn - region->start_pfn;
238
239 if (fault_rel_pfn < kbase_reg_current_backed_size(region)) {
240 dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring",
241 faulting_as->fault_addr, region->start_pfn,
242 region->start_pfn +
243 kbase_reg_current_backed_size(region));
244
245 mutex_lock(&kbdev->mmu_hw_mutex);
246
247 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
248 KBASE_MMU_FAULT_TYPE_PAGE);
249 /* [1] in case another page fault occurred while we were
250 * handling the (duplicate) page fault we need to ensure we
251 * don't loose the other page fault as result of us clearing
252 * the MMU IRQ. Therefore, after we clear the MMU IRQ we send
253 * an UNLOCK command that will retry any stalled memory
254 * transaction (which should cause the other page fault to be
255 * raised again).
256 */
257 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
258 AS_COMMAND_UNLOCK, 1);
259
260 mutex_unlock(&kbdev->mmu_hw_mutex);
261
262 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
263 KBASE_MMU_FAULT_TYPE_PAGE);
264 kbase_gpu_vm_unlock(kctx);
265
266 goto fault_done;
267 }
268
269 new_pages = make_multiple(fault_rel_pfn -
270 kbase_reg_current_backed_size(region) + 1,
271 region->extent);
272
273 /* cap to max vsize */
274 if (new_pages + kbase_reg_current_backed_size(region) >
275 region->nr_pages)
276 new_pages = region->nr_pages -
277 kbase_reg_current_backed_size(region);
278
279 if (0 == new_pages) {
280 mutex_lock(&kbdev->mmu_hw_mutex);
281
282 /* Duplicate of a fault we've already handled, nothing to do */
283 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
284 KBASE_MMU_FAULT_TYPE_PAGE);
285 /* See comment [1] about UNLOCK usage */
286 kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
287 AS_COMMAND_UNLOCK, 1);
288
289 mutex_unlock(&kbdev->mmu_hw_mutex);
290
291 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
292 KBASE_MMU_FAULT_TYPE_PAGE);
293 kbase_gpu_vm_unlock(kctx);
294 goto fault_done;
295 }
296
297 if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
298 if (region->gpu_alloc != region->cpu_alloc) {
299 if (kbase_alloc_phy_pages_helper(
300 region->cpu_alloc, new_pages) == 0) {
301 grown = true;
302 } else {
303 kbase_free_phy_pages_helper(region->gpu_alloc,
304 new_pages);
305 }
306 } else {
307 grown = true;
308 }
309 }
310
311
312 if (grown) {
313 u64 pfn_offset;
314 u32 op;
315
316 /* alloc success */
317 KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages);
318
319 /* set up the new pages */
320 pfn_offset = kbase_reg_current_backed_size(region) - new_pages;
321 /*
322 * Note:
323 * Issuing an MMU operation will unlock the MMU and cause the
324 * translation to be replayed. If the page insertion fails then
325 * rather then trying to continue the context should be killed
326 * so the no_flush version of insert_pages is used which allows
327 * us to unlock the MMU as we see fit.
328 */
329 err = kbase_mmu_insert_pages_no_flush(kctx,
330 region->start_pfn + pfn_offset,
331 &kbase_get_gpu_phy_pages(region)[pfn_offset],
332 new_pages, region->flags);
333 if (err) {
334 kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
335 if (region->gpu_alloc != region->cpu_alloc)
336 kbase_free_phy_pages_helper(region->cpu_alloc,
337 new_pages);
338 kbase_gpu_vm_unlock(kctx);
339 /* The locked VA region will be unlocked and the cache invalidated in here */
340 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
341 "Page table update failure");
342 goto fault_done;
343 }
344 #if defined(CONFIG_MALI_GATOR_SUPPORT)
345 kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
346 #endif
347 KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages);
348
349 /* AS transaction begin */
350 mutex_lock(&kbdev->mmu_hw_mutex);
351
352 /* flush L2 and unlock the VA (resumes the MMU) */
353 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
354 op = AS_COMMAND_FLUSH;
355 else
356 op = AS_COMMAND_FLUSH_PT;
357
358 /* clear MMU interrupt - this needs to be done after updating
359 * the page tables but before issuing a FLUSH command. The
360 * FLUSH cmd has a side effect that it restarts stalled memory
361 * transactions in other address spaces which may cause
362 * another fault to occur. If we didn't clear the interrupt at
363 * this stage a new IRQ might not be raised when the GPU finds
364 * a MMU IRQ is already pending.
365 */
366 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
367 KBASE_MMU_FAULT_TYPE_PAGE);
368
369 kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx,
370 faulting_as->fault_addr >> PAGE_SHIFT,
371 new_pages,
372 op, 1);
373
374 mutex_unlock(&kbdev->mmu_hw_mutex);
375 /* AS transaction end */
376
377 /* reenable this in the mask */
378 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
379 KBASE_MMU_FAULT_TYPE_PAGE);
380 kbase_gpu_vm_unlock(kctx);
381 } else {
382 /* failed to extend, handle as a normal PF */
383 kbase_gpu_vm_unlock(kctx);
384 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
385 "Page allocation failure");
386 }
387
388 fault_done:
389 /*
390 * By this point, the fault was handled in some way,
391 * so release the ctx refcount
392 */
393 kbasep_js_runpool_release_ctx(kbdev, kctx);
394
395 atomic_dec(&kbdev->faults_pending);
396 }
397
kbase_mmu_alloc_pgd(struct kbase_context * kctx)398 phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx)
399 {
400 u64 *page;
401 int i;
402 struct page *p;
403 int new_page_count __maybe_unused;
404
405 KBASE_DEBUG_ASSERT(NULL != kctx);
406 new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages);
407 kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages);
408
409 p = kbase_mem_pool_alloc(&kctx->mem_pool);
410 if (!p)
411 goto sub_pages;
412
413 KBASE_TLSTREAM_AUX_PAGESALLOC(
414 (u32)kctx->id,
415 (u64)new_page_count);
416
417 page = kmap(p);
418 if (NULL == page)
419 goto alloc_free;
420
421 kbase_process_page_usage_inc(kctx, 1);
422
423 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++)
424 kctx->kbdev->mmu_mode->entry_invalidate(&page[i]);
425
426 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
427
428 kunmap(p);
429 return page_to_phys(p);
430
431 alloc_free:
432 kbase_mem_pool_free(&kctx->mem_pool, p, false);
433 sub_pages:
434 kbase_atomic_sub_pages(1, &kctx->used_pages);
435 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
436
437 return 0;
438 }
439
440 KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd);
441
442 /* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the
443 * new table from the pool if needed and possible
444 */
mmu_get_next_pgd(struct kbase_context * kctx,phys_addr_t * pgd,u64 vpfn,int level)445 static int mmu_get_next_pgd(struct kbase_context *kctx,
446 phys_addr_t *pgd, u64 vpfn, int level)
447 {
448 u64 *page;
449 phys_addr_t target_pgd;
450 struct page *p;
451
452 KBASE_DEBUG_ASSERT(*pgd);
453 KBASE_DEBUG_ASSERT(NULL != kctx);
454
455 lockdep_assert_held(&kctx->mmu_lock);
456
457 /*
458 * Architecture spec defines level-0 as being the top-most.
459 * This is a bit unfortunate here, but we keep the same convention.
460 */
461 vpfn >>= (3 - level) * 9;
462 vpfn &= 0x1FF;
463
464 p = pfn_to_page(PFN_DOWN(*pgd));
465 page = kmap(p);
466 if (NULL == page) {
467 dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n");
468 return -EINVAL;
469 }
470
471 target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
472
473 if (!target_pgd) {
474 target_pgd = kbase_mmu_alloc_pgd(kctx);
475 if (!target_pgd) {
476 dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n");
477 kunmap(p);
478 return -ENOMEM;
479 }
480
481 kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd);
482
483 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
484 /* Rely on the caller to update the address space flags. */
485 }
486
487 kunmap(p);
488 *pgd = target_pgd;
489
490 return 0;
491 }
492
mmu_get_bottom_pgd(struct kbase_context * kctx,u64 vpfn,phys_addr_t * out_pgd)493 static int mmu_get_bottom_pgd(struct kbase_context *kctx,
494 u64 vpfn, phys_addr_t *out_pgd)
495 {
496 phys_addr_t pgd;
497 int l;
498
499 lockdep_assert_held(&kctx->mmu_lock);
500
501 pgd = kctx->pgd;
502 for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
503 int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l);
504 /* Handle failure condition */
505 if (err) {
506 dev_dbg(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n");
507 return err;
508 }
509 }
510
511 *out_pgd = pgd;
512
513 return 0;
514 }
515
mmu_insert_pages_recover_get_next_pgd(struct kbase_context * kctx,phys_addr_t pgd,u64 vpfn,int level)516 static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
517 {
518 u64 *page;
519 phys_addr_t target_pgd;
520
521 KBASE_DEBUG_ASSERT(pgd);
522 KBASE_DEBUG_ASSERT(NULL != kctx);
523
524 lockdep_assert_held(&kctx->mmu_lock);
525 lockdep_assert_held(&kctx->reg_lock);
526
527 /*
528 * Architecture spec defines level-0 as being the top-most.
529 * This is a bit unfortunate here, but we keep the same convention.
530 */
531 vpfn >>= (3 - level) * 9;
532 vpfn &= 0x1FF;
533
534 page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
535 /* kmap_atomic should NEVER fail */
536 KBASE_DEBUG_ASSERT(NULL != page);
537
538 target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
539 /* As we are recovering from what has already been set up, we should have a target_pgd */
540 KBASE_DEBUG_ASSERT(0 != target_pgd);
541 kunmap_atomic(page);
542 return target_pgd;
543 }
544
mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context * kctx,u64 vpfn)545 static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn)
546 {
547 phys_addr_t pgd;
548 int l;
549
550 lockdep_assert_held(&kctx->mmu_lock);
551
552 pgd = kctx->pgd;
553
554 for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) {
555 pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l);
556 /* Should never fail */
557 KBASE_DEBUG_ASSERT(0 != pgd);
558 }
559
560 return pgd;
561 }
562
mmu_insert_pages_failure_recovery(struct kbase_context * kctx,u64 vpfn,size_t nr)563 static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn,
564 size_t nr)
565 {
566 phys_addr_t pgd;
567 u64 *pgd_page;
568 struct kbase_mmu_mode const *mmu_mode;
569
570 KBASE_DEBUG_ASSERT(NULL != kctx);
571 KBASE_DEBUG_ASSERT(0 != vpfn);
572 /* 64-bit address range is the max */
573 KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
574
575 lockdep_assert_held(&kctx->mmu_lock);
576 lockdep_assert_held(&kctx->reg_lock);
577
578 mmu_mode = kctx->kbdev->mmu_mode;
579
580 while (nr) {
581 unsigned int i;
582 unsigned int index = vpfn & 0x1FF;
583 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
584 struct page *p;
585
586 if (count > nr)
587 count = nr;
588
589 pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn);
590 KBASE_DEBUG_ASSERT(0 != pgd);
591
592 p = pfn_to_page(PFN_DOWN(pgd));
593
594 pgd_page = kmap_atomic(p);
595 KBASE_DEBUG_ASSERT(NULL != pgd_page);
596
597 /* Invalidate the entries we added */
598 for (i = 0; i < count; i++)
599 mmu_mode->entry_invalidate(&pgd_page[index + i]);
600
601 vpfn += count;
602 nr -= count;
603
604 kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE);
605
606 kunmap_atomic(pgd_page);
607 }
608 }
609
610 /*
611 * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn'
612 */
kbase_mmu_insert_single_page(struct kbase_context * kctx,u64 vpfn,phys_addr_t phys,size_t nr,unsigned long flags)613 int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
614 phys_addr_t phys, size_t nr,
615 unsigned long flags)
616 {
617 phys_addr_t pgd;
618 u64 *pgd_page;
619 /* In case the insert_single_page only partially completes we need to be
620 * able to recover */
621 bool recover_required = false;
622 u64 recover_vpfn = vpfn;
623 size_t recover_count = 0;
624 size_t remain = nr;
625 int err;
626
627 KBASE_DEBUG_ASSERT(NULL != kctx);
628 KBASE_DEBUG_ASSERT(0 != vpfn);
629 /* 64-bit address range is the max */
630 KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
631
632 /* Early out if there is nothing to do */
633 if (nr == 0)
634 return 0;
635
636 mutex_lock(&kctx->mmu_lock);
637
638 while (remain) {
639 unsigned int i;
640 unsigned int index = vpfn & 0x1FF;
641 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
642 struct page *p;
643
644 if (count > remain)
645 count = remain;
646
647 /*
648 * Repeatedly calling mmu_get_bottom_pte() is clearly
649 * suboptimal. We don't have to re-parse the whole tree
650 * each time (just cache the l0-l2 sequence).
651 * On the other hand, it's only a gain when we map more than
652 * 256 pages at once (on average). Do we really care?
653 */
654 do {
655 err = mmu_get_bottom_pgd(kctx, vpfn, &pgd);
656 if (err != -ENOMEM)
657 break;
658 /* Fill the memory pool with enough pages for
659 * the page walk to succeed
660 */
661 mutex_unlock(&kctx->mmu_lock);
662 err = kbase_mem_pool_grow(&kctx->mem_pool,
663 MIDGARD_MMU_BOTTOMLEVEL);
664 mutex_lock(&kctx->mmu_lock);
665 } while (!err);
666 if (err) {
667 dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
668 if (recover_required) {
669 /* Invalidate the pages we have partially
670 * completed */
671 mmu_insert_pages_failure_recovery(kctx,
672 recover_vpfn,
673 recover_count);
674 }
675 goto fail_unlock;
676 }
677
678 p = pfn_to_page(PFN_DOWN(pgd));
679 pgd_page = kmap(p);
680 if (!pgd_page) {
681 dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
682 if (recover_required) {
683 /* Invalidate the pages we have partially
684 * completed */
685 mmu_insert_pages_failure_recovery(kctx,
686 recover_vpfn,
687 recover_count);
688 }
689 err = -ENOMEM;
690 goto fail_unlock;
691 }
692
693 for (i = 0; i < count; i++) {
694 unsigned int ofs = index + i;
695
696 KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
697 kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
698 phys, flags);
699 }
700
701 vpfn += count;
702 remain -= count;
703
704 kbase_mmu_sync_pgd(kctx->kbdev,
705 kbase_dma_addr(p) + (index * sizeof(u64)),
706 count * sizeof(u64));
707
708 kunmap(p);
709 /* We have started modifying the page table.
710 * If further pages need inserting and fail we need to undo what
711 * has already taken place */
712 recover_required = true;
713 recover_count += count;
714 }
715 mutex_unlock(&kctx->mmu_lock);
716 kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
717 return 0;
718
719 fail_unlock:
720 mutex_unlock(&kctx->mmu_lock);
721 kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
722 return err;
723 }
724
kbase_mmu_insert_pages_no_flush(struct kbase_context * kctx,u64 vpfn,phys_addr_t * phys,size_t nr,unsigned long flags)725 int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn,
726 phys_addr_t *phys, size_t nr,
727 unsigned long flags)
728 {
729 phys_addr_t pgd;
730 u64 *pgd_page;
731 /* In case the insert_pages only partially completes we need to be able
732 * to recover */
733 bool recover_required = false;
734 u64 recover_vpfn = vpfn;
735 size_t recover_count = 0;
736 size_t remain = nr;
737 int err;
738
739 KBASE_DEBUG_ASSERT(NULL != kctx);
740 KBASE_DEBUG_ASSERT(0 != vpfn);
741 /* 64-bit address range is the max */
742 KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
743
744 /* Early out if there is nothing to do */
745 if (nr == 0)
746 return 0;
747
748 mutex_lock(&kctx->mmu_lock);
749
750 while (remain) {
751 unsigned int i;
752 unsigned int index = vpfn & 0x1FF;
753 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
754 struct page *p;
755
756 if (count > remain)
757 count = remain;
758
759 /*
760 * Repeatedly calling mmu_get_bottom_pte() is clearly
761 * suboptimal. We don't have to re-parse the whole tree
762 * each time (just cache the l0-l2 sequence).
763 * On the other hand, it's only a gain when we map more than
764 * 256 pages at once (on average). Do we really care?
765 */
766 do {
767 err = mmu_get_bottom_pgd(kctx, vpfn, &pgd);
768 if (err != -ENOMEM)
769 break;
770 /* Fill the memory pool with enough pages for
771 * the page walk to succeed
772 */
773 mutex_unlock(&kctx->mmu_lock);
774 err = kbase_mem_pool_grow(&kctx->mem_pool,
775 MIDGARD_MMU_BOTTOMLEVEL);
776 mutex_lock(&kctx->mmu_lock);
777 } while (!err);
778 if (err) {
779 dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
780 if (recover_required) {
781 /* Invalidate the pages we have partially
782 * completed */
783 mmu_insert_pages_failure_recovery(kctx,
784 recover_vpfn,
785 recover_count);
786 }
787 goto fail_unlock;
788 }
789
790 p = pfn_to_page(PFN_DOWN(pgd));
791 pgd_page = kmap(p);
792 if (!pgd_page) {
793 dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
794 if (recover_required) {
795 /* Invalidate the pages we have partially
796 * completed */
797 mmu_insert_pages_failure_recovery(kctx,
798 recover_vpfn,
799 recover_count);
800 }
801 err = -ENOMEM;
802 goto fail_unlock;
803 }
804
805 for (i = 0; i < count; i++) {
806 unsigned int ofs = index + i;
807
808 KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
809 kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
810 phys[i], flags);
811 }
812
813 phys += count;
814 vpfn += count;
815 remain -= count;
816
817 kbase_mmu_sync_pgd(kctx->kbdev,
818 kbase_dma_addr(p) + (index * sizeof(u64)),
819 count * sizeof(u64));
820
821 kunmap(p);
822 /* We have started modifying the page table. If further pages
823 * need inserting and fail we need to undo what has already
824 * taken place */
825 recover_required = true;
826 recover_count += count;
827 }
828
829 mutex_unlock(&kctx->mmu_lock);
830 return 0;
831
832 fail_unlock:
833 mutex_unlock(&kctx->mmu_lock);
834 return err;
835 }
836
837 /*
838 * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn'
839 */
kbase_mmu_insert_pages(struct kbase_context * kctx,u64 vpfn,phys_addr_t * phys,size_t nr,unsigned long flags)840 int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
841 phys_addr_t *phys, size_t nr,
842 unsigned long flags)
843 {
844 int err;
845
846 err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags);
847 kbase_mmu_flush_invalidate(kctx, vpfn, nr, false);
848 return err;
849 }
850
851 KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
852
853 /**
854 * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches
855 * without retaining the kbase context.
856 * @kctx: The KBase context.
857 * @vpfn: The virtual page frame number to start the flush on.
858 * @nr: The number of pages to flush.
859 * @sync: Set if the operation should be synchronous or not.
860 *
861 * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any
862 * other locking.
863 */
kbase_mmu_flush_invalidate_noretain(struct kbase_context * kctx,u64 vpfn,size_t nr,bool sync)864 static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx,
865 u64 vpfn, size_t nr, bool sync)
866 {
867 struct kbase_device *kbdev = kctx->kbdev;
868 int err;
869 u32 op;
870
871 /* Early out if there is nothing to do */
872 if (nr == 0)
873 return;
874
875 if (sync)
876 op = AS_COMMAND_FLUSH_MEM;
877 else
878 op = AS_COMMAND_FLUSH_PT;
879
880 err = kbase_mmu_hw_do_operation(kbdev,
881 &kbdev->as[kctx->as_nr],
882 kctx, vpfn, nr, op, 0);
883 #if KBASE_GPU_RESET_EN
884 if (err) {
885 /* Flush failed to complete, assume the
886 * GPU has hung and perform a reset to
887 * recover */
888 dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n");
889
890 if (kbase_prepare_to_reset_gpu_locked(kbdev))
891 kbase_reset_gpu_locked(kbdev);
892 }
893 #endif /* KBASE_GPU_RESET_EN */
894
895 #ifndef CONFIG_MALI_NO_MALI
896 /*
897 * As this function could be called in interrupt context the sync
898 * request can't block. Instead log the request and the next flush
899 * request will pick it up.
900 */
901 if ((!err) && sync &&
902 kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367))
903 atomic_set(&kctx->drain_pending, 1);
904 #endif /* !CONFIG_MALI_NO_MALI */
905 }
906
kbase_mmu_flush_invalidate(struct kbase_context * kctx,u64 vpfn,size_t nr,bool sync)907 static void kbase_mmu_flush_invalidate(struct kbase_context *kctx,
908 u64 vpfn, size_t nr, bool sync)
909 {
910 struct kbase_device *kbdev;
911 bool ctx_is_in_runpool;
912 #ifndef CONFIG_MALI_NO_MALI
913 bool drain_pending = false;
914
915 if (atomic_xchg(&kctx->drain_pending, 0))
916 drain_pending = true;
917 #endif /* !CONFIG_MALI_NO_MALI */
918
919 /* Early out if there is nothing to do */
920 if (nr == 0)
921 return;
922
923 kbdev = kctx->kbdev;
924 mutex_lock(&kbdev->js_data.queue_mutex);
925 ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx);
926 mutex_unlock(&kbdev->js_data.queue_mutex);
927
928 if (ctx_is_in_runpool) {
929 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
930
931 if (!kbase_pm_context_active_handle_suspend(kbdev,
932 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
933 int err;
934 u32 op;
935
936 /* AS transaction begin */
937 mutex_lock(&kbdev->mmu_hw_mutex);
938
939 if (sync)
940 op = AS_COMMAND_FLUSH_MEM;
941 else
942 op = AS_COMMAND_FLUSH_PT;
943
944 err = kbase_mmu_hw_do_operation(kbdev,
945 &kbdev->as[kctx->as_nr],
946 kctx, vpfn, nr, op, 0);
947
948 #if KBASE_GPU_RESET_EN
949 if (err) {
950 /* Flush failed to complete, assume the
951 * GPU has hung and perform a reset to
952 * recover */
953 dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
954
955 if (kbase_prepare_to_reset_gpu(kbdev))
956 kbase_reset_gpu(kbdev);
957 }
958 #endif /* KBASE_GPU_RESET_EN */
959
960 mutex_unlock(&kbdev->mmu_hw_mutex);
961 /* AS transaction end */
962
963 #ifndef CONFIG_MALI_NO_MALI
964 /*
965 * The transaction lock must be dropped before here
966 * as kbase_wait_write_flush could take it if
967 * the GPU was powered down (static analysis doesn't
968 * know this can't happen).
969 */
970 drain_pending |= (!err) && sync &&
971 kbase_hw_has_issue(kctx->kbdev,
972 BASE_HW_ISSUE_6367);
973 if (drain_pending) {
974 /* Wait for GPU to flush write buffer */
975 kbase_wait_write_flush(kctx);
976 }
977 #endif /* !CONFIG_MALI_NO_MALI */
978
979 kbase_pm_context_idle(kbdev);
980 }
981 kbasep_js_runpool_release_ctx(kbdev, kctx);
982 }
983 }
984
kbase_mmu_update(struct kbase_context * kctx)985 void kbase_mmu_update(struct kbase_context *kctx)
986 {
987 lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
988 lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex);
989 /* ASSERT that the context has a valid as_nr, which is only the case
990 * when it's scheduled in.
991 *
992 * as_nr won't change because the caller has the hwaccess_lock */
993 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
994
995 kctx->kbdev->mmu_mode->update(kctx);
996 }
997 KBASE_EXPORT_TEST_API(kbase_mmu_update);
998
kbase_mmu_disable_as(struct kbase_device * kbdev,int as_nr)999 void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr)
1000 {
1001 lockdep_assert_held(&kbdev->hwaccess_lock);
1002 lockdep_assert_held(&kbdev->mmu_hw_mutex);
1003
1004 kbdev->mmu_mode->disable_as(kbdev, as_nr);
1005 }
1006
kbase_mmu_disable(struct kbase_context * kctx)1007 void kbase_mmu_disable(struct kbase_context *kctx)
1008 {
1009 /* ASSERT that the context has a valid as_nr, which is only the case
1010 * when it's scheduled in.
1011 *
1012 * as_nr won't change because the caller has the hwaccess_lock */
1013 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1014
1015 lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
1016
1017 /*
1018 * The address space is being disabled, drain all knowledge of it out
1019 * from the caches as pages and page tables might be freed after this.
1020 *
1021 * The job scheduler code will already be holding the locks and context
1022 * so just do the flush.
1023 */
1024 kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true);
1025
1026 kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr);
1027 }
1028 KBASE_EXPORT_TEST_API(kbase_mmu_disable);
1029
1030 /*
1031 * We actually only discard the ATE, and not the page table
1032 * pages. There is a potential DoS here, as we'll leak memory by
1033 * having PTEs that are potentially unused. Will require physical
1034 * page accounting, so MMU pages are part of the process allocation.
1035 *
1036 * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
1037 * currently scheduled into the runpool, and so potentially uses a lot of locks.
1038 * These locks must be taken in the correct order with respect to others
1039 * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
1040 * information.
1041 */
kbase_mmu_teardown_pages(struct kbase_context * kctx,u64 vpfn,size_t nr)1042 int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
1043 {
1044 phys_addr_t pgd;
1045 u64 *pgd_page;
1046 struct kbase_device *kbdev;
1047 size_t requested_nr = nr;
1048 struct kbase_mmu_mode const *mmu_mode;
1049 int err;
1050
1051 KBASE_DEBUG_ASSERT(NULL != kctx);
1052 beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr);
1053
1054 if (0 == nr) {
1055 /* early out if nothing to do */
1056 return 0;
1057 }
1058
1059 mutex_lock(&kctx->mmu_lock);
1060
1061 kbdev = kctx->kbdev;
1062 mmu_mode = kbdev->mmu_mode;
1063
1064 while (nr) {
1065 unsigned int i;
1066 unsigned int index = vpfn & 0x1FF;
1067 unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
1068 struct page *p;
1069
1070 if (count > nr)
1071 count = nr;
1072
1073 err = mmu_get_bottom_pgd(kctx, vpfn, &pgd);
1074 if (err) {
1075 dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n");
1076 err = -EINVAL;
1077 goto fail_unlock;
1078 }
1079
1080 p = pfn_to_page(PFN_DOWN(pgd));
1081 pgd_page = kmap(p);
1082 if (!pgd_page) {
1083 dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n");
1084 err = -ENOMEM;
1085 goto fail_unlock;
1086 }
1087
1088 for (i = 0; i < count; i++)
1089 mmu_mode->entry_invalidate(&pgd_page[index + i]);
1090
1091 vpfn += count;
1092 nr -= count;
1093
1094 kbase_mmu_sync_pgd(kctx->kbdev,
1095 kbase_dma_addr(p) + (index * sizeof(u64)),
1096 count * sizeof(u64));
1097
1098 kunmap(p);
1099 }
1100
1101 mutex_unlock(&kctx->mmu_lock);
1102 kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1103 return 0;
1104
1105 fail_unlock:
1106 mutex_unlock(&kctx->mmu_lock);
1107 kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1108 return err;
1109 }
1110
1111 KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
1112
1113 /**
1114 * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'.
1115 * This call is being triggered as a response to the changes of the mem attributes
1116 *
1117 * @pre : The caller is responsible for validating the memory attributes
1118 *
1119 * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is
1120 * currently scheduled into the runpool, and so potentially uses a lot of locks.
1121 * These locks must be taken in the correct order with respect to others
1122 * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
1123 * information.
1124 */
kbase_mmu_update_pages(struct kbase_context * kctx,u64 vpfn,phys_addr_t * phys,size_t nr,unsigned long flags)1125 int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags)
1126 {
1127 phys_addr_t pgd;
1128 u64 *pgd_page;
1129 size_t requested_nr = nr;
1130 struct kbase_mmu_mode const *mmu_mode;
1131 int err;
1132
1133 KBASE_DEBUG_ASSERT(NULL != kctx);
1134 KBASE_DEBUG_ASSERT(0 != vpfn);
1135 KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
1136
1137 /* Early out if there is nothing to do */
1138 if (nr == 0)
1139 return 0;
1140
1141 mutex_lock(&kctx->mmu_lock);
1142
1143 mmu_mode = kctx->kbdev->mmu_mode;
1144
1145 dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages",
1146 vpfn, phys, nr);
1147
1148 while (nr) {
1149 unsigned int i;
1150 unsigned int index = vpfn & 0x1FF;
1151 size_t count = KBASE_MMU_PAGE_ENTRIES - index;
1152 struct page *p;
1153
1154 if (count > nr)
1155 count = nr;
1156
1157 do {
1158 err = mmu_get_bottom_pgd(kctx, vpfn, &pgd);
1159 if (err != -ENOMEM)
1160 break;
1161 /* Fill the memory pool with enough pages for
1162 * the page walk to succeed
1163 */
1164 mutex_unlock(&kctx->mmu_lock);
1165 err = kbase_mem_pool_grow(&kctx->mem_pool,
1166 MIDGARD_MMU_BOTTOMLEVEL);
1167 mutex_lock(&kctx->mmu_lock);
1168 } while (!err);
1169 if (err) {
1170 dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n");
1171 goto fail_unlock;
1172 }
1173
1174 p = pfn_to_page(PFN_DOWN(pgd));
1175 pgd_page = kmap(p);
1176 if (!pgd_page) {
1177 dev_warn(kctx->kbdev->dev, "kmap failure\n");
1178 err = -ENOMEM;
1179 goto fail_unlock;
1180 }
1181
1182 for (i = 0; i < count; i++)
1183 mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i],
1184 flags);
1185
1186 phys += count;
1187 vpfn += count;
1188 nr -= count;
1189
1190 kbase_mmu_sync_pgd(kctx->kbdev,
1191 kbase_dma_addr(p) + (index * sizeof(u64)),
1192 count * sizeof(u64));
1193
1194 kunmap(pfn_to_page(PFN_DOWN(pgd)));
1195 }
1196
1197 mutex_unlock(&kctx->mmu_lock);
1198 kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1199 return 0;
1200
1201 fail_unlock:
1202 mutex_unlock(&kctx->mmu_lock);
1203 kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true);
1204 return err;
1205 }
1206
1207 /* This is a debug feature only */
mmu_check_unused(struct kbase_context * kctx,phys_addr_t pgd)1208 static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd)
1209 {
1210 u64 *page;
1211 int i;
1212
1213 lockdep_assert_held(&kctx->reg_lock);
1214
1215 page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
1216 /* kmap_atomic should NEVER fail. */
1217 KBASE_DEBUG_ASSERT(NULL != page);
1218
1219 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1220 if (kctx->kbdev->mmu_mode->ate_is_valid(page[i]))
1221 beenthere(kctx, "live pte %016lx", (unsigned long)page[i]);
1222 }
1223 kunmap_atomic(page);
1224 }
1225
mmu_teardown_level(struct kbase_context * kctx,phys_addr_t pgd,int level,int zap,u64 * pgd_page_buffer)1226 static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer)
1227 {
1228 phys_addr_t target_pgd;
1229 u64 *pgd_page;
1230 int i;
1231 struct kbase_mmu_mode const *mmu_mode;
1232
1233 KBASE_DEBUG_ASSERT(NULL != kctx);
1234 lockdep_assert_held(&kctx->mmu_lock);
1235 lockdep_assert_held(&kctx->reg_lock);
1236
1237 pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd)));
1238 /* kmap_atomic should NEVER fail. */
1239 KBASE_DEBUG_ASSERT(NULL != pgd_page);
1240 /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */
1241 memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE);
1242 kunmap_atomic(pgd_page);
1243 pgd_page = pgd_page_buffer;
1244
1245 mmu_mode = kctx->kbdev->mmu_mode;
1246
1247 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1248 target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
1249
1250 if (target_pgd) {
1251 if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) {
1252 mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64)));
1253 } else {
1254 /*
1255 * So target_pte is a level-3 page.
1256 * As a leaf, it is safe to free it.
1257 * Unless we have live pages attached to it!
1258 */
1259 mmu_check_unused(kctx, target_pgd);
1260 }
1261
1262 beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1);
1263 if (zap) {
1264 struct page *p = phys_to_page(target_pgd);
1265
1266 kbase_mem_pool_free(&kctx->mem_pool, p, true);
1267 kbase_process_page_usage_dec(kctx, 1);
1268 kbase_atomic_sub_pages(1, &kctx->used_pages);
1269 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1270 }
1271 }
1272 }
1273 }
1274
kbase_mmu_init(struct kbase_context * kctx)1275 int kbase_mmu_init(struct kbase_context *kctx)
1276 {
1277 KBASE_DEBUG_ASSERT(NULL != kctx);
1278 KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages);
1279
1280 mutex_init(&kctx->mmu_lock);
1281
1282 /* Preallocate MMU depth of four pages for mmu_teardown_level to use */
1283 kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
1284
1285 if (NULL == kctx->mmu_teardown_pages)
1286 return -ENOMEM;
1287
1288 return 0;
1289 }
1290
kbase_mmu_term(struct kbase_context * kctx)1291 void kbase_mmu_term(struct kbase_context *kctx)
1292 {
1293 KBASE_DEBUG_ASSERT(NULL != kctx);
1294 KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1295
1296 kfree(kctx->mmu_teardown_pages);
1297 kctx->mmu_teardown_pages = NULL;
1298 }
1299
kbase_mmu_free_pgd(struct kbase_context * kctx)1300 void kbase_mmu_free_pgd(struct kbase_context *kctx)
1301 {
1302 int new_page_count __maybe_unused;
1303
1304 KBASE_DEBUG_ASSERT(NULL != kctx);
1305 KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages);
1306
1307 mutex_lock(&kctx->mmu_lock);
1308 mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages);
1309 mutex_unlock(&kctx->mmu_lock);
1310
1311 beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd);
1312 kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true);
1313 kbase_process_page_usage_dec(kctx, 1);
1314 new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages);
1315 kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
1316
1317 KBASE_TLSTREAM_AUX_PAGESALLOC(
1318 (u32)kctx->id,
1319 (u64)new_page_count);
1320 }
1321
1322 KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd);
1323
kbasep_mmu_dump_level(struct kbase_context * kctx,phys_addr_t pgd,int level,char ** const buffer,size_t * size_left)1324 static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left)
1325 {
1326 phys_addr_t target_pgd;
1327 u64 *pgd_page;
1328 int i;
1329 size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64);
1330 size_t dump_size;
1331 struct kbase_mmu_mode const *mmu_mode;
1332
1333 KBASE_DEBUG_ASSERT(NULL != kctx);
1334 lockdep_assert_held(&kctx->mmu_lock);
1335
1336 mmu_mode = kctx->kbdev->mmu_mode;
1337
1338 pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd)));
1339 if (!pgd_page) {
1340 dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n");
1341 return 0;
1342 }
1343
1344 if (*size_left >= size) {
1345 /* A modified physical address that contains the page table level */
1346 u64 m_pgd = pgd | level;
1347
1348 /* Put the modified physical address in the output buffer */
1349 memcpy(*buffer, &m_pgd, sizeof(m_pgd));
1350 *buffer += sizeof(m_pgd);
1351
1352 /* Followed by the page table itself */
1353 memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES);
1354 *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES;
1355
1356 *size_left -= size;
1357 }
1358
1359 if (level < MIDGARD_MMU_BOTTOMLEVEL) {
1360 for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
1361 if (mmu_mode->pte_is_valid(pgd_page[i])) {
1362 target_pgd = mmu_mode->pte_to_phy_addr(
1363 pgd_page[i]);
1364
1365 dump_size = kbasep_mmu_dump_level(kctx,
1366 target_pgd, level + 1,
1367 buffer, size_left);
1368 if (!dump_size) {
1369 kunmap(pfn_to_page(PFN_DOWN(pgd)));
1370 return 0;
1371 }
1372 size += dump_size;
1373 }
1374 }
1375 }
1376
1377 kunmap(pfn_to_page(PFN_DOWN(pgd)));
1378
1379 return size;
1380 }
1381
kbase_mmu_dump(struct kbase_context * kctx,int nr_pages)1382 void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages)
1383 {
1384 void *kaddr;
1385 size_t size_left;
1386
1387 KBASE_DEBUG_ASSERT(kctx);
1388
1389 if (0 == nr_pages) {
1390 /* can't dump in a 0 sized buffer, early out */
1391 return NULL;
1392 }
1393
1394 size_left = nr_pages * PAGE_SIZE;
1395
1396 KBASE_DEBUG_ASSERT(0 != size_left);
1397 kaddr = vmalloc_user(size_left);
1398
1399 mutex_lock(&kctx->mmu_lock);
1400
1401 if (kaddr) {
1402 u64 end_marker = 0xFFULL;
1403 char *buffer;
1404 char *mmu_dump_buffer;
1405 u64 config[3];
1406 size_t size;
1407
1408 buffer = (char *)kaddr;
1409 mmu_dump_buffer = buffer;
1410
1411 if (kctx->api_version >= KBASE_API_VERSION(8, 4)) {
1412 struct kbase_mmu_setup as_setup;
1413
1414 kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup);
1415 config[0] = as_setup.transtab;
1416 config[1] = as_setup.memattr;
1417 config[2] = as_setup.transcfg;
1418 memcpy(buffer, &config, sizeof(config));
1419 mmu_dump_buffer += sizeof(config);
1420 size_left -= sizeof(config);
1421 }
1422
1423
1424
1425 size = kbasep_mmu_dump_level(kctx,
1426 kctx->pgd,
1427 MIDGARD_MMU_TOPLEVEL,
1428 &mmu_dump_buffer,
1429 &size_left);
1430
1431 if (!size)
1432 goto fail_free;
1433
1434 /* Add on the size for the end marker */
1435 size += sizeof(u64);
1436 /* Add on the size for the config */
1437 if (kctx->api_version >= KBASE_API_VERSION(8, 4))
1438 size += sizeof(config);
1439
1440
1441 if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) {
1442 /* The buffer isn't big enough - free the memory and return failure */
1443 goto fail_free;
1444 }
1445
1446 /* Add the end marker */
1447 memcpy(mmu_dump_buffer, &end_marker, sizeof(u64));
1448 }
1449
1450 mutex_unlock(&kctx->mmu_lock);
1451 return kaddr;
1452
1453 fail_free:
1454 vfree(kaddr);
1455 mutex_unlock(&kctx->mmu_lock);
1456 return NULL;
1457 }
1458 KBASE_EXPORT_TEST_API(kbase_mmu_dump);
1459
bus_fault_worker(struct work_struct * data)1460 void bus_fault_worker(struct work_struct *data)
1461 {
1462 struct kbase_as *faulting_as;
1463 int as_no;
1464 struct kbase_context *kctx;
1465 struct kbase_device *kbdev;
1466 #if KBASE_GPU_RESET_EN
1467 bool reset_status = false;
1468 #endif /* KBASE_GPU_RESET_EN */
1469
1470 faulting_as = container_of(data, struct kbase_as, work_busfault);
1471
1472 as_no = faulting_as->number;
1473
1474 kbdev = container_of(faulting_as, struct kbase_device, as[as_no]);
1475
1476 /* Grab the context that was already refcounted in kbase_mmu_interrupt().
1477 * Therefore, it cannot be scheduled out of this AS until we explicitly release it
1478 */
1479 kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no);
1480 if (WARN_ON(!kctx)) {
1481 atomic_dec(&kbdev->faults_pending);
1482 return;
1483 }
1484
1485 if (unlikely(faulting_as->protected_mode))
1486 {
1487 kbase_mmu_report_fault_and_kill(kctx, faulting_as,
1488 "Permission failure");
1489 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
1490 KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1491 kbasep_js_runpool_release_ctx(kbdev, kctx);
1492 atomic_dec(&kbdev->faults_pending);
1493 return;
1494
1495 }
1496
1497 #if KBASE_GPU_RESET_EN
1498 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1499 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1500 * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1501 * are evicted from the GPU before the switch.
1502 */
1503 dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
1504 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1505 }
1506 #endif /* KBASE_GPU_RESET_EN */
1507 /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */
1508 if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1509 unsigned long flags;
1510
1511 /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1512 /* AS transaction begin */
1513 mutex_lock(&kbdev->mmu_hw_mutex);
1514
1515 /* Set the MMU into unmapped mode */
1516 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1517 kbase_mmu_disable(kctx);
1518 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1519
1520 mutex_unlock(&kbdev->mmu_hw_mutex);
1521 /* AS transaction end */
1522
1523 kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
1524 KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1525 kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
1526 KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
1527
1528 kbase_pm_context_idle(kbdev);
1529 }
1530
1531 #if KBASE_GPU_RESET_EN
1532 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1533 kbase_reset_gpu(kbdev);
1534 #endif /* KBASE_GPU_RESET_EN */
1535
1536 kbasep_js_runpool_release_ctx(kbdev, kctx);
1537
1538 atomic_dec(&kbdev->faults_pending);
1539 }
1540
kbase_exception_name(struct kbase_device * kbdev,u32 exception_code)1541 const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
1542 {
1543 const char *e;
1544
1545 switch (exception_code) {
1546 /* Non-Fault Status code */
1547 case 0x00:
1548 e = "NOT_STARTED/IDLE/OK";
1549 break;
1550 case 0x01:
1551 e = "DONE";
1552 break;
1553 case 0x02:
1554 e = "INTERRUPTED";
1555 break;
1556 case 0x03:
1557 e = "STOPPED";
1558 break;
1559 case 0x04:
1560 e = "TERMINATED";
1561 break;
1562 case 0x08:
1563 e = "ACTIVE";
1564 break;
1565 /* Job exceptions */
1566 case 0x40:
1567 e = "JOB_CONFIG_FAULT";
1568 break;
1569 case 0x41:
1570 e = "JOB_POWER_FAULT";
1571 break;
1572 case 0x42:
1573 e = "JOB_READ_FAULT";
1574 break;
1575 case 0x43:
1576 e = "JOB_WRITE_FAULT";
1577 break;
1578 case 0x44:
1579 e = "JOB_AFFINITY_FAULT";
1580 break;
1581 case 0x48:
1582 e = "JOB_BUS_FAULT";
1583 break;
1584 case 0x50:
1585 e = "INSTR_INVALID_PC";
1586 break;
1587 case 0x51:
1588 e = "INSTR_INVALID_ENC";
1589 break;
1590 case 0x52:
1591 e = "INSTR_TYPE_MISMATCH";
1592 break;
1593 case 0x53:
1594 e = "INSTR_OPERAND_FAULT";
1595 break;
1596 case 0x54:
1597 e = "INSTR_TLS_FAULT";
1598 break;
1599 case 0x55:
1600 e = "INSTR_BARRIER_FAULT";
1601 break;
1602 case 0x56:
1603 e = "INSTR_ALIGN_FAULT";
1604 break;
1605 case 0x58:
1606 e = "DATA_INVALID_FAULT";
1607 break;
1608 case 0x59:
1609 e = "TILE_RANGE_FAULT";
1610 break;
1611 case 0x5A:
1612 e = "ADDR_RANGE_FAULT";
1613 break;
1614 case 0x60:
1615 e = "OUT_OF_MEMORY";
1616 break;
1617 /* GPU exceptions */
1618 case 0x80:
1619 e = "DELAYED_BUS_FAULT";
1620 break;
1621 case 0x88:
1622 e = "SHAREABILITY_FAULT";
1623 break;
1624 /* MMU exceptions */
1625 case 0xC0:
1626 case 0xC1:
1627 case 0xC2:
1628 case 0xC3:
1629 case 0xC4:
1630 case 0xC5:
1631 case 0xC6:
1632 case 0xC7:
1633 e = "TRANSLATION_FAULT";
1634 break;
1635 case 0xC8:
1636 e = "PERMISSION_FAULT";
1637 break;
1638 case 0xC9:
1639 case 0xCA:
1640 case 0xCB:
1641 case 0xCC:
1642 case 0xCD:
1643 case 0xCE:
1644 case 0xCF:
1645 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
1646 e = "PERMISSION_FAULT";
1647 else
1648 e = "UNKNOWN";
1649 break;
1650 case 0xD0:
1651 case 0xD1:
1652 case 0xD2:
1653 case 0xD3:
1654 case 0xD4:
1655 case 0xD5:
1656 case 0xD6:
1657 case 0xD7:
1658 e = "TRANSTAB_BUS_FAULT";
1659 break;
1660 case 0xD8:
1661 e = "ACCESS_FLAG";
1662 break;
1663 case 0xD9:
1664 case 0xDA:
1665 case 0xDB:
1666 case 0xDC:
1667 case 0xDD:
1668 case 0xDE:
1669 case 0xDF:
1670 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
1671 e = "ACCESS_FLAG";
1672 else
1673 e = "UNKNOWN";
1674 break;
1675 case 0xE0:
1676 case 0xE1:
1677 case 0xE2:
1678 case 0xE3:
1679 case 0xE4:
1680 case 0xE5:
1681 case 0xE6:
1682 case 0xE7:
1683 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
1684 e = "ADDRESS_SIZE_FAULT";
1685 else
1686 e = "UNKNOWN";
1687 break;
1688 case 0xE8:
1689 case 0xE9:
1690 case 0xEA:
1691 case 0xEB:
1692 case 0xEC:
1693 case 0xED:
1694 case 0xEE:
1695 case 0xEF:
1696 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
1697 e = "MEMORY_ATTRIBUTES_FAULT";
1698 else
1699 e = "UNKNOWN";
1700 break;
1701 default:
1702 e = "UNKNOWN";
1703 break;
1704 };
1705
1706 return e;
1707 }
1708
access_type_name(struct kbase_device * kbdev,u32 fault_status)1709 static const char *access_type_name(struct kbase_device *kbdev,
1710 u32 fault_status)
1711 {
1712 switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1713 case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
1714 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
1715 return "ATOMIC";
1716 else
1717 return "UNKNOWN";
1718 case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1719 return "READ";
1720 case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1721 return "WRITE";
1722 case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1723 return "EXECUTE";
1724 default:
1725 WARN_ON(1);
1726 return NULL;
1727 }
1728 }
1729
1730 /**
1731 * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on.
1732 */
kbase_mmu_report_fault_and_kill(struct kbase_context * kctx,struct kbase_as * as,const char * reason_str)1733 static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx,
1734 struct kbase_as *as, const char *reason_str)
1735 {
1736 unsigned long flags;
1737 int exception_type;
1738 int access_type;
1739 int source_id;
1740 int as_no;
1741 struct kbase_device *kbdev;
1742 struct kbasep_js_device_data *js_devdata;
1743
1744 #if KBASE_GPU_RESET_EN
1745 bool reset_status = false;
1746 #endif
1747
1748 as_no = as->number;
1749 kbdev = kctx->kbdev;
1750 js_devdata = &kbdev->js_data;
1751
1752 /* ASSERT that the context won't leave the runpool */
1753 KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0);
1754
1755 /* decode the fault status */
1756 exception_type = as->fault_status & 0xFF;
1757 access_type = (as->fault_status >> 8) & 0x3;
1758 source_id = (as->fault_status >> 16);
1759
1760 /* terminal fault, print info about the fault */
1761 dev_err(kbdev->dev,
1762 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
1763 "Reason: %s\n"
1764 "raw fault status: 0x%X\n"
1765 "decoded fault status: %s\n"
1766 "exception type 0x%X: %s\n"
1767 "access type 0x%X: %s\n"
1768 "source id 0x%X\n"
1769 "pid: %d\n",
1770 as_no, as->fault_addr,
1771 reason_str,
1772 as->fault_status,
1773 (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1774 exception_type, kbase_exception_name(kbdev, exception_type),
1775 access_type, access_type_name(kbdev, as->fault_status),
1776 source_id,
1777 kctx->pid);
1778
1779 /* hardware counters dump fault handling */
1780 if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) &&
1781 (kbdev->hwcnt.backend.state ==
1782 KBASE_INSTR_STATE_DUMPING)) {
1783 unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
1784
1785 if ((as->fault_addr >= kbdev->hwcnt.addr) &&
1786 (as->fault_addr < (kbdev->hwcnt.addr +
1787 (num_core_groups * 2048))))
1788 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT;
1789 }
1790
1791 /* Stop the kctx from submitting more jobs and cause it to be scheduled
1792 * out/rescheduled - this will occur on releasing the context's refcount */
1793 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1794 kbasep_js_clear_submit_allowed(js_devdata, kctx);
1795 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1796
1797 /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this
1798 * context can appear in the job slots from this point on */
1799 kbase_backend_jm_kill_jobs_from_kctx(kctx);
1800 /* AS transaction begin */
1801 mutex_lock(&kbdev->mmu_hw_mutex);
1802 #if KBASE_GPU_RESET_EN
1803 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
1804 /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode.
1805 * We start the reset before switching to UNMAPPED to ensure that unrelated jobs
1806 * are evicted from the GPU before the switch.
1807 */
1808 dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery.");
1809 reset_status = kbase_prepare_to_reset_gpu(kbdev);
1810 }
1811 #endif /* KBASE_GPU_RESET_EN */
1812 /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
1813 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1814 kbase_mmu_disable(kctx);
1815 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1816
1817 mutex_unlock(&kbdev->mmu_hw_mutex);
1818 /* AS transaction end */
1819 /* Clear down the fault */
1820 kbase_mmu_hw_clear_fault(kbdev, as, kctx,
1821 KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1822 kbase_mmu_hw_enable_fault(kbdev, as, kctx,
1823 KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
1824
1825 #if KBASE_GPU_RESET_EN
1826 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status)
1827 kbase_reset_gpu(kbdev);
1828 #endif /* KBASE_GPU_RESET_EN */
1829 }
1830
kbasep_as_do_poke(struct work_struct * work)1831 void kbasep_as_do_poke(struct work_struct *work)
1832 {
1833 struct kbase_as *as;
1834 struct kbase_device *kbdev;
1835 struct kbase_context *kctx;
1836 unsigned long flags;
1837
1838 KBASE_DEBUG_ASSERT(work);
1839 as = container_of(work, struct kbase_as, poke_work);
1840 kbdev = container_of(as, struct kbase_device, as[as->number]);
1841 KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1842
1843 /* GPU power will already be active by virtue of the caller holding a JS
1844 * reference on the address space, and will not release it until this worker
1845 * has finished */
1846
1847 /* Further to the comment above, we know that while this function is running
1848 * the AS will not be released as before the atom is released this workqueue
1849 * is flushed (in kbase_as_poking_timer_release_atom)
1850 */
1851 kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number);
1852
1853 /* AS transaction begin */
1854 mutex_lock(&kbdev->mmu_hw_mutex);
1855 /* Force a uTLB invalidate */
1856 kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0,
1857 AS_COMMAND_UNLOCK, 0);
1858 mutex_unlock(&kbdev->mmu_hw_mutex);
1859 /* AS transaction end */
1860
1861 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1862 if (as->poke_refcount &&
1863 !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) {
1864 /* Only queue up the timer if we need it, and we're not trying to kill it */
1865 hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL);
1866 }
1867 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1868 }
1869
kbasep_as_poke_timer_callback(struct hrtimer * timer)1870 enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer)
1871 {
1872 struct kbase_as *as;
1873 int queue_work_ret;
1874
1875 KBASE_DEBUG_ASSERT(NULL != timer);
1876 as = container_of(timer, struct kbase_as, poke_timer);
1877 KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1878
1879 queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1880 KBASE_DEBUG_ASSERT(queue_work_ret);
1881 return HRTIMER_NORESTART;
1882 }
1883
1884 /**
1885 * Retain the poking timer on an atom's context (if the atom hasn't already
1886 * done so), and start the timer (if it's not already started).
1887 *
1888 * This must only be called on a context that's scheduled in, and an atom
1889 * that's running on the GPU.
1890 *
1891 * The caller must hold hwaccess_lock
1892 *
1893 * This can be called safely from atomic context
1894 */
kbase_as_poking_timer_retain_atom(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbase_jd_atom * katom)1895 void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1896 {
1897 struct kbase_as *as;
1898
1899 KBASE_DEBUG_ASSERT(kbdev);
1900 KBASE_DEBUG_ASSERT(kctx);
1901 KBASE_DEBUG_ASSERT(katom);
1902 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1903 lockdep_assert_held(&kbdev->hwaccess_lock);
1904
1905 if (katom->poking)
1906 return;
1907
1908 katom->poking = 1;
1909
1910 /* It's safe to work on the as/as_nr without an explicit reference,
1911 * because the caller holds the hwaccess_lock, and the atom itself
1912 * was also running and had already taken a reference */
1913 as = &kbdev->as[kctx->as_nr];
1914
1915 if (++(as->poke_refcount) == 1) {
1916 /* First refcount for poke needed: check if not already in flight */
1917 if (!as->poke_state) {
1918 /* need to start poking */
1919 as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT;
1920 queue_work(as->poke_wq, &as->poke_work);
1921 }
1922 }
1923 }
1924
1925 /**
1926 * If an atom holds a poking timer, release it and wait for it to finish
1927 *
1928 * This must only be called on a context that's scheduled in, and an atom
1929 * that still has a JS reference on the context
1930 *
1931 * This must \b not be called from atomic context, since it can sleep.
1932 */
kbase_as_poking_timer_release_atom(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbase_jd_atom * katom)1933 void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
1934 {
1935 struct kbase_as *as;
1936 unsigned long flags;
1937
1938 KBASE_DEBUG_ASSERT(kbdev);
1939 KBASE_DEBUG_ASSERT(kctx);
1940 KBASE_DEBUG_ASSERT(katom);
1941 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
1942
1943 if (!katom->poking)
1944 return;
1945
1946 as = &kbdev->as[kctx->as_nr];
1947
1948 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1949 KBASE_DEBUG_ASSERT(as->poke_refcount > 0);
1950 KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT);
1951
1952 if (--(as->poke_refcount) == 0) {
1953 as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE;
1954 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1955
1956 hrtimer_cancel(&as->poke_timer);
1957 flush_workqueue(as->poke_wq);
1958
1959 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1960
1961 /* Re-check whether it's still needed */
1962 if (as->poke_refcount) {
1963 int queue_work_ret;
1964 /* Poking still needed:
1965 * - Another retain will not be starting the timer or queueing work,
1966 * because it's still marked as in-flight
1967 * - The hrtimer has finished, and has not started a new timer or
1968 * queued work because it's been marked as killing
1969 *
1970 * So whatever happens now, just queue the work again */
1971 as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE);
1972 queue_work_ret = queue_work(as->poke_wq, &as->poke_work);
1973 KBASE_DEBUG_ASSERT(queue_work_ret);
1974 } else {
1975 /* It isn't - so mark it as not in flight, and not killing */
1976 as->poke_state = 0u;
1977
1978 /* The poke associated with the atom has now finished. If this is
1979 * also the last atom on the context, then we can guarentee no more
1980 * pokes (and thus no more poking register accesses) will occur on
1981 * the context until new atoms are run */
1982 }
1983 }
1984 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1985
1986 katom->poking = 0;
1987 }
1988
kbase_mmu_interrupt_process(struct kbase_device * kbdev,struct kbase_context * kctx,struct kbase_as * as)1989 void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as)
1990 {
1991 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1992
1993 lockdep_assert_held(&kbdev->hwaccess_lock);
1994
1995 if (!kctx) {
1996 dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n",
1997 kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault",
1998 as->number, as->fault_addr);
1999
2000 /* Since no ctx was found, the MMU must be disabled. */
2001 WARN_ON(as->current_setup.transtab);
2002
2003 if (kbase_as_has_bus_fault(as)) {
2004 kbase_mmu_hw_clear_fault(kbdev, as, kctx,
2005 KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
2006 kbase_mmu_hw_enable_fault(kbdev, as, kctx,
2007 KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED);
2008 } else if (kbase_as_has_page_fault(as)) {
2009 kbase_mmu_hw_clear_fault(kbdev, as, kctx,
2010 KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
2011 kbase_mmu_hw_enable_fault(kbdev, as, kctx,
2012 KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED);
2013 }
2014
2015 #if KBASE_GPU_RESET_EN
2016 if (kbase_as_has_bus_fault(as) &&
2017 kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) {
2018 bool reset_status;
2019 /*
2020 * Reset the GPU, like in bus_fault_worker, in case an
2021 * earlier error hasn't been properly cleared by this
2022 * point.
2023 */
2024 dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n");
2025 reset_status = kbase_prepare_to_reset_gpu_locked(kbdev);
2026 if (reset_status)
2027 kbase_reset_gpu_locked(kbdev);
2028 }
2029 #endif /* KBASE_GPU_RESET_EN */
2030
2031 return;
2032 }
2033
2034 if (kbase_as_has_bus_fault(as)) {
2035 /*
2036 * hw counters dumping in progress, signal the
2037 * other thread that it failed
2038 */
2039 if ((kbdev->hwcnt.kctx == kctx) &&
2040 (kbdev->hwcnt.backend.state ==
2041 KBASE_INSTR_STATE_DUMPING))
2042 kbdev->hwcnt.backend.state =
2043 KBASE_INSTR_STATE_FAULT;
2044
2045 /*
2046 * Stop the kctx from submitting more jobs and cause it
2047 * to be scheduled out/rescheduled when all references
2048 * to it are released
2049 */
2050 kbasep_js_clear_submit_allowed(js_devdata, kctx);
2051
2052 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
2053 dev_warn(kbdev->dev,
2054 "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n",
2055 as->number, as->fault_addr,
2056 as->fault_extra_addr);
2057 else
2058 dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
2059 as->number, as->fault_addr);
2060
2061 /*
2062 * We need to switch to UNMAPPED mode - but we do this in a
2063 * worker so that we can sleep
2064 */
2065 kbdev->kbase_group_error++;
2066 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault));
2067 WARN_ON(work_pending(&as->work_busfault));
2068 queue_work(as->pf_wq, &as->work_busfault);
2069 atomic_inc(&kbdev->faults_pending);
2070 } else {
2071 kbdev->kbase_group_error++;
2072 KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault));
2073 WARN_ON(work_pending(&as->work_pagefault));
2074 queue_work(as->pf_wq, &as->work_pagefault);
2075 atomic_inc(&kbdev->faults_pending);
2076 }
2077 }
2078
kbase_flush_mmu_wqs(struct kbase_device * kbdev)2079 void kbase_flush_mmu_wqs(struct kbase_device *kbdev)
2080 {
2081 int i;
2082
2083 for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
2084 struct kbase_as *as = &kbdev->as[i];
2085
2086 flush_workqueue(as->pf_wq);
2087 }
2088 }
2089