Lines Matching refs:rdev
68 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) in radeon_fence_write() argument
70 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_write()
71 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { in radeon_fence_write()
89 static u32 radeon_fence_read(struct radeon_device *rdev, int ring) in radeon_fence_read() argument
91 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_read()
94 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { in radeon_fence_read()
114 static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) in radeon_fence_schedule_check() argument
121 &rdev->fence_drv[ring].lockup_work, in radeon_fence_schedule_check()
135 int radeon_fence_emit(struct radeon_device *rdev, in radeon_fence_emit() argument
146 (*fence)->rdev = rdev; in radeon_fence_emit()
147 (*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_emit()
151 &rdev->fence_queue.lock, in radeon_fence_emit()
152 rdev->fence_context + ring, in radeon_fence_emit()
154 radeon_fence_ring_emit(rdev, ring, *fence); in radeon_fence_emit()
155 trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); in radeon_fence_emit()
156 radeon_fence_schedule_check(rdev, ring); in radeon_fence_emit()
178 seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); in radeon_fence_check_signaled()
187 radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); in radeon_fence_check_signaled()
188 __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); in radeon_fence_check_signaled()
205 static bool radeon_fence_activity(struct radeon_device *rdev, int ring) in radeon_fence_activity() argument
232 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_activity()
234 last_emitted = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_activity()
235 seq = radeon_fence_read(rdev, ring); in radeon_fence_activity()
259 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); in radeon_fence_activity()
262 radeon_fence_schedule_check(rdev, ring); in radeon_fence_activity()
278 struct radeon_device *rdev; in radeon_fence_check_lockup() local
283 rdev = fence_drv->rdev; in radeon_fence_check_lockup()
284 ring = fence_drv - &rdev->fence_drv[0]; in radeon_fence_check_lockup()
286 if (!down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_check_lockup()
288 radeon_fence_schedule_check(rdev, ring); in radeon_fence_check_lockup()
292 if (fence_drv->delayed_irq && rdev->ddev->irq_enabled) { in radeon_fence_check_lockup()
296 spin_lock_irqsave(&rdev->irq.lock, irqflags); in radeon_fence_check_lockup()
297 radeon_irq_set(rdev); in radeon_fence_check_lockup()
298 spin_unlock_irqrestore(&rdev->irq.lock, irqflags); in radeon_fence_check_lockup()
301 if (radeon_fence_activity(rdev, ring)) in radeon_fence_check_lockup()
302 wake_up_all(&rdev->fence_queue); in radeon_fence_check_lockup()
304 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { in radeon_fence_check_lockup()
307 dev_warn(rdev->dev, "GPU lockup (current fence id " in radeon_fence_check_lockup()
313 rdev->needs_reset = true; in radeon_fence_check_lockup()
314 wake_up_all(&rdev->fence_queue); in radeon_fence_check_lockup()
316 up_read(&rdev->exclusive_lock); in radeon_fence_check_lockup()
328 void radeon_fence_process(struct radeon_device *rdev, int ring) in radeon_fence_process() argument
330 if (radeon_fence_activity(rdev, ring)) in radeon_fence_process()
331 wake_up_all(&rdev->fence_queue); in radeon_fence_process()
348 static bool radeon_fence_seq_signaled(struct radeon_device *rdev, in radeon_fence_seq_signaled() argument
351 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
355 radeon_fence_process(rdev, ring); in radeon_fence_seq_signaled()
356 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
365 struct radeon_device *rdev = fence->rdev; in radeon_fence_is_signaled() local
369 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
373 if (down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_is_signaled()
374 radeon_fence_process(rdev, ring); in radeon_fence_is_signaled()
375 up_read(&rdev->exclusive_lock); in radeon_fence_is_signaled()
377 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
395 struct radeon_device *rdev = fence->rdev; in radeon_fence_enable_signaling() local
397 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) in radeon_fence_enable_signaling()
400 if (down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_enable_signaling()
401 radeon_irq_kms_sw_irq_get(rdev, fence->ring); in radeon_fence_enable_signaling()
403 if (radeon_fence_activity(rdev, fence->ring)) in radeon_fence_enable_signaling()
404 wake_up_all_locked(&rdev->fence_queue); in radeon_fence_enable_signaling()
407 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { in radeon_fence_enable_signaling()
408 radeon_irq_kms_sw_irq_put(rdev, fence->ring); in radeon_fence_enable_signaling()
409 up_read(&rdev->exclusive_lock); in radeon_fence_enable_signaling()
413 up_read(&rdev->exclusive_lock); in radeon_fence_enable_signaling()
416 if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) in radeon_fence_enable_signaling()
417 rdev->fence_drv[fence->ring].delayed_irq = true; in radeon_fence_enable_signaling()
418 radeon_fence_schedule_check(rdev, fence->ring); in radeon_fence_enable_signaling()
424 __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); in radeon_fence_enable_signaling()
444 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { in radeon_fence_signaled()
466 static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) in radeon_fence_any_seq_signaled() argument
471 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) in radeon_fence_any_seq_signaled()
494 static long radeon_fence_wait_seq_timeout(struct radeon_device *rdev, in radeon_fence_wait_seq_timeout() argument
501 if (radeon_fence_any_seq_signaled(rdev, target_seq)) in radeon_fence_wait_seq_timeout()
509 trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); in radeon_fence_wait_seq_timeout()
510 radeon_irq_kms_sw_irq_get(rdev, i); in radeon_fence_wait_seq_timeout()
514 r = wait_event_interruptible_timeout(rdev->fence_queue, ( in radeon_fence_wait_seq_timeout()
515 radeon_fence_any_seq_signaled(rdev, target_seq) in radeon_fence_wait_seq_timeout()
516 || rdev->needs_reset), timeout); in radeon_fence_wait_seq_timeout()
518 r = wait_event_timeout(rdev->fence_queue, ( in radeon_fence_wait_seq_timeout()
519 radeon_fence_any_seq_signaled(rdev, target_seq) in radeon_fence_wait_seq_timeout()
520 || rdev->needs_reset), timeout); in radeon_fence_wait_seq_timeout()
523 if (rdev->needs_reset) in radeon_fence_wait_seq_timeout()
530 radeon_irq_kms_sw_irq_put(rdev, i); in radeon_fence_wait_seq_timeout()
531 trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); in radeon_fence_wait_seq_timeout()
566 r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout); in radeon_fence_wait_timeout()
611 int radeon_fence_wait_any(struct radeon_device *rdev, in radeon_fence_wait_any() argument
634 r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_any()
651 int radeon_fence_wait_next(struct radeon_device *rdev, int ring) in radeon_fence_wait_next() argument
656 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; in radeon_fence_wait_next()
657 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { in radeon_fence_wait_next()
662 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_next()
678 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) in radeon_fence_wait_empty() argument
683 seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_wait_empty()
687 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_empty()
692 dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n", in radeon_fence_wait_empty()
739 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) in radeon_fence_count_emitted() argument
746 radeon_fence_process(rdev, ring); in radeon_fence_count_emitted()
747 emitted = rdev->fence_drv[ring].sync_seq[ring] in radeon_fence_count_emitted()
748 - atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_count_emitted()
780 fdrv = &fence->rdev->fence_drv[dst_ring]; in radeon_fence_need_sync()
811 src = &fence->rdev->fence_drv[fence->ring]; in radeon_fence_note_sync()
812 dst = &fence->rdev->fence_drv[dst_ring]; in radeon_fence_note_sync()
833 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_start_ring() argument
838 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
839 if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { in radeon_fence_driver_start_ring()
840 rdev->fence_drv[ring].scratch_reg = 0; in radeon_fence_driver_start_ring()
843 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
844 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + in radeon_fence_driver_start_ring()
849 index = ALIGN(rdev->uvd_fw->size, 8); in radeon_fence_driver_start_ring()
850 rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index; in radeon_fence_driver_start_ring()
851 rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; in radeon_fence_driver_start_ring()
855 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
857 dev_err(rdev->dev, "fence failed to get scratch register\n"); in radeon_fence_driver_start_ring()
861 rdev->fence_drv[ring].scratch_reg - in radeon_fence_driver_start_ring()
862 rdev->scratch.reg_base; in radeon_fence_driver_start_ring()
863 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
864 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; in radeon_fence_driver_start_ring()
866 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); in radeon_fence_driver_start_ring()
867 rdev->fence_drv[ring].initialized = true; in radeon_fence_driver_start_ring()
868 dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", in radeon_fence_driver_start_ring()
869 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); in radeon_fence_driver_start_ring()
883 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_init_ring() argument
887 rdev->fence_drv[ring].scratch_reg = -1; in radeon_fence_driver_init_ring()
888 rdev->fence_drv[ring].cpu_addr = NULL; in radeon_fence_driver_init_ring()
889 rdev->fence_drv[ring].gpu_addr = 0; in radeon_fence_driver_init_ring()
891 rdev->fence_drv[ring].sync_seq[i] = 0; in radeon_fence_driver_init_ring()
892 atomic64_set(&rdev->fence_drv[ring].last_seq, 0); in radeon_fence_driver_init_ring()
893 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_init_ring()
894 INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, in radeon_fence_driver_init_ring()
896 rdev->fence_drv[ring].rdev = rdev; in radeon_fence_driver_init_ring()
911 int radeon_fence_driver_init(struct radeon_device *rdev) in radeon_fence_driver_init() argument
915 init_waitqueue_head(&rdev->fence_queue); in radeon_fence_driver_init()
917 radeon_fence_driver_init_ring(rdev, ring); in radeon_fence_driver_init()
919 if (radeon_debugfs_fence_init(rdev)) { in radeon_fence_driver_init()
920 dev_err(rdev->dev, "fence debugfs file creation failed\n"); in radeon_fence_driver_init()
933 void radeon_fence_driver_fini(struct radeon_device *rdev) in radeon_fence_driver_fini() argument
937 mutex_lock(&rdev->ring_lock); in radeon_fence_driver_fini()
939 if (!rdev->fence_drv[ring].initialized) in radeon_fence_driver_fini()
941 r = radeon_fence_wait_empty(rdev, ring); in radeon_fence_driver_fini()
944 radeon_fence_driver_force_completion(rdev, ring); in radeon_fence_driver_fini()
946 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_fini()
947 wake_up_all(&rdev->fence_queue); in radeon_fence_driver_fini()
948 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_fini()
949 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_fini()
951 mutex_unlock(&rdev->ring_lock); in radeon_fence_driver_fini()
963 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) in radeon_fence_driver_force_completion() argument
965 if (rdev->fence_drv[ring].initialized) { in radeon_fence_driver_force_completion()
966 radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); in radeon_fence_driver_force_completion()
967 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_force_completion()
980 struct radeon_device *rdev = dev->dev_private; in radeon_debugfs_fence_info() local
984 if (!rdev->fence_drv[i].initialized) in radeon_debugfs_fence_info()
987 radeon_fence_process(rdev, i); in radeon_debugfs_fence_info()
991 (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq)); in radeon_debugfs_fence_info()
993 rdev->fence_drv[i].sync_seq[i]); in radeon_debugfs_fence_info()
996 if (i != j && rdev->fence_drv[j].initialized) in radeon_debugfs_fence_info()
998 j, rdev->fence_drv[i].sync_seq[j]); in radeon_debugfs_fence_info()
1013 struct radeon_device *rdev = dev->dev_private; in radeon_debugfs_gpu_reset() local
1015 down_read(&rdev->exclusive_lock); in radeon_debugfs_gpu_reset()
1016 seq_printf(m, "%d\n", rdev->needs_reset); in radeon_debugfs_gpu_reset()
1017 rdev->needs_reset = true; in radeon_debugfs_gpu_reset()
1018 wake_up_all(&rdev->fence_queue); in radeon_debugfs_gpu_reset()
1019 up_read(&rdev->exclusive_lock); in radeon_debugfs_gpu_reset()
1030 int radeon_debugfs_fence_init(struct radeon_device *rdev) in radeon_debugfs_fence_init() argument
1033 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); in radeon_debugfs_fence_init()
1083 struct radeon_device *rdev = fence->rdev; in radeon_fence_default_wait() local
1104 if (rdev->needs_reset) { in radeon_fence_default_wait()