54 	struct amdgpu_ring		*ring;  member
96 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)  in amdgpu_fence_write()  argument
98 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
112 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)  in amdgpu_fence_read()  argument
114 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
134 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)  in amdgpu_fence_emit()  argument
136 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_emit()
145 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
146 	fence->ring = ring;  in amdgpu_fence_emit()
148 		   &ring->fence_drv.lock,  in amdgpu_fence_emit()
149 		   adev->fence_context + ring->idx,  in amdgpu_fence_emit()
151 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
154 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
178 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)  in amdgpu_fence_schedule_fallback()  argument
180 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
193 void amdgpu_fence_process(struct amdgpu_ring *ring)  in amdgpu_fence_process()  argument
195 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
200 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
201 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_process()
205 	if (seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
206 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_process()
247 	struct amdgpu_ring *ring = (void *)arg;  in amdgpu_fence_fallback()  local
249 	amdgpu_fence_process(ring);  in amdgpu_fence_fallback()
261 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)  in amdgpu_fence_wait_empty()  argument
263 	uint64_t seq = ACCESS_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
270 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
293 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)  in amdgpu_fence_count_emitted()  argument
300 	amdgpu_fence_process(ring);  in amdgpu_fence_count_emitted()
302 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
303 	emitted += ACCESS_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
320 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_start_ring()  argument
324 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_start_ring()
327 	if (ring != &adev->uvd.ring) {  in amdgpu_fence_driver_start_ring()
328 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
329 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
333 		ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;  in amdgpu_fence_driver_start_ring()
334 		ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;  in amdgpu_fence_driver_start_ring()
336 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
339 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
340 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
341 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
344 		 "cpu addr 0x%p\n", ring->idx,  in amdgpu_fence_driver_start_ring()
345 		 ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);  in amdgpu_fence_driver_start_ring()
359 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_init_ring()  argument
369 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
370 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
371 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
372 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
373 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
375 	setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,  in amdgpu_fence_driver_init_ring()
376 		    (unsigned long)ring);  in amdgpu_fence_driver_init_ring()
378 	ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
379 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
380 	ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
382 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
396 	r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,  in amdgpu_fence_driver_init_ring()
398 			   timeout, ring->name);  in amdgpu_fence_driver_init_ring()
401 			  ring->name);  in amdgpu_fence_driver_init_ring()
442 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_fini()  local
444 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_fini()
446 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_fini()
451 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_fini()
452 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_fini()
453 		amd_sched_fini(&ring->sched);  in amdgpu_fence_driver_fini()
454 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_fini()
455 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_fini()
456 			fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_fini()
457 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_fini()
458 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_fini()
459 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_fini()
476 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_suspend()  local
477 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_suspend()
481 		r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_suspend()
488 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_suspend()
489 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_suspend()
510 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_resume()  local
511 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_resume()
515 		amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_resume()
516 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_resume()
533 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_force_completion()  local
534 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_force_completion()
537 		amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
553 	return (const char *)fence->ring->name;  in amdgpu_fence_get_timeline_name()
567 	struct amdgpu_ring *ring = fence->ring;  in amdgpu_fence_enable_signaling()  local
569 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
570 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_enable_signaling()
572 	FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);  in amdgpu_fence_enable_signaling()
624 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_debugfs_fence_info()  local
625 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info()
628 		amdgpu_fence_process(ring);  in amdgpu_debugfs_fence_info()
630 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);  in amdgpu_debugfs_fence_info()
632 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info()
634 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info()