79 	return (rptr & 0x3fffc) >> 2;  in cik_sdma_get_rptr()
100 	return (RREG32(reg) & 0x3fffc) >> 2;  in cik_sdma_get_wptr()
121 	WREG32(reg, (ring->wptr << 2) & 0x3fffc);  in cik_sdma_set_wptr()
137 	u32 extra_bits = (ib->vm ? ib->vm->ids[ib->ring].id : 0) & 0xf;  in cik_sdma_ring_ib_execute()
144 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_ib_execute()
145 		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);  in cik_sdma_ring_ib_execute()
153 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));  in cik_sdma_ring_ib_execute()
154 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));  in cik_sdma_ring_ib_execute()
155 	radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */  in cik_sdma_ring_ib_execute()
182 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_sdma_hdp_flush_ring_emit()
187 	radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_sdma_hdp_flush_ring_emit()
207 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));  in cik_sdma_fence_ring_emit()
212 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));  in cik_sdma_fence_ring_emit()
234 	u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;  in cik_sdma_semaphore_ring_emit()
236 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));  in cik_sdma_semaphore_ring_emit()
237 	radeon_ring_write(ring, addr & 0xfffffff8);  in cik_sdma_semaphore_ring_emit()
259 	for (i = 0; i < 2; i++) {  in cik_sdma_gfx_stop()
260 		if (i == 0)  in cik_sdma_gfx_stop()
267 		WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);  in cik_sdma_gfx_stop()
280 	WREG32(SRBM_SOFT_RESET, 0);  in cik_sdma_gfx_stop()
309 	for (i = 0; i < 2; i++) {  in cik_sdma_ctx_switch_enable()
310 		if (i == 0)  in cik_sdma_ctx_switch_enable()
341 	for (i = 0; i < 2; i++) {  in cik_sdma_enable()
342 		if (i == 0)  in cik_sdma_enable()
363  * Returns 0 for success, error for failure.
373 	for (i = 0; i < 2; i++) {  in cik_sdma_gfx_resume()
374 		if (i == 0) {  in cik_sdma_gfx_resume()
384 		WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);  in cik_sdma_gfx_resume()
385 		WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);  in cik_sdma_gfx_resume()
396 		WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);  in cik_sdma_gfx_resume()
397 		WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);  in cik_sdma_gfx_resume()
401 		       upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);  in cik_sdma_gfx_resume()
403 		       ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));  in cik_sdma_gfx_resume()
411 		ring->wptr = 0;  in cik_sdma_gfx_resume()
437 	return 0;  in cik_sdma_gfx_resume()
446  * Returns 0 for success, error for failure.
451 	return 0;  in cik_sdma_rlc_resume()
460  * Returns 0 for success, -EINVAL if the ucode is not available.
484 		WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
485 		for (i = 0; i < fw_size; i++)  in cik_sdma_load_microcode()
493 		WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
494 		for (i = 0; i < fw_size; i++)  in cik_sdma_load_microcode()
502 		WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
503 		for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)  in cik_sdma_load_microcode()
509 		WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
510 		for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)  in cik_sdma_load_microcode()
515 	WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
516 	WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);  in cik_sdma_load_microcode()
517 	return 0;  in cik_sdma_load_microcode()
526  * Returns 0 for success, error for failure.
547 	return 0;  in cik_sdma_resume()
590 	int r = 0;  in cik_copy_dma()
595 	num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);  in cik_copy_dma()
606 	for (i = 0; i < num_loops; i++) {  in cik_copy_dma()
608 		if (cur_size_in_bytes > 0x1fffff)  in cik_copy_dma()
609 			cur_size_in_bytes = 0x1fffff;  in cik_copy_dma()
611 		radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));  in cik_copy_dma()
613 		radeon_ring_write(ring, 0); /* src/dst endian swap */  in cik_copy_dma()
643  * Returns 0 for success, error for failure.
661 	tmp = 0xCAFEDEAD;  in cik_sdma_ring_test()
669 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_test()
673 	radeon_ring_write(ring, 0xDEADBEEF);  in cik_sdma_ring_test()
676 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_sdma_ring_test()
678 		if (tmp == 0xDEADBEEF)  in cik_sdma_ring_test()
686 		DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",  in cik_sdma_ring_test()
700  * Returns 0 on success, error on failure.
708 	u32 tmp = 0;  in cik_sdma_ib_test()
718 	tmp = 0xCAFEDEAD;  in cik_sdma_ib_test()
727 	ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);  in cik_sdma_ib_test()
731 	ib.ptr[4] = 0xDEADBEEF;  in cik_sdma_ib_test()
742 	if (r < 0) {  in cik_sdma_ib_test()
745 	} else if (r == 0) {  in cik_sdma_ib_test()
749 	r = 0;  in cik_sdma_ib_test()
750 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_sdma_ib_test()
752 		if (tmp == 0xDEADBEEF)  in cik_sdma_ib_test()
759 		DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);  in cik_sdma_ib_test()
810 		if (bytes > 0x1FFFF8)  in cik_sdma_vm_copy_pages()
811 			bytes = 0x1FFFF8;  in cik_sdma_vm_copy_pages()
814 			SDMA_WRITE_SUB_OPCODE_LINEAR, 0);  in cik_sdma_vm_copy_pages()
816 		ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */  in cik_sdma_vm_copy_pages()
852 		if (ndw > 0xFFFFE)  in cik_sdma_vm_write_pages()
853 			ndw = 0xFFFFE;  in cik_sdma_vm_write_pages()
857 			SDMA_WRITE_SUB_OPCODE_LINEAR, 0);  in cik_sdma_vm_write_pages()
861 		for (; ndw > 0; ndw -= 2, --count, pe += 8) {  in cik_sdma_vm_write_pages()
867 				value = 0;  in cik_sdma_vm_write_pages()
901 		if (ndw > 0x7FFFF)  in cik_sdma_vm_set_pages()
902 			ndw = 0x7FFFF;  in cik_sdma_vm_set_pages()
907 			value = 0;  in cik_sdma_vm_set_pages()
910 		ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);  in cik_sdma_vm_set_pages()
914 		ib->ptr[ib->length_dw++] = 0;  in cik_sdma_vm_set_pages()
918 		ib->ptr[ib->length_dw++] = 0;  in cik_sdma_vm_set_pages()
935 	while (ib->length_dw & 0x7)  in cik_sdma_vm_pad_ib()
936 		ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);  in cik_sdma_vm_pad_ib()
950 	u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |  in cik_dma_vm_flush()
951 			  SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */  in cik_dma_vm_flush()
953 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
962 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
966 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
968 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
970 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
972 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
974 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
978 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
980 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
982 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
984 	radeon_ring_write(ring, VMID(0));  in cik_dma_vm_flush()
990 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_dma_vm_flush()
994 	radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_dma_vm_flush()
996 	radeon_ring_write(ring, 0);  in cik_dma_vm_flush()
997 	radeon_ring_write(ring, 0); /* reference */  in cik_dma_vm_flush()
998 	radeon_ring_write(ring, 0); /* mask */  in cik_dma_vm_flush()
999 	radeon_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_dma_vm_flush()