37 static void update_mqd(struct mqd_manager *mm, void *mqd,
41 static uint64_t mqd_stride_v9(struct mqd_manager *mm,  in mqd_stride_v9()  argument
44 	if (mm->dev->kfd->cwsr_enabled &&  in mqd_stride_v9()
49 	return mm->mqd_size;  in mqd_stride_v9()
62 static void update_cu_mask(struct mqd_manager *mm, void *mqd,  in update_cu_mask()  argument
71 	mqd_symmetrically_map_cu_mask(mm,  in update_cu_mask()
80 	if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3)) {  in update_cu_mask()
158 static void init_mqd(struct mqd_manager *mm, void **mqd,  in init_mqd()  argument
207 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {  in init_mqd()
223 	update_mqd(mm, m, q, NULL);  in init_mqd()
226 static int load_mqd(struct mqd_manager *mm, void *mqd,  in load_mqd()  argument
233 	return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,  in load_mqd()
238 static void update_mqd(struct mqd_manager *mm, void *mqd,  in update_mqd()  argument
299 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)  in update_mqd()
302 	if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3))  in update_mqd()
303 		update_cu_mask(mm, mqd, minfo, 0);  in update_mqd()
317 static int get_wave_state(struct mqd_manager *mm, void *mqd,  in get_wave_state()  argument
353 static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)  in get_checkpoint_info()  argument
360 static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)  in checkpoint_mqd()  argument
372 static void restore_mqd(struct mqd_manager *mm, void **mqd,  in restore_mqd()  argument
404 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,  in init_mqd_hiq()  argument
410 	init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);  in init_mqd_hiq()
418 static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,  in destroy_hiq_mqd()  argument
430 	err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);  in destroy_hiq_mqd()
437 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,  in init_mqd_sdma()  argument
451 	mm->update_mqd(mm, m, q, NULL);  in init_mqd_sdma()
456 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,  in update_mqd_sdma()  argument
483 static void checkpoint_mqd_sdma(struct mqd_manager *mm,  in checkpoint_mqd_sdma()  argument
495 static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,  in restore_mqd_sdma()  argument
519 static void init_mqd_hiq_v9_4_3(struct mqd_manager *mm, void **mqd,  in init_mqd_hiq_v9_4_3()  argument
530 	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {  in init_mqd_hiq_v9_4_3()
531 		kfd_get_hiq_xcc_mqd(mm->dev, &xcc_mqd_mem_obj, xcc);  in init_mqd_hiq_v9_4_3()
533 		init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);  in init_mqd_hiq_v9_4_3()
538 		m->cp_mqd_stride_size = kfd_hiq_mqd_stride(mm->dev);  in init_mqd_hiq_v9_4_3()
550 static int hiq_load_mqd_kiq_v9_4_3(struct mqd_manager *mm, void *mqd,  in hiq_load_mqd_kiq_v9_4_3()  argument
554 	uint32_t xcc_mask = mm->dev->xcc_mask;  in hiq_load_mqd_kiq_v9_4_3()
557 	uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);  in hiq_load_mqd_kiq_v9_4_3()
561 		err = mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, xcc_mqd,  in hiq_load_mqd_kiq_v9_4_3()
574 static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,  in destroy_hiq_mqd_v9_4_3()  argument
578 	uint32_t xcc_mask = mm->dev->xcc_mask;  in destroy_hiq_mqd_v9_4_3()
580 	uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);  in destroy_hiq_mqd_v9_4_3()
590 		err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, xcc_id);  in destroy_hiq_mqd_v9_4_3()
612 static void init_mqd_v9_4_3(struct mqd_manager *mm, void **mqd,  in init_mqd_v9_4_3()  argument
621 	uint64_t offset = mm->mqd_stride(mm, q);  in init_mqd_v9_4_3()
622 	uint32_t local_xcc_start = mm->dev->dqm->current_logical_xcc_start++;  in init_mqd_v9_4_3()
625 	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {  in init_mqd_v9_4_3()
628 		init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);  in init_mqd_v9_4_3()
636 		if (mm->dev->kfd->cwsr_enabled &&  in init_mqd_v9_4_3()
652 					NUM_XCC(mm->dev->xcc_mask);  in init_mqd_v9_4_3()
678 static void update_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,  in update_mqd_v9_4_3()  argument
683 	uint64_t size = mm->mqd_stride(mm, q);  in update_mqd_v9_4_3()
685 	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {  in update_mqd_v9_4_3()
687 		update_mqd(mm, m, q, minfo);  in update_mqd_v9_4_3()
689 		update_cu_mask(mm, m, minfo, xcc);  in update_mqd_v9_4_3()
711 static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,  in destroy_mqd_v9_4_3()  argument
715 	uint32_t xcc_mask = mm->dev->xcc_mask;  in destroy_mqd_v9_4_3()
726 		err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd,  in destroy_mqd_v9_4_3()
739 static int load_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,  in load_mqd_v9_4_3()  argument
745 	uint32_t xcc_mask = mm->dev->xcc_mask;  in load_mqd_v9_4_3()
748 	uint64_t mqd_stride_size = mm->mqd_stride(mm, p);  in load_mqd_v9_4_3()
752 		err = mm->dev->kfd2kgd->hqd_load(  in load_mqd_v9_4_3()
753 			mm->dev->adev, xcc_mqd, pipe_id, queue_id,  in load_mqd_v9_4_3()
766 static int get_wave_state_v9_4_3(struct mqd_manager *mm, void *mqd,  in get_wave_state_v9_4_3()  argument
775 	uint64_t mqd_stride_size = mm->mqd_stride(mm, q);  in get_wave_state_v9_4_3()
778 	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {  in get_wave_state_v9_4_3()
783 		err = get_wave_state(mm, xcc_mqd, q, xcc_ctl_stack,  in get_wave_state_v9_4_3()