124 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
125 extern void r600_ih_ring_fini(struct radeon_device *rdev);
126 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
127 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
128 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
129 extern void sumo_rlc_fini(struct radeon_device *rdev);
130 extern int sumo_rlc_init(struct radeon_device *rdev);
131 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
132 extern void si_rlc_reset(struct radeon_device *rdev);
133 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
134 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
135 extern int cik_sdma_resume(struct radeon_device *rdev);
136 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
137 extern void cik_sdma_fini(struct radeon_device *rdev);
138 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
139 static void cik_rlc_stop(struct radeon_device *rdev);
140 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
141 static void cik_program_aspm(struct radeon_device *rdev);
142 static void cik_init_pg(struct radeon_device *rdev);
143 static void cik_init_cg(struct radeon_device *rdev);
144 static void cik_fini_pg(struct radeon_device *rdev);
145 static void cik_fini_cg(struct radeon_device *rdev);
146 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
159 int cik_get_allowed_info_register(struct radeon_device *rdev,  in cik_get_allowed_info_register()  argument
185 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)  in cik_didt_rreg()  argument
190 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
193 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
197 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in cik_didt_wreg()  argument
201 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
204 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
208 int ci_get_temp(struct radeon_device *rdev)  in ci_get_temp()  argument
227 int kv_get_temp(struct radeon_device *rdev)  in kv_get_temp()  argument
247 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)  in cik_pciep_rreg()  argument
252 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
256 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
260 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in cik_pciep_wreg()  argument
264 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
269 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
1629 static void cik_init_golden_registers(struct radeon_device *rdev)  in cik_init_golden_registers()  argument
1632 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_init_golden_registers()
1633 	switch (rdev->family) {  in cik_init_golden_registers()
1635 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1638 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1641 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1644 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1649 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1652 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1655 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1658 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1663 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1666 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1669 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1672 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1677 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1680 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1683 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1686 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1691 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1694 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1697 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1700 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1707 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_init_golden_registers()
1718 u32 cik_get_xclk(struct radeon_device *rdev)  in cik_get_xclk()  argument
1720 	u32 reference_clock = rdev->clock.spll.reference_freq;  in cik_get_xclk()
1722 	if (rdev->flags & RADEON_IS_IGP) {  in cik_get_xclk()
1741 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)  in cik_mm_rdoorbell()  argument
1743 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_rdoorbell()
1744 		return readl(rdev->doorbell.ptr + index);  in cik_mm_rdoorbell()
1761 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)  in cik_mm_wdoorbell()  argument
1763 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_wdoorbell()
1764 		writel(v, rdev->doorbell.ptr + index);  in cik_mm_wdoorbell()
1854 static void cik_srbm_select(struct radeon_device *rdev,  in cik_srbm_select()  argument
1873 int ci_mc_load_microcode(struct radeon_device *rdev)  in ci_mc_load_microcode()  argument
1882 	if (!rdev->mc_fw)  in ci_mc_load_microcode()
1885 	if (rdev->new_fw) {  in ci_mc_load_microcode()
1887 			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1893 			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in ci_mc_load_microcode()
1896 			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in ci_mc_load_microcode()
1898 		ucode_size = rdev->mc_fw->size / 4;  in ci_mc_load_microcode()
1900 		switch (rdev->family) {  in ci_mc_load_microcode()
1912 		fw_data = (const __be32 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1924 			if (rdev->new_fw) {  in ci_mc_load_microcode()
1934 		if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {  in ci_mc_load_microcode()
1943 			if (rdev->new_fw)  in ci_mc_load_microcode()
1955 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1960 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1979 static int cik_init_microcode(struct radeon_device *rdev)  in cik_init_microcode()  argument
1994 	switch (rdev->family) {  in cik_init_microcode()
1997 		if ((rdev->pdev->revision == 0x80) ||  in cik_init_microcode()
1998 		    (rdev->pdev->revision == 0x81) ||  in cik_init_microcode()
1999 		    (rdev->pdev->device == 0x665f))  in cik_init_microcode()
2015 		if (rdev->pdev->revision == 0x80)  in cik_init_microcode()
2068 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2071 		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2074 		if (rdev->pfp_fw->size != pfp_req_size) {  in cik_init_microcode()
2076 			       rdev->pfp_fw->size, fw_name);  in cik_init_microcode()
2081 		err = radeon_ucode_validate(rdev->pfp_fw);  in cik_init_microcode()
2092 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2095 		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2098 		if (rdev->me_fw->size != me_req_size) {  in cik_init_microcode()
2100 			       rdev->me_fw->size, fw_name);  in cik_init_microcode()
2104 		err = radeon_ucode_validate(rdev->me_fw);  in cik_init_microcode()
2115 	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2118 		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2121 		if (rdev->ce_fw->size != ce_req_size) {  in cik_init_microcode()
2123 			       rdev->ce_fw->size, fw_name);  in cik_init_microcode()
2127 		err = radeon_ucode_validate(rdev->ce_fw);  in cik_init_microcode()
2138 	err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2141 		err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2144 		if (rdev->mec_fw->size != mec_req_size) {  in cik_init_microcode()
2146 			       rdev->mec_fw->size, fw_name);  in cik_init_microcode()
2150 		err = radeon_ucode_validate(rdev->mec_fw);  in cik_init_microcode()
2160 	if (rdev->family == CHIP_KAVERI) {  in cik_init_microcode()
2162 		err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);  in cik_init_microcode()
2166 			err = radeon_ucode_validate(rdev->mec2_fw);  in cik_init_microcode()
2176 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2179 		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2182 		if (rdev->rlc_fw->size != rlc_req_size) {  in cik_init_microcode()
2184 			       rdev->rlc_fw->size, fw_name);  in cik_init_microcode()
2188 		err = radeon_ucode_validate(rdev->rlc_fw);  in cik_init_microcode()
2199 	err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2202 		err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2205 		if (rdev->sdma_fw->size != sdma_req_size) {  in cik_init_microcode()
2207 			       rdev->sdma_fw->size, fw_name);  in cik_init_microcode()
2211 		err = radeon_ucode_validate(rdev->sdma_fw);  in cik_init_microcode()
2222 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_init_microcode()
2224 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2227 			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2230 				err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2234 			if ((rdev->mc_fw->size != mc_req_size) &&  in cik_init_microcode()
2235 			    (rdev->mc_fw->size != mc2_req_size)){  in cik_init_microcode()
2237 				       rdev->mc_fw->size, fw_name);  in cik_init_microcode()
2240 			DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);  in cik_init_microcode()
2242 			err = radeon_ucode_validate(rdev->mc_fw);  in cik_init_microcode()
2256 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2259 			err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2263 				release_firmware(rdev->smc_fw);  in cik_init_microcode()
2264 				rdev->smc_fw = NULL;  in cik_init_microcode()
2266 			} else if (rdev->smc_fw->size != smc_req_size) {  in cik_init_microcode()
2268 				       rdev->smc_fw->size, fw_name);  in cik_init_microcode()
2272 			err = radeon_ucode_validate(rdev->smc_fw);  in cik_init_microcode()
2284 		rdev->new_fw = false;  in cik_init_microcode()
2289 		rdev->new_fw = true;  in cik_init_microcode()
2297 		release_firmware(rdev->pfp_fw);  in cik_init_microcode()
2298 		rdev->pfp_fw = NULL;  in cik_init_microcode()
2299 		release_firmware(rdev->me_fw);  in cik_init_microcode()
2300 		rdev->me_fw = NULL;  in cik_init_microcode()
2301 		release_firmware(rdev->ce_fw);  in cik_init_microcode()
2302 		rdev->ce_fw = NULL;  in cik_init_microcode()
2303 		release_firmware(rdev->mec_fw);  in cik_init_microcode()
2304 		rdev->mec_fw = NULL;  in cik_init_microcode()
2305 		release_firmware(rdev->mec2_fw);  in cik_init_microcode()
2306 		rdev->mec2_fw = NULL;  in cik_init_microcode()
2307 		release_firmware(rdev->rlc_fw);  in cik_init_microcode()
2308 		rdev->rlc_fw = NULL;  in cik_init_microcode()
2309 		release_firmware(rdev->sdma_fw);  in cik_init_microcode()
2310 		rdev->sdma_fw = NULL;  in cik_init_microcode()
2311 		release_firmware(rdev->mc_fw);  in cik_init_microcode()
2312 		rdev->mc_fw = NULL;  in cik_init_microcode()
2313 		release_firmware(rdev->smc_fw);  in cik_init_microcode()
2314 		rdev->smc_fw = NULL;  in cik_init_microcode()
2333 static void cik_tiling_mode_table_init(struct radeon_device *rdev)  in cik_tiling_mode_table_init()  argument
2335 	u32 *tile = rdev->config.cik.tile_mode_array;  in cik_tiling_mode_table_init()
2336 	u32 *macrotile = rdev->config.cik.macrotile_mode_array;  in cik_tiling_mode_table_init()
2338 			ARRAY_SIZE(rdev->config.cik.tile_mode_array);  in cik_tiling_mode_table_init()
2340 			ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);  in cik_tiling_mode_table_init()
2343 	u32 num_rbs = rdev->config.cik.max_backends_per_se *  in cik_tiling_mode_table_init()
2344 		rdev->config.cik.max_shader_engines;  in cik_tiling_mode_table_init()
2346 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_tiling_mode_table_init()
2359 	num_pipe_configs = rdev->config.cik.max_tile_pipes;  in cik_tiling_mode_table_init()
3039 static void cik_select_se_sh(struct radeon_device *rdev,  in cik_select_se_sh()  argument
3085 static u32 cik_get_rb_disabled(struct radeon_device *rdev,  in cik_get_rb_disabled()  argument
3115 static void cik_setup_rb(struct radeon_device *rdev,  in cik_setup_rb()  argument
3124 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3127 			cik_select_se_sh(rdev, i, j);  in cik_setup_rb()
3128 			data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);  in cik_setup_rb()
3129 			if (rdev->family == CHIP_HAWAII)  in cik_setup_rb()
3135 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3136 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3145 	rdev->config.cik.backend_enable_mask = enabled_rbs;  in cik_setup_rb()
3147 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3149 		cik_select_se_sh(rdev, i, 0xffffffff);  in cik_setup_rb()
3174 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3175 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3186 static void cik_gpu_init(struct radeon_device *rdev)  in cik_gpu_init()  argument
3194 	switch (rdev->family) {  in cik_gpu_init()
3196 		rdev->config.cik.max_shader_engines = 2;  in cik_gpu_init()
3197 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3198 		rdev->config.cik.max_cu_per_sh = 7;  in cik_gpu_init()
3199 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3200 		rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3201 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3202 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3203 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3204 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3206 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3207 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3208 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3209 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3213 		rdev->config.cik.max_shader_engines = 4;  in cik_gpu_init()
3214 		rdev->config.cik.max_tile_pipes = 16;  in cik_gpu_init()
3215 		rdev->config.cik.max_cu_per_sh = 11;  in cik_gpu_init()
3216 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3217 		rdev->config.cik.max_backends_per_se = 4;  in cik_gpu_init()
3218 		rdev->config.cik.max_texture_channel_caches = 16;  in cik_gpu_init()
3219 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3220 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3221 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3223 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3224 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3225 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3226 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3230 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3231 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3232 		rdev->config.cik.max_cu_per_sh = 8;  in cik_gpu_init()
3233 		rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3234 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3235 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3236 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3237 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3238 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3240 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3241 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3242 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3243 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3249 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3250 		rdev->config.cik.max_tile_pipes = 2;  in cik_gpu_init()
3251 		rdev->config.cik.max_cu_per_sh = 2;  in cik_gpu_init()
3252 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3253 		rdev->config.cik.max_backends_per_se = 1;  in cik_gpu_init()
3254 		rdev->config.cik.max_texture_channel_caches = 2;  in cik_gpu_init()
3255 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3256 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3257 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3259 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3260 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3261 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3262 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3285 	rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;  in cik_gpu_init()
3286 	rdev->config.cik.mem_max_burst_length_bytes = 256;  in cik_gpu_init()
3288 	rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;  in cik_gpu_init()
3289 	if (rdev->config.cik.mem_row_size_in_kb > 4)  in cik_gpu_init()
3290 		rdev->config.cik.mem_row_size_in_kb = 4;  in cik_gpu_init()
3292 	rdev->config.cik.shader_engine_tile_size = 32;  in cik_gpu_init()
3293 	rdev->config.cik.num_gpus = 1;  in cik_gpu_init()
3294 	rdev->config.cik.multi_gpu_tile_size = 64;  in cik_gpu_init()
3298 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_gpu_init()
3318 	rdev->config.cik.tile_config = 0;  in cik_gpu_init()
3319 	switch (rdev->config.cik.num_tile_pipes) {  in cik_gpu_init()
3321 		rdev->config.cik.tile_config |= (0 << 0);  in cik_gpu_init()
3324 		rdev->config.cik.tile_config |= (1 << 0);  in cik_gpu_init()
3327 		rdev->config.cik.tile_config |= (2 << 0);  in cik_gpu_init()
3332 		rdev->config.cik.tile_config |= (3 << 0);  in cik_gpu_init()
3335 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3337 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3339 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3351 	cik_tiling_mode_table_init(rdev);  in cik_gpu_init()
3353 	cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,  in cik_gpu_init()
3354 		     rdev->config.cik.max_sh_per_se,  in cik_gpu_init()
3355 		     rdev->config.cik.max_backends_per_se);  in cik_gpu_init()
3357 	rdev->config.cik.active_cus = 0;  in cik_gpu_init()
3358 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_gpu_init()
3359 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_gpu_init()
3360 			rdev->config.cik.active_cus +=  in cik_gpu_init()
3361 				hweight32(cik_get_cu_active_bitmap(rdev, i, j));  in cik_gpu_init()
3368 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_gpu_init()
3373 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_gpu_init()
3400 	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |  in cik_gpu_init()
3401 				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |  in cik_gpu_init()
3402 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |  in cik_gpu_init()
3403 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));  in cik_gpu_init()
3429 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_gpu_init()
3447 static void cik_scratch_init(struct radeon_device *rdev)  in cik_scratch_init()  argument
3451 	rdev->scratch.num_reg = 7;  in cik_scratch_init()
3452 	rdev->scratch.reg_base = SCRATCH_REG0;  in cik_scratch_init()
3453 	for (i = 0; i < rdev->scratch.num_reg; i++) {  in cik_scratch_init()
3454 		rdev->scratch.free[i] = true;  in cik_scratch_init()
3455 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);  in cik_scratch_init()
3470 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_ring_test()  argument
3477 	r = radeon_scratch_get(rdev, &scratch);  in cik_ring_test()
3483 	r = radeon_ring_lock(rdev, ring, 3);  in cik_ring_test()
3486 		radeon_scratch_free(rdev, scratch);  in cik_ring_test()
3492 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_ring_test()
3494 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ring_test()
3500 	if (i < rdev->usec_timeout) {  in cik_ring_test()
3507 	radeon_scratch_free(rdev, scratch);  in cik_ring_test()
3519 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,  in cik_hdp_flush_cp_ring_emit()  argument
3522 	struct radeon_ring *ring = &rdev->ring[ridx];  in cik_hdp_flush_cp_ring_emit()
3565 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,  in cik_fence_gfx_ring_emit()  argument
3568 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_gfx_ring_emit()
3569 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_gfx_ring_emit()
3606 void cik_fence_compute_ring_emit(struct radeon_device *rdev,  in cik_fence_compute_ring_emit()  argument
3609 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_compute_ring_emit()
3610 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_compute_ring_emit()
3636 bool cik_semaphore_ring_emit(struct radeon_device *rdev,  in cik_semaphore_ring_emit()  argument
3670 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,  in cik_copy_cpdma()  argument
3677 	int ring_index = rdev->asic->copy.blit_ring_index;  in cik_copy_cpdma()
3678 	struct radeon_ring *ring = &rdev->ring[ring_index];  in cik_copy_cpdma()
3687 	r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);  in cik_copy_cpdma()
3690 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_cpdma()
3694 	radeon_sync_resv(rdev, &sync, resv, false);  in cik_copy_cpdma()
3695 	radeon_sync_rings(rdev, &sync, ring->idx);  in cik_copy_cpdma()
3716 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in cik_copy_cpdma()
3718 		radeon_ring_unlock_undo(rdev, ring);  in cik_copy_cpdma()
3719 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_cpdma()
3723 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_copy_cpdma()
3724 	radeon_sync_free(rdev, &sync, fence);  in cik_copy_cpdma()
3744 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in cik_ring_ib_execute()  argument
3746 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cik_ring_ib_execute()
3764 		} else if (rdev->wb.enabled) {  in cik_ring_ib_execute()
3794 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_ib_test()  argument
3802 	r = radeon_scratch_get(rdev, &scratch);  in cik_ib_test()
3808 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in cik_ib_test()
3811 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
3818 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in cik_ib_test()
3820 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
3821 		radeon_ib_free(rdev, &ib);  in cik_ib_test()
3829 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
3830 		radeon_ib_free(rdev, &ib);  in cik_ib_test()
3834 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
3835 		radeon_ib_free(rdev, &ib);  in cik_ib_test()
3839 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ib_test()
3845 	if (i < rdev->usec_timeout) {  in cik_ib_test()
3852 	radeon_scratch_free(rdev, scratch);  in cik_ib_test()
3853 	radeon_ib_free(rdev, &ib);  in cik_ib_test()
3888 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)  in cik_cp_gfx_enable()  argument
3893 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_enable()
3894 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in cik_cp_gfx_enable()
3896 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_enable()
3909 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)  in cik_cp_gfx_load_microcode()  argument
3913 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)  in cik_cp_gfx_load_microcode()
3916 	cik_cp_gfx_enable(rdev, false);  in cik_cp_gfx_load_microcode()
3918 	if (rdev->new_fw) {  in cik_cp_gfx_load_microcode()
3920 			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
3922 			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
3924 			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
3934 			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3943 			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3952 			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3963 		fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
3970 		fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
3977 		fw_data = (const __be32 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
3996 static int cik_cp_gfx_start(struct radeon_device *rdev)  in cik_cp_gfx_start()  argument
3998 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_start()
4002 	WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);  in cik_cp_gfx_start()
4006 	cik_cp_gfx_enable(rdev, true);  in cik_cp_gfx_start()
4008 	r = radeon_ring_lock(rdev, ring, cik_default_size + 17);  in cik_cp_gfx_start()
4043 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_cp_gfx_start()
4056 static void cik_cp_gfx_fini(struct radeon_device *rdev)  in cik_cp_gfx_fini()  argument
4058 	cik_cp_gfx_enable(rdev, false);  in cik_cp_gfx_fini()
4059 	radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_fini()
4071 static int cik_cp_gfx_resume(struct radeon_device *rdev)  in cik_cp_gfx_resume()  argument
4080 	if (rdev->family != CHIP_HAWAII)  in cik_cp_gfx_resume()
4089 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in cik_cp_gfx_resume()
4093 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_resume()
4107 	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);  in cik_cp_gfx_resume()
4108 	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in cik_cp_gfx_resume()
4113 	if (!rdev->wb.enabled)  in cik_cp_gfx_resume()
4124 	cik_cp_gfx_start(rdev);  in cik_cp_gfx_resume()
4125 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;  in cik_cp_gfx_resume()
4126 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_resume()
4128 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_resume()
4132 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_resume()
4133 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in cik_cp_gfx_resume()
4138 u32 cik_gfx_get_rptr(struct radeon_device *rdev,  in cik_gfx_get_rptr()  argument
4143 	if (rdev->wb.enabled)  in cik_gfx_get_rptr()
4144 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_gfx_get_rptr()
4151 u32 cik_gfx_get_wptr(struct radeon_device *rdev,  in cik_gfx_get_wptr()  argument
4157 void cik_gfx_set_wptr(struct radeon_device *rdev,  in cik_gfx_set_wptr()  argument
4164 u32 cik_compute_get_rptr(struct radeon_device *rdev,  in cik_compute_get_rptr()  argument
4169 	if (rdev->wb.enabled) {  in cik_compute_get_rptr()
4170 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_compute_get_rptr()
4172 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4173 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_rptr()
4175 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_rptr()
4176 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4182 u32 cik_compute_get_wptr(struct radeon_device *rdev,  in cik_compute_get_wptr()  argument
4187 	if (rdev->wb.enabled) {  in cik_compute_get_wptr()
4189 		wptr = rdev->wb.wb[ring->wptr_offs/4];  in cik_compute_get_wptr()
4191 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4192 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_wptr()
4194 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_wptr()
4195 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4201 void cik_compute_set_wptr(struct radeon_device *rdev,  in cik_compute_set_wptr()  argument
4205 	rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;  in cik_compute_set_wptr()
4209 static void cik_compute_stop(struct radeon_device *rdev,  in cik_compute_stop()  argument
4214 	cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_stop()
4222 		for (j = 0; j < rdev->usec_timeout; j++) {  in cik_compute_stop()
4231 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_stop()
4242 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)  in cik_cp_compute_enable()  argument
4251 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4252 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);  in cik_cp_compute_enable()
4253 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);  in cik_cp_compute_enable()
4254 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4257 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cik_cp_compute_enable()
4258 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cik_cp_compute_enable()
4271 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)  in cik_cp_compute_load_microcode()  argument
4275 	if (!rdev->mec_fw)  in cik_cp_compute_load_microcode()
4278 	cik_cp_compute_enable(rdev, false);  in cik_cp_compute_load_microcode()
4280 	if (rdev->new_fw) {  in cik_cp_compute_load_microcode()
4282 			(const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4290 			(rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));  in cik_cp_compute_load_microcode()
4298 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4300 				(const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_cp_compute_load_microcode()
4303 				(rdev->mec2_fw->data +  in cik_cp_compute_load_microcode()
4315 		fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4321 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4323 			fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4342 static int cik_cp_compute_start(struct radeon_device *rdev)  in cik_cp_compute_start()  argument
4344 	cik_cp_compute_enable(rdev, true);  in cik_cp_compute_start()
4357 static void cik_cp_compute_fini(struct radeon_device *rdev)  in cik_cp_compute_fini()  argument
4361 	cik_cp_compute_enable(rdev, false);  in cik_cp_compute_fini()
4369 		if (rdev->ring[idx].mqd_obj) {  in cik_cp_compute_fini()
4370 			r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_fini()
4372 				dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);  in cik_cp_compute_fini()
4374 			radeon_bo_unpin(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4375 			radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4377 			radeon_bo_unref(&rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4378 			rdev->ring[idx].mqd_obj = NULL;  in cik_cp_compute_fini()
4383 static void cik_mec_fini(struct radeon_device *rdev)  in cik_mec_fini()  argument
4387 	if (rdev->mec.hpd_eop_obj) {  in cik_mec_fini()
4388 		r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_fini()
4390 			dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);  in cik_mec_fini()
4391 		radeon_bo_unpin(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4392 		radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4394 		radeon_bo_unref(&rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4395 		rdev->mec.hpd_eop_obj = NULL;  in cik_mec_fini()
4401 static int cik_mec_init(struct radeon_device *rdev)  in cik_mec_init()  argument
4412 	rdev->mec.num_mec = 1;  in cik_mec_init()
4413 	rdev->mec.num_pipe = 1;  in cik_mec_init()
4414 	rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;  in cik_mec_init()
4416 	if (rdev->mec.hpd_eop_obj == NULL) {  in cik_mec_init()
4417 		r = radeon_bo_create(rdev,  in cik_mec_init()
4418 				     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,  in cik_mec_init()
4421 				     &rdev->mec.hpd_eop_obj);  in cik_mec_init()
4423 			dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);  in cik_mec_init()
4428 	r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_init()
4430 		cik_mec_fini(rdev);  in cik_mec_init()
4433 	r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,  in cik_mec_init()
4434 			  &rdev->mec.hpd_eop_gpu_addr);  in cik_mec_init()
4436 		dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);  in cik_mec_init()
4437 		cik_mec_fini(rdev);  in cik_mec_init()
4440 	r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);  in cik_mec_init()
4442 		dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);  in cik_mec_init()
4443 		cik_mec_fini(rdev);  in cik_mec_init()
4448 	memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);  in cik_mec_init()
4450 	radeon_bo_kunmap(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4451 	radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4532 static int cik_cp_compute_resume(struct radeon_device *rdev)  in cik_cp_compute_resume()  argument
4544 	r = cik_cp_compute_start(rdev);  in cik_cp_compute_resume()
4554 	mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4556 	for (i = 0; i < rdev->mec.num_pipe; ++i) {  in cik_cp_compute_resume()
4557 		cik_srbm_select(rdev, 0, i, 0, 0);  in cik_cp_compute_resume()
4559 		eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;  in cik_cp_compute_resume()
4574 	mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4583 		if (rdev->ring[idx].mqd_obj == NULL) {  in cik_cp_compute_resume()
4584 			r = radeon_bo_create(rdev,  in cik_cp_compute_resume()
4588 					     NULL, &rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4590 				dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);  in cik_cp_compute_resume()
4595 		r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_resume()
4597 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
4600 		r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,  in cik_cp_compute_resume()
4603 			dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);  in cik_cp_compute_resume()
4604 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
4607 		r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);  in cik_cp_compute_resume()
4609 			dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);  in cik_cp_compute_resume()
4610 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
4624 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4625 		cik_srbm_select(rdev, rdev->ring[idx].me,  in cik_cp_compute_resume()
4626 				rdev->ring[idx].pipe,  in cik_cp_compute_resume()
4627 				rdev->ring[idx].queue, 0);  in cik_cp_compute_resume()
4650 			for (j = 0; j < rdev->usec_timeout; j++) {  in cik_cp_compute_resume()
4671 		hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;  in cik_cp_compute_resume()
4683 			order_base_2(rdev->ring[idx].ring_size / 8);  in cik_cp_compute_resume()
4697 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;  in cik_cp_compute_resume()
4699 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;  in cik_cp_compute_resume()
4708 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;  in cik_cp_compute_resume()
4710 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;  in cik_cp_compute_resume()
4725 				DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);  in cik_cp_compute_resume()
4737 		rdev->ring[idx].wptr = 0;  in cik_cp_compute_resume()
4738 		mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;  in cik_cp_compute_resume()
4750 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_cp_compute_resume()
4751 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4753 		radeon_bo_kunmap(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4754 		radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4756 		rdev->ring[idx].ready = true;  in cik_cp_compute_resume()
4757 		r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);  in cik_cp_compute_resume()
4759 			rdev->ring[idx].ready = false;  in cik_cp_compute_resume()
4765 static void cik_cp_enable(struct radeon_device *rdev, bool enable)  in cik_cp_enable()  argument
4767 	cik_cp_gfx_enable(rdev, enable);  in cik_cp_enable()
4768 	cik_cp_compute_enable(rdev, enable);  in cik_cp_enable()
4771 static int cik_cp_load_microcode(struct radeon_device *rdev)  in cik_cp_load_microcode()  argument
4775 	r = cik_cp_gfx_load_microcode(rdev);  in cik_cp_load_microcode()
4778 	r = cik_cp_compute_load_microcode(rdev);  in cik_cp_load_microcode()
4785 static void cik_cp_fini(struct radeon_device *rdev)  in cik_cp_fini()  argument
4787 	cik_cp_gfx_fini(rdev);  in cik_cp_fini()
4788 	cik_cp_compute_fini(rdev);  in cik_cp_fini()
4791 static int cik_cp_resume(struct radeon_device *rdev)  in cik_cp_resume()  argument
4795 	cik_enable_gui_idle_interrupt(rdev, false);  in cik_cp_resume()
4797 	r = cik_cp_load_microcode(rdev);  in cik_cp_resume()
4801 	r = cik_cp_gfx_resume(rdev);  in cik_cp_resume()
4804 	r = cik_cp_compute_resume(rdev);  in cik_cp_resume()
4808 	cik_enable_gui_idle_interrupt(rdev, true);  in cik_cp_resume()
4813 static void cik_print_gpu_status_regs(struct radeon_device *rdev)  in cik_print_gpu_status_regs()  argument
4815 	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
4817 	dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
4819 	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",  in cik_print_gpu_status_regs()
4821 	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",  in cik_print_gpu_status_regs()
4823 	dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",  in cik_print_gpu_status_regs()
4825 	dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",  in cik_print_gpu_status_regs()
4827 	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
4829 	dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
4831 	dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
4833 	dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
4835 	dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));  in cik_print_gpu_status_regs()
4836 	dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4838 	dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",  in cik_print_gpu_status_regs()
4840 	dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",  in cik_print_gpu_status_regs()
4842 	dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",  in cik_print_gpu_status_regs()
4844 	dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4846 	dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));  in cik_print_gpu_status_regs()
4847 	dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));  in cik_print_gpu_status_regs()
4848 	dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4850 	dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));  in cik_print_gpu_status_regs()
4862 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)  in cik_gpu_check_soft_reset()  argument
4922 	if (evergreen_is_display_hung(rdev))  in cik_gpu_check_soft_reset()
4942 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in cik_gpu_soft_reset()  argument
4951 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in cik_gpu_soft_reset()
4953 	cik_print_gpu_status_regs(rdev);  in cik_gpu_soft_reset()
4954 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_gpu_soft_reset()
4956 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_gpu_soft_reset()
4960 	cik_fini_pg(rdev);  in cik_gpu_soft_reset()
4961 	cik_fini_cg(rdev);  in cik_gpu_soft_reset()
4964 	cik_rlc_stop(rdev);  in cik_gpu_soft_reset()
4985 	evergreen_mc_stop(rdev, &save);  in cik_gpu_soft_reset()
4986 	if (evergreen_mc_wait_for_idle(rdev)) {  in cik_gpu_soft_reset()
4987 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_gpu_soft_reset()
5023 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_gpu_soft_reset()
5031 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5045 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5059 	evergreen_mc_resume(rdev, &save);  in cik_gpu_soft_reset()
5062 	cik_print_gpu_status_regs(rdev);  in cik_gpu_soft_reset()
5071 static void kv_save_regs_for_reset(struct radeon_device *rdev,  in kv_save_regs_for_reset()  argument
5083 static void kv_restore_regs_for_reset(struct radeon_device *rdev,  in kv_restore_regs_for_reset()  argument
5156 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)  in cik_gpu_pci_config_reset()  argument
5162 	dev_info(rdev->dev, "GPU pci config reset\n");  in cik_gpu_pci_config_reset()
5167 	cik_fini_pg(rdev);  in cik_gpu_pci_config_reset()
5168 	cik_fini_cg(rdev);  in cik_gpu_pci_config_reset()
5187 	cik_rlc_stop(rdev);  in cik_gpu_pci_config_reset()
5192 	evergreen_mc_stop(rdev, &save);  in cik_gpu_pci_config_reset()
5193 	if (evergreen_mc_wait_for_idle(rdev)) {  in cik_gpu_pci_config_reset()
5194 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");  in cik_gpu_pci_config_reset()
5197 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5198 		kv_save_regs_for_reset(rdev, &kv_save);  in cik_gpu_pci_config_reset()
5201 	pci_clear_master(rdev->pdev);  in cik_gpu_pci_config_reset()
5203 	radeon_pci_config_reset(rdev);  in cik_gpu_pci_config_reset()
5208 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_gpu_pci_config_reset()
5215 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5216 		kv_restore_regs_for_reset(rdev, &kv_save);  in cik_gpu_pci_config_reset()
5229 int cik_asic_reset(struct radeon_device *rdev, bool hard)  in cik_asic_reset()  argument
5234 		cik_gpu_pci_config_reset(rdev);  in cik_asic_reset()
5238 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5241 		r600_set_bios_scratch_engine_hung(rdev, true);  in cik_asic_reset()
5244 	cik_gpu_soft_reset(rdev, reset_mask);  in cik_asic_reset()
5246 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5250 		cik_gpu_pci_config_reset(rdev);  in cik_asic_reset()
5252 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5255 		r600_set_bios_scratch_engine_hung(rdev, false);  in cik_asic_reset()
5269 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_gfx_is_lockup()  argument
5271 	u32 reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_gfx_is_lockup()
5276 		radeon_ring_lockup_update(rdev, ring);  in cik_gfx_is_lockup()
5279 	return radeon_ring_test_lockup(rdev, ring);  in cik_gfx_is_lockup()
5291 static void cik_mc_program(struct radeon_device *rdev)  in cik_mc_program()  argument
5307 	evergreen_mc_stop(rdev, &save);  in cik_mc_program()
5308 	if (radeon_mc_wait_for_idle(rdev)) {  in cik_mc_program()
5309 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5315 	       rdev->mc.vram_start >> 12);  in cik_mc_program()
5317 	       rdev->mc.vram_end >> 12);  in cik_mc_program()
5319 	       rdev->vram_scratch.gpu_addr >> 12);  in cik_mc_program()
5320 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in cik_mc_program()
5321 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in cik_mc_program()
5324 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in cik_mc_program()
5330 	if (radeon_mc_wait_for_idle(rdev)) {  in cik_mc_program()
5331 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5333 	evergreen_mc_resume(rdev, &save);  in cik_mc_program()
5336 	rv515_vga_render_disable(rdev);  in cik_mc_program()
5348 static int cik_mc_init(struct radeon_device *rdev)  in cik_mc_init()  argument
5354 	rdev->mc.vram_is_ddr = true;  in cik_mc_init()
5392 	rdev->mc.vram_width = numchan * chansize;  in cik_mc_init()
5394 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in cik_mc_init()
5395 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in cik_mc_init()
5397 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5398 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5399 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in cik_mc_init()
5400 	si_vram_gtt_location(rdev, &rdev->mc);  in cik_mc_init()
5401 	radeon_update_bandwidth_info(rdev);  in cik_mc_init()
5419 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)  in cik_pcie_gart_tlb_flush()  argument
5428 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)  in cik_pcie_init_compute_vmid()  argument
5437 	mutex_lock(&rdev->srbm_mutex);  in cik_pcie_init_compute_vmid()
5439 		cik_srbm_select(rdev, 0, 0, 0, i);  in cik_pcie_init_compute_vmid()
5446 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_pcie_init_compute_vmid()
5447 	mutex_unlock(&rdev->srbm_mutex);  in cik_pcie_init_compute_vmid()
5461 static int cik_pcie_gart_enable(struct radeon_device *rdev)  in cik_pcie_gart_enable()  argument
5465 	if (rdev->gart.robj == NULL) {  in cik_pcie_gart_enable()
5466 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in cik_pcie_gart_enable()
5469 	r = radeon_gart_table_vram_pin(rdev);  in cik_pcie_gart_enable()
5492 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in cik_pcie_gart_enable()
5493 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in cik_pcie_gart_enable()
5494 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in cik_pcie_gart_enable()
5496 			(u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5508 	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);  in cik_pcie_gart_enable()
5512 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5515 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5520 	       (u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5537 	if (rdev->family == CHIP_KAVERI) {  in cik_pcie_gart_enable()
5545 	mutex_lock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5547 		cik_srbm_select(rdev, 0, 0, 0, i);  in cik_pcie_gart_enable()
5560 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_pcie_gart_enable()
5561 	mutex_unlock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5563 	cik_pcie_init_compute_vmid(rdev);  in cik_pcie_gart_enable()
5565 	cik_pcie_gart_tlb_flush(rdev);  in cik_pcie_gart_enable()
5567 		 (unsigned)(rdev->mc.gtt_size >> 20),  in cik_pcie_gart_enable()
5568 		 (unsigned long long)rdev->gart.table_addr);  in cik_pcie_gart_enable()
5569 	rdev->gart.ready = true;  in cik_pcie_gart_enable()
5580 static void cik_pcie_gart_disable(struct radeon_device *rdev)  in cik_pcie_gart_disable()  argument
5590 		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);  in cik_pcie_gart_disable()
5609 	radeon_gart_table_vram_unpin(rdev);  in cik_pcie_gart_disable()
5619 static void cik_pcie_gart_fini(struct radeon_device *rdev)  in cik_pcie_gart_fini()  argument
5621 	cik_pcie_gart_disable(rdev);  in cik_pcie_gart_fini()
5622 	radeon_gart_table_vram_free(rdev);  in cik_pcie_gart_fini()
5623 	radeon_gart_fini(rdev);  in cik_pcie_gart_fini()
5635 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)  in cik_ib_parse()  argument
5655 int cik_vm_init(struct radeon_device *rdev)  in cik_vm_init()  argument
5663 	rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;  in cik_vm_init()
5665 	if (rdev->flags & RADEON_IS_IGP) {  in cik_vm_init()
5668 		rdev->vm_manager.vram_base_offset = tmp;  in cik_vm_init()
5670 		rdev->vm_manager.vram_base_offset = 0;  in cik_vm_init()
5682 void cik_vm_fini(struct radeon_device *rdev)  in cik_vm_fini()  argument
5695 static void cik_vm_decode_fault(struct radeon_device *rdev,  in cik_vm_decode_fault()  argument
5704 	if (rdev->family == CHIP_HAWAII)  in cik_vm_decode_fault()
5723 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,  in cik_vm_flush()  argument
5768 	cik_hdp_flush_cp_ring_emit(rdev, ring->idx);  in cik_vm_flush()
5803 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,  in cik_enable_gui_idle_interrupt()  argument
5815 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)  in cik_enable_lbpw()  argument
5827 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)  in cik_wait_for_rlc_serdes()  argument
5832 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_wait_for_rlc_serdes()
5833 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_wait_for_rlc_serdes()
5834 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_wait_for_rlc_serdes()
5835 			cik_select_se_sh(rdev, i, j);  in cik_wait_for_rlc_serdes()
5836 			for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
5843 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_wait_for_rlc_serdes()
5844 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_wait_for_rlc_serdes()
5847 	for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
5854 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)  in cik_update_rlc()  argument
5863 static u32 cik_halt_rlc(struct radeon_device *rdev)  in cik_halt_rlc()  argument
5875 		for (i = 0; i < rdev->usec_timeout; i++) {  in cik_halt_rlc()
5881 		cik_wait_for_rlc_serdes(rdev);  in cik_halt_rlc()
5887 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)  in cik_enter_rlc_safe_mode()  argument
5895 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
5901 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
5908 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)  in cik_exit_rlc_safe_mode()  argument
5923 static void cik_rlc_stop(struct radeon_device *rdev)  in cik_rlc_stop()  argument
5927 	cik_enable_gui_idle_interrupt(rdev, false);  in cik_rlc_stop()
5929 	cik_wait_for_rlc_serdes(rdev);  in cik_rlc_stop()
5939 static void cik_rlc_start(struct radeon_device *rdev)  in cik_rlc_start()  argument
5943 	cik_enable_gui_idle_interrupt(rdev, true);  in cik_rlc_start()
5957 static int cik_rlc_resume(struct radeon_device *rdev)  in cik_rlc_resume()  argument
5961 	if (!rdev->rlc_fw)  in cik_rlc_resume()
5964 	cik_rlc_stop(rdev);  in cik_rlc_resume()
5970 	si_rlc_reset(rdev);  in cik_rlc_resume()
5972 	cik_init_pg(rdev);  in cik_rlc_resume()
5974 	cik_init_cg(rdev);  in cik_rlc_resume()
5979 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_rlc_resume()
5980 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_rlc_resume()
5984 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_rlc_resume()
5989 	if (rdev->new_fw) {  in cik_rlc_resume()
5991 			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;  in cik_rlc_resume()
5993 			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_rlc_resume()
6005 		switch (rdev->family) {  in cik_rlc_resume()
6022 		fw_data = (const __be32 *)rdev->rlc_fw->data;  in cik_rlc_resume()
6030 	cik_enable_lbpw(rdev, false);  in cik_rlc_resume()
6032 	if (rdev->family == CHIP_BONAIRE)  in cik_rlc_resume()
6035 	cik_rlc_start(rdev);  in cik_rlc_resume()
6040 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)  in cik_enable_cgcg()  argument
6046 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {  in cik_enable_cgcg()
6047 		cik_enable_gui_idle_interrupt(rdev, true);  in cik_enable_cgcg()
6049 		tmp = cik_halt_rlc(rdev);  in cik_enable_cgcg()
6051 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_cgcg()
6052 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_cgcg()
6057 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_cgcg()
6059 		cik_update_rlc(rdev, tmp);  in cik_enable_cgcg()
6063 		cik_enable_gui_idle_interrupt(rdev, false);  in cik_enable_cgcg()
6078 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)  in cik_enable_mgcg()  argument
6082 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {  in cik_enable_mgcg()
6083 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {  in cik_enable_mgcg()
6084 			if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {  in cik_enable_mgcg()
6098 		tmp = cik_halt_rlc(rdev);  in cik_enable_mgcg()
6100 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6101 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6106 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6108 		cik_update_rlc(rdev, tmp);  in cik_enable_mgcg()
6110 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {  in cik_enable_mgcg()
6116 			if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&  in cik_enable_mgcg()
6117 			    (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))  in cik_enable_mgcg()
6148 		tmp = cik_halt_rlc(rdev);  in cik_enable_mgcg()
6150 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6151 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6156 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6158 		cik_update_rlc(rdev, tmp);  in cik_enable_mgcg()
6175 static void cik_enable_mc_ls(struct radeon_device *rdev,  in cik_enable_mc_ls()  argument
6183 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))  in cik_enable_mc_ls()
6192 static void cik_enable_mc_mgcg(struct radeon_device *rdev,  in cik_enable_mc_mgcg()  argument
6200 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))  in cik_enable_mc_mgcg()
6209 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,  in cik_enable_sdma_mgcg()  argument
6214 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {  in cik_enable_sdma_mgcg()
6230 static void cik_enable_sdma_mgls(struct radeon_device *rdev,  in cik_enable_sdma_mgls()  argument
6235 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {  in cik_enable_sdma_mgls()
6258 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,  in cik_enable_uvd_mgcg()  argument
6263 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {  in cik_enable_uvd_mgcg()
6284 static void cik_enable_bif_mgls(struct radeon_device *rdev,  in cik_enable_bif_mgls()  argument
6291 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))  in cik_enable_bif_mgls()
6302 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,  in cik_enable_hdp_mgcg()  argument
6309 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))  in cik_enable_hdp_mgcg()
6318 static void cik_enable_hdp_ls(struct radeon_device *rdev,  in cik_enable_hdp_ls()  argument
6325 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))  in cik_enable_hdp_ls()
6334 void cik_update_cg(struct radeon_device *rdev,  in cik_update_cg()  argument
6339 		cik_enable_gui_idle_interrupt(rdev, false);  in cik_update_cg()
6342 			cik_enable_mgcg(rdev, true);  in cik_update_cg()
6343 			cik_enable_cgcg(rdev, true);  in cik_update_cg()
6345 			cik_enable_cgcg(rdev, false);  in cik_update_cg()
6346 			cik_enable_mgcg(rdev, false);  in cik_update_cg()
6348 		cik_enable_gui_idle_interrupt(rdev, true);  in cik_update_cg()
6352 		if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_update_cg()
6353 			cik_enable_mc_mgcg(rdev, enable);  in cik_update_cg()
6354 			cik_enable_mc_ls(rdev, enable);  in cik_update_cg()
6359 		cik_enable_sdma_mgcg(rdev, enable);  in cik_update_cg()
6360 		cik_enable_sdma_mgls(rdev, enable);  in cik_update_cg()
6364 		cik_enable_bif_mgls(rdev, enable);  in cik_update_cg()
6368 		if (rdev->has_uvd)  in cik_update_cg()
6369 			cik_enable_uvd_mgcg(rdev, enable);  in cik_update_cg()
6373 		cik_enable_hdp_mgcg(rdev, enable);  in cik_update_cg()
6374 		cik_enable_hdp_ls(rdev, enable);  in cik_update_cg()
6378 		vce_v2_0_enable_mgcg(rdev, enable);  in cik_update_cg()
6382 static void cik_init_cg(struct radeon_device *rdev)  in cik_init_cg()  argument
6385 	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);  in cik_init_cg()
6387 	if (rdev->has_uvd)  in cik_init_cg()
6388 		si_init_uvd_internal_cg(rdev);  in cik_init_cg()
6390 	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |  in cik_init_cg()
6397 static void cik_fini_cg(struct radeon_device *rdev)  in cik_fini_cg()  argument
6399 	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |  in cik_fini_cg()
6405 	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);  in cik_fini_cg()
6408 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,  in cik_enable_sck_slowdown_on_pu()  argument
6414 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pu()
6422 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,  in cik_enable_sck_slowdown_on_pd()  argument
6428 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pd()
6436 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)  in cik_enable_cp_pg()  argument
6441 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))  in cik_enable_cp_pg()
6449 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)  in cik_enable_gds_pg()  argument
6454 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))  in cik_enable_gds_pg()
6466 void cik_init_cp_pg_table(struct radeon_device *rdev)  in cik_init_cp_pg_table()  argument
6473 	if (rdev->family == CHIP_KAVERI)  in cik_init_cp_pg_table()
6476 	if (rdev->rlc.cp_table_ptr == NULL)  in cik_init_cp_pg_table()
6480 	dst_ptr = rdev->rlc.cp_table_ptr;  in cik_init_cp_pg_table()
6482 		if (rdev->new_fw) {  in cik_init_cp_pg_table()
6487 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6489 					(rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6493 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6495 					(rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6499 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6501 					(rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6505 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6507 					(rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6511 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_init_cp_pg_table()
6513 					(rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6528 				fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6531 				fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6534 				fw_data = (const __be32 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6537 				fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6550 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,  in cik_enable_gfx_cgpg()  argument
6555 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {  in cik_enable_gfx_cgpg()
6580 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)  in cik_get_cu_active_bitmap()  argument
6585 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_get_cu_active_bitmap()
6586 	cik_select_se_sh(rdev, se, sh);  in cik_get_cu_active_bitmap()
6589 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_get_cu_active_bitmap()
6590 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_get_cu_active_bitmap()
6597 	for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {  in cik_get_cu_active_bitmap()
6605 static void cik_init_ao_cu_mask(struct radeon_device *rdev)  in cik_init_ao_cu_mask()  argument
6611 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_init_ao_cu_mask()
6612 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_init_ao_cu_mask()
6616 			for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {  in cik_init_ao_cu_mask()
6617 				if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {  in cik_init_ao_cu_mask()
6638 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,  in cik_enable_gfx_static_mgpg()  argument
6644 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))  in cik_enable_gfx_static_mgpg()
6652 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,  in cik_enable_gfx_dynamic_mgpg()  argument
6658 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))  in cik_enable_gfx_dynamic_mgpg()
6669 static void cik_init_gfx_cgpg(struct radeon_device *rdev)  in cik_init_gfx_cgpg()  argument
6674 	if (rdev->rlc.cs_data) {  in cik_init_gfx_cgpg()
6676 		WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
6677 		WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
6678 		WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);  in cik_init_gfx_cgpg()
6684 	if (rdev->rlc.reg_list) {  in cik_init_gfx_cgpg()
6686 		for (i = 0; i < rdev->rlc.reg_list_size; i++)  in cik_init_gfx_cgpg()
6687 			WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);  in cik_init_gfx_cgpg()
6695 	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);  in cik_init_gfx_cgpg()
6696 	WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);  in cik_init_gfx_cgpg()
6718 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)  in cik_update_gfx_pg()  argument
6720 	cik_enable_gfx_cgpg(rdev, enable);  in cik_update_gfx_pg()
6721 	cik_enable_gfx_static_mgpg(rdev, enable);  in cik_update_gfx_pg()
6722 	cik_enable_gfx_dynamic_mgpg(rdev, enable);  in cik_update_gfx_pg()
6725 u32 cik_get_csb_size(struct radeon_device *rdev)  in cik_get_csb_size()  argument
6731 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_size()
6739 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_size()
6757 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)  in cik_get_csb_buffer()  argument
6763 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_buffer()
6775 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_buffer()
6791 	switch (rdev->family) {  in cik_get_csb_buffer()
6822 static void cik_init_pg(struct radeon_device *rdev)  in cik_init_pg()  argument
6824 	if (rdev->pg_flags) {  in cik_init_pg()
6825 		cik_enable_sck_slowdown_on_pu(rdev, true);  in cik_init_pg()
6826 		cik_enable_sck_slowdown_on_pd(rdev, true);  in cik_init_pg()
6827 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_init_pg()
6828 			cik_init_gfx_cgpg(rdev);  in cik_init_pg()
6829 			cik_enable_cp_pg(rdev, true);  in cik_init_pg()
6830 			cik_enable_gds_pg(rdev, true);  in cik_init_pg()
6832 		cik_init_ao_cu_mask(rdev);  in cik_init_pg()
6833 		cik_update_gfx_pg(rdev, true);  in cik_init_pg()
6837 static void cik_fini_pg(struct radeon_device *rdev)  in cik_fini_pg()  argument
6839 	if (rdev->pg_flags) {  in cik_fini_pg()
6840 		cik_update_gfx_pg(rdev, false);  in cik_fini_pg()
6841 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_fini_pg()
6842 			cik_enable_cp_pg(rdev, false);  in cik_fini_pg()
6843 			cik_enable_gds_pg(rdev, false);  in cik_fini_pg()
6870 static void cik_enable_interrupts(struct radeon_device *rdev)  in cik_enable_interrupts()  argument
6879 	rdev->ih.enabled = true;  in cik_enable_interrupts()
6889 static void cik_disable_interrupts(struct radeon_device *rdev)  in cik_disable_interrupts()  argument
6901 	rdev->ih.enabled = false;  in cik_disable_interrupts()
6902 	rdev->ih.rptr = 0;  in cik_disable_interrupts()
6912 static void cik_disable_interrupt_state(struct radeon_device *rdev)  in cik_disable_interrupt_state()  argument
6941 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
6945 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
6950 	if (rdev->num_crtc >= 2) {  in cik_disable_interrupt_state()
6954 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
6958 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
6993 static int cik_irq_init(struct radeon_device *rdev)  in cik_irq_init()  argument
7000 	ret = r600_ih_ring_alloc(rdev);  in cik_irq_init()
7005 	cik_disable_interrupts(rdev);  in cik_irq_init()
7008 	ret = cik_rlc_resume(rdev);  in cik_irq_init()
7010 		r600_ih_ring_fini(rdev);  in cik_irq_init()
7016 	WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);  in cik_irq_init()
7026 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);  in cik_irq_init()
7027 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);  in cik_irq_init()
7033 	if (rdev->wb.enabled)  in cik_irq_init()
7037 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);  in cik_irq_init()
7038 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);  in cik_irq_init()
7049 	if (rdev->msi_enabled)  in cik_irq_init()
7054 	cik_disable_interrupt_state(rdev);  in cik_irq_init()
7056 	pci_set_master(rdev->pdev);  in cik_irq_init()
7059 	cik_enable_interrupts(rdev);  in cik_irq_init()
7073 int cik_irq_set(struct radeon_device *rdev)  in cik_irq_set()  argument
7082 	if (!rdev->irq.installed) {  in cik_irq_set()
7087 	if (!rdev->ih.enabled) {  in cik_irq_set()
7088 		cik_disable_interrupts(rdev);  in cik_irq_set()
7090 		cik_disable_interrupt_state(rdev);  in cik_irq_set()
7111 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in cik_irq_set()
7115 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {  in cik_irq_set()
7116 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_set()
7131 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {  in cik_irq_set()
7132 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_set()
7148 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in cik_irq_set()
7153 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {  in cik_irq_set()
7158 	if (rdev->irq.crtc_vblank_int[0] ||  in cik_irq_set()
7159 	    atomic_read(&rdev->irq.pflip[0])) {  in cik_irq_set()
7163 	if (rdev->irq.crtc_vblank_int[1] ||  in cik_irq_set()
7164 	    atomic_read(&rdev->irq.pflip[1])) {  in cik_irq_set()
7168 	if (rdev->irq.crtc_vblank_int[2] ||  in cik_irq_set()
7169 	    atomic_read(&rdev->irq.pflip[2])) {  in cik_irq_set()
7173 	if (rdev->irq.crtc_vblank_int[3] ||  in cik_irq_set()
7174 	    atomic_read(&rdev->irq.pflip[3])) {  in cik_irq_set()
7178 	if (rdev->irq.crtc_vblank_int[4] ||  in cik_irq_set()
7179 	    atomic_read(&rdev->irq.pflip[4])) {  in cik_irq_set()
7183 	if (rdev->irq.crtc_vblank_int[5] ||  in cik_irq_set()
7184 	    atomic_read(&rdev->irq.pflip[5])) {  in cik_irq_set()
7188 	if (rdev->irq.hpd[0]) {  in cik_irq_set()
7192 	if (rdev->irq.hpd[1]) {  in cik_irq_set()
7196 	if (rdev->irq.hpd[2]) {  in cik_irq_set()
7200 	if (rdev->irq.hpd[3]) {  in cik_irq_set()
7204 	if (rdev->irq.hpd[4]) {  in cik_irq_set()
7208 	if (rdev->irq.hpd[5]) {  in cik_irq_set()
7224 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7228 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7233 	if (rdev->num_crtc >= 2) {  in cik_irq_set()
7239 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7245 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7274 static inline void cik_irq_ack(struct radeon_device *rdev)  in cik_irq_ack()  argument
7278 	rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);  in cik_irq_ack()
7279 	rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);  in cik_irq_ack()
7280 	rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);  in cik_irq_ack()
7281 	rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);  in cik_irq_ack()
7282 	rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);  in cik_irq_ack()
7283 	rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);  in cik_irq_ack()
7284 	rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);  in cik_irq_ack()
7286 	rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7288 	rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7290 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7291 		rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7293 		rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7296 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7297 		rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7299 		rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7303 	if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7306 	if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7309 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)  in cik_irq_ack()
7311 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)  in cik_irq_ack()
7313 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)  in cik_irq_ack()
7315 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)  in cik_irq_ack()
7318 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7319 		if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7322 		if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7325 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)  in cik_irq_ack()
7327 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)  in cik_irq_ack()
7329 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)  in cik_irq_ack()
7331 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)  in cik_irq_ack()
7335 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7336 		if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7339 		if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7342 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)  in cik_irq_ack()
7344 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)  in cik_irq_ack()
7346 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)  in cik_irq_ack()
7348 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)  in cik_irq_ack()
7352 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {  in cik_irq_ack()
7357 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {  in cik_irq_ack()
7362 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {  in cik_irq_ack()
7367 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {  in cik_irq_ack()
7372 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {  in cik_irq_ack()
7377 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {  in cik_irq_ack()
7382 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {  in cik_irq_ack()
7387 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {  in cik_irq_ack()
7392 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {  in cik_irq_ack()
7397 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {  in cik_irq_ack()
7402 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {  in cik_irq_ack()
7407 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {  in cik_irq_ack()
7421 static void cik_irq_disable(struct radeon_device *rdev)  in cik_irq_disable()  argument
7423 	cik_disable_interrupts(rdev);  in cik_irq_disable()
7426 	cik_irq_ack(rdev);  in cik_irq_disable()
7427 	cik_disable_interrupt_state(rdev);  in cik_irq_disable()
7438 static void cik_irq_suspend(struct radeon_device *rdev)  in cik_irq_suspend()  argument
7440 	cik_irq_disable(rdev);  in cik_irq_suspend()
7441 	cik_rlc_stop(rdev);  in cik_irq_suspend()
7453 static void cik_irq_fini(struct radeon_device *rdev)  in cik_irq_fini()  argument
7455 	cik_irq_suspend(rdev);  in cik_irq_fini()
7456 	r600_ih_ring_fini(rdev);  in cik_irq_fini()
7470 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)  in cik_get_ih_wptr()  argument
7474 	if (rdev->wb.enabled)  in cik_get_ih_wptr()
7475 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in cik_get_ih_wptr()
7485 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in cik_get_ih_wptr()
7486 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7487 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in cik_get_ih_wptr()
7492 	return (wptr & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7527 int cik_irq_process(struct radeon_device *rdev)  in cik_irq_process()  argument
7529 	struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_process()
7530 	struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_process()
7542 	if (!rdev->ih.enabled || rdev->shutdown)  in cik_irq_process()
7545 	wptr = cik_get_ih_wptr(rdev);  in cik_irq_process()
7549 	if (atomic_xchg(&rdev->ih.lock, 1))  in cik_irq_process()
7552 	rptr = rdev->ih.rptr;  in cik_irq_process()
7559 	cik_irq_ack(rdev);  in cik_irq_process()
7565 		radeon_kfd_interrupt(rdev,  in cik_irq_process()
7566 				(const void *) &rdev->ih.ring[ring_index]);  in cik_irq_process()
7568 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in cik_irq_process()
7569 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in cik_irq_process()
7570 		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;  in cik_irq_process()
7576 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))  in cik_irq_process()
7579 				if (rdev->irq.crtc_vblank_int[0]) {  in cik_irq_process()
7580 					drm_handle_vblank(rdev->ddev, 0);  in cik_irq_process()
7581 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7582 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7584 				if (atomic_read(&rdev->irq.pflip[0]))  in cik_irq_process()
7585 					radeon_crtc_handle_vblank(rdev, 0);  in cik_irq_process()
7586 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;  in cik_irq_process()
7591 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))  in cik_irq_process()
7594 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;  in cik_irq_process()
7606 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))  in cik_irq_process()
7609 				if (rdev->irq.crtc_vblank_int[1]) {  in cik_irq_process()
7610 					drm_handle_vblank(rdev->ddev, 1);  in cik_irq_process()
7611 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7612 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7614 				if (atomic_read(&rdev->irq.pflip[1]))  in cik_irq_process()
7615 					radeon_crtc_handle_vblank(rdev, 1);  in cik_irq_process()
7616 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;  in cik_irq_process()
7621 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))  in cik_irq_process()
7624 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;  in cik_irq_process()
7636 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))  in cik_irq_process()
7639 				if (rdev->irq.crtc_vblank_int[2]) {  in cik_irq_process()
7640 					drm_handle_vblank(rdev->ddev, 2);  in cik_irq_process()
7641 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7642 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7644 				if (atomic_read(&rdev->irq.pflip[2]))  in cik_irq_process()
7645 					radeon_crtc_handle_vblank(rdev, 2);  in cik_irq_process()
7646 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;  in cik_irq_process()
7651 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))  in cik_irq_process()
7654 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;  in cik_irq_process()
7666 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))  in cik_irq_process()
7669 				if (rdev->irq.crtc_vblank_int[3]) {  in cik_irq_process()
7670 					drm_handle_vblank(rdev->ddev, 3);  in cik_irq_process()
7671 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7672 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7674 				if (atomic_read(&rdev->irq.pflip[3]))  in cik_irq_process()
7675 					radeon_crtc_handle_vblank(rdev, 3);  in cik_irq_process()
7676 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;  in cik_irq_process()
7681 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))  in cik_irq_process()
7684 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;  in cik_irq_process()
7696 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))  in cik_irq_process()
7699 				if (rdev->irq.crtc_vblank_int[4]) {  in cik_irq_process()
7700 					drm_handle_vblank(rdev->ddev, 4);  in cik_irq_process()
7701 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7702 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7704 				if (atomic_read(&rdev->irq.pflip[4]))  in cik_irq_process()
7705 					radeon_crtc_handle_vblank(rdev, 4);  in cik_irq_process()
7706 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;  in cik_irq_process()
7711 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))  in cik_irq_process()
7714 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;  in cik_irq_process()
7726 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))  in cik_irq_process()
7729 				if (rdev->irq.crtc_vblank_int[5]) {  in cik_irq_process()
7730 					drm_handle_vblank(rdev->ddev, 5);  in cik_irq_process()
7731 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7732 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7734 				if (atomic_read(&rdev->irq.pflip[5]))  in cik_irq_process()
7735 					radeon_crtc_handle_vblank(rdev, 5);  in cik_irq_process()
7736 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;  in cik_irq_process()
7741 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))  in cik_irq_process()
7744 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;  in cik_irq_process()
7761 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);  in cik_irq_process()
7766 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))  in cik_irq_process()
7769 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;  in cik_irq_process()
7775 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))  in cik_irq_process()
7778 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;  in cik_irq_process()
7784 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))  in cik_irq_process()
7787 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;  in cik_irq_process()
7793 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))  in cik_irq_process()
7796 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;  in cik_irq_process()
7802 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))  in cik_irq_process()
7805 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;  in cik_irq_process()
7811 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))  in cik_irq_process()
7814 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;  in cik_irq_process()
7820 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))  in cik_irq_process()
7823 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;  in cik_irq_process()
7829 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))  in cik_irq_process()
7832 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;  in cik_irq_process()
7838 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))  in cik_irq_process()
7841 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;  in cik_irq_process()
7847 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))  in cik_irq_process()
7850 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;  in cik_irq_process()
7856 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))  in cik_irq_process()
7859 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;  in cik_irq_process()
7865 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))  in cik_irq_process()
7868 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;  in cik_irq_process()
7884 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);  in cik_irq_process()
7895 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);  in cik_irq_process()
7896 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_irq_process()
7898 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_irq_process()
7900 			cik_vm_decode_fault(rdev, status, addr, mc_client);  in cik_irq_process()
7906 				radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);  in cik_irq_process()
7909 				radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);  in cik_irq_process()
7918 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_irq_process()
7928 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_irq_process()
7933 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in cik_irq_process()
7935 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in cik_irq_process()
7994 					radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);  in cik_irq_process()
8007 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in cik_irq_process()
8021 			rdev->pm.dpm.thermal.high_to_low = false;  in cik_irq_process()
8026 			rdev->pm.dpm.thermal.high_to_low = true;  in cik_irq_process()
8078 		rptr &= rdev->ih.ptr_mask;  in cik_irq_process()
8082 		schedule_work(&rdev->dp_work);  in cik_irq_process()
8084 		schedule_delayed_work(&rdev->hotplug_work, 0);  in cik_irq_process()
8086 		rdev->needs_reset = true;  in cik_irq_process()
8087 		wake_up_all(&rdev->fence_queue);  in cik_irq_process()
8090 		schedule_work(&rdev->pm.dpm.thermal.work);  in cik_irq_process()
8091 	rdev->ih.rptr = rptr;  in cik_irq_process()
8092 	atomic_set(&rdev->ih.lock, 0);  in cik_irq_process()
8095 	wptr = cik_get_ih_wptr(rdev);  in cik_irq_process()
8105 static void cik_uvd_init(struct radeon_device *rdev)  in cik_uvd_init()  argument
8109 	if (!rdev->has_uvd)  in cik_uvd_init()
8112 	r = radeon_uvd_init(rdev);  in cik_uvd_init()
8114 		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);  in cik_uvd_init()
8121 		rdev->has_uvd = 0;  in cik_uvd_init()
8124 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;  in cik_uvd_init()
8125 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);  in cik_uvd_init()
8128 static void cik_uvd_start(struct radeon_device *rdev)  in cik_uvd_start()  argument
8132 	if (!rdev->has_uvd)  in cik_uvd_start()
8135 	r = radeon_uvd_resume(rdev);  in cik_uvd_start()
8137 		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);  in cik_uvd_start()
8140 	r = uvd_v4_2_resume(rdev);  in cik_uvd_start()
8142 		dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);  in cik_uvd_start()
8145 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);  in cik_uvd_start()
8147 		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);  in cik_uvd_start()
8153 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in cik_uvd_start()
8156 static void cik_uvd_resume(struct radeon_device *rdev)  in cik_uvd_resume()  argument
8161 	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)  in cik_uvd_resume()
8164 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cik_uvd_resume()
8165 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));  in cik_uvd_resume()
8167 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);  in cik_uvd_resume()
8170 	r = uvd_v1_0_init(rdev);  in cik_uvd_resume()
8172 		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);  in cik_uvd_resume()
8177 static void cik_vce_init(struct radeon_device *rdev)  in cik_vce_init()  argument
8181 	if (!rdev->has_vce)  in cik_vce_init()
8184 	r = radeon_vce_init(rdev);  in cik_vce_init()
8186 		dev_err(rdev->dev, "failed VCE (%d) init.\n", r);  in cik_vce_init()
8193 		rdev->has_vce = 0;  in cik_vce_init()
8196 	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;  in cik_vce_init()
8197 	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);  in cik_vce_init()
8198 	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;  in cik_vce_init()
8199 	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);  in cik_vce_init()
8202 static void cik_vce_start(struct radeon_device *rdev)  in cik_vce_start()  argument
8206 	if (!rdev->has_vce)  in cik_vce_start()
8209 	r = radeon_vce_resume(rdev);  in cik_vce_start()
8211 		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);  in cik_vce_start()
8214 	r = vce_v2_0_resume(rdev);  in cik_vce_start()
8216 		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);  in cik_vce_start()
8219 	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);  in cik_vce_start()
8221 		dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);  in cik_vce_start()
8224 	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);  in cik_vce_start()
8226 		dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);  in cik_vce_start()
8232 	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;  in cik_vce_start()
8233 	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;  in cik_vce_start()
8236 static void cik_vce_resume(struct radeon_device *rdev)  in cik_vce_resume()  argument
8241 	if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)  in cik_vce_resume()
8244 	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];  in cik_vce_resume()
8245 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);  in cik_vce_resume()
8247 		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);  in cik_vce_resume()
8250 	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];  in cik_vce_resume()
8251 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);  in cik_vce_resume()
8253 		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);  in cik_vce_resume()
8256 	r = vce_v1_0_init(rdev);  in cik_vce_resume()
8258 		dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);  in cik_vce_resume()
8272 static int cik_startup(struct radeon_device *rdev)  in cik_startup()  argument
8279 	cik_pcie_gen3_enable(rdev);  in cik_startup()
8281 	cik_program_aspm(rdev);  in cik_startup()
8284 	r = r600_vram_scratch_init(rdev);  in cik_startup()
8288 	cik_mc_program(rdev);  in cik_startup()
8290 	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {  in cik_startup()
8291 		r = ci_mc_load_microcode(rdev);  in cik_startup()
8298 	r = cik_pcie_gart_enable(rdev);  in cik_startup()
8301 	cik_gpu_init(rdev);  in cik_startup()
8304 	if (rdev->flags & RADEON_IS_IGP) {  in cik_startup()
8305 		if (rdev->family == CHIP_KAVERI) {  in cik_startup()
8306 			rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;  in cik_startup()
8307 			rdev->rlc.reg_list_size =  in cik_startup()
8310 			rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;  in cik_startup()
8311 			rdev->rlc.reg_list_size =  in cik_startup()
8315 	rdev->rlc.cs_data = ci_cs_data;  in cik_startup()
8316 	rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */  in cik_startup()
8317 	rdev->rlc.cp_table_size += 64 * 1024; /* GDS */  in cik_startup()
8318 	r = sumo_rlc_init(rdev);  in cik_startup()
8325 	r = radeon_wb_init(rdev);  in cik_startup()
8330 	r = cik_mec_init(rdev);  in cik_startup()
8336 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_startup()
8338 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8342 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in cik_startup()
8344 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8348 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in cik_startup()
8350 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8354 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);  in cik_startup()
8356 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8360 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in cik_startup()
8362 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8366 	cik_uvd_start(rdev);  in cik_startup()
8367 	cik_vce_start(rdev);  in cik_startup()
8370 	if (!rdev->irq.installed) {  in cik_startup()
8371 		r = radeon_irq_kms_init(rdev);  in cik_startup()
8376 	r = cik_irq_init(rdev);  in cik_startup()
8379 		radeon_irq_kms_fini(rdev);  in cik_startup()
8382 	cik_irq_set(rdev);  in cik_startup()
8384 	if (rdev->family == CHIP_HAWAII) {  in cik_startup()
8385 		if (rdev->new_fw)  in cik_startup()
8393 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_startup()
8394 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in cik_startup()
8401 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_startup()
8402 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,  in cik_startup()
8412 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_startup()
8413 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,  in cik_startup()
8423 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_startup()
8424 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in cik_startup()
8429 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_startup()
8430 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,  in cik_startup()
8435 	r = cik_cp_resume(rdev);  in cik_startup()
8439 	r = cik_sdma_resume(rdev);  in cik_startup()
8443 	cik_uvd_resume(rdev);  in cik_startup()
8444 	cik_vce_resume(rdev);  in cik_startup()
8446 	r = radeon_ib_pool_init(rdev);  in cik_startup()
8448 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in cik_startup()
8452 	r = radeon_vm_manager_init(rdev);  in cik_startup()
8454 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);  in cik_startup()
8458 	r = radeon_audio_init(rdev);  in cik_startup()
8462 	r = radeon_kfd_resume(rdev);  in cik_startup()
8478 int cik_resume(struct radeon_device *rdev)  in cik_resume()  argument
8483 	atom_asic_init(rdev->mode_info.atom_context);  in cik_resume()
8486 	cik_init_golden_registers(rdev);  in cik_resume()
8488 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in cik_resume()
8489 		radeon_pm_resume(rdev);  in cik_resume()
8491 	rdev->accel_working = true;  in cik_resume()
8492 	r = cik_startup(rdev);  in cik_resume()
8495 		rdev->accel_working = false;  in cik_resume()
8512 int cik_suspend(struct radeon_device *rdev)  in cik_suspend()  argument
8514 	radeon_kfd_suspend(rdev);  in cik_suspend()
8515 	radeon_pm_suspend(rdev);  in cik_suspend()
8516 	radeon_audio_fini(rdev);  in cik_suspend()
8517 	radeon_vm_manager_fini(rdev);  in cik_suspend()
8518 	cik_cp_enable(rdev, false);  in cik_suspend()
8519 	cik_sdma_enable(rdev, false);  in cik_suspend()
8520 	if (rdev->has_uvd) {  in cik_suspend()
8521 		uvd_v1_0_fini(rdev);  in cik_suspend()
8522 		radeon_uvd_suspend(rdev);  in cik_suspend()
8524 	if (rdev->has_vce)  in cik_suspend()
8525 		radeon_vce_suspend(rdev);  in cik_suspend()
8526 	cik_fini_pg(rdev);  in cik_suspend()
8527 	cik_fini_cg(rdev);  in cik_suspend()
8528 	cik_irq_suspend(rdev);  in cik_suspend()
8529 	radeon_wb_disable(rdev);  in cik_suspend()
8530 	cik_pcie_gart_disable(rdev);  in cik_suspend()
8550 int cik_init(struct radeon_device *rdev)  in cik_init()  argument
8556 	if (!radeon_get_bios(rdev)) {  in cik_init()
8557 		if (ASIC_IS_AVIVO(rdev))  in cik_init()
8561 	if (!rdev->is_atom_bios) {  in cik_init()
8562 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");  in cik_init()
8565 	r = radeon_atombios_init(rdev);  in cik_init()
8570 	if (!radeon_card_posted(rdev)) {  in cik_init()
8571 		if (!rdev->bios) {  in cik_init()
8572 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in cik_init()
8576 		atom_asic_init(rdev->mode_info.atom_context);  in cik_init()
8579 	cik_init_golden_registers(rdev);  in cik_init()
8581 	cik_scratch_init(rdev);  in cik_init()
8583 	radeon_surface_init(rdev);  in cik_init()
8585 	radeon_get_clock_info(rdev->ddev);  in cik_init()
8588 	r = radeon_fence_driver_init(rdev);  in cik_init()
8593 	r = cik_mc_init(rdev);  in cik_init()
8597 	r = radeon_bo_init(rdev);  in cik_init()
8601 	if (rdev->flags & RADEON_IS_IGP) {  in cik_init()
8602 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8603 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {  in cik_init()
8604 			r = cik_init_microcode(rdev);  in cik_init()
8611 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8612 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||  in cik_init()
8613 		    !rdev->mc_fw) {  in cik_init()
8614 			r = cik_init_microcode(rdev);  in cik_init()
8623 	radeon_pm_init(rdev);  in cik_init()
8625 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_init()
8627 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8629 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_init()
8631 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8632 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8636 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_init()
8638 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8639 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8643 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_init()
8645 	r600_ring_init(rdev, ring, 256 * 1024);  in cik_init()
8647 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_init()
8649 	r600_ring_init(rdev, ring, 256 * 1024);  in cik_init()
8651 	cik_uvd_init(rdev);  in cik_init()
8652 	cik_vce_init(rdev);  in cik_init()
8654 	rdev->ih.ring_obj = NULL;  in cik_init()
8655 	r600_ih_ring_init(rdev, 64 * 1024);  in cik_init()
8657 	r = r600_pcie_gart_init(rdev);  in cik_init()
8661 	rdev->accel_working = true;  in cik_init()
8662 	r = cik_startup(rdev);  in cik_init()
8664 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in cik_init()
8665 		cik_cp_fini(rdev);  in cik_init()
8666 		cik_sdma_fini(rdev);  in cik_init()
8667 		cik_irq_fini(rdev);  in cik_init()
8668 		sumo_rlc_fini(rdev);  in cik_init()
8669 		cik_mec_fini(rdev);  in cik_init()
8670 		radeon_wb_fini(rdev);  in cik_init()
8671 		radeon_ib_pool_fini(rdev);  in cik_init()
8672 		radeon_vm_manager_fini(rdev);  in cik_init()
8673 		radeon_irq_kms_fini(rdev);  in cik_init()
8674 		cik_pcie_gart_fini(rdev);  in cik_init()
8675 		rdev->accel_working = false;  in cik_init()
8682 	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in cik_init()
8699 void cik_fini(struct radeon_device *rdev)  in cik_fini()  argument
8701 	radeon_pm_fini(rdev);  in cik_fini()
8702 	cik_cp_fini(rdev);  in cik_fini()
8703 	cik_sdma_fini(rdev);  in cik_fini()
8704 	cik_fini_pg(rdev);  in cik_fini()
8705 	cik_fini_cg(rdev);  in cik_fini()
8706 	cik_irq_fini(rdev);  in cik_fini()
8707 	sumo_rlc_fini(rdev);  in cik_fini()
8708 	cik_mec_fini(rdev);  in cik_fini()
8709 	radeon_wb_fini(rdev);  in cik_fini()
8710 	radeon_vm_manager_fini(rdev);  in cik_fini()
8711 	radeon_ib_pool_fini(rdev);  in cik_fini()
8712 	radeon_irq_kms_fini(rdev);  in cik_fini()
8713 	uvd_v1_0_fini(rdev);  in cik_fini()
8714 	radeon_uvd_fini(rdev);  in cik_fini()
8715 	radeon_vce_fini(rdev);  in cik_fini()
8716 	cik_pcie_gart_fini(rdev);  in cik_fini()
8717 	r600_vram_scratch_fini(rdev);  in cik_fini()
8718 	radeon_gem_fini(rdev);  in cik_fini()
8719 	radeon_fence_driver_fini(rdev);  in cik_fini()
8720 	radeon_bo_fini(rdev);  in cik_fini()
8721 	radeon_atombios_fini(rdev);  in cik_fini()
8722 	kfree(rdev->bios);  in cik_fini()
8723 	rdev->bios = NULL;  in cik_fini()
8729 	struct radeon_device *rdev = dev->dev_private;  in dce8_program_fmt()  local
8803 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,  in dce8_line_buffer_adjust()  argument
8826 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
8830 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
8842 	for (i = 0; i < rdev->usec_timeout; i++) {  in dce8_line_buffer_adjust()
8874 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)  in cik_get_number_of_dram_channels()  argument
9232 static void dce8_program_watermarks(struct radeon_device *rdev,  in dce8_program_watermarks()  argument
9251 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9252 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9254 				radeon_dpm_get_mclk(rdev, false) * 10;  in dce8_program_watermarks()
9256 				radeon_dpm_get_sclk(rdev, false) * 10;  in dce8_program_watermarks()
9258 			wm_high.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9259 			wm_high.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9275 		wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);  in dce8_program_watermarks()
9286 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9291 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9292 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9294 				radeon_dpm_get_mclk(rdev, true) * 10;  in dce8_program_watermarks()
9296 				radeon_dpm_get_sclk(rdev, true) * 10;  in dce8_program_watermarks()
9298 			wm_low.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9299 			wm_low.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9315 		wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);  in dce8_program_watermarks()
9326 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9368 void dce8_bandwidth_update(struct radeon_device *rdev)  in dce8_bandwidth_update()  argument
9374 	if (!rdev->mode_info.mode_config_initialized)  in dce8_bandwidth_update()
9377 	radeon_update_display_priority(rdev);  in dce8_bandwidth_update()
9379 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9380 		if (rdev->mode_info.crtcs[i]->base.enabled)  in dce8_bandwidth_update()
9383 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9384 		mode = &rdev->mode_info.crtcs[i]->base.mode;  in dce8_bandwidth_update()
9385 		lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);  in dce8_bandwidth_update()
9386 		dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);  in dce8_bandwidth_update()
9398 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)  in cik_get_gpu_clock_counter()  argument
9402 	mutex_lock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9406 	mutex_unlock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9410 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,  in cik_set_uvd_clock()  argument
9417 	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,  in cik_set_uvd_clock()
9438 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in cik_set_uvd_clocks()  argument
9442 	r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);  in cik_set_uvd_clocks()
9446 	r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);  in cik_set_uvd_clocks()
9450 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)  in cik_set_vce_clocks()  argument
9456 	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,  in cik_set_vce_clocks()
9485 static void cik_pcie_gen3_enable(struct radeon_device *rdev)  in cik_pcie_gen3_enable()  argument
9487 	struct pci_dev *root = rdev->pdev->bus->self;  in cik_pcie_gen3_enable()
9493 	if (pci_is_root_bus(rdev->pdev->bus))  in cik_pcie_gen3_enable()
9499 	if (rdev->flags & RADEON_IS_IGP)  in cik_pcie_gen3_enable()
9502 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_pcie_gen3_enable()
9505 	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);  in cik_pcie_gen3_enable()
9533 	gpu_pos = pci_pcie_cap(rdev->pdev);  in cik_pcie_gen3_enable()
9545 			pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);  in cik_pcie_gen3_enable()
9551 			pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);  in cik_pcie_gen3_enable()
9569 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);  in cik_pcie_gen3_enable()
9574 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);  in cik_pcie_gen3_enable()
9577 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);  in cik_pcie_gen3_enable()
9595 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);  in cik_pcie_gen3_enable()
9598 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);  in cik_pcie_gen3_enable()
9606 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);  in cik_pcie_gen3_enable()
9609 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);  in cik_pcie_gen3_enable()
9623 	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);  in cik_pcie_gen3_enable()
9631 	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);  in cik_pcie_gen3_enable()
9637 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_pcie_gen3_enable()
9645 static void cik_program_aspm(struct radeon_device *rdev)  in cik_program_aspm()  argument
9655 	if (rdev->flags & RADEON_IS_IGP)  in cik_program_aspm()
9658 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_program_aspm()
9723 			    !pci_is_root_bus(rdev->pdev->bus)) {  in cik_program_aspm()
9724 				struct pci_dev *root = rdev->pdev->bus->self;  in cik_program_aspm()