Lines Matching refs:rdev
128 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
129 static void si_pcie_gen3_enable(struct radeon_device *rdev);
130 static void si_program_aspm(struct radeon_device *rdev);
131 extern void sumo_rlc_fini(struct radeon_device *rdev);
132 extern int sumo_rlc_init(struct radeon_device *rdev);
133 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
135 static void si_init_pg(struct radeon_device *rdev);
136 static void si_init_cg(struct radeon_device *rdev);
137 static void si_fini_pg(struct radeon_device *rdev);
138 static void si_fini_cg(struct radeon_device *rdev);
139 static void si_rlc_stop(struct radeon_device *rdev);
1228 static void si_init_golden_registers(struct radeon_device *rdev) in si_init_golden_registers() argument
1230 switch (rdev->family) { in si_init_golden_registers()
1232 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1235 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1238 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1241 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1246 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1249 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1252 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1257 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1260 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1263 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1266 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1271 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1274 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1277 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1282 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1285 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1288 radeon_program_register_sequence(rdev, in si_init_golden_registers()
1307 int si_get_allowed_info_register(struct radeon_device *rdev, in si_get_allowed_info_register() argument
1338 u32 si_get_xclk(struct radeon_device *rdev) in si_get_xclk() argument
1340 u32 reference_clock = rdev->clock.spll.reference_freq; in si_get_xclk()
1355 int si_get_temp(struct radeon_device *rdev) in si_get_temp() argument
1571 int si_mc_load_microcode(struct radeon_device *rdev) in si_mc_load_microcode() argument
1580 if (!rdev->mc_fw) in si_mc_load_microcode()
1583 if (rdev->new_fw) { in si_mc_load_microcode()
1585 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; in si_mc_load_microcode()
1590 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); in si_mc_load_microcode()
1593 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in si_mc_load_microcode()
1595 ucode_size = rdev->mc_fw->size / 4; in si_mc_load_microcode()
1597 switch (rdev->family) { in si_mc_load_microcode()
1620 fw_data = (const __be32 *)rdev->mc_fw->data; in si_mc_load_microcode()
1632 if (rdev->new_fw) { in si_mc_load_microcode()
1642 if (rdev->new_fw) in si_mc_load_microcode()
1654 for (i = 0; i < rdev->usec_timeout; i++) { in si_mc_load_microcode()
1659 for (i = 0; i < rdev->usec_timeout; i++) { in si_mc_load_microcode()
1669 static int si_init_microcode(struct radeon_device *rdev) in si_init_microcode() argument
1684 switch (rdev->family) { in si_init_microcode()
1698 if ((rdev->pdev->revision == 0x81) && in si_init_microcode()
1699 ((rdev->pdev->device == 0x6810) || in si_init_microcode()
1700 (rdev->pdev->device == 0x6811))) in si_init_microcode()
1713 if (((rdev->pdev->device == 0x6820) && in si_init_microcode()
1714 ((rdev->pdev->revision == 0x81) || in si_init_microcode()
1715 (rdev->pdev->revision == 0x83))) || in si_init_microcode()
1716 ((rdev->pdev->device == 0x6821) && in si_init_microcode()
1717 ((rdev->pdev->revision == 0x83) || in si_init_microcode()
1718 (rdev->pdev->revision == 0x87))) || in si_init_microcode()
1719 ((rdev->pdev->revision == 0x87) && in si_init_microcode()
1720 ((rdev->pdev->device == 0x6823) || in si_init_microcode()
1721 (rdev->pdev->device == 0x682b)))) in si_init_microcode()
1734 if (((rdev->pdev->revision == 0x81) && in si_init_microcode()
1735 ((rdev->pdev->device == 0x6600) || in si_init_microcode()
1736 (rdev->pdev->device == 0x6604) || in si_init_microcode()
1737 (rdev->pdev->device == 0x6605) || in si_init_microcode()
1738 (rdev->pdev->device == 0x6610))) || in si_init_microcode()
1739 ((rdev->pdev->revision == 0x83) && in si_init_microcode()
1740 (rdev->pdev->device == 0x6610))) in si_init_microcode()
1752 if (((rdev->pdev->revision == 0x81) && in si_init_microcode()
1753 (rdev->pdev->device == 0x6660)) || in si_init_microcode()
1754 ((rdev->pdev->revision == 0x83) && in si_init_microcode()
1755 ((rdev->pdev->device == 0x6660) || in si_init_microcode()
1756 (rdev->pdev->device == 0x6663) || in si_init_microcode()
1757 (rdev->pdev->device == 0x6665) || in si_init_microcode()
1758 (rdev->pdev->device == 0x6667)))) in si_init_microcode()
1760 else if ((rdev->pdev->revision == 0xc3) && in si_init_microcode()
1761 (rdev->pdev->device == 0x6665)) in si_init_microcode()
1781 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in si_init_microcode()
1784 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in si_init_microcode()
1787 if (rdev->pfp_fw->size != pfp_req_size) { in si_init_microcode()
1789 rdev->pfp_fw->size, fw_name); in si_init_microcode()
1794 err = radeon_ucode_validate(rdev->pfp_fw); in si_init_microcode()
1805 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in si_init_microcode()
1808 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in si_init_microcode()
1811 if (rdev->me_fw->size != me_req_size) { in si_init_microcode()
1813 rdev->me_fw->size, fw_name); in si_init_microcode()
1817 err = radeon_ucode_validate(rdev->me_fw); in si_init_microcode()
1828 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in si_init_microcode()
1831 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in si_init_microcode()
1834 if (rdev->ce_fw->size != ce_req_size) { in si_init_microcode()
1836 rdev->ce_fw->size, fw_name); in si_init_microcode()
1840 err = radeon_ucode_validate(rdev->ce_fw); in si_init_microcode()
1851 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in si_init_microcode()
1854 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in si_init_microcode()
1857 if (rdev->rlc_fw->size != rlc_req_size) { in si_init_microcode()
1859 rdev->rlc_fw->size, fw_name); in si_init_microcode()
1863 err = radeon_ucode_validate(rdev->rlc_fw); in si_init_microcode()
1877 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in si_init_microcode()
1880 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in si_init_microcode()
1883 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in si_init_microcode()
1887 if ((rdev->mc_fw->size != mc_req_size) && in si_init_microcode()
1888 (rdev->mc_fw->size != mc2_req_size)) { in si_init_microcode()
1890 rdev->mc_fw->size, fw_name); in si_init_microcode()
1893 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); in si_init_microcode()
1895 err = radeon_ucode_validate(rdev->mc_fw); in si_init_microcode()
1911 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in si_init_microcode()
1914 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in si_init_microcode()
1917 release_firmware(rdev->smc_fw); in si_init_microcode()
1918 rdev->smc_fw = NULL; in si_init_microcode()
1920 } else if (rdev->smc_fw->size != smc_req_size) { in si_init_microcode()
1922 rdev->smc_fw->size, fw_name); in si_init_microcode()
1926 err = radeon_ucode_validate(rdev->smc_fw); in si_init_microcode()
1937 rdev->new_fw = false; in si_init_microcode()
1942 rdev->new_fw = true; in si_init_microcode()
1949 release_firmware(rdev->pfp_fw); in si_init_microcode()
1950 rdev->pfp_fw = NULL; in si_init_microcode()
1951 release_firmware(rdev->me_fw); in si_init_microcode()
1952 rdev->me_fw = NULL; in si_init_microcode()
1953 release_firmware(rdev->ce_fw); in si_init_microcode()
1954 rdev->ce_fw = NULL; in si_init_microcode()
1955 release_firmware(rdev->rlc_fw); in si_init_microcode()
1956 rdev->rlc_fw = NULL; in si_init_microcode()
1957 release_firmware(rdev->mc_fw); in si_init_microcode()
1958 rdev->mc_fw = NULL; in si_init_microcode()
1959 release_firmware(rdev->smc_fw); in si_init_microcode()
1960 rdev->smc_fw = NULL; in si_init_microcode()
1966 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, in dce6_line_buffer_adjust() argument
2004 for (i = 0; i < rdev->usec_timeout; i++) { in dce6_line_buffer_adjust()
2025 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev) in si_get_number_of_dram_channels() argument
2293 static void dce6_program_watermarks(struct radeon_device *rdev, in dce6_program_watermarks() argument
2318 if (rdev->family == CHIP_ARUBA) in dce6_program_watermarks()
2319 dram_channels = evergreen_get_number_of_dram_channels(rdev); in dce6_program_watermarks()
2321 dram_channels = si_get_number_of_dram_channels(rdev); in dce6_program_watermarks()
2324 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { in dce6_program_watermarks()
2326 radeon_dpm_get_mclk(rdev, false) * 10; in dce6_program_watermarks()
2328 radeon_dpm_get_sclk(rdev, false) * 10; in dce6_program_watermarks()
2330 wm_high.yclk = rdev->pm.current_mclk * 10; in dce6_program_watermarks()
2331 wm_high.sclk = rdev->pm.current_sclk * 10; in dce6_program_watermarks()
2351 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { in dce6_program_watermarks()
2353 radeon_dpm_get_mclk(rdev, true) * 10; in dce6_program_watermarks()
2355 radeon_dpm_get_sclk(rdev, true) * 10; in dce6_program_watermarks()
2357 wm_low.yclk = rdev->pm.current_mclk * 10; in dce6_program_watermarks()
2358 wm_low.sclk = rdev->pm.current_sclk * 10; in dce6_program_watermarks()
2387 (rdev->disp_priority == 2)) { in dce6_program_watermarks()
2395 (rdev->disp_priority == 2)) { in dce6_program_watermarks()
2459 void dce6_bandwidth_update(struct radeon_device *rdev) in dce6_bandwidth_update() argument
2466 if (!rdev->mode_info.mode_config_initialized) in dce6_bandwidth_update()
2469 radeon_update_display_priority(rdev); in dce6_bandwidth_update()
2471 for (i = 0; i < rdev->num_crtc; i++) { in dce6_bandwidth_update()
2472 if (rdev->mode_info.crtcs[i]->base.enabled) in dce6_bandwidth_update()
2475 for (i = 0; i < rdev->num_crtc; i += 2) { in dce6_bandwidth_update()
2476 mode0 = &rdev->mode_info.crtcs[i]->base.mode; in dce6_bandwidth_update()
2477 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; in dce6_bandwidth_update()
2478 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); in dce6_bandwidth_update()
2479 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); in dce6_bandwidth_update()
2480 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); in dce6_bandwidth_update()
2481 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); in dce6_bandwidth_update()
2488 static void si_tiling_mode_table_init(struct radeon_device *rdev) in si_tiling_mode_table_init() argument
2490 u32 *tile = rdev->config.si.tile_mode_array; in si_tiling_mode_table_init()
2492 ARRAY_SIZE(rdev->config.si.tile_mode_array); in si_tiling_mode_table_init()
2495 switch (rdev->config.si.mem_row_size_in_kb) { in si_tiling_mode_table_init()
2511 switch(rdev->family) { in si_tiling_mode_table_init()
2942 DRM_ERROR("unknown asic: 0x%x\n", rdev->family); in si_tiling_mode_table_init()
2946 static void si_select_se_sh(struct radeon_device *rdev, in si_select_se_sh() argument
2973 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) in si_get_cu_enabled() argument
2991 static void si_setup_spi(struct radeon_device *rdev, in si_setup_spi() argument
3000 si_select_se_sh(rdev, i, j); in si_setup_spi()
3002 active_cu = si_get_cu_enabled(rdev, cu_per_sh); in si_setup_spi()
3015 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); in si_setup_spi()
3018 static u32 si_get_rb_disabled(struct radeon_device *rdev, in si_get_rb_disabled() argument
3038 static void si_setup_rb(struct radeon_device *rdev, in si_setup_rb() argument
3049 si_select_se_sh(rdev, i, j); in si_setup_rb()
3050 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); in si_setup_rb()
3054 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); in si_setup_rb()
3063 rdev->config.si.backend_enable_mask = enabled_rbs; in si_setup_rb()
3066 si_select_se_sh(rdev, i, 0xffffffff); in si_setup_rb()
3085 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); in si_setup_rb()
3088 static void si_gpu_init(struct radeon_device *rdev) in si_gpu_init() argument
3097 switch (rdev->family) { in si_gpu_init()
3099 rdev->config.si.max_shader_engines = 2; in si_gpu_init()
3100 rdev->config.si.max_tile_pipes = 12; in si_gpu_init()
3101 rdev->config.si.max_cu_per_sh = 8; in si_gpu_init()
3102 rdev->config.si.max_sh_per_se = 2; in si_gpu_init()
3103 rdev->config.si.max_backends_per_se = 4; in si_gpu_init()
3104 rdev->config.si.max_texture_channel_caches = 12; in si_gpu_init()
3105 rdev->config.si.max_gprs = 256; in si_gpu_init()
3106 rdev->config.si.max_gs_threads = 32; in si_gpu_init()
3107 rdev->config.si.max_hw_contexts = 8; in si_gpu_init()
3109 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; in si_gpu_init()
3110 rdev->config.si.sc_prim_fifo_size_backend = 0x100; in si_gpu_init()
3111 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; in si_gpu_init()
3112 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; in si_gpu_init()
3116 rdev->config.si.max_shader_engines = 2; in si_gpu_init()
3117 rdev->config.si.max_tile_pipes = 8; in si_gpu_init()
3118 rdev->config.si.max_cu_per_sh = 5; in si_gpu_init()
3119 rdev->config.si.max_sh_per_se = 2; in si_gpu_init()
3120 rdev->config.si.max_backends_per_se = 4; in si_gpu_init()
3121 rdev->config.si.max_texture_channel_caches = 8; in si_gpu_init()
3122 rdev->config.si.max_gprs = 256; in si_gpu_init()
3123 rdev->config.si.max_gs_threads = 32; in si_gpu_init()
3124 rdev->config.si.max_hw_contexts = 8; in si_gpu_init()
3126 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; in si_gpu_init()
3127 rdev->config.si.sc_prim_fifo_size_backend = 0x100; in si_gpu_init()
3128 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; in si_gpu_init()
3129 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; in si_gpu_init()
3134 rdev->config.si.max_shader_engines = 1; in si_gpu_init()
3135 rdev->config.si.max_tile_pipes = 4; in si_gpu_init()
3136 rdev->config.si.max_cu_per_sh = 5; in si_gpu_init()
3137 rdev->config.si.max_sh_per_se = 2; in si_gpu_init()
3138 rdev->config.si.max_backends_per_se = 4; in si_gpu_init()
3139 rdev->config.si.max_texture_channel_caches = 4; in si_gpu_init()
3140 rdev->config.si.max_gprs = 256; in si_gpu_init()
3141 rdev->config.si.max_gs_threads = 32; in si_gpu_init()
3142 rdev->config.si.max_hw_contexts = 8; in si_gpu_init()
3144 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; in si_gpu_init()
3145 rdev->config.si.sc_prim_fifo_size_backend = 0x40; in si_gpu_init()
3146 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; in si_gpu_init()
3147 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; in si_gpu_init()
3151 rdev->config.si.max_shader_engines = 1; in si_gpu_init()
3152 rdev->config.si.max_tile_pipes = 4; in si_gpu_init()
3153 rdev->config.si.max_cu_per_sh = 6; in si_gpu_init()
3154 rdev->config.si.max_sh_per_se = 1; in si_gpu_init()
3155 rdev->config.si.max_backends_per_se = 2; in si_gpu_init()
3156 rdev->config.si.max_texture_channel_caches = 4; in si_gpu_init()
3157 rdev->config.si.max_gprs = 256; in si_gpu_init()
3158 rdev->config.si.max_gs_threads = 16; in si_gpu_init()
3159 rdev->config.si.max_hw_contexts = 8; in si_gpu_init()
3161 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; in si_gpu_init()
3162 rdev->config.si.sc_prim_fifo_size_backend = 0x40; in si_gpu_init()
3163 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; in si_gpu_init()
3164 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; in si_gpu_init()
3168 rdev->config.si.max_shader_engines = 1; in si_gpu_init()
3169 rdev->config.si.max_tile_pipes = 4; in si_gpu_init()
3170 rdev->config.si.max_cu_per_sh = 5; in si_gpu_init()
3171 rdev->config.si.max_sh_per_se = 1; in si_gpu_init()
3172 rdev->config.si.max_backends_per_se = 1; in si_gpu_init()
3173 rdev->config.si.max_texture_channel_caches = 2; in si_gpu_init()
3174 rdev->config.si.max_gprs = 256; in si_gpu_init()
3175 rdev->config.si.max_gs_threads = 16; in si_gpu_init()
3176 rdev->config.si.max_hw_contexts = 8; in si_gpu_init()
3178 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; in si_gpu_init()
3179 rdev->config.si.sc_prim_fifo_size_backend = 0x40; in si_gpu_init()
3180 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; in si_gpu_init()
3181 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; in si_gpu_init()
3199 evergreen_fix_pci_max_read_req_size(rdev); in si_gpu_init()
3206 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; in si_gpu_init()
3207 rdev->config.si.mem_max_burst_length_bytes = 256; in si_gpu_init()
3209 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; in si_gpu_init()
3210 if (rdev->config.si.mem_row_size_in_kb > 4) in si_gpu_init()
3211 rdev->config.si.mem_row_size_in_kb = 4; in si_gpu_init()
3213 rdev->config.si.shader_engine_tile_size = 32; in si_gpu_init()
3214 rdev->config.si.num_gpus = 1; in si_gpu_init()
3215 rdev->config.si.multi_gpu_tile_size = 64; in si_gpu_init()
3219 switch (rdev->config.si.mem_row_size_in_kb) { in si_gpu_init()
3239 rdev->config.si.tile_config = 0; in si_gpu_init()
3240 switch (rdev->config.si.num_tile_pipes) { in si_gpu_init()
3242 rdev->config.si.tile_config |= (0 << 0); in si_gpu_init()
3245 rdev->config.si.tile_config |= (1 << 0); in si_gpu_init()
3248 rdev->config.si.tile_config |= (2 << 0); in si_gpu_init()
3253 rdev->config.si.tile_config |= (3 << 0); in si_gpu_init()
3258 rdev->config.si.tile_config |= 0 << 4; in si_gpu_init()
3261 rdev->config.si.tile_config |= 1 << 4; in si_gpu_init()
3265 rdev->config.si.tile_config |= 2 << 4; in si_gpu_init()
3268 rdev->config.si.tile_config |= in si_gpu_init()
3270 rdev->config.si.tile_config |= in si_gpu_init()
3279 if (rdev->has_uvd) { in si_gpu_init()
3285 si_tiling_mode_table_init(rdev); in si_gpu_init()
3287 si_setup_rb(rdev, rdev->config.si.max_shader_engines, in si_gpu_init()
3288 rdev->config.si.max_sh_per_se, in si_gpu_init()
3289 rdev->config.si.max_backends_per_se); in si_gpu_init()
3291 si_setup_spi(rdev, rdev->config.si.max_shader_engines, in si_gpu_init()
3292 rdev->config.si.max_sh_per_se, in si_gpu_init()
3293 rdev->config.si.max_cu_per_sh); in si_gpu_init()
3295 rdev->config.si.active_cus = 0; in si_gpu_init()
3296 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { in si_gpu_init()
3297 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { in si_gpu_init()
3298 rdev->config.si.active_cus += in si_gpu_init()
3299 hweight32(si_get_cu_active_bitmap(rdev, i, j)); in si_gpu_init()
3313 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | in si_gpu_init()
3314 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | in si_gpu_init()
3315 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | in si_gpu_init()
3316 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); in si_gpu_init()
3357 static void si_scratch_init(struct radeon_device *rdev) in si_scratch_init() argument
3361 rdev->scratch.num_reg = 7; in si_scratch_init()
3362 rdev->scratch.reg_base = SCRATCH_REG0; in si_scratch_init()
3363 for (i = 0; i < rdev->scratch.num_reg; i++) { in si_scratch_init()
3364 rdev->scratch.free[i] = true; in si_scratch_init()
3365 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); in si_scratch_init()
3369 void si_fence_ring_emit(struct radeon_device *rdev, in si_fence_ring_emit() argument
3372 struct radeon_ring *ring = &rdev->ring[fence->ring]; in si_fence_ring_emit()
3373 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; in si_fence_ring_emit()
3399 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) in si_ring_ib_execute() argument
3401 struct radeon_ring *ring = &rdev->ring[ib->ring]; in si_ring_ib_execute()
3419 } else if (rdev->wb.enabled) { in si_ring_ib_execute()
3459 static void si_cp_enable(struct radeon_device *rdev, bool enable) in si_cp_enable() argument
3464 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in si_cp_enable()
3465 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); in si_cp_enable()
3468 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in si_cp_enable()
3469 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; in si_cp_enable()
3470 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; in si_cp_enable()
3475 static int si_cp_load_microcode(struct radeon_device *rdev) in si_cp_load_microcode() argument
3479 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) in si_cp_load_microcode()
3482 si_cp_enable(rdev, false); in si_cp_load_microcode()
3484 if (rdev->new_fw) { in si_cp_load_microcode()
3486 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; in si_cp_load_microcode()
3488 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; in si_cp_load_microcode()
3490 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; in si_cp_load_microcode()
3500 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); in si_cp_load_microcode()
3509 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); in si_cp_load_microcode()
3518 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); in si_cp_load_microcode()
3528 fw_data = (const __be32 *)rdev->pfp_fw->data; in si_cp_load_microcode()
3535 fw_data = (const __be32 *)rdev->ce_fw->data; in si_cp_load_microcode()
3542 fw_data = (const __be32 *)rdev->me_fw->data; in si_cp_load_microcode()
3556 static int si_cp_start(struct radeon_device *rdev) in si_cp_start() argument
3558 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_cp_start()
3561 r = radeon_ring_lock(rdev, ring, 7 + 4); in si_cp_start()
3570 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); in si_cp_start()
3580 radeon_ring_unlock_commit(rdev, ring, false); in si_cp_start()
3582 si_cp_enable(rdev, true); in si_cp_start()
3584 r = radeon_ring_lock(rdev, ring, si_default_size + 10); in si_cp_start()
3609 radeon_ring_unlock_commit(rdev, ring, false); in si_cp_start()
3612 ring = &rdev->ring[i]; in si_cp_start()
3613 r = radeon_ring_lock(rdev, ring, 2); in si_cp_start()
3623 radeon_ring_unlock_commit(rdev, ring, false); in si_cp_start()
3629 static void si_cp_fini(struct radeon_device *rdev) in si_cp_fini() argument
3632 si_cp_enable(rdev, false); in si_cp_fini()
3634 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_cp_fini()
3635 radeon_ring_fini(rdev, ring); in si_cp_fini()
3636 radeon_scratch_free(rdev, ring->rptr_save_reg); in si_cp_fini()
3638 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in si_cp_fini()
3639 radeon_ring_fini(rdev, ring); in si_cp_fini()
3640 radeon_scratch_free(rdev, ring->rptr_save_reg); in si_cp_fini()
3642 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in si_cp_fini()
3643 radeon_ring_fini(rdev, ring); in si_cp_fini()
3644 radeon_scratch_free(rdev, ring->rptr_save_reg); in si_cp_fini()
3647 static int si_cp_resume(struct radeon_device *rdev) in si_cp_resume() argument
3654 si_enable_gui_idle_interrupt(rdev, false); in si_cp_resume()
3663 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); in si_cp_resume()
3667 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_cp_resume()
3681 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); in si_cp_resume()
3682 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); in si_cp_resume()
3684 if (rdev->wb.enabled) in si_cp_resume()
3698 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in si_cp_resume()
3712 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); in si_cp_resume()
3713 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); in si_cp_resume()
3722 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in si_cp_resume()
3736 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); in si_cp_resume()
3737 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); in si_cp_resume()
3745 si_cp_start(rdev); in si_cp_resume()
3746 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; in si_cp_resume()
3747 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; in si_cp_resume()
3748 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; in si_cp_resume()
3749 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); in si_cp_resume()
3751 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in si_cp_resume()
3752 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; in si_cp_resume()
3753 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; in si_cp_resume()
3756 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); in si_cp_resume()
3758 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; in si_cp_resume()
3760 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); in si_cp_resume()
3762 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; in si_cp_resume()
3765 si_enable_gui_idle_interrupt(rdev, true); in si_cp_resume()
3767 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in si_cp_resume()
3768 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); in si_cp_resume()
3773 u32 si_gpu_check_soft_reset(struct radeon_device *rdev) in si_gpu_check_soft_reset() argument
3837 if (evergreen_is_display_hung(rdev)) in si_gpu_check_soft_reset()
3854 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) in si_gpu_soft_reset() argument
3863 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); in si_gpu_soft_reset()
3865 evergreen_print_gpu_status_regs(rdev); in si_gpu_soft_reset()
3866 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in si_gpu_soft_reset()
3868 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in si_gpu_soft_reset()
3872 si_fini_pg(rdev); in si_gpu_soft_reset()
3873 si_fini_cg(rdev); in si_gpu_soft_reset()
3876 si_rlc_stop(rdev); in si_gpu_soft_reset()
3896 evergreen_mc_stop(rdev, &save); in si_gpu_soft_reset()
3897 if (evergreen_mc_wait_for_idle(rdev)) { in si_gpu_soft_reset()
3898 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in si_gpu_soft_reset()
3952 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in si_gpu_soft_reset()
3966 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); in si_gpu_soft_reset()
3980 evergreen_mc_resume(rdev, &save); in si_gpu_soft_reset()
3983 evergreen_print_gpu_status_regs(rdev); in si_gpu_soft_reset()
3986 static void si_set_clk_bypass_mode(struct radeon_device *rdev) in si_set_clk_bypass_mode() argument
3998 for (i = 0; i < rdev->usec_timeout; i++) { in si_set_clk_bypass_mode()
4013 static void si_spll_powerdown(struct radeon_device *rdev) in si_spll_powerdown() argument
4034 static void si_gpu_pci_config_reset(struct radeon_device *rdev) in si_gpu_pci_config_reset() argument
4039 dev_info(rdev->dev, "GPU pci config reset\n"); in si_gpu_pci_config_reset()
4044 si_fini_pg(rdev); in si_gpu_pci_config_reset()
4045 si_fini_cg(rdev); in si_gpu_pci_config_reset()
4060 si_rlc_stop(rdev); in si_gpu_pci_config_reset()
4065 evergreen_mc_stop(rdev, &save); in si_gpu_pci_config_reset()
4066 if (evergreen_mc_wait_for_idle(rdev)) { in si_gpu_pci_config_reset()
4067 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); in si_gpu_pci_config_reset()
4071 si_set_clk_bypass_mode(rdev); in si_gpu_pci_config_reset()
4073 si_spll_powerdown(rdev); in si_gpu_pci_config_reset()
4075 pci_clear_master(rdev->pdev); in si_gpu_pci_config_reset()
4077 radeon_pci_config_reset(rdev); in si_gpu_pci_config_reset()
4079 for (i = 0; i < rdev->usec_timeout; i++) { in si_gpu_pci_config_reset()
4086 int si_asic_reset(struct radeon_device *rdev, bool hard) in si_asic_reset() argument
4091 si_gpu_pci_config_reset(rdev); in si_asic_reset()
4095 reset_mask = si_gpu_check_soft_reset(rdev); in si_asic_reset()
4098 r600_set_bios_scratch_engine_hung(rdev, true); in si_asic_reset()
4101 si_gpu_soft_reset(rdev, reset_mask); in si_asic_reset()
4103 reset_mask = si_gpu_check_soft_reset(rdev); in si_asic_reset()
4107 si_gpu_pci_config_reset(rdev); in si_asic_reset()
4109 reset_mask = si_gpu_check_soft_reset(rdev); in si_asic_reset()
4112 r600_set_bios_scratch_engine_hung(rdev, false); in si_asic_reset()
4126 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) in si_gfx_is_lockup() argument
4128 u32 reset_mask = si_gpu_check_soft_reset(rdev); in si_gfx_is_lockup()
4133 radeon_ring_lockup_update(rdev, ring); in si_gfx_is_lockup()
4136 return radeon_ring_test_lockup(rdev, ring); in si_gfx_is_lockup()
4140 static void si_mc_program(struct radeon_device *rdev) in si_mc_program() argument
4156 evergreen_mc_stop(rdev, &save); in si_mc_program()
4157 if (radeon_mc_wait_for_idle(rdev)) { in si_mc_program()
4158 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in si_mc_program()
4160 if (!ASIC_IS_NODCE(rdev)) in si_mc_program()
4165 rdev->mc.vram_start >> 12); in si_mc_program()
4167 rdev->mc.vram_end >> 12); in si_mc_program()
4169 rdev->vram_scratch.gpu_addr >> 12); in si_mc_program()
4170 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; in si_mc_program()
4171 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); in si_mc_program()
4174 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); in si_mc_program()
4180 if (radeon_mc_wait_for_idle(rdev)) { in si_mc_program()
4181 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in si_mc_program()
4183 evergreen_mc_resume(rdev, &save); in si_mc_program()
4184 if (!ASIC_IS_NODCE(rdev)) { in si_mc_program()
4187 rv515_vga_render_disable(rdev); in si_mc_program()
4191 void si_vram_gtt_location(struct radeon_device *rdev, in si_vram_gtt_location() argument
4196 dev_warn(rdev->dev, "limiting VRAM\n"); in si_vram_gtt_location()
4200 radeon_vram_location(rdev, &rdev->mc, 0); in si_vram_gtt_location()
4201 rdev->mc.gtt_base_align = 0; in si_vram_gtt_location()
4202 radeon_gtt_location(rdev, mc); in si_vram_gtt_location()
4205 static int si_mc_init(struct radeon_device *rdev) in si_mc_init() argument
4211 rdev->mc.vram_is_ddr = true; in si_mc_init()
4251 rdev->mc.vram_width = numchan * chansize; in si_mc_init()
4253 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); in si_mc_init()
4254 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); in si_mc_init()
4263 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL; in si_mc_init()
4264 rdev->mc.real_vram_size = rdev->mc.mc_vram_size; in si_mc_init()
4265 rdev->mc.visible_vram_size = rdev->mc.aper_size; in si_mc_init()
4266 si_vram_gtt_location(rdev, &rdev->mc); in si_mc_init()
4267 radeon_update_bandwidth_info(rdev); in si_mc_init()
4275 void si_pcie_gart_tlb_flush(struct radeon_device *rdev) in si_pcie_gart_tlb_flush() argument
4284 static int si_pcie_gart_enable(struct radeon_device *rdev) in si_pcie_gart_enable() argument
4288 if (rdev->gart.robj == NULL) { in si_pcie_gart_enable()
4289 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); in si_pcie_gart_enable()
4292 r = radeon_gart_table_vram_pin(rdev); in si_pcie_gart_enable()
4315 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); in si_pcie_gart_enable()
4316 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); in si_pcie_gart_enable()
4317 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); in si_pcie_gart_enable()
4319 (u32)(rdev->dummy_page.addr >> 12)); in si_pcie_gart_enable()
4331 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); in si_pcie_gart_enable()
4339 rdev->vm_manager.saved_table_addr[i]); in si_pcie_gart_enable()
4342 rdev->vm_manager.saved_table_addr[i]); in si_pcie_gart_enable()
4347 (u32)(rdev->dummy_page.addr >> 12)); in si_pcie_gart_enable()
4364 si_pcie_gart_tlb_flush(rdev); in si_pcie_gart_enable()
4366 (unsigned)(rdev->mc.gtt_size >> 20), in si_pcie_gart_enable()
4367 (unsigned long long)rdev->gart.table_addr); in si_pcie_gart_enable()
4368 rdev->gart.ready = true; in si_pcie_gart_enable()
4372 static void si_pcie_gart_disable(struct radeon_device *rdev) in si_pcie_gart_disable() argument
4382 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); in si_pcie_gart_disable()
4399 radeon_gart_table_vram_unpin(rdev); in si_pcie_gart_disable()
4402 static void si_pcie_gart_fini(struct radeon_device *rdev) in si_pcie_gart_fini() argument
4404 si_pcie_gart_disable(rdev); in si_pcie_gart_fini()
4405 radeon_gart_table_vram_free(rdev); in si_pcie_gart_fini()
4406 radeon_gart_fini(rdev); in si_pcie_gart_fini()
4456 static int si_vm_packet3_ce_check(struct radeon_device *rdev, in si_vm_packet3_ce_check() argument
4529 static int si_vm_packet3_gfx_check(struct radeon_device *rdev, in si_vm_packet3_gfx_check() argument
4647 static int si_vm_packet3_compute_check(struct radeon_device *rdev, in si_vm_packet3_compute_check() argument
4735 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) in si_ib_parse() argument
4748 dev_err(rdev->dev, "Packet0 not allowed!\n"); in si_ib_parse()
4757 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); in si_ib_parse()
4761 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); in si_ib_parse()
4765 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); in si_ib_parse()
4768 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); in si_ib_parse()
4776 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); in si_ib_parse()
4797 int si_vm_init(struct radeon_device *rdev) in si_vm_init() argument
4800 rdev->vm_manager.nvm = 16; in si_vm_init()
4802 rdev->vm_manager.vram_base_offset = 0; in si_vm_init()
4807 void si_vm_fini(struct radeon_device *rdev) in si_vm_fini() argument
4820 static void si_vm_decode_fault(struct radeon_device *rdev, in si_vm_decode_fault() argument
4828 if (rdev->family == CHIP_TAHITI) { in si_vm_decode_fault()
5075 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, in si_vm_flush() argument
5127 static void si_wait_for_rlc_serdes(struct radeon_device *rdev) in si_wait_for_rlc_serdes() argument
5131 for (i = 0; i < rdev->usec_timeout; i++) { in si_wait_for_rlc_serdes()
5137 for (i = 0; i < rdev->usec_timeout; i++) { in si_wait_for_rlc_serdes()
5144 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, in si_enable_gui_idle_interrupt() argument
5162 for (i = 0; i < rdev->usec_timeout; i++) { in si_enable_gui_idle_interrupt()
5170 static void si_set_uvd_dcm(struct radeon_device *rdev, in si_set_uvd_dcm() argument
5191 void si_init_uvd_internal_cg(struct radeon_device *rdev) in si_init_uvd_internal_cg() argument
5196 si_set_uvd_dcm(rdev, false); in si_init_uvd_internal_cg()
5204 static u32 si_halt_rlc(struct radeon_device *rdev) in si_halt_rlc() argument
5214 si_wait_for_rlc_serdes(rdev); in si_halt_rlc()
5220 static void si_update_rlc(struct radeon_device *rdev, u32 rlc) in si_update_rlc() argument
5229 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) in si_enable_dma_pg() argument
5234 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA)) in si_enable_dma_pg()
5242 static void si_init_dma_pg(struct radeon_device *rdev) in si_init_dma_pg() argument
5253 static void si_enable_gfx_cgpg(struct radeon_device *rdev, in si_enable_gfx_cgpg() argument
5258 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { in si_enable_gfx_cgpg()
5278 static void si_init_gfx_cgpg(struct radeon_device *rdev) in si_init_gfx_cgpg() argument
5282 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); in si_init_gfx_cgpg()
5288 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); in si_init_gfx_cgpg()
5298 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) in si_get_cu_active_bitmap() argument
5303 si_select_se_sh(rdev, se, sh); in si_get_cu_active_bitmap()
5306 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); in si_get_cu_active_bitmap()
5313 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { in si_get_cu_active_bitmap()
5321 static void si_init_ao_cu_mask(struct radeon_device *rdev) in si_init_ao_cu_mask() argument
5327 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { in si_init_ao_cu_mask()
5328 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { in si_init_ao_cu_mask()
5332 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { in si_init_ao_cu_mask()
5333 if (si_get_cu_active_bitmap(rdev, i, j) & mask) { in si_init_ao_cu_mask()
5354 static void si_enable_cgcg(struct radeon_device *rdev, in si_enable_cgcg() argument
5361 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { in si_enable_cgcg()
5362 si_enable_gui_idle_interrupt(rdev, true); in si_enable_cgcg()
5366 tmp = si_halt_rlc(rdev); in si_enable_cgcg()
5372 si_wait_for_rlc_serdes(rdev); in si_enable_cgcg()
5374 si_update_rlc(rdev, tmp); in si_enable_cgcg()
5380 si_enable_gui_idle_interrupt(rdev, false); in si_enable_cgcg()
5394 static void si_enable_mgcg(struct radeon_device *rdev, in si_enable_mgcg() argument
5399 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { in si_enable_mgcg()
5405 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { in si_enable_mgcg()
5417 tmp = si_halt_rlc(rdev); in si_enable_mgcg()
5423 si_update_rlc(rdev, tmp); in si_enable_mgcg()
5440 tmp = si_halt_rlc(rdev); in si_enable_mgcg()
5446 si_update_rlc(rdev, tmp); in si_enable_mgcg()
5450 static void si_enable_uvd_mgcg(struct radeon_device *rdev, in si_enable_uvd_mgcg() argument
5455 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { in si_enable_uvd_mgcg()
5495 static void si_enable_mc_ls(struct radeon_device *rdev, in si_enable_mc_ls() argument
5503 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) in si_enable_mc_ls()
5512 static void si_enable_mc_mgcg(struct radeon_device *rdev, in si_enable_mc_mgcg() argument
5520 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) in si_enable_mc_mgcg()
5529 static void si_enable_dma_mgcg(struct radeon_device *rdev, in si_enable_dma_mgcg() argument
5535 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { in si_enable_dma_mgcg()
5566 static void si_enable_bif_mgls(struct radeon_device *rdev, in si_enable_bif_mgls() argument
5573 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) in si_enable_bif_mgls()
5584 static void si_enable_hdp_mgcg(struct radeon_device *rdev, in si_enable_hdp_mgcg() argument
5591 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) in si_enable_hdp_mgcg()
5600 static void si_enable_hdp_ls(struct radeon_device *rdev, in si_enable_hdp_ls() argument
5607 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) in si_enable_hdp_ls()
5616 static void si_update_cg(struct radeon_device *rdev, in si_update_cg() argument
5620 si_enable_gui_idle_interrupt(rdev, false); in si_update_cg()
5623 si_enable_mgcg(rdev, true); in si_update_cg()
5624 si_enable_cgcg(rdev, true); in si_update_cg()
5626 si_enable_cgcg(rdev, false); in si_update_cg()
5627 si_enable_mgcg(rdev, false); in si_update_cg()
5629 si_enable_gui_idle_interrupt(rdev, true); in si_update_cg()
5633 si_enable_mc_mgcg(rdev, enable); in si_update_cg()
5634 si_enable_mc_ls(rdev, enable); in si_update_cg()
5638 si_enable_dma_mgcg(rdev, enable); in si_update_cg()
5642 si_enable_bif_mgls(rdev, enable); in si_update_cg()
5646 if (rdev->has_uvd) { in si_update_cg()
5647 si_enable_uvd_mgcg(rdev, enable); in si_update_cg()
5652 si_enable_hdp_mgcg(rdev, enable); in si_update_cg()
5653 si_enable_hdp_ls(rdev, enable); in si_update_cg()
5657 static void si_init_cg(struct radeon_device *rdev) in si_init_cg() argument
5659 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | in si_init_cg()
5664 if (rdev->has_uvd) { in si_init_cg()
5665 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); in si_init_cg()
5666 si_init_uvd_internal_cg(rdev); in si_init_cg()
5670 static void si_fini_cg(struct radeon_device *rdev) in si_fini_cg() argument
5672 if (rdev->has_uvd) { in si_fini_cg()
5673 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); in si_fini_cg()
5675 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | in si_fini_cg()
5682 u32 si_get_csb_size(struct radeon_device *rdev) in si_get_csb_size() argument
5688 if (rdev->rlc.cs_data == NULL) in si_get_csb_size()
5696 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in si_get_csb_size()
5714 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) in si_get_csb_buffer() argument
5720 if (rdev->rlc.cs_data == NULL) in si_get_csb_buffer()
5732 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in si_get_csb_buffer()
5748 switch (rdev->family) { in si_get_csb_buffer()
5774 static void si_init_pg(struct radeon_device *rdev) in si_init_pg() argument
5776 if (rdev->pg_flags) { in si_init_pg()
5777 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) { in si_init_pg()
5778 si_init_dma_pg(rdev); in si_init_pg()
5780 si_init_ao_cu_mask(rdev); in si_init_pg()
5781 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { in si_init_pg()
5782 si_init_gfx_cgpg(rdev); in si_init_pg()
5784 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); in si_init_pg()
5785 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); in si_init_pg()
5787 si_enable_dma_pg(rdev, true); in si_init_pg()
5788 si_enable_gfx_cgpg(rdev, true); in si_init_pg()
5790 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); in si_init_pg()
5791 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); in si_init_pg()
5795 static void si_fini_pg(struct radeon_device *rdev) in si_fini_pg() argument
5797 if (rdev->pg_flags) { in si_fini_pg()
5798 si_enable_dma_pg(rdev, false); in si_fini_pg()
5799 si_enable_gfx_cgpg(rdev, false); in si_fini_pg()
5806 void si_rlc_reset(struct radeon_device *rdev) in si_rlc_reset() argument
5818 static void si_rlc_stop(struct radeon_device *rdev) in si_rlc_stop() argument
5822 si_enable_gui_idle_interrupt(rdev, false); in si_rlc_stop()
5824 si_wait_for_rlc_serdes(rdev); in si_rlc_stop()
5827 static void si_rlc_start(struct radeon_device *rdev) in si_rlc_start() argument
5831 si_enable_gui_idle_interrupt(rdev, true); in si_rlc_start()
5836 static bool si_lbpw_supported(struct radeon_device *rdev) in si_lbpw_supported() argument
5847 static void si_enable_lbpw(struct radeon_device *rdev, bool enable) in si_enable_lbpw() argument
5859 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); in si_enable_lbpw()
5864 static int si_rlc_resume(struct radeon_device *rdev) in si_rlc_resume() argument
5868 if (!rdev->rlc_fw) in si_rlc_resume()
5871 si_rlc_stop(rdev); in si_rlc_resume()
5873 si_rlc_reset(rdev); in si_rlc_resume()
5875 si_init_pg(rdev); in si_rlc_resume()
5877 si_init_cg(rdev); in si_rlc_resume()
5889 if (rdev->new_fw) { in si_rlc_resume()
5891 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; in si_rlc_resume()
5894 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in si_rlc_resume()
5904 (const __be32 *)rdev->rlc_fw->data; in si_rlc_resume()
5912 si_enable_lbpw(rdev, si_lbpw_supported(rdev)); in si_rlc_resume()
5914 si_rlc_start(rdev); in si_rlc_resume()
5919 static void si_enable_interrupts(struct radeon_device *rdev) in si_enable_interrupts() argument
5928 rdev->ih.enabled = true; in si_enable_interrupts()
5931 static void si_disable_interrupts(struct radeon_device *rdev) in si_disable_interrupts() argument
5943 rdev->ih.enabled = false; in si_disable_interrupts()
5944 rdev->ih.rptr = 0; in si_disable_interrupts()
5947 static void si_disable_interrupt_state(struct radeon_device *rdev) in si_disable_interrupt_state() argument
5963 for (i = 0; i < rdev->num_crtc; i++) in si_disable_interrupt_state()
5965 for (i = 0; i < rdev->num_crtc; i++) in si_disable_interrupt_state()
5968 if (!ASIC_IS_NODCE(rdev)) { in si_disable_interrupt_state()
5977 static int si_irq_init(struct radeon_device *rdev) in si_irq_init() argument
5984 ret = r600_ih_ring_alloc(rdev); in si_irq_init()
5989 si_disable_interrupts(rdev); in si_irq_init()
5992 ret = si_rlc_resume(rdev); in si_irq_init()
5994 r600_ih_ring_fini(rdev); in si_irq_init()
6000 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8); in si_irq_init()
6010 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); in si_irq_init()
6011 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); in si_irq_init()
6017 if (rdev->wb.enabled) in si_irq_init()
6021 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); in si_irq_init()
6022 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); in si_irq_init()
6033 if (rdev->msi_enabled) in si_irq_init()
6038 si_disable_interrupt_state(rdev); in si_irq_init()
6040 pci_set_master(rdev->pdev); in si_irq_init()
6043 si_enable_interrupts(rdev); in si_irq_init()
6049 int si_irq_set(struct radeon_device *rdev) in si_irq_set() argument
6058 if (!rdev->irq.installed) { in si_irq_set()
6063 if (!rdev->ih.enabled) { in si_irq_set()
6064 si_disable_interrupts(rdev); in si_irq_set()
6066 si_disable_interrupt_state(rdev); in si_irq_set()
6080 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { in si_irq_set()
6084 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { in si_irq_set()
6088 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { in si_irq_set()
6092 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { in si_irq_set()
6097 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { in si_irq_set()
6111 if (rdev->irq.dpm_thermal) { in si_irq_set()
6116 for (i = 0; i < rdev->num_crtc; i++) { in si_irq_set()
6118 rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK, in si_irq_set()
6119 rdev->irq.crtc_vblank_int[i] || in si_irq_set()
6120 atomic_read(&rdev->irq.pflip[i]), "vblank", i); in si_irq_set()
6123 for (i = 0; i < rdev->num_crtc; i++) in si_irq_set()
6126 if (!ASIC_IS_NODCE(rdev)) { in si_irq_set()
6129 rdev, DC_HPDx_INT_CONTROL(i), in si_irq_set()
6131 rdev->irq.hpd[i], "HPD", i); in si_irq_set()
6144 static inline void si_irq_ack(struct radeon_device *rdev) in si_irq_ack() argument
6147 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; in si_irq_ack()
6148 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int; in si_irq_ack()
6150 if (ASIC_IS_NODCE(rdev)) in si_irq_ack()
6155 if (i < rdev->num_crtc) in si_irq_ack()
6160 for (i = 0; i < rdev->num_crtc; i += 2) { in si_irq_ack()
6188 static void si_irq_disable(struct radeon_device *rdev) in si_irq_disable() argument
6190 si_disable_interrupts(rdev); in si_irq_disable()
6193 si_irq_ack(rdev); in si_irq_disable()
6194 si_disable_interrupt_state(rdev); in si_irq_disable()
6197 static void si_irq_suspend(struct radeon_device *rdev) in si_irq_suspend() argument
6199 si_irq_disable(rdev); in si_irq_suspend()
6200 si_rlc_stop(rdev); in si_irq_suspend()
6203 static void si_irq_fini(struct radeon_device *rdev) in si_irq_fini() argument
6205 si_irq_suspend(rdev); in si_irq_fini()
6206 r600_ih_ring_fini(rdev); in si_irq_fini()
6209 static inline u32 si_get_ih_wptr(struct radeon_device *rdev) in si_get_ih_wptr() argument
6213 if (rdev->wb.enabled) in si_get_ih_wptr()
6214 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); in si_get_ih_wptr()
6224 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", in si_get_ih_wptr()
6225 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); in si_get_ih_wptr()
6226 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; in si_get_ih_wptr()
6231 return (wptr & rdev->ih.ptr_mask); in si_get_ih_wptr()
6244 int si_irq_process(struct radeon_device *rdev) in si_irq_process() argument
6246 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; in si_irq_process()
6259 if (!rdev->ih.enabled || rdev->shutdown) in si_irq_process()
6262 wptr = si_get_ih_wptr(rdev); in si_irq_process()
6266 if (atomic_xchg(&rdev->ih.lock, 1)) in si_irq_process()
6269 rptr = rdev->ih.rptr; in si_irq_process()
6276 si_irq_ack(rdev); in si_irq_process()
6281 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; in si_irq_process()
6282 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; in si_irq_process()
6283 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; in si_irq_process()
6298 if (rdev->irq.crtc_vblank_int[crtc_idx]) { in si_irq_process()
6299 drm_handle_vblank(rdev->ddev, crtc_idx); in si_irq_process()
6300 rdev->pm.vblank_sync = true; in si_irq_process()
6301 wake_up(&rdev->irq.vblank_queue); in si_irq_process()
6303 if (atomic_read(&rdev->irq.pflip[crtc_idx])) { in si_irq_process()
6304 radeon_crtc_handle_vblank(rdev, in si_irq_process()
6334 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); in si_irq_process()
6367 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); in si_irq_process()
6377 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); in si_irq_process()
6378 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in si_irq_process()
6380 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in si_irq_process()
6382 si_vm_decode_fault(rdev, status, addr); in si_irq_process()
6385 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in si_irq_process()
6388 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in si_irq_process()
6391 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in si_irq_process()
6397 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in si_irq_process()
6400 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in si_irq_process()
6403 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in si_irq_process()
6409 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); in si_irq_process()
6413 rdev->pm.dpm.thermal.high_to_low = false; in si_irq_process()
6418 rdev->pm.dpm.thermal.high_to_low = true; in si_irq_process()
6426 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in si_irq_process()
6435 rptr &= rdev->ih.ptr_mask; in si_irq_process()
6439 schedule_work(&rdev->dp_work); in si_irq_process()
6441 schedule_delayed_work(&rdev->hotplug_work, 0); in si_irq_process()
6442 if (queue_thermal && rdev->pm.dpm_enabled) in si_irq_process()
6443 schedule_work(&rdev->pm.dpm.thermal.work); in si_irq_process()
6444 rdev->ih.rptr = rptr; in si_irq_process()
6445 atomic_set(&rdev->ih.lock, 0); in si_irq_process()
6448 wptr = si_get_ih_wptr(rdev); in si_irq_process()
6458 static void si_uvd_init(struct radeon_device *rdev) in si_uvd_init() argument
6462 if (!rdev->has_uvd) in si_uvd_init()
6465 r = radeon_uvd_init(rdev); in si_uvd_init()
6467 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); in si_uvd_init()
6474 rdev->has_uvd = false; in si_uvd_init()
6477 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; in si_uvd_init()
6478 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); in si_uvd_init()
6481 static void si_uvd_start(struct radeon_device *rdev) in si_uvd_start() argument
6485 if (!rdev->has_uvd) in si_uvd_start()
6488 r = uvd_v2_2_resume(rdev); in si_uvd_start()
6490 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); in si_uvd_start()
6493 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); in si_uvd_start()
6495 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); in si_uvd_start()
6501 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; in si_uvd_start()
6504 static void si_uvd_resume(struct radeon_device *rdev) in si_uvd_resume() argument
6509 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) in si_uvd_resume()
6512 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; in si_uvd_resume()
6513 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); in si_uvd_resume()
6515 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); in si_uvd_resume()
6518 r = uvd_v1_0_init(rdev); in si_uvd_resume()
6520 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); in si_uvd_resume()
6525 static void si_vce_init(struct radeon_device *rdev) in si_vce_init() argument
6529 if (!rdev->has_vce) in si_vce_init()
6532 r = radeon_vce_init(rdev); in si_vce_init()
6534 dev_err(rdev->dev, "failed VCE (%d) init.\n", r); in si_vce_init()
6541 rdev->has_vce = false; in si_vce_init()
6544 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL; in si_vce_init()
6545 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096); in si_vce_init()
6546 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL; in si_vce_init()
6547 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096); in si_vce_init()
6550 static void si_vce_start(struct radeon_device *rdev) in si_vce_start() argument
6554 if (!rdev->has_vce) in si_vce_start()
6557 r = radeon_vce_resume(rdev); in si_vce_start()
6559 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); in si_vce_start()
6562 r = vce_v1_0_resume(rdev); in si_vce_start()
6564 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); in si_vce_start()
6567 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX); in si_vce_start()
6569 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r); in si_vce_start()
6572 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX); in si_vce_start()
6574 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r); in si_vce_start()
6580 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; in si_vce_start()
6581 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; in si_vce_start()
6584 static void si_vce_resume(struct radeon_device *rdev) in si_vce_resume() argument
6589 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size) in si_vce_resume()
6592 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; in si_vce_resume()
6593 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); in si_vce_resume()
6595 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); in si_vce_resume()
6598 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; in si_vce_resume()
6599 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); in si_vce_resume()
6601 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); in si_vce_resume()
6604 r = vce_v1_0_init(rdev); in si_vce_resume()
6606 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r); in si_vce_resume()
6611 static int si_startup(struct radeon_device *rdev) in si_startup() argument
6617 si_pcie_gen3_enable(rdev); in si_startup()
6619 si_program_aspm(rdev); in si_startup()
6622 r = r600_vram_scratch_init(rdev); in si_startup()
6626 si_mc_program(rdev); in si_startup()
6628 if (!rdev->pm.dpm_enabled) { in si_startup()
6629 r = si_mc_load_microcode(rdev); in si_startup()
6636 r = si_pcie_gart_enable(rdev); in si_startup()
6639 si_gpu_init(rdev); in si_startup()
6642 if (rdev->family == CHIP_VERDE) { in si_startup()
6643 rdev->rlc.reg_list = verde_rlc_save_restore_register_list; in si_startup()
6644 rdev->rlc.reg_list_size = in si_startup()
6647 rdev->rlc.cs_data = si_cs_data; in si_startup()
6648 r = sumo_rlc_init(rdev); in si_startup()
6655 r = radeon_wb_init(rdev); in si_startup()
6659 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); in si_startup()
6661 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in si_startup()
6665 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in si_startup()
6667 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in si_startup()
6671 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in si_startup()
6673 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in si_startup()
6677 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); in si_startup()
6679 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in si_startup()
6683 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in si_startup()
6685 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in si_startup()
6689 si_uvd_start(rdev); in si_startup()
6690 si_vce_start(rdev); in si_startup()
6693 if (!rdev->irq.installed) { in si_startup()
6694 r = radeon_irq_kms_init(rdev); in si_startup()
6699 r = si_irq_init(rdev); in si_startup()
6702 radeon_irq_kms_fini(rdev); in si_startup()
6705 si_irq_set(rdev); in si_startup()
6707 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_startup()
6708 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, in si_startup()
6713 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in si_startup()
6714 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, in si_startup()
6719 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in si_startup()
6720 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, in si_startup()
6725 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in si_startup()
6726 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, in si_startup()
6731 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in si_startup()
6732 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, in si_startup()
6737 r = si_cp_load_microcode(rdev); in si_startup()
6740 r = si_cp_resume(rdev); in si_startup()
6744 r = cayman_dma_resume(rdev); in si_startup()
6748 si_uvd_resume(rdev); in si_startup()
6749 si_vce_resume(rdev); in si_startup()
6751 r = radeon_ib_pool_init(rdev); in si_startup()
6753 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); in si_startup()
6757 r = radeon_vm_manager_init(rdev); in si_startup()
6759 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); in si_startup()
6763 r = radeon_audio_init(rdev); in si_startup()
6770 int si_resume(struct radeon_device *rdev) in si_resume() argument
6779 atom_asic_init(rdev->mode_info.atom_context); in si_resume()
6782 si_init_golden_registers(rdev); in si_resume()
6784 if (rdev->pm.pm_method == PM_METHOD_DPM) in si_resume()
6785 radeon_pm_resume(rdev); in si_resume()
6787 rdev->accel_working = true; in si_resume()
6788 r = si_startup(rdev); in si_resume()
6791 rdev->accel_working = false; in si_resume()
6799 int si_suspend(struct radeon_device *rdev) in si_suspend() argument
6801 radeon_pm_suspend(rdev); in si_suspend()
6802 radeon_audio_fini(rdev); in si_suspend()
6803 radeon_vm_manager_fini(rdev); in si_suspend()
6804 si_cp_enable(rdev, false); in si_suspend()
6805 cayman_dma_stop(rdev); in si_suspend()
6806 if (rdev->has_uvd) { in si_suspend()
6807 uvd_v1_0_fini(rdev); in si_suspend()
6808 radeon_uvd_suspend(rdev); in si_suspend()
6810 if (rdev->has_vce) in si_suspend()
6811 radeon_vce_suspend(rdev); in si_suspend()
6812 si_fini_pg(rdev); in si_suspend()
6813 si_fini_cg(rdev); in si_suspend()
6814 si_irq_suspend(rdev); in si_suspend()
6815 radeon_wb_disable(rdev); in si_suspend()
6816 si_pcie_gart_disable(rdev); in si_suspend()
6826 int si_init(struct radeon_device *rdev) in si_init() argument
6828 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_init()
6832 if (!radeon_get_bios(rdev)) { in si_init()
6833 if (ASIC_IS_AVIVO(rdev)) in si_init()
6837 if (!rdev->is_atom_bios) { in si_init()
6838 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); in si_init()
6841 r = radeon_atombios_init(rdev); in si_init()
6846 if (!radeon_card_posted(rdev)) { in si_init()
6847 if (!rdev->bios) { in si_init()
6848 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); in si_init()
6852 atom_asic_init(rdev->mode_info.atom_context); in si_init()
6855 si_init_golden_registers(rdev); in si_init()
6857 si_scratch_init(rdev); in si_init()
6859 radeon_surface_init(rdev); in si_init()
6861 radeon_get_clock_info(rdev->ddev); in si_init()
6864 radeon_fence_driver_init(rdev); in si_init()
6867 r = si_mc_init(rdev); in si_init()
6871 r = radeon_bo_init(rdev); in si_init()
6875 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || in si_init()
6876 !rdev->rlc_fw || !rdev->mc_fw) { in si_init()
6877 r = si_init_microcode(rdev); in si_init()
6885 radeon_pm_init(rdev); in si_init()
6887 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in si_init()
6889 r600_ring_init(rdev, ring, 1024 * 1024); in si_init()
6891 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in si_init()
6893 r600_ring_init(rdev, ring, 1024 * 1024); in si_init()
6895 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in si_init()
6897 r600_ring_init(rdev, ring, 1024 * 1024); in si_init()
6899 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in si_init()
6901 r600_ring_init(rdev, ring, 64 * 1024); in si_init()
6903 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in si_init()
6905 r600_ring_init(rdev, ring, 64 * 1024); in si_init()
6907 si_uvd_init(rdev); in si_init()
6908 si_vce_init(rdev); in si_init()
6910 rdev->ih.ring_obj = NULL; in si_init()
6911 r600_ih_ring_init(rdev, 64 * 1024); in si_init()
6913 r = r600_pcie_gart_init(rdev); in si_init()
6917 rdev->accel_working = true; in si_init()
6918 r = si_startup(rdev); in si_init()
6920 dev_err(rdev->dev, "disabling GPU acceleration\n"); in si_init()
6921 si_cp_fini(rdev); in si_init()
6922 cayman_dma_fini(rdev); in si_init()
6923 si_irq_fini(rdev); in si_init()
6924 sumo_rlc_fini(rdev); in si_init()
6925 radeon_wb_fini(rdev); in si_init()
6926 radeon_ib_pool_fini(rdev); in si_init()
6927 radeon_vm_manager_fini(rdev); in si_init()
6928 radeon_irq_kms_fini(rdev); in si_init()
6929 si_pcie_gart_fini(rdev); in si_init()
6930 rdev->accel_working = false; in si_init()
6937 if (!rdev->mc_fw) { in si_init()
6945 void si_fini(struct radeon_device *rdev) in si_fini() argument
6947 radeon_pm_fini(rdev); in si_fini()
6948 si_cp_fini(rdev); in si_fini()
6949 cayman_dma_fini(rdev); in si_fini()
6950 si_fini_pg(rdev); in si_fini()
6951 si_fini_cg(rdev); in si_fini()
6952 si_irq_fini(rdev); in si_fini()
6953 sumo_rlc_fini(rdev); in si_fini()
6954 radeon_wb_fini(rdev); in si_fini()
6955 radeon_vm_manager_fini(rdev); in si_fini()
6956 radeon_ib_pool_fini(rdev); in si_fini()
6957 radeon_irq_kms_fini(rdev); in si_fini()
6958 if (rdev->has_uvd) { in si_fini()
6959 uvd_v1_0_fini(rdev); in si_fini()
6960 radeon_uvd_fini(rdev); in si_fini()
6962 if (rdev->has_vce) in si_fini()
6963 radeon_vce_fini(rdev); in si_fini()
6964 si_pcie_gart_fini(rdev); in si_fini()
6965 r600_vram_scratch_fini(rdev); in si_fini()
6966 radeon_gem_fini(rdev); in si_fini()
6967 radeon_fence_driver_fini(rdev); in si_fini()
6968 radeon_bo_fini(rdev); in si_fini()
6969 radeon_atombios_fini(rdev); in si_fini()
6970 kfree(rdev->bios); in si_fini()
6971 rdev->bios = NULL; in si_fini()
6982 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) in si_get_gpu_clock_counter() argument
6986 mutex_lock(&rdev->gpu_clock_mutex); in si_get_gpu_clock_counter()
6990 mutex_unlock(&rdev->gpu_clock_mutex); in si_get_gpu_clock_counter()
6994 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) in si_set_uvd_clocks() argument
7012 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, in si_set_uvd_clocks()
7032 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); in si_set_uvd_clocks()
7069 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); in si_set_uvd_clocks()
7083 static void si_pcie_gen3_enable(struct radeon_device *rdev) in si_pcie_gen3_enable() argument
7085 struct pci_dev *root = rdev->pdev->bus->self; in si_pcie_gen3_enable()
7091 if (pci_is_root_bus(rdev->pdev->bus)) in si_pcie_gen3_enable()
7097 if (rdev->flags & RADEON_IS_IGP) in si_pcie_gen3_enable()
7100 if (!(rdev->flags & RADEON_IS_PCIE)) in si_pcie_gen3_enable()
7128 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev)) in si_pcie_gen3_enable()
7139 pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD); in si_pcie_gen3_enable()
7157 pcie_capability_read_word(rdev->pdev, in si_pcie_gen3_enable()
7165 pcie_capability_read_word(rdev->pdev, in si_pcie_gen3_enable()
7171 pcie_capability_read_word(rdev->pdev, in si_pcie_gen3_enable()
7190 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL, in si_pcie_gen3_enable()
7207 pcie_capability_read_word(rdev->pdev, in si_pcie_gen3_enable()
7215 pcie_capability_write_word(rdev->pdev, in si_pcie_gen3_enable()
7231 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16); in si_pcie_gen3_enable()
7239 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16); in si_pcie_gen3_enable()
7245 for (i = 0; i < rdev->usec_timeout; i++) { in si_pcie_gen3_enable()
7253 static void si_program_aspm(struct radeon_device *rdev) in si_program_aspm() argument
7262 if (!(rdev->flags & RADEON_IS_PCIE)) in si_program_aspm()
7320 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { in si_program_aspm()
7369 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) in si_program_aspm()
7376 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) in si_program_aspm()
7382 !pci_is_root_bus(rdev->pdev->bus)) { in si_program_aspm()
7383 struct pci_dev *root = rdev->pdev->bus->self; in si_program_aspm()
7458 static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev) in si_vce_send_vcepll_ctlreq() argument
7489 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) in si_set_vce_clocks() argument
7510 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000, in si_set_vce_clocks()
7533 r = si_vce_send_vcepll_ctlreq(rdev); in si_set_vce_clocks()
7565 r = si_vce_send_vcepll_ctlreq(rdev); in si_set_vce_clocks()