49 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)  in eg_cg_rreg()  argument
54 	spin_lock_irqsave(&rdev->cg_idx_lock, flags);  in eg_cg_rreg()
57 	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);  in eg_cg_rreg()
61 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_cg_wreg()  argument
65 	spin_lock_irqsave(&rdev->cg_idx_lock, flags);  in eg_cg_wreg()
68 	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);  in eg_cg_wreg()
71 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)  in eg_pif_phy0_rreg()  argument
76 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_rreg()
79 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_rreg()
83 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_pif_phy0_wreg()  argument
87 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_wreg()
90 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_wreg()
93 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)  in eg_pif_phy1_rreg()  argument
98 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_rreg()
101 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_rreg()
105 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_pif_phy1_wreg()  argument
109 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_wreg()
112 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_wreg()
212 static void evergreen_gpu_init(struct radeon_device *rdev);
213 void evergreen_fini(struct radeon_device *rdev);
214 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
215 void evergreen_program_aspm(struct radeon_device *rdev);
216 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
218 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
220 void cik_init_cp_pg_table(struct radeon_device *rdev);
222 extern u32 si_get_csb_size(struct radeon_device *rdev);
223 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
224 extern u32 cik_get_csb_size(struct radeon_device *rdev);
225 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
226 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
1000 static void evergreen_init_golden_registers(struct radeon_device *rdev)  in evergreen_init_golden_registers()  argument
1002 	switch (rdev->family) {  in evergreen_init_golden_registers()
1005 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1008 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1011 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1016 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1019 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1022 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1027 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1030 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1033 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1038 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1041 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1044 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1049 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1054 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1059 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1062 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1067 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1072 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1077 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1096 int evergreen_get_allowed_info_register(struct radeon_device *rdev,  in evergreen_get_allowed_info_register()  argument
1145 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,  in sumo_set_uvd_clock()  argument
1151 	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,  in sumo_set_uvd_clock()
1169 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in sumo_set_uvd_clocks()  argument
1174 	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);  in sumo_set_uvd_clocks()
1180 	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);  in sumo_set_uvd_clocks()
1192 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in evergreen_set_uvd_clocks()  argument
1212 	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,  in evergreen_set_uvd_clocks()
1230 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in evergreen_set_uvd_clocks()
1267 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in evergreen_set_uvd_clocks()
1281 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)  in evergreen_fix_pci_max_read_req_size()  argument
1286 	readrq = pcie_get_readrq(rdev->pdev);  in evergreen_fix_pci_max_read_req_size()
1292 		pcie_set_readrq(rdev->pdev, 512);  in evergreen_fix_pci_max_read_req_size()
1298 	struct radeon_device *rdev = dev->dev_private;  in dce4_program_fmt()  local
1351 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)  in dce4_is_in_vblank()  argument
1359 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)  in dce4_is_counter_moving()  argument
1380 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)  in dce4_wait_for_vblank()  argument
1384 	if (crtc >= rdev->num_crtc)  in dce4_wait_for_vblank()
1393 	while (dce4_is_in_vblank(rdev, crtc)) {  in dce4_wait_for_vblank()
1395 			if (!dce4_is_counter_moving(rdev, crtc))  in dce4_wait_for_vblank()
1400 	while (!dce4_is_in_vblank(rdev, crtc)) {  in dce4_wait_for_vblank()
1402 			if (!dce4_is_counter_moving(rdev, crtc))  in dce4_wait_for_vblank()
1418 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,  in evergreen_page_flip()  argument
1421 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];  in evergreen_page_flip()
1442 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)  in evergreen_page_flip_pending()  argument
1444 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];  in evergreen_page_flip_pending()
1452 int evergreen_get_temp(struct radeon_device *rdev)  in evergreen_get_temp()  argument
1457 	if (rdev->family == CHIP_JUNIPER) {  in evergreen_get_temp()
1490 int sumo_get_temp(struct radeon_device *rdev)  in sumo_get_temp()  argument
1507 void sumo_pm_init_profile(struct radeon_device *rdev)  in sumo_pm_init_profile()  argument
1512 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in sumo_pm_init_profile()
1513 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in sumo_pm_init_profile()
1514 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1515 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1518 	if (rdev->flags & RADEON_IS_MOBILITY)  in sumo_pm_init_profile()
1519 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);  in sumo_pm_init_profile()
1521 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in sumo_pm_init_profile()
1523 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1524 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1525 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1526 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1528 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1529 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1530 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1531 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1533 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1534 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1535 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1536 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1538 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1539 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1540 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1541 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1544 	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in sumo_pm_init_profile()
1545 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1546 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1547 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1548 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =  in sumo_pm_init_profile()
1549 		rdev->pm.power_state[idx].num_clock_modes - 1;  in sumo_pm_init_profile()
1551 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1552 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1553 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1554 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =  in sumo_pm_init_profile()
1555 		rdev->pm.power_state[idx].num_clock_modes - 1;  in sumo_pm_init_profile()
1567 void btc_pm_init_profile(struct radeon_device *rdev)  in btc_pm_init_profile()  argument
1572 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in btc_pm_init_profile()
1573 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in btc_pm_init_profile()
1574 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1575 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1580 	if (rdev->flags & RADEON_IS_MOBILITY)  in btc_pm_init_profile()
1581 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);  in btc_pm_init_profile()
1583 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in btc_pm_init_profile()
1585 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1586 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1587 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1588 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in btc_pm_init_profile()
1590 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1591 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1592 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1593 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;  in btc_pm_init_profile()
1595 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1596 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1597 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1598 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1600 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1601 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1602 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1603 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in btc_pm_init_profile()
1605 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1606 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1607 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1608 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;  in btc_pm_init_profile()
1610 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1611 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1612 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1613 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1624 void evergreen_pm_misc(struct radeon_device *rdev)  in evergreen_pm_misc()  argument
1626 	int req_ps_idx = rdev->pm.requested_power_state_index;  in evergreen_pm_misc()
1627 	int req_cm_idx = rdev->pm.requested_clock_mode_index;  in evergreen_pm_misc()
1628 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];  in evergreen_pm_misc()
1635 		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {  in evergreen_pm_misc()
1636 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);  in evergreen_pm_misc()
1637 			rdev->pm.current_vddc = voltage->voltage;  in evergreen_pm_misc()
1645 		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&  in evergreen_pm_misc()
1646 		    (rdev->family >= CHIP_BARTS) &&  in evergreen_pm_misc()
1647 		    rdev->pm.active_crtc_count &&  in evergreen_pm_misc()
1648 		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||  in evergreen_pm_misc()
1649 		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))  in evergreen_pm_misc()
1650 			voltage = &rdev->pm.power_state[req_ps_idx].  in evergreen_pm_misc()
1651 				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;  in evergreen_pm_misc()
1656 		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {  in evergreen_pm_misc()
1657 			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);  in evergreen_pm_misc()
1658 			rdev->pm.current_vddci = voltage->vddci;  in evergreen_pm_misc()
1671 void evergreen_pm_prepare(struct radeon_device *rdev)  in evergreen_pm_prepare()  argument
1673 	struct drm_device *ddev = rdev->ddev;  in evergreen_pm_prepare()
1696 void evergreen_pm_finish(struct radeon_device *rdev)  in evergreen_pm_finish()  argument
1698 	struct drm_device *ddev = rdev->ddev;  in evergreen_pm_finish()
1723 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)  in evergreen_hpd_sense()  argument
1739 void evergreen_hpd_set_polarity(struct radeon_device *rdev,  in evergreen_hpd_set_polarity()  argument
1742 	bool connected = evergreen_hpd_sense(rdev, hpd);  in evergreen_hpd_set_polarity()
1761 void evergreen_hpd_init(struct radeon_device *rdev)  in evergreen_hpd_init()  argument
1763 	struct drm_device *dev = rdev->ddev;  in evergreen_hpd_init()
1789 		radeon_hpd_set_polarity(rdev, hpd);  in evergreen_hpd_init()
1791 	radeon_irq_kms_enable_hpd(rdev, enabled);  in evergreen_hpd_init()
1802 void evergreen_hpd_fini(struct radeon_device *rdev)  in evergreen_hpd_fini()  argument
1804 	struct drm_device *dev = rdev->ddev;  in evergreen_hpd_fini()
1818 	radeon_irq_kms_disable_hpd(rdev, disabled);  in evergreen_hpd_fini()
1823 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,  in evergreen_line_buffer_adjust()  argument
1869 	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {  in evergreen_line_buffer_adjust()
1872 		for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_line_buffer_adjust()
1885 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1891 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1897 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1903 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1914 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)  in evergreen_get_number_of_dram_channels()  argument
2152 static void evergreen_program_watermarks(struct radeon_device *rdev,  in evergreen_program_watermarks()  argument
2177 		dram_channels = evergreen_get_number_of_dram_channels(rdev);  in evergreen_program_watermarks()
2180 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {  in evergreen_program_watermarks()
2182 				radeon_dpm_get_mclk(rdev, false) * 10;  in evergreen_program_watermarks()
2184 				radeon_dpm_get_sclk(rdev, false) * 10;  in evergreen_program_watermarks()
2186 			wm_high.yclk = rdev->pm.current_mclk * 10;  in evergreen_program_watermarks()
2187 			wm_high.sclk = rdev->pm.current_sclk * 10;  in evergreen_program_watermarks()
2207 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {  in evergreen_program_watermarks()
2209 				radeon_dpm_get_mclk(rdev, true) * 10;  in evergreen_program_watermarks()
2211 				radeon_dpm_get_sclk(rdev, true) * 10;  in evergreen_program_watermarks()
2213 			wm_low.yclk = rdev->pm.current_mclk * 10;  in evergreen_program_watermarks()
2214 			wm_low.sclk = rdev->pm.current_sclk * 10;  in evergreen_program_watermarks()
2243 		    (rdev->disp_priority == 2)) {  in evergreen_program_watermarks()
2250 		    (rdev->disp_priority == 2)) {  in evergreen_program_watermarks()
2321 void evergreen_bandwidth_update(struct radeon_device *rdev)  in evergreen_bandwidth_update()  argument
2328 	if (!rdev->mode_info.mode_config_initialized)  in evergreen_bandwidth_update()
2331 	radeon_update_display_priority(rdev);  in evergreen_bandwidth_update()
2333 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_bandwidth_update()
2334 		if (rdev->mode_info.crtcs[i]->base.enabled)  in evergreen_bandwidth_update()
2337 	for (i = 0; i < rdev->num_crtc; i += 2) {  in evergreen_bandwidth_update()
2338 		mode0 = &rdev->mode_info.crtcs[i]->base.mode;  in evergreen_bandwidth_update()
2339 		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;  in evergreen_bandwidth_update()
2340 		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);  in evergreen_bandwidth_update()
2341 		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);  in evergreen_bandwidth_update()
2342 		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);  in evergreen_bandwidth_update()
2343 		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);  in evergreen_bandwidth_update()
2356 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)  in evergreen_mc_wait_for_idle()  argument
2361 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_mc_wait_for_idle()
2374 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)  in evergreen_pcie_gart_tlb_flush()  argument
2382 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_pcie_gart_tlb_flush()
2397 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)  in evergreen_pcie_gart_enable()  argument
2402 	if (rdev->gart.robj == NULL) {  in evergreen_pcie_gart_enable()
2403 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in evergreen_pcie_gart_enable()
2406 	r = radeon_gart_table_vram_pin(rdev);  in evergreen_pcie_gart_enable()
2420 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_pcie_gart_enable()
2428 		if ((rdev->family == CHIP_JUNIPER) ||  in evergreen_pcie_gart_enable()
2429 		    (rdev->family == CHIP_CYPRESS) ||  in evergreen_pcie_gart_enable()
2430 		    (rdev->family == CHIP_HEMLOCK) ||  in evergreen_pcie_gart_enable()
2431 		    (rdev->family == CHIP_BARTS))  in evergreen_pcie_gart_enable()
2438 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in evergreen_pcie_gart_enable()
2439 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in evergreen_pcie_gart_enable()
2440 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in evergreen_pcie_gart_enable()
2444 			(u32)(rdev->dummy_page.addr >> 12));  in evergreen_pcie_gart_enable()
2447 	evergreen_pcie_gart_tlb_flush(rdev);  in evergreen_pcie_gart_enable()
2449 		 (unsigned)(rdev->mc.gtt_size >> 20),  in evergreen_pcie_gart_enable()
2450 		 (unsigned long long)rdev->gart.table_addr);  in evergreen_pcie_gart_enable()
2451 	rdev->gart.ready = true;  in evergreen_pcie_gart_enable()
2455 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)  in evergreen_pcie_gart_disable()  argument
2477 	radeon_gart_table_vram_unpin(rdev);  in evergreen_pcie_gart_disable()
2480 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)  in evergreen_pcie_gart_fini()  argument
2482 	evergreen_pcie_gart_disable(rdev);  in evergreen_pcie_gart_fini()
2483 	radeon_gart_table_vram_free(rdev);  in evergreen_pcie_gart_fini()
2484 	radeon_gart_fini(rdev);  in evergreen_pcie_gart_fini()
2488 static void evergreen_agp_enable(struct radeon_device *rdev)  in evergreen_agp_enable()  argument
2564 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,  in evergreen_is_dp_sst_stream_enabled()  argument
2624 static void evergreen_blank_dp_output(struct radeon_device *rdev,  in evergreen_blank_dp_output()  argument
2664 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)  in evergreen_mc_stop()  argument
2670 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_stop()
2678 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_stop()
2682 			if (ASIC_IS_DCE6(rdev)) {  in evergreen_mc_stop()
2685 					radeon_wait_for_vblank(rdev, i);  in evergreen_mc_stop()
2694 					radeon_wait_for_vblank(rdev, i);  in evergreen_mc_stop()
2702 			frame_count = radeon_get_vblank_counter(rdev, i);  in evergreen_mc_stop()
2703 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_stop()
2704 				if (radeon_get_vblank_counter(rdev, i) != frame_count)  in evergreen_mc_stop()
2715 			if (ASIC_IS_DCE5(rdev) &&  in evergreen_mc_stop()
2716 			    evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))  in evergreen_mc_stop()
2717 				evergreen_blank_dp_output(rdev, dig_fe);  in evergreen_mc_stop()
2732 	radeon_mc_wait_for_idle(rdev);  in evergreen_mc_stop()
2746 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_stop()
2762 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)  in evergreen_mc_resume()  argument
2768 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2770 		       upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2772 		       upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2774 		       (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2776 		       (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2779 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_resume()
2780 		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2781 		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2785 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2802 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_resume()
2818 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2820 			if (ASIC_IS_DCE6(rdev)) {  in evergreen_mc_resume()
2834 			frame_count = radeon_get_vblank_counter(rdev, i);  in evergreen_mc_resume()
2835 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_resume()
2836 				if (radeon_get_vblank_counter(rdev, i) != frame_count)  in evergreen_mc_resume()
2842 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_resume()
2850 void evergreen_mc_program(struct radeon_device *rdev)  in evergreen_mc_program()  argument
2866 	evergreen_mc_stop(rdev, &save);  in evergreen_mc_program()
2867 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_mc_program()
2868 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_mc_program()
2873 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_mc_program()
2874 		if (rdev->mc.vram_start < rdev->mc.gtt_start) {  in evergreen_mc_program()
2877 				rdev->mc.vram_start >> 12);  in evergreen_mc_program()
2879 				rdev->mc.gtt_end >> 12);  in evergreen_mc_program()
2883 				rdev->mc.gtt_start >> 12);  in evergreen_mc_program()
2885 				rdev->mc.vram_end >> 12);  in evergreen_mc_program()
2889 			rdev->mc.vram_start >> 12);  in evergreen_mc_program()
2891 			rdev->mc.vram_end >> 12);  in evergreen_mc_program()
2893 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);  in evergreen_mc_program()
2895 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_program()
2896 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_program()
2897 	    (rdev->family == CHIP_SUMO2)) {  in evergreen_mc_program()
2899 		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;  in evergreen_mc_program()
2900 		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;  in evergreen_mc_program()
2903 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in evergreen_mc_program()
2904 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in evergreen_mc_program()
2906 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in evergreen_mc_program()
2909 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_mc_program()
2910 		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);  in evergreen_mc_program()
2911 		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);  in evergreen_mc_program()
2912 		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);  in evergreen_mc_program()
2918 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_mc_program()
2919 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_mc_program()
2921 	evergreen_mc_resume(rdev, &save);  in evergreen_mc_program()
2924 	rv515_vga_render_disable(rdev);  in evergreen_mc_program()
2930 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in evergreen_ring_ib_execute()  argument
2932 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in evergreen_ring_ib_execute()
2945 	} else if (rdev->wb.enabled) {  in evergreen_ring_ib_execute()
2965 static int evergreen_cp_load_microcode(struct radeon_device *rdev)  in evergreen_cp_load_microcode()  argument
2970 	if (!rdev->me_fw || !rdev->pfp_fw)  in evergreen_cp_load_microcode()
2973 	r700_cp_stop(rdev);  in evergreen_cp_load_microcode()
2980 	fw_data = (const __be32 *)rdev->pfp_fw->data;  in evergreen_cp_load_microcode()
2986 	fw_data = (const __be32 *)rdev->me_fw->data;  in evergreen_cp_load_microcode()
2997 static int evergreen_cp_start(struct radeon_device *rdev)  in evergreen_cp_start()  argument
2999 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_cp_start()
3003 	r = radeon_ring_lock(rdev, ring, 7);  in evergreen_cp_start()
3011 	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);  in evergreen_cp_start()
3015 	radeon_ring_unlock_commit(rdev, ring, false);  in evergreen_cp_start()
3020 	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);  in evergreen_cp_start()
3058 	radeon_ring_unlock_commit(rdev, ring, false);  in evergreen_cp_start()
3063 static int evergreen_cp_resume(struct radeon_device *rdev)  in evergreen_cp_resume()  argument
3065 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_cp_resume()
3103 	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));  in evergreen_cp_resume()
3104 	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in evergreen_cp_resume()
3105 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in evergreen_cp_resume()
3107 	if (rdev->wb.enabled)  in evergreen_cp_resume()
3120 	evergreen_cp_start(rdev);  in evergreen_cp_resume()
3122 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);  in evergreen_cp_resume()
3133 static void evergreen_gpu_init(struct radeon_device *rdev)  in evergreen_gpu_init()  argument
3154 	switch (rdev->family) {  in evergreen_gpu_init()
3157 		rdev->config.evergreen.num_ses = 2;  in evergreen_gpu_init()
3158 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3159 		rdev->config.evergreen.max_tile_pipes = 8;  in evergreen_gpu_init()
3160 		rdev->config.evergreen.max_simds = 10;  in evergreen_gpu_init()
3161 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3162 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3163 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3164 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3165 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3166 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3167 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3168 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3169 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3170 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3171 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3173 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3174 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3175 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3179 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3180 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3181 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3182 		rdev->config.evergreen.max_simds = 10;  in evergreen_gpu_init()
3183 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3184 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3185 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3186 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3187 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3188 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3189 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3190 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3191 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3192 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3193 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3195 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3196 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3197 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3201 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3202 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3203 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3204 		rdev->config.evergreen.max_simds = 5;  in evergreen_gpu_init()
3205 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3206 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3207 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3208 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3209 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3210 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3211 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3212 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3213 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3214 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3215 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3217 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3218 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3219 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3224 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3225 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3226 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3227 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3228 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3229 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3230 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3231 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3232 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3233 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3234 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3235 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3236 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3237 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3238 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3240 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3241 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3242 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3246 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3247 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3248 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3249 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3250 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3251 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3252 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3253 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3254 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3255 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3256 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3257 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3258 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3259 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3260 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3262 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3263 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3264 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3268 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3269 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3270 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3271 		if (rdev->pdev->device == 0x9648)  in evergreen_gpu_init()
3272 			rdev->config.evergreen.max_simds = 3;  in evergreen_gpu_init()
3273 		else if ((rdev->pdev->device == 0x9647) ||  in evergreen_gpu_init()
3274 			 (rdev->pdev->device == 0x964a))  in evergreen_gpu_init()
3275 			rdev->config.evergreen.max_simds = 4;  in evergreen_gpu_init()
3277 			rdev->config.evergreen.max_simds = 5;  in evergreen_gpu_init()
3278 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3279 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3280 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3281 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3282 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3283 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3284 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3285 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3286 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3287 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3288 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3290 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3291 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3292 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3296 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3297 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3298 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3299 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3300 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3301 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3302 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3303 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3304 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3305 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3306 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3307 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3308 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3309 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3310 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3312 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3313 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3314 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3318 		rdev->config.evergreen.num_ses = 2;  in evergreen_gpu_init()
3319 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3320 		rdev->config.evergreen.max_tile_pipes = 8;  in evergreen_gpu_init()
3321 		rdev->config.evergreen.max_simds = 7;  in evergreen_gpu_init()
3322 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3323 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3324 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3325 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3326 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3327 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3328 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3329 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3330 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3331 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3332 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3334 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3335 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3336 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3340 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3341 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3342 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3343 		rdev->config.evergreen.max_simds = 6;  in evergreen_gpu_init()
3344 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3345 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3346 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3347 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3348 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3349 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3350 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3351 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3352 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3353 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3354 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3356 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3357 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3358 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3362 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3363 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3364 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3365 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3366 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3367 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3368 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3369 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3370 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3371 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3372 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3373 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3374 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3375 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3376 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3378 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3379 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3380 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3398 	evergreen_fix_pci_max_read_req_size(rdev);  in evergreen_gpu_init()
3401 	if ((rdev->family == CHIP_PALM) ||  in evergreen_gpu_init()
3402 	    (rdev->family == CHIP_SUMO) ||  in evergreen_gpu_init()
3403 	    (rdev->family == CHIP_SUMO2))  in evergreen_gpu_init()
3415 	rdev->config.evergreen.tile_config = 0;  in evergreen_gpu_init()
3416 	switch (rdev->config.evergreen.max_tile_pipes) {  in evergreen_gpu_init()
3419 		rdev->config.evergreen.tile_config |= (0 << 0);  in evergreen_gpu_init()
3422 		rdev->config.evergreen.tile_config |= (1 << 0);  in evergreen_gpu_init()
3425 		rdev->config.evergreen.tile_config |= (2 << 0);  in evergreen_gpu_init()
3428 		rdev->config.evergreen.tile_config |= (3 << 0);  in evergreen_gpu_init()
3432 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_gpu_init()
3433 		rdev->config.evergreen.tile_config |= 1 << 4;  in evergreen_gpu_init()
3437 			rdev->config.evergreen.tile_config |= 0 << 4;  in evergreen_gpu_init()
3440 			rdev->config.evergreen.tile_config |= 1 << 4;  in evergreen_gpu_init()
3444 			rdev->config.evergreen.tile_config |= 2 << 4;  in evergreen_gpu_init()
3448 	rdev->config.evergreen.tile_config |= 0 << 8;  in evergreen_gpu_init()
3449 	rdev->config.evergreen.tile_config |=  in evergreen_gpu_init()
3452 	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {  in evergreen_gpu_init()
3462 		for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {  in evergreen_gpu_init()
3475 	for (i = 0; i < rdev->config.evergreen.max_backends; i++)  in evergreen_gpu_init()
3479 		for (i = 0; i < rdev->config.evergreen.max_backends; i++)  in evergreen_gpu_init()
3483 	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {  in evergreen_gpu_init()
3489 		simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;  in evergreen_gpu_init()
3493 	rdev->config.evergreen.active_simds = hweight32(~tmp);  in evergreen_gpu_init()
3506 	if ((rdev->config.evergreen.max_backends == 1) &&  in evergreen_gpu_init()
3507 	    (rdev->flags & RADEON_IS_IGP)) {  in evergreen_gpu_init()
3517 		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,  in evergreen_gpu_init()
3520 	rdev->config.evergreen.backend_map = tmp;  in evergreen_gpu_init()
3546 	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);  in evergreen_gpu_init()
3549 	if (rdev->family <= CHIP_SUMO2)  in evergreen_gpu_init()
3552 …WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) …  in evergreen_gpu_init()
3553 					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |  in evergreen_gpu_init()
3554 					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));  in evergreen_gpu_init()
3556 	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |  in evergreen_gpu_init()
3557 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |  in evergreen_gpu_init()
3558 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));  in evergreen_gpu_init()
3565 	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |  in evergreen_gpu_init()
3582 	switch (rdev->family) {  in evergreen_gpu_init()
3597 	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);  in evergreen_gpu_init()
3598 	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);  in evergreen_gpu_init()
3600 	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);  in evergreen_gpu_init()
3601 	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);  in evergreen_gpu_init()
3602 	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);  in evergreen_gpu_init()
3603 	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);  in evergreen_gpu_init()
3605 	switch (rdev->family) {  in evergreen_gpu_init()
3618 …sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3619 …sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3620 …sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3621 …sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count…  in evergreen_gpu_init()
3622 …sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_coun…  in evergreen_gpu_init()
3624 …sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3625 …sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3626 …sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3627 …sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3628 …sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3629 …sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3646 	switch (rdev->family) {  in evergreen_gpu_init()
3710 int evergreen_mc_init(struct radeon_device *rdev)  in evergreen_mc_init()  argument
3716 	rdev->mc.vram_is_ddr = true;  in evergreen_mc_init()
3717 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_init()
3718 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_init()
3719 	    (rdev->family == CHIP_SUMO2))  in evergreen_mc_init()
3746 	rdev->mc.vram_width = numchan * chansize;  in evergreen_mc_init()
3748 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in evergreen_mc_init()
3749 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in evergreen_mc_init()
3751 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_init()
3752 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_init()
3753 	    (rdev->family == CHIP_SUMO2)) {  in evergreen_mc_init()
3755 		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);  in evergreen_mc_init()
3756 		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);  in evergreen_mc_init()
3759 		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in evergreen_mc_init()
3760 		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in evergreen_mc_init()
3762 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in evergreen_mc_init()
3763 	r700_vram_gtt_location(rdev, &rdev->mc);  in evergreen_mc_init()
3764 	radeon_update_bandwidth_info(rdev);  in evergreen_mc_init()
3769 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)  in evergreen_print_gpu_status_regs()  argument
3771 	dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3773 	dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3775 	dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3777 	dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3779 	dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3781 	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3783 	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3785 	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3787 	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3789 	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3791 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_print_gpu_status_regs()
3792 		dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3797 bool evergreen_is_display_hung(struct radeon_device *rdev)  in evergreen_is_display_hung()  argument
3803 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_is_display_hung()
3811 		for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_is_display_hung()
3826 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)  in evergreen_gpu_check_soft_reset()  argument
3878 	if (evergreen_is_display_hung(rdev))  in evergreen_gpu_check_soft_reset()
3895 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in evergreen_gpu_soft_reset()  argument
3904 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in evergreen_gpu_soft_reset()
3906 	evergreen_print_gpu_status_regs(rdev);  in evergreen_gpu_soft_reset()
3920 	evergreen_mc_stop(rdev, &save);  in evergreen_gpu_soft_reset()
3921 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_gpu_soft_reset()
3922 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_gpu_soft_reset()
3967 	if (!(rdev->flags & RADEON_IS_IGP)) {  in evergreen_gpu_soft_reset()
3975 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in evergreen_gpu_soft_reset()
3989 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in evergreen_gpu_soft_reset()
4003 	evergreen_mc_resume(rdev, &save);  in evergreen_gpu_soft_reset()
4006 	evergreen_print_gpu_status_regs(rdev);  in evergreen_gpu_soft_reset()
4009 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)  in evergreen_gpu_pci_config_reset()  argument
4014 	dev_info(rdev->dev, "GPU pci config reset\n");  in evergreen_gpu_pci_config_reset()
4028 	r600_rlc_stop(rdev);  in evergreen_gpu_pci_config_reset()
4033 	rv770_set_clk_bypass_mode(rdev);  in evergreen_gpu_pci_config_reset()
4035 	pci_clear_master(rdev->pdev);  in evergreen_gpu_pci_config_reset()
4037 	evergreen_mc_stop(rdev, &save);  in evergreen_gpu_pci_config_reset()
4038 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_gpu_pci_config_reset()
4039 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");  in evergreen_gpu_pci_config_reset()
4042 	radeon_pci_config_reset(rdev);  in evergreen_gpu_pci_config_reset()
4044 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_gpu_pci_config_reset()
4051 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)  in evergreen_asic_reset()  argument
4056 		evergreen_gpu_pci_config_reset(rdev);  in evergreen_asic_reset()
4060 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4063 		r600_set_bios_scratch_engine_hung(rdev, true);  in evergreen_asic_reset()
4066 	evergreen_gpu_soft_reset(rdev, reset_mask);  in evergreen_asic_reset()
4068 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4072 		evergreen_gpu_pci_config_reset(rdev);  in evergreen_asic_reset()
4074 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4077 		r600_set_bios_scratch_engine_hung(rdev, false);  in evergreen_asic_reset()
4091 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in evergreen_gfx_is_lockup()  argument
4093 	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_gfx_is_lockup()
4098 		radeon_ring_lockup_update(rdev, ring);  in evergreen_gfx_is_lockup()
4101 	return radeon_ring_test_lockup(rdev, ring);  in evergreen_gfx_is_lockup()
4110 void sumo_rlc_fini(struct radeon_device *rdev)  in sumo_rlc_fini()  argument
4115 	if (rdev->rlc.save_restore_obj) {  in sumo_rlc_fini()
4116 		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);  in sumo_rlc_fini()
4118 			dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);  in sumo_rlc_fini()
4119 		radeon_bo_unpin(rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4120 		radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4122 		radeon_bo_unref(&rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4123 		rdev->rlc.save_restore_obj = NULL;  in sumo_rlc_fini()
4127 	if (rdev->rlc.clear_state_obj) {  in sumo_rlc_fini()
4128 		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);  in sumo_rlc_fini()
4130 			dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);  in sumo_rlc_fini()
4131 		radeon_bo_unpin(rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4132 		radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4134 		radeon_bo_unref(&rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4135 		rdev->rlc.clear_state_obj = NULL;  in sumo_rlc_fini()
4139 	if (rdev->rlc.cp_table_obj) {  in sumo_rlc_fini()
4140 		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);  in sumo_rlc_fini()
4142 			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);  in sumo_rlc_fini()
4143 		radeon_bo_unpin(rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4144 		radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4146 		radeon_bo_unref(&rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4147 		rdev->rlc.cp_table_obj = NULL;  in sumo_rlc_fini()
4153 int sumo_rlc_init(struct radeon_device *rdev)  in sumo_rlc_init()  argument
4163 	src_ptr = rdev->rlc.reg_list;  in sumo_rlc_init()
4164 	dws = rdev->rlc.reg_list_size;  in sumo_rlc_init()
4165 	if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4168 	cs_data = rdev->rlc.cs_data;  in sumo_rlc_init()
4172 		if (rdev->rlc.save_restore_obj == NULL) {  in sumo_rlc_init()
4173 			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,  in sumo_rlc_init()
4175 					     NULL, &rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4177 				dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);  in sumo_rlc_init()
4182 		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);  in sumo_rlc_init()
4184 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4187 		r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4188 				  &rdev->rlc.save_restore_gpu_addr);  in sumo_rlc_init()
4190 			radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4191 			dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);  in sumo_rlc_init()
4192 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4196 		r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);  in sumo_rlc_init()
4198 			dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);  in sumo_rlc_init()
4199 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4203 		dst_ptr = rdev->rlc.sr_ptr;  in sumo_rlc_init()
4204 		if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4206 			for (i = 0; i < rdev->rlc.reg_list_size; i++)  in sumo_rlc_init()
4226 		radeon_bo_kunmap(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4227 		radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4232 		if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4233 			rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);  in sumo_rlc_init()
4234 		} else if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4235 			rdev->rlc.clear_state_size = si_get_csb_size(rdev);  in sumo_rlc_init()
4236 			dws = rdev->rlc.clear_state_size + (256 / 4);  in sumo_rlc_init()
4248 			rdev->rlc.clear_state_size = dws;  in sumo_rlc_init()
4251 		if (rdev->rlc.clear_state_obj == NULL) {  in sumo_rlc_init()
4252 			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,  in sumo_rlc_init()
4254 					     NULL, &rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4256 				dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);  in sumo_rlc_init()
4257 				sumo_rlc_fini(rdev);  in sumo_rlc_init()
4261 		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);  in sumo_rlc_init()
4263 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4266 		r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4267 				  &rdev->rlc.clear_state_gpu_addr);  in sumo_rlc_init()
4269 			radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4270 			dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);  in sumo_rlc_init()
4271 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4275 		r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);  in sumo_rlc_init()
4277 			dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);  in sumo_rlc_init()
4278 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4282 		dst_ptr = rdev->rlc.cs_ptr;  in sumo_rlc_init()
4283 		if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4284 			cik_get_csb_buffer(rdev, dst_ptr);  in sumo_rlc_init()
4285 		} else if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4286 			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;  in sumo_rlc_init()
4289 			dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);  in sumo_rlc_init()
4290 			si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);  in sumo_rlc_init()
4293 			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);  in sumo_rlc_init()
4322 		radeon_bo_kunmap(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4323 		radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4326 	if (rdev->rlc.cp_table_size) {  in sumo_rlc_init()
4327 		if (rdev->rlc.cp_table_obj == NULL) {  in sumo_rlc_init()
4328 			r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,  in sumo_rlc_init()
4331 					     NULL, &rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4333 				dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);  in sumo_rlc_init()
4334 				sumo_rlc_fini(rdev);  in sumo_rlc_init()
4339 		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);  in sumo_rlc_init()
4341 			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);  in sumo_rlc_init()
4342 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4345 		r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4346 				  &rdev->rlc.cp_table_gpu_addr);  in sumo_rlc_init()
4348 			radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4349 			dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);  in sumo_rlc_init()
4350 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4353 		r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);  in sumo_rlc_init()
4355 			dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);  in sumo_rlc_init()
4356 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4360 		cik_init_cp_pg_table(rdev);  in sumo_rlc_init()
4362 		radeon_bo_kunmap(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4363 		radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4370 static void evergreen_rlc_start(struct radeon_device *rdev)  in evergreen_rlc_start()  argument
4374 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_rlc_start()
4381 int evergreen_rlc_resume(struct radeon_device *rdev)  in evergreen_rlc_resume()  argument
4386 	if (!rdev->rlc_fw)  in evergreen_rlc_resume()
4389 	r600_rlc_stop(rdev);  in evergreen_rlc_resume()
4393 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_rlc_resume()
4394 		if (rdev->family == CHIP_ARUBA) {  in evergreen_rlc_resume()
4396 				3 | (3 << (16 * rdev->config.cayman.max_shader_engines));  in evergreen_rlc_resume()
4399 			tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;  in evergreen_rlc_resume()
4401 			if (tmp == rdev->config.cayman.max_simds_per_se) {  in evergreen_rlc_resume()
4412 		WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);  in evergreen_rlc_resume()
4413 		WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);  in evergreen_rlc_resume()
4424 	fw_data = (const __be32 *)rdev->rlc_fw->data;  in evergreen_rlc_resume()
4425 	if (rdev->family >= CHIP_ARUBA) {  in evergreen_rlc_resume()
4430 	} else if (rdev->family >= CHIP_CAYMAN) {  in evergreen_rlc_resume()
4443 	evergreen_rlc_start(rdev);  in evergreen_rlc_resume()
4450 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)  in evergreen_get_vblank_counter()  argument
4452 	if (crtc >= rdev->num_crtc)  in evergreen_get_vblank_counter()
4458 void evergreen_disable_interrupt_state(struct radeon_device *rdev)  in evergreen_disable_interrupt_state()  argument
4463 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_disable_interrupt_state()
4464 		cayman_cp_int_cntl_setup(rdev, 0,  in evergreen_disable_interrupt_state()
4466 		cayman_cp_int_cntl_setup(rdev, 1, 0);  in evergreen_disable_interrupt_state()
4467 		cayman_cp_int_cntl_setup(rdev, 2, 0);  in evergreen_disable_interrupt_state()
4476 	for (i = 0; i < rdev->num_crtc; i++)  in evergreen_disable_interrupt_state()
4478 	for (i = 0; i < rdev->num_crtc; i++)  in evergreen_disable_interrupt_state()
4482 	if (!ASIC_IS_DCE5(rdev))  in evergreen_disable_interrupt_state()
4491 int evergreen_irq_set(struct radeon_device *rdev)  in evergreen_irq_set()  argument
4500 	if (!rdev->irq.installed) {  in evergreen_irq_set()
4505 	if (!rdev->ih.enabled) {  in evergreen_irq_set()
4506 		r600_disable_interrupts(rdev);  in evergreen_irq_set()
4508 		evergreen_disable_interrupt_state(rdev);  in evergreen_irq_set()
4512 	if (rdev->family == CHIP_ARUBA)  in evergreen_irq_set()
4521 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4523 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in evergreen_irq_set()
4527 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {  in evergreen_irq_set()
4531 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {  in evergreen_irq_set()
4536 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in evergreen_irq_set()
4543 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in evergreen_irq_set()
4548 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4550 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {  in evergreen_irq_set()
4556 	if (rdev->irq.dpm_thermal) {  in evergreen_irq_set()
4561 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4562 		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);  in evergreen_irq_set()
4563 		cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);  in evergreen_irq_set()
4564 		cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);  in evergreen_irq_set()
4570 	if (rdev->family >= CHIP_CAYMAN)  in evergreen_irq_set()
4575 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_irq_set()
4577 		    rdev, INT_MASK + crtc_offsets[i],  in evergreen_irq_set()
4579 		    rdev->irq.crtc_vblank_int[i] ||  in evergreen_irq_set()
4580 		    atomic_read(&rdev->irq.pflip[i]), "vblank", i);  in evergreen_irq_set()
4583 	for (i = 0; i < rdev->num_crtc; i++)  in evergreen_irq_set()
4588 		    rdev, DC_HPDx_INT_CONTROL(i),  in evergreen_irq_set()
4590 		    rdev->irq.hpd[i], "HPD", i);  in evergreen_irq_set()
4593 	if (rdev->family == CHIP_ARUBA)  in evergreen_irq_set()
4600 		    rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],  in evergreen_irq_set()
4602 		    rdev->irq.afmt[i], "HDMI", i);  in evergreen_irq_set()
4612 static void evergreen_irq_ack(struct radeon_device *rdev)  in evergreen_irq_ack()  argument
4615 	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;  in evergreen_irq_ack()
4616 	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;  in evergreen_irq_ack()
4617 	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;  in evergreen_irq_ack()
4622 		if (i < rdev->num_crtc)  in evergreen_irq_ack()
4627 	for (i = 0; i < rdev->num_crtc; i += 2) {  in evergreen_irq_ack()
4661 static void evergreen_irq_disable(struct radeon_device *rdev)  in evergreen_irq_disable()  argument
4663 	r600_disable_interrupts(rdev);  in evergreen_irq_disable()
4666 	evergreen_irq_ack(rdev);  in evergreen_irq_disable()
4667 	evergreen_disable_interrupt_state(rdev);  in evergreen_irq_disable()
4670 void evergreen_irq_suspend(struct radeon_device *rdev)  in evergreen_irq_suspend()  argument
4672 	evergreen_irq_disable(rdev);  in evergreen_irq_suspend()
4673 	r600_rlc_stop(rdev);  in evergreen_irq_suspend()
4676 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)  in evergreen_get_ih_wptr()  argument
4680 	if (rdev->wb.enabled)  in evergreen_get_ih_wptr()
4681 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in evergreen_get_ih_wptr()
4691 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in evergreen_get_ih_wptr()
4692 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in evergreen_get_ih_wptr()
4693 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in evergreen_get_ih_wptr()
4698 	return (wptr & rdev->ih.ptr_mask);  in evergreen_get_ih_wptr()
4701 int evergreen_irq_process(struct radeon_device *rdev)  in evergreen_irq_process()  argument
4703 	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;  in evergreen_irq_process()
4704 	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;  in evergreen_irq_process()
4718 	if (!rdev->ih.enabled || rdev->shutdown)  in evergreen_irq_process()
4721 	wptr = evergreen_get_ih_wptr(rdev);  in evergreen_irq_process()
4725 	if (atomic_xchg(&rdev->ih.lock, 1))  in evergreen_irq_process()
4728 	rptr = rdev->ih.rptr;  in evergreen_irq_process()
4735 	evergreen_irq_ack(rdev);  in evergreen_irq_process()
4740 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in evergreen_irq_process()
4741 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in evergreen_irq_process()
4756 				if (rdev->irq.crtc_vblank_int[crtc_idx]) {  in evergreen_irq_process()
4757 					drm_handle_vblank(rdev->ddev, crtc_idx);  in evergreen_irq_process()
4758 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
4759 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
4761 				if (atomic_read(&rdev->irq.pflip[crtc_idx])) {  in evergreen_irq_process()
4762 					radeon_crtc_handle_vblank(rdev,  in evergreen_irq_process()
4792 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);  in evergreen_irq_process()
4840 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);  in evergreen_irq_process()
4850 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);  in evergreen_irq_process()
4851 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in evergreen_irq_process()
4853 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in evergreen_irq_process()
4855 			cayman_vm_decode_fault(rdev, status, addr);  in evergreen_irq_process()
4861 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
4865 			if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_process()
4868 					radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
4871 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in evergreen_irq_process()
4874 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in evergreen_irq_process()
4878 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
4882 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);  in evergreen_irq_process()
4886 			rdev->pm.dpm.thermal.high_to_low = false;  in evergreen_irq_process()
4891 			rdev->pm.dpm.thermal.high_to_low = true;  in evergreen_irq_process()
4898 			if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_process()
4900 				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in evergreen_irq_process()
4910 		rptr &= rdev->ih.ptr_mask;  in evergreen_irq_process()
4914 		schedule_work(&rdev->dp_work);  in evergreen_irq_process()
4916 		schedule_delayed_work(&rdev->hotplug_work, 0);  in evergreen_irq_process()
4918 		schedule_work(&rdev->audio_work);  in evergreen_irq_process()
4919 	if (queue_thermal && rdev->pm.dpm_enabled)  in evergreen_irq_process()
4920 		schedule_work(&rdev->pm.dpm.thermal.work);  in evergreen_irq_process()
4921 	rdev->ih.rptr = rptr;  in evergreen_irq_process()
4922 	atomic_set(&rdev->ih.lock, 0);  in evergreen_irq_process()
4925 	wptr = evergreen_get_ih_wptr(rdev);  in evergreen_irq_process()
4932 static void evergreen_uvd_init(struct radeon_device *rdev)  in evergreen_uvd_init()  argument
4936 	if (!rdev->has_uvd)  in evergreen_uvd_init()
4939 	r = radeon_uvd_init(rdev);  in evergreen_uvd_init()
4941 		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);  in evergreen_uvd_init()
4948 		rdev->has_uvd = 0;  in evergreen_uvd_init()
4951 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;  in evergreen_uvd_init()
4952 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);  in evergreen_uvd_init()
4955 static void evergreen_uvd_start(struct radeon_device *rdev)  in evergreen_uvd_start()  argument
4959 	if (!rdev->has_uvd)  in evergreen_uvd_start()
4962 	r = uvd_v2_2_resume(rdev);  in evergreen_uvd_start()
4964 		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);  in evergreen_uvd_start()
4967 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);  in evergreen_uvd_start()
4969 		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);  in evergreen_uvd_start()
4975 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in evergreen_uvd_start()
4978 static void evergreen_uvd_resume(struct radeon_device *rdev)  in evergreen_uvd_resume()  argument
4983 	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)  in evergreen_uvd_resume()
4986 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in evergreen_uvd_resume()
4987 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));  in evergreen_uvd_resume()
4989 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);  in evergreen_uvd_resume()
4992 	r = uvd_v1_0_init(rdev);  in evergreen_uvd_resume()
4994 		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);  in evergreen_uvd_resume()
4999 static int evergreen_startup(struct radeon_device *rdev)  in evergreen_startup()  argument
5005 	evergreen_pcie_gen2_enable(rdev);  in evergreen_startup()
5007 	evergreen_program_aspm(rdev);  in evergreen_startup()
5010 	r = r600_vram_scratch_init(rdev);  in evergreen_startup()
5014 	evergreen_mc_program(rdev);  in evergreen_startup()
5016 	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {  in evergreen_startup()
5017 		r = ni_mc_load_microcode(rdev);  in evergreen_startup()
5024 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_startup()
5025 		evergreen_agp_enable(rdev);  in evergreen_startup()
5027 		r = evergreen_pcie_gart_enable(rdev);  in evergreen_startup()
5031 	evergreen_gpu_init(rdev);  in evergreen_startup()
5034 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_startup()
5035 		rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;  in evergreen_startup()
5036 		rdev->rlc.reg_list_size =  in evergreen_startup()
5038 		rdev->rlc.cs_data = evergreen_cs_data;  in evergreen_startup()
5039 		r = sumo_rlc_init(rdev);  in evergreen_startup()
5047 	r = radeon_wb_init(rdev);  in evergreen_startup()
5051 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_startup()
5053 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in evergreen_startup()
5057 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);  in evergreen_startup()
5059 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in evergreen_startup()
5063 	evergreen_uvd_start(rdev);  in evergreen_startup()
5066 	if (!rdev->irq.installed) {  in evergreen_startup()
5067 		r = radeon_irq_kms_init(rdev);  in evergreen_startup()
5072 	r = r600_irq_init(rdev);  in evergreen_startup()
5075 		radeon_irq_kms_fini(rdev);  in evergreen_startup()
5078 	evergreen_irq_set(rdev);  in evergreen_startup()
5080 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_startup()
5081 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in evergreen_startup()
5086 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in evergreen_startup()
5087 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in evergreen_startup()
5092 	r = evergreen_cp_load_microcode(rdev);  in evergreen_startup()
5095 	r = evergreen_cp_resume(rdev);  in evergreen_startup()
5098 	r = r600_dma_resume(rdev);  in evergreen_startup()
5102 	evergreen_uvd_resume(rdev);  in evergreen_startup()
5104 	r = radeon_ib_pool_init(rdev);  in evergreen_startup()
5106 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in evergreen_startup()
5110 	r = radeon_audio_init(rdev);  in evergreen_startup()
5119 int evergreen_resume(struct radeon_device *rdev)  in evergreen_resume()  argument
5126 	if (radeon_asic_reset(rdev))  in evergreen_resume()
5127 		dev_warn(rdev->dev, "GPU reset failed !\n");  in evergreen_resume()
5133 	atom_asic_init(rdev->mode_info.atom_context);  in evergreen_resume()
5136 	evergreen_init_golden_registers(rdev);  in evergreen_resume()
5138 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in evergreen_resume()
5139 		radeon_pm_resume(rdev);  in evergreen_resume()
5141 	rdev->accel_working = true;  in evergreen_resume()
5142 	r = evergreen_startup(rdev);  in evergreen_resume()
5145 		rdev->accel_working = false;  in evergreen_resume()
5153 int evergreen_suspend(struct radeon_device *rdev)  in evergreen_suspend()  argument
5155 	radeon_pm_suspend(rdev);  in evergreen_suspend()
5156 	radeon_audio_fini(rdev);  in evergreen_suspend()
5157 	if (rdev->has_uvd) {  in evergreen_suspend()
5158 		uvd_v1_0_fini(rdev);  in evergreen_suspend()
5159 		radeon_uvd_suspend(rdev);  in evergreen_suspend()
5161 	r700_cp_stop(rdev);  in evergreen_suspend()
5162 	r600_dma_stop(rdev);  in evergreen_suspend()
5163 	evergreen_irq_suspend(rdev);  in evergreen_suspend()
5164 	radeon_wb_disable(rdev);  in evergreen_suspend()
5165 	evergreen_pcie_gart_disable(rdev);  in evergreen_suspend()
5176 int evergreen_init(struct radeon_device *rdev)  in evergreen_init()  argument
5181 	if (!radeon_get_bios(rdev)) {  in evergreen_init()
5182 		if (ASIC_IS_AVIVO(rdev))  in evergreen_init()
5186 	if (!rdev->is_atom_bios) {  in evergreen_init()
5187 		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");  in evergreen_init()
5190 	r = radeon_atombios_init(rdev);  in evergreen_init()
5196 	if (radeon_asic_reset(rdev))  in evergreen_init()
5197 		dev_warn(rdev->dev, "GPU reset failed !\n");  in evergreen_init()
5199 	if (!radeon_card_posted(rdev)) {  in evergreen_init()
5200 		if (!rdev->bios) {  in evergreen_init()
5201 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in evergreen_init()
5205 		atom_asic_init(rdev->mode_info.atom_context);  in evergreen_init()
5208 	evergreen_init_golden_registers(rdev);  in evergreen_init()
5210 	r600_scratch_init(rdev);  in evergreen_init()
5212 	radeon_surface_init(rdev);  in evergreen_init()
5214 	radeon_get_clock_info(rdev->ddev);  in evergreen_init()
5216 	r = radeon_fence_driver_init(rdev);  in evergreen_init()
5220 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_init()
5221 		r = radeon_agp_init(rdev);  in evergreen_init()
5223 			radeon_agp_disable(rdev);  in evergreen_init()
5226 	r = evergreen_mc_init(rdev);  in evergreen_init()
5230 	r = radeon_bo_init(rdev);  in evergreen_init()
5234 	if (ASIC_IS_DCE5(rdev)) {  in evergreen_init()
5235 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {  in evergreen_init()
5236 			r = ni_init_microcode(rdev);  in evergreen_init()
5243 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {  in evergreen_init()
5244 			r = r600_init_microcode(rdev);  in evergreen_init()
5253 	radeon_pm_init(rdev);  in evergreen_init()
5255 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;  in evergreen_init()
5256 	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);  in evergreen_init()
5258 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;  in evergreen_init()
5259 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);  in evergreen_init()
5261 	evergreen_uvd_init(rdev);  in evergreen_init()
5263 	rdev->ih.ring_obj = NULL;  in evergreen_init()
5264 	r600_ih_ring_init(rdev, 64 * 1024);  in evergreen_init()
5266 	r = r600_pcie_gart_init(rdev);  in evergreen_init()
5270 	rdev->accel_working = true;  in evergreen_init()
5271 	r = evergreen_startup(rdev);  in evergreen_init()
5273 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in evergreen_init()
5274 		r700_cp_fini(rdev);  in evergreen_init()
5275 		r600_dma_fini(rdev);  in evergreen_init()
5276 		r600_irq_fini(rdev);  in evergreen_init()
5277 		if (rdev->flags & RADEON_IS_IGP)  in evergreen_init()
5278 			sumo_rlc_fini(rdev);  in evergreen_init()
5279 		radeon_wb_fini(rdev);  in evergreen_init()
5280 		radeon_ib_pool_fini(rdev);  in evergreen_init()
5281 		radeon_irq_kms_fini(rdev);  in evergreen_init()
5282 		evergreen_pcie_gart_fini(rdev);  in evergreen_init()
5283 		rdev->accel_working = false;  in evergreen_init()
5290 	if (ASIC_IS_DCE5(rdev)) {  in evergreen_init()
5291 		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in evergreen_init()
5300 void evergreen_fini(struct radeon_device *rdev)  in evergreen_fini()  argument
5302 	radeon_pm_fini(rdev);  in evergreen_fini()
5303 	radeon_audio_fini(rdev);  in evergreen_fini()
5304 	r700_cp_fini(rdev);  in evergreen_fini()
5305 	r600_dma_fini(rdev);  in evergreen_fini()
5306 	r600_irq_fini(rdev);  in evergreen_fini()
5307 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_fini()
5308 		sumo_rlc_fini(rdev);  in evergreen_fini()
5309 	radeon_wb_fini(rdev);  in evergreen_fini()
5310 	radeon_ib_pool_fini(rdev);  in evergreen_fini()
5311 	radeon_irq_kms_fini(rdev);  in evergreen_fini()
5312 	uvd_v1_0_fini(rdev);  in evergreen_fini()
5313 	radeon_uvd_fini(rdev);  in evergreen_fini()
5314 	evergreen_pcie_gart_fini(rdev);  in evergreen_fini()
5315 	r600_vram_scratch_fini(rdev);  in evergreen_fini()
5316 	radeon_gem_fini(rdev);  in evergreen_fini()
5317 	radeon_fence_driver_fini(rdev);  in evergreen_fini()
5318 	radeon_agp_fini(rdev);  in evergreen_fini()
5319 	radeon_bo_fini(rdev);  in evergreen_fini()
5320 	radeon_atombios_fini(rdev);  in evergreen_fini()
5321 	kfree(rdev->bios);  in evergreen_fini()
5322 	rdev->bios = NULL;  in evergreen_fini()
5325 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)  in evergreen_pcie_gen2_enable()  argument
5332 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_pcie_gen2_enable()
5335 	if (!(rdev->flags & RADEON_IS_PCIE))  in evergreen_pcie_gen2_enable()
5339 	if (ASIC_IS_X2(rdev))  in evergreen_pcie_gen2_enable()
5342 	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&  in evergreen_pcie_gen2_enable()
5343 		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))  in evergreen_pcie_gen2_enable()
5388 void evergreen_program_aspm(struct radeon_device *rdev)  in evergreen_program_aspm()  argument
5403 	if (!(rdev->flags & RADEON_IS_PCIE))  in evergreen_program_aspm()
5406 	switch (rdev->family) {  in evergreen_program_aspm()
5423 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_program_aspm()
5445 		if (rdev->family >= CHIP_BARTS)  in evergreen_program_aspm()
5452 		if (rdev->family >= CHIP_BARTS)  in evergreen_program_aspm()
5482 			if (rdev->family >= CHIP_BARTS) {  in evergreen_program_aspm()
5514 			if (rdev->family >= CHIP_BARTS) {  in evergreen_program_aspm()
5531 	if (rdev->family < CHIP_BARTS)  in evergreen_program_aspm()