100 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
103 int r600_mc_wait_for_idle(struct radeon_device *rdev);
104 static void r600_gpu_init(struct radeon_device *rdev);
105 void r600_fini(struct radeon_device *rdev);
106 void r600_irq_disable(struct radeon_device *rdev);
107 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
108 extern int evergreen_rlc_resume(struct radeon_device *rdev);
109 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
114 u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg)  in r600_rcu_rreg()  argument
119 	spin_lock_irqsave(&rdev->rcu_idx_lock, flags);  in r600_rcu_rreg()
122 	spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);  in r600_rcu_rreg()
126 void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in r600_rcu_wreg()  argument
130 	spin_lock_irqsave(&rdev->rcu_idx_lock, flags);  in r600_rcu_wreg()
133 	spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);  in r600_rcu_wreg()
136 u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg)  in r600_uvd_ctx_rreg()  argument
141 	spin_lock_irqsave(&rdev->uvd_idx_lock, flags);  in r600_uvd_ctx_rreg()
144 	spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);  in r600_uvd_ctx_rreg()
148 void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in r600_uvd_ctx_wreg()  argument
152 	spin_lock_irqsave(&rdev->uvd_idx_lock, flags);  in r600_uvd_ctx_wreg()
155 	spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);  in r600_uvd_ctx_wreg()
168 int r600_get_allowed_info_register(struct radeon_device *rdev,  in r600_get_allowed_info_register()  argument
192 u32 r600_get_xclk(struct radeon_device *rdev)  in r600_get_xclk()  argument
194 	return rdev->clock.spll.reference_freq;  in r600_get_xclk()
197 int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in r600_set_uvd_clocks()  argument
211 	if (rdev->family >= CHIP_RS780)  in r600_set_uvd_clocks()
221 	if (rdev->clock.spll.reference_freq == 10000)  in r600_set_uvd_clocks()
226 	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,  in r600_set_uvd_clocks()
232 	if (rdev->family >= CHIP_RV670 && rdev->family < CHIP_RS780)  in r600_set_uvd_clocks()
237 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in r600_set_uvd_clocks()
245 	if (rdev->family >= CHIP_RS780)  in r600_set_uvd_clocks()
273 	if (rdev->family >= CHIP_RS780)  in r600_set_uvd_clocks()
276 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in r600_set_uvd_clocks()
293 	struct radeon_device *rdev = dev->dev_private;  in dce3_program_fmt()  local
344 int rv6xx_get_temp(struct radeon_device *rdev)  in rv6xx_get_temp()  argument
356 void r600_pm_get_dynpm_state(struct radeon_device *rdev)  in r600_pm_get_dynpm_state()  argument
360 	rdev->pm.dynpm_can_upclock = true;  in r600_pm_get_dynpm_state()
361 	rdev->pm.dynpm_can_downclock = true;  in r600_pm_get_dynpm_state()
364 	if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {  in r600_pm_get_dynpm_state()
367 		if (rdev->pm.num_power_states > 2)  in r600_pm_get_dynpm_state()
370 		switch (rdev->pm.dynpm_planned_action) {  in r600_pm_get_dynpm_state()
372 			rdev->pm.requested_power_state_index = min_power_state_index;  in r600_pm_get_dynpm_state()
373 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
374 			rdev->pm.dynpm_can_downclock = false;  in r600_pm_get_dynpm_state()
377 			if (rdev->pm.current_power_state_index == min_power_state_index) {  in r600_pm_get_dynpm_state()
378 				rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;  in r600_pm_get_dynpm_state()
379 				rdev->pm.dynpm_can_downclock = false;  in r600_pm_get_dynpm_state()
381 				if (rdev->pm.active_crtc_count > 1) {  in r600_pm_get_dynpm_state()
382 					for (i = 0; i < rdev->pm.num_power_states; i++) {  in r600_pm_get_dynpm_state()
383 						if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)  in r600_pm_get_dynpm_state()
385 						else if (i >= rdev->pm.current_power_state_index) {  in r600_pm_get_dynpm_state()
386 							rdev->pm.requested_power_state_index =  in r600_pm_get_dynpm_state()
387 								rdev->pm.current_power_state_index;  in r600_pm_get_dynpm_state()
390 							rdev->pm.requested_power_state_index = i;  in r600_pm_get_dynpm_state()
395 					if (rdev->pm.current_power_state_index == 0)  in r600_pm_get_dynpm_state()
396 						rdev->pm.requested_power_state_index =  in r600_pm_get_dynpm_state()
397 							rdev->pm.num_power_states - 1;  in r600_pm_get_dynpm_state()
399 						rdev->pm.requested_power_state_index =  in r600_pm_get_dynpm_state()
400 							rdev->pm.current_power_state_index - 1;  in r600_pm_get_dynpm_state()
403 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
405 			if ((rdev->pm.active_crtc_count > 0) &&  in r600_pm_get_dynpm_state()
406 			    (rdev->pm.power_state[rdev->pm.requested_power_state_index].  in r600_pm_get_dynpm_state()
407 			     clock_info[rdev->pm.requested_clock_mode_index].flags &  in r600_pm_get_dynpm_state()
409 				rdev->pm.requested_power_state_index++;  in r600_pm_get_dynpm_state()
413 			if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {  in r600_pm_get_dynpm_state()
414 				rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;  in r600_pm_get_dynpm_state()
415 				rdev->pm.dynpm_can_upclock = false;  in r600_pm_get_dynpm_state()
417 				if (rdev->pm.active_crtc_count > 1) {  in r600_pm_get_dynpm_state()
418 					for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {  in r600_pm_get_dynpm_state()
419 						if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)  in r600_pm_get_dynpm_state()
421 						else if (i <= rdev->pm.current_power_state_index) {  in r600_pm_get_dynpm_state()
422 							rdev->pm.requested_power_state_index =  in r600_pm_get_dynpm_state()
423 								rdev->pm.current_power_state_index;  in r600_pm_get_dynpm_state()
426 							rdev->pm.requested_power_state_index = i;  in r600_pm_get_dynpm_state()
431 					rdev->pm.requested_power_state_index =  in r600_pm_get_dynpm_state()
432 						rdev->pm.current_power_state_index + 1;  in r600_pm_get_dynpm_state()
434 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
437 			rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;  in r600_pm_get_dynpm_state()
438 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
439 			rdev->pm.dynpm_can_upclock = false;  in r600_pm_get_dynpm_state()
450 		if (rdev->pm.active_crtc_count > 1) {  in r600_pm_get_dynpm_state()
451 			rdev->pm.requested_power_state_index = -1;  in r600_pm_get_dynpm_state()
453 			for (i = 1; i < rdev->pm.num_power_states; i++) {  in r600_pm_get_dynpm_state()
454 				if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)  in r600_pm_get_dynpm_state()
456 				else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||  in r600_pm_get_dynpm_state()
457 					 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {  in r600_pm_get_dynpm_state()
458 					rdev->pm.requested_power_state_index = i;  in r600_pm_get_dynpm_state()
463 			if (rdev->pm.requested_power_state_index == -1)  in r600_pm_get_dynpm_state()
464 				rdev->pm.requested_power_state_index = 0;  in r600_pm_get_dynpm_state()
466 			rdev->pm.requested_power_state_index = 1;  in r600_pm_get_dynpm_state()
468 		switch (rdev->pm.dynpm_planned_action) {  in r600_pm_get_dynpm_state()
470 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
471 			rdev->pm.dynpm_can_downclock = false;  in r600_pm_get_dynpm_state()
474 			if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {  in r600_pm_get_dynpm_state()
475 				if (rdev->pm.current_clock_mode_index == 0) {  in r600_pm_get_dynpm_state()
476 					rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
477 					rdev->pm.dynpm_can_downclock = false;  in r600_pm_get_dynpm_state()
479 					rdev->pm.requested_clock_mode_index =  in r600_pm_get_dynpm_state()
480 						rdev->pm.current_clock_mode_index - 1;  in r600_pm_get_dynpm_state()
482 				rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
483 				rdev->pm.dynpm_can_downclock = false;  in r600_pm_get_dynpm_state()
486 			if ((rdev->pm.active_crtc_count > 0) &&  in r600_pm_get_dynpm_state()
487 			    (rdev->pm.power_state[rdev->pm.requested_power_state_index].  in r600_pm_get_dynpm_state()
488 			     clock_info[rdev->pm.requested_clock_mode_index].flags &  in r600_pm_get_dynpm_state()
490 				rdev->pm.requested_clock_mode_index++;  in r600_pm_get_dynpm_state()
494 			if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {  in r600_pm_get_dynpm_state()
495 				if (rdev->pm.current_clock_mode_index ==  in r600_pm_get_dynpm_state()
496 				    (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {  in r600_pm_get_dynpm_state()
497 					rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;  in r600_pm_get_dynpm_state()
498 					rdev->pm.dynpm_can_upclock = false;  in r600_pm_get_dynpm_state()
500 					rdev->pm.requested_clock_mode_index =  in r600_pm_get_dynpm_state()
501 						rdev->pm.current_clock_mode_index + 1;  in r600_pm_get_dynpm_state()
503 				rdev->pm.requested_clock_mode_index =  in r600_pm_get_dynpm_state()
504 					rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;  in r600_pm_get_dynpm_state()
505 				rdev->pm.dynpm_can_upclock = false;  in r600_pm_get_dynpm_state()
509 			rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;  in r600_pm_get_dynpm_state()
510 			rdev->pm.requested_clock_mode_index = 0;  in r600_pm_get_dynpm_state()
511 			rdev->pm.dynpm_can_upclock = false;  in r600_pm_get_dynpm_state()
521 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].  in r600_pm_get_dynpm_state()
522 		  clock_info[rdev->pm.requested_clock_mode_index].sclk,  in r600_pm_get_dynpm_state()
523 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].  in r600_pm_get_dynpm_state()
524 		  clock_info[rdev->pm.requested_clock_mode_index].mclk,  in r600_pm_get_dynpm_state()
525 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].  in r600_pm_get_dynpm_state()
529 void rs780_pm_init_profile(struct radeon_device *rdev)  in rs780_pm_init_profile()  argument
531 	if (rdev->pm.num_power_states == 2) {  in rs780_pm_init_profile()
533 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
534 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
535 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
536 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
538 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
539 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
540 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
541 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
543 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
544 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
545 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
546 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
548 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
549 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
550 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
551 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
553 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
554 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
555 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
556 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
558 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
559 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
560 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
561 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
563 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;  in rs780_pm_init_profile()
564 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
565 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
566 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
567 	} else if (rdev->pm.num_power_states == 3) {  in rs780_pm_init_profile()
569 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
570 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
571 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
572 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
574 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
575 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
576 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
577 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
579 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
580 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
581 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
582 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
584 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
585 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;  in rs780_pm_init_profile()
586 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
587 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
589 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
590 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
591 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
592 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
594 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
595 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;  in rs780_pm_init_profile()
596 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
597 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
599 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;  in rs780_pm_init_profile()
600 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;  in rs780_pm_init_profile()
601 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
602 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
605 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
606 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in rs780_pm_init_profile()
607 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
608 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
610 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
611 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;  in rs780_pm_init_profile()
612 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
613 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
615 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
616 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;  in rs780_pm_init_profile()
617 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
618 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
620 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
621 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;  in rs780_pm_init_profile()
622 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
623 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
625 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
626 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
627 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
628 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
630 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
631 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;  in rs780_pm_init_profile()
632 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
633 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
635 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;  in rs780_pm_init_profile()
636 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;  in rs780_pm_init_profile()
637 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in rs780_pm_init_profile()
638 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;  in rs780_pm_init_profile()
642 void r600_pm_init_profile(struct radeon_device *rdev)  in r600_pm_init_profile()  argument
646 	if (rdev->family == CHIP_R600) {  in r600_pm_init_profile()
649 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
650 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
651 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
652 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
654 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
655 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
656 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
657 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
659 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
660 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
661 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
662 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
664 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
665 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
666 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
667 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
669 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
670 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
671 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
672 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
674 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
675 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
676 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
677 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
679 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
680 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
681 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
682 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
684 		if (rdev->pm.num_power_states < 4) {  in r600_pm_init_profile()
686 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
687 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
688 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
689 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
691 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;  in r600_pm_init_profile()
692 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;  in r600_pm_init_profile()
693 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
694 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
696 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;  in r600_pm_init_profile()
697 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;  in r600_pm_init_profile()
698 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
699 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;  in r600_pm_init_profile()
701 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;  in r600_pm_init_profile()
702 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;  in r600_pm_init_profile()
703 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
704 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
706 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;  in r600_pm_init_profile()
707 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;  in r600_pm_init_profile()
708 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
709 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
711 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;  in r600_pm_init_profile()
712 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;  in r600_pm_init_profile()
713 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
714 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;  in r600_pm_init_profile()
716 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;  in r600_pm_init_profile()
717 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;  in r600_pm_init_profile()
718 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
719 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
722 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
723 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in r600_pm_init_profile()
724 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
725 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
727 			if (rdev->flags & RADEON_IS_MOBILITY)  in r600_pm_init_profile()
728 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);  in r600_pm_init_profile()
730 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in r600_pm_init_profile()
731 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
732 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
733 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
734 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
736 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
737 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
738 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
739 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;  in r600_pm_init_profile()
741 			idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in r600_pm_init_profile()
742 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
743 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
744 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
745 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
747 			if (rdev->flags & RADEON_IS_MOBILITY)  in r600_pm_init_profile()
748 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);  in r600_pm_init_profile()
750 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);  in r600_pm_init_profile()
751 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
752 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
753 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
754 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in r600_pm_init_profile()
756 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
757 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
758 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
759 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;  in r600_pm_init_profile()
761 			idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);  in r600_pm_init_profile()
762 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;  in r600_pm_init_profile()
763 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;  in r600_pm_init_profile()
764 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in r600_pm_init_profile()
765 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;  in r600_pm_init_profile()
770 void r600_pm_misc(struct radeon_device *rdev)  in r600_pm_misc()  argument
772 	int req_ps_idx = rdev->pm.requested_power_state_index;  in r600_pm_misc()
773 	int req_cm_idx = rdev->pm.requested_clock_mode_index;  in r600_pm_misc()
774 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];  in r600_pm_misc()
781 		if (voltage->voltage != rdev->pm.current_vddc) {  in r600_pm_misc()
782 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);  in r600_pm_misc()
783 			rdev->pm.current_vddc = voltage->voltage;  in r600_pm_misc()
789 bool r600_gui_idle(struct radeon_device *rdev)  in r600_gui_idle()  argument
798 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)  in r600_hpd_sense()  argument
802 	if (ASIC_IS_DCE3(rdev)) {  in r600_hpd_sense()
853 void r600_hpd_set_polarity(struct radeon_device *rdev,  in r600_hpd_set_polarity()  argument
857 	bool connected = r600_hpd_sense(rdev, hpd);  in r600_hpd_set_polarity()
859 	if (ASIC_IS_DCE3(rdev)) {  in r600_hpd_set_polarity()
945 void r600_hpd_init(struct radeon_device *rdev)  in r600_hpd_init()  argument
947 	struct drm_device *dev = rdev->ddev;  in r600_hpd_init()
962 		if (ASIC_IS_DCE3(rdev)) {  in r600_hpd_init()
964 			if (ASIC_IS_DCE32(rdev))  in r600_hpd_init()
1007 		radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);  in r600_hpd_init()
1009 	radeon_irq_kms_enable_hpd(rdev, enable);  in r600_hpd_init()
1012 void r600_hpd_fini(struct radeon_device *rdev)  in r600_hpd_fini()  argument
1014 	struct drm_device *dev = rdev->ddev;  in r600_hpd_fini()
1020 		if (ASIC_IS_DCE3(rdev)) {  in r600_hpd_fini()
1062 	radeon_irq_kms_disable_hpd(rdev, disable);  in r600_hpd_fini()
1068 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)  in r600_pcie_gart_tlb_flush()  argument
1074 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&  in r600_pcie_gart_tlb_flush()
1075 	    !(rdev->flags & RADEON_IS_AGP)) {  in r600_pcie_gart_tlb_flush()
1076 		void __iomem *ptr = (void *)rdev->gart.ptr;  in r600_pcie_gart_tlb_flush()
1089 	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);  in r600_pcie_gart_tlb_flush()
1090 	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);  in r600_pcie_gart_tlb_flush()
1092 	for (i = 0; i < rdev->usec_timeout; i++) {  in r600_pcie_gart_tlb_flush()
1107 int r600_pcie_gart_init(struct radeon_device *rdev)  in r600_pcie_gart_init()  argument
1111 	if (rdev->gart.robj) {  in r600_pcie_gart_init()
1116 	r = radeon_gart_init(rdev);  in r600_pcie_gart_init()
1119 	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;  in r600_pcie_gart_init()
1120 	return radeon_gart_table_vram_alloc(rdev);  in r600_pcie_gart_init()
1123 static int r600_pcie_gart_enable(struct radeon_device *rdev)  in r600_pcie_gart_enable()  argument
1128 	if (rdev->gart.robj == NULL) {  in r600_pcie_gart_enable()
1129 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in r600_pcie_gart_enable()
1132 	r = radeon_gart_table_vram_pin(rdev);  in r600_pcie_gart_enable()
1163 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in r600_pcie_gart_enable()
1164 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in r600_pcie_gart_enable()
1165 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in r600_pcie_gart_enable()
1169 			(u32)(rdev->dummy_page.addr >> 12));  in r600_pcie_gart_enable()
1173 	r600_pcie_gart_tlb_flush(rdev);  in r600_pcie_gart_enable()
1175 		 (unsigned)(rdev->mc.gtt_size >> 20),  in r600_pcie_gart_enable()
1176 		 (unsigned long long)rdev->gart.table_addr);  in r600_pcie_gart_enable()
1177 	rdev->gart.ready = true;  in r600_pcie_gart_enable()
1181 static void r600_pcie_gart_disable(struct radeon_device *rdev)  in r600_pcie_gart_disable()  argument
1213 	radeon_gart_table_vram_unpin(rdev);  in r600_pcie_gart_disable()
1216 static void r600_pcie_gart_fini(struct radeon_device *rdev)  in r600_pcie_gart_fini()  argument
1218 	radeon_gart_fini(rdev);  in r600_pcie_gart_fini()
1219 	r600_pcie_gart_disable(rdev);  in r600_pcie_gart_fini()
1220 	radeon_gart_table_vram_free(rdev);  in r600_pcie_gart_fini()
1223 static void r600_agp_enable(struct radeon_device *rdev)  in r600_agp_enable()  argument
1257 int r600_mc_wait_for_idle(struct radeon_device *rdev)  in r600_mc_wait_for_idle()  argument
1262 	for (i = 0; i < rdev->usec_timeout; i++) {  in r600_mc_wait_for_idle()
1272 uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)  in rs780_mc_rreg()  argument
1277 	spin_lock_irqsave(&rdev->mc_idx_lock, flags);  in rs780_mc_rreg()
1281 	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);  in rs780_mc_rreg()
1285 void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)  in rs780_mc_wreg()  argument
1289 	spin_lock_irqsave(&rdev->mc_idx_lock, flags);  in rs780_mc_wreg()
1294 	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);  in rs780_mc_wreg()
1297 static void r600_mc_program(struct radeon_device *rdev)  in r600_mc_program()  argument
1313 	rv515_mc_stop(rdev, &save);  in r600_mc_program()
1314 	if (r600_mc_wait_for_idle(rdev)) {  in r600_mc_program()
1315 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in r600_mc_program()
1320 	if (rdev->flags & RADEON_IS_AGP) {  in r600_mc_program()
1321 		if (rdev->mc.vram_start < rdev->mc.gtt_start) {  in r600_mc_program()
1324 				rdev->mc.vram_start >> 12);  in r600_mc_program()
1326 				rdev->mc.gtt_end >> 12);  in r600_mc_program()
1330 				rdev->mc.gtt_start >> 12);  in r600_mc_program()
1332 				rdev->mc.vram_end >> 12);  in r600_mc_program()
1335 		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);  in r600_mc_program()
1336 		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);  in r600_mc_program()
1338 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);  in r600_mc_program()
1339 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in r600_mc_program()
1340 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in r600_mc_program()
1342 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in r600_mc_program()
1345 	if (rdev->flags & RADEON_IS_AGP) {  in r600_mc_program()
1346 		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);  in r600_mc_program()
1347 		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);  in r600_mc_program()
1348 		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);  in r600_mc_program()
1354 	if (r600_mc_wait_for_idle(rdev)) {  in r600_mc_program()
1355 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in r600_mc_program()
1357 	rv515_mc_resume(rdev, &save);  in r600_mc_program()
1360 	rv515_vga_render_disable(rdev);  in r600_mc_program()
1384 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)  in r600_vram_gtt_location()  argument
1390 		dev_warn(rdev->dev, "limiting VRAM\n");  in r600_vram_gtt_location()
1394 	if (rdev->flags & RADEON_IS_AGP) {  in r600_vram_gtt_location()
1399 				dev_warn(rdev->dev, "limiting VRAM\n");  in r600_vram_gtt_location()
1406 				dev_warn(rdev->dev, "limiting VRAM\n");  in r600_vram_gtt_location()
1413 		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",  in r600_vram_gtt_location()
1418 		if (rdev->flags & RADEON_IS_IGP) {  in r600_vram_gtt_location()
1422 		radeon_vram_location(rdev, &rdev->mc, base);  in r600_vram_gtt_location()
1423 		rdev->mc.gtt_base_align = 0;  in r600_vram_gtt_location()
1424 		radeon_gtt_location(rdev, mc);  in r600_vram_gtt_location()
1428 static int r600_mc_init(struct radeon_device *rdev)  in r600_mc_init()  argument
1436 	rdev->mc.vram_is_ddr = true;  in r600_mc_init()
1461 	rdev->mc.vram_width = numchan * chansize;  in r600_mc_init()
1463 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in r600_mc_init()
1464 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in r600_mc_init()
1466 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);  in r600_mc_init()
1467 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);  in r600_mc_init()
1468 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in r600_mc_init()
1469 	r600_vram_gtt_location(rdev, &rdev->mc);  in r600_mc_init()
1471 	if (rdev->flags & RADEON_IS_IGP) {  in r600_mc_init()
1472 		rs690_pm_info(rdev);  in r600_mc_init()
1473 		rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);  in r600_mc_init()
1475 		if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {  in r600_mc_init()
1477 			rdev->fastfb_working = false;  in r600_mc_init()
1482 			if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)  in r600_mc_init()
1488 				if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {  in r600_mc_init()
1490 						(unsigned long long)rdev->mc.aper_base, k8_addr);  in r600_mc_init()
1491 					rdev->mc.aper_base = (resource_size_t)k8_addr;  in r600_mc_init()
1492 					rdev->fastfb_working = true;  in r600_mc_init()
1498 	radeon_update_bandwidth_info(rdev);  in r600_mc_init()
1502 int r600_vram_scratch_init(struct radeon_device *rdev)  in r600_vram_scratch_init()  argument
1506 	if (rdev->vram_scratch.robj == NULL) {  in r600_vram_scratch_init()
1507 		r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,  in r600_vram_scratch_init()
1509 				     0, NULL, NULL, &rdev->vram_scratch.robj);  in r600_vram_scratch_init()
1515 	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);  in r600_vram_scratch_init()
1518 	r = radeon_bo_pin(rdev->vram_scratch.robj,  in r600_vram_scratch_init()
1519 			  RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);  in r600_vram_scratch_init()
1521 		radeon_bo_unreserve(rdev->vram_scratch.robj);  in r600_vram_scratch_init()
1524 	r = radeon_bo_kmap(rdev->vram_scratch.robj,  in r600_vram_scratch_init()
1525 				(void **)&rdev->vram_scratch.ptr);  in r600_vram_scratch_init()
1527 		radeon_bo_unpin(rdev->vram_scratch.robj);  in r600_vram_scratch_init()
1528 	radeon_bo_unreserve(rdev->vram_scratch.robj);  in r600_vram_scratch_init()
1533 void r600_vram_scratch_fini(struct radeon_device *rdev)  in r600_vram_scratch_fini()  argument
1537 	if (rdev->vram_scratch.robj == NULL) {  in r600_vram_scratch_fini()
1540 	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);  in r600_vram_scratch_fini()
1542 		radeon_bo_kunmap(rdev->vram_scratch.robj);  in r600_vram_scratch_fini()
1543 		radeon_bo_unpin(rdev->vram_scratch.robj);  in r600_vram_scratch_fini()
1544 		radeon_bo_unreserve(rdev->vram_scratch.robj);  in r600_vram_scratch_fini()
1546 	radeon_bo_unref(&rdev->vram_scratch.robj);  in r600_vram_scratch_fini()
1549 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)  in r600_set_bios_scratch_engine_hung()  argument
1561 static void r600_print_gpu_status_regs(struct radeon_device *rdev)  in r600_print_gpu_status_regs()  argument
1563 	dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",  in r600_print_gpu_status_regs()
1565 	dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",  in r600_print_gpu_status_regs()
1567 	dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",  in r600_print_gpu_status_regs()
1569 	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",  in r600_print_gpu_status_regs()
1571 	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",  in r600_print_gpu_status_regs()
1573 	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",  in r600_print_gpu_status_regs()
1575 	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",  in r600_print_gpu_status_regs()
1577 	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",  in r600_print_gpu_status_regs()
1581 static bool r600_is_display_hung(struct radeon_device *rdev)  in r600_is_display_hung()  argument
1587 	for (i = 0; i < rdev->num_crtc; i++) {  in r600_is_display_hung()
1595 		for (i = 0; i < rdev->num_crtc; i++) {  in r600_is_display_hung()
1610 u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)  in r600_gpu_check_soft_reset()  argument
1617 	if (rdev->family >= CHIP_RV770) {  in r600_gpu_check_soft_reset()
1667 	if (r600_is_display_hung(rdev))  in r600_gpu_check_soft_reset()
1679 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in r600_gpu_soft_reset()  argument
1688 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in r600_gpu_soft_reset()
1690 	r600_print_gpu_status_regs(rdev);  in r600_gpu_soft_reset()
1693 	if (rdev->family >= CHIP_RV770)  in r600_gpu_soft_reset()
1710 	rv515_mc_stop(rdev, &save);  in r600_gpu_soft_reset()
1711 	if (r600_mc_wait_for_idle(rdev)) {  in r600_gpu_soft_reset()
1712 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in r600_gpu_soft_reset()
1716 		if (rdev->family >= CHIP_RV770)  in r600_gpu_soft_reset()
1752 		if (rdev->family >= CHIP_RV770)  in r600_gpu_soft_reset()
1770 	if (!(rdev->flags & RADEON_IS_IGP)) {  in r600_gpu_soft_reset()
1781 		dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);  in r600_gpu_soft_reset()
1795 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in r600_gpu_soft_reset()
1809 	rv515_mc_resume(rdev, &save);  in r600_gpu_soft_reset()
1812 	r600_print_gpu_status_regs(rdev);  in r600_gpu_soft_reset()
1815 static void r600_gpu_pci_config_reset(struct radeon_device *rdev)  in r600_gpu_pci_config_reset()  argument
1820 	dev_info(rdev->dev, "GPU pci config reset\n");  in r600_gpu_pci_config_reset()
1825 	if (rdev->family >= CHIP_RV770)  in r600_gpu_pci_config_reset()
1841 	if (rdev->family >= CHIP_RV770)  in r600_gpu_pci_config_reset()
1842 		rv770_set_clk_bypass_mode(rdev);  in r600_gpu_pci_config_reset()
1844 	pci_clear_master(rdev->pdev);  in r600_gpu_pci_config_reset()
1846 	rv515_mc_stop(rdev, &save);  in r600_gpu_pci_config_reset()
1847 	if (r600_mc_wait_for_idle(rdev)) {  in r600_gpu_pci_config_reset()
1848 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in r600_gpu_pci_config_reset()
1859 	radeon_pci_config_reset(rdev);  in r600_gpu_pci_config_reset()
1869 	for (i = 0; i < rdev->usec_timeout; i++) {  in r600_gpu_pci_config_reset()
1876 int r600_asic_reset(struct radeon_device *rdev, bool hard)  in r600_asic_reset()  argument
1881 		r600_gpu_pci_config_reset(rdev);  in r600_asic_reset()
1885 	reset_mask = r600_gpu_check_soft_reset(rdev);  in r600_asic_reset()
1888 		r600_set_bios_scratch_engine_hung(rdev, true);  in r600_asic_reset()
1891 	r600_gpu_soft_reset(rdev, reset_mask);  in r600_asic_reset()
1893 	reset_mask = r600_gpu_check_soft_reset(rdev);  in r600_asic_reset()
1897 		r600_gpu_pci_config_reset(rdev);  in r600_asic_reset()
1899 	reset_mask = r600_gpu_check_soft_reset(rdev);  in r600_asic_reset()
1902 		r600_set_bios_scratch_engine_hung(rdev, false);  in r600_asic_reset()
1916 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in r600_gfx_is_lockup()  argument
1918 	u32 reset_mask = r600_gpu_check_soft_reset(rdev);  in r600_gfx_is_lockup()
1923 		radeon_ring_lockup_update(rdev, ring);  in r600_gfx_is_lockup()
1926 	return radeon_ring_test_lockup(rdev, ring);  in r600_gfx_is_lockup()
1929 u32 r6xx_remap_render_backend(struct radeon_device *rdev,  in r6xx_remap_render_backend()  argument
1953 	if (rdev->family <= CHIP_RV740) {  in r6xx_remap_render_backend()
1984 static void r600_gpu_init(struct radeon_device *rdev)  in r600_gpu_init()  argument
1999 	rdev->config.r600.tiling_group_size = 256;  in r600_gpu_init()
2000 	switch (rdev->family) {  in r600_gpu_init()
2002 		rdev->config.r600.max_pipes = 4;  in r600_gpu_init()
2003 		rdev->config.r600.max_tile_pipes = 8;  in r600_gpu_init()
2004 		rdev->config.r600.max_simds = 4;  in r600_gpu_init()
2005 		rdev->config.r600.max_backends = 4;  in r600_gpu_init()
2006 		rdev->config.r600.max_gprs = 256;  in r600_gpu_init()
2007 		rdev->config.r600.max_threads = 192;  in r600_gpu_init()
2008 		rdev->config.r600.max_stack_entries = 256;  in r600_gpu_init()
2009 		rdev->config.r600.max_hw_contexts = 8;  in r600_gpu_init()
2010 		rdev->config.r600.max_gs_threads = 16;  in r600_gpu_init()
2011 		rdev->config.r600.sx_max_export_size = 128;  in r600_gpu_init()
2012 		rdev->config.r600.sx_max_export_pos_size = 16;  in r600_gpu_init()
2013 		rdev->config.r600.sx_max_export_smx_size = 128;  in r600_gpu_init()
2014 		rdev->config.r600.sq_num_cf_insts = 2;  in r600_gpu_init()
2018 		rdev->config.r600.max_pipes = 2;  in r600_gpu_init()
2019 		rdev->config.r600.max_tile_pipes = 2;  in r600_gpu_init()
2020 		rdev->config.r600.max_simds = 3;  in r600_gpu_init()
2021 		rdev->config.r600.max_backends = 1;  in r600_gpu_init()
2022 		rdev->config.r600.max_gprs = 128;  in r600_gpu_init()
2023 		rdev->config.r600.max_threads = 192;  in r600_gpu_init()
2024 		rdev->config.r600.max_stack_entries = 128;  in r600_gpu_init()
2025 		rdev->config.r600.max_hw_contexts = 8;  in r600_gpu_init()
2026 		rdev->config.r600.max_gs_threads = 4;  in r600_gpu_init()
2027 		rdev->config.r600.sx_max_export_size = 128;  in r600_gpu_init()
2028 		rdev->config.r600.sx_max_export_pos_size = 16;  in r600_gpu_init()
2029 		rdev->config.r600.sx_max_export_smx_size = 128;  in r600_gpu_init()
2030 		rdev->config.r600.sq_num_cf_insts = 2;  in r600_gpu_init()
2036 		rdev->config.r600.max_pipes = 1;  in r600_gpu_init()
2037 		rdev->config.r600.max_tile_pipes = 1;  in r600_gpu_init()
2038 		rdev->config.r600.max_simds = 2;  in r600_gpu_init()
2039 		rdev->config.r600.max_backends = 1;  in r600_gpu_init()
2040 		rdev->config.r600.max_gprs = 128;  in r600_gpu_init()
2041 		rdev->config.r600.max_threads = 192;  in r600_gpu_init()
2042 		rdev->config.r600.max_stack_entries = 128;  in r600_gpu_init()
2043 		rdev->config.r600.max_hw_contexts = 4;  in r600_gpu_init()
2044 		rdev->config.r600.max_gs_threads = 4;  in r600_gpu_init()
2045 		rdev->config.r600.sx_max_export_size = 128;  in r600_gpu_init()
2046 		rdev->config.r600.sx_max_export_pos_size = 16;  in r600_gpu_init()
2047 		rdev->config.r600.sx_max_export_smx_size = 128;  in r600_gpu_init()
2048 		rdev->config.r600.sq_num_cf_insts = 1;  in r600_gpu_init()
2051 		rdev->config.r600.max_pipes = 4;  in r600_gpu_init()
2052 		rdev->config.r600.max_tile_pipes = 4;  in r600_gpu_init()
2053 		rdev->config.r600.max_simds = 4;  in r600_gpu_init()
2054 		rdev->config.r600.max_backends = 4;  in r600_gpu_init()
2055 		rdev->config.r600.max_gprs = 192;  in r600_gpu_init()
2056 		rdev->config.r600.max_threads = 192;  in r600_gpu_init()
2057 		rdev->config.r600.max_stack_entries = 256;  in r600_gpu_init()
2058 		rdev->config.r600.max_hw_contexts = 8;  in r600_gpu_init()
2059 		rdev->config.r600.max_gs_threads = 16;  in r600_gpu_init()
2060 		rdev->config.r600.sx_max_export_size = 128;  in r600_gpu_init()
2061 		rdev->config.r600.sx_max_export_pos_size = 16;  in r600_gpu_init()
2062 		rdev->config.r600.sx_max_export_smx_size = 128;  in r600_gpu_init()
2063 		rdev->config.r600.sq_num_cf_insts = 2;  in r600_gpu_init()
2083 	switch (rdev->config.r600.max_tile_pipes) {  in r600_gpu_init()
2099 	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;  in r600_gpu_init()
2100 	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);  in r600_gpu_init()
2115 	tmp = rdev->config.r600.max_simds -  in r600_gpu_init()
2117 	rdev->config.r600.active_simds = tmp;  in r600_gpu_init()
2121 	for (i = 0; i < rdev->config.r600.max_backends; i++)  in r600_gpu_init()
2125 		for (i = 0; i < rdev->config.r600.max_backends; i++)  in r600_gpu_init()
2129 	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,  in r600_gpu_init()
2132 	rdev->config.r600.backend_map = tmp;  in r600_gpu_init()
2134 	rdev->config.r600.tile_config = tiling_config;  in r600_gpu_init()
2151 	if (rdev->family == CHIP_RV670)  in r600_gpu_init()
2156 	if ((rdev->family > CHIP_R600))  in r600_gpu_init()
2160 	if (((rdev->family) == CHIP_R600) ||  in r600_gpu_init()
2161 	    ((rdev->family) == CHIP_RV630) ||  in r600_gpu_init()
2162 	    ((rdev->family) == CHIP_RV610) ||  in r600_gpu_init()
2163 	    ((rdev->family) == CHIP_RV620) ||  in r600_gpu_init()
2164 	    ((rdev->family) == CHIP_RS780) ||  in r600_gpu_init()
2165 	    ((rdev->family) == CHIP_RS880)) {  in r600_gpu_init()
2180 	if (((rdev->family) == CHIP_RV610) ||  in r600_gpu_init()
2181 	    ((rdev->family) == CHIP_RV620) ||  in r600_gpu_init()
2182 	    ((rdev->family) == CHIP_RS780) ||  in r600_gpu_init()
2183 	    ((rdev->family) == CHIP_RS880)) {  in r600_gpu_init()
2188 	} else if (((rdev->family) == CHIP_R600) ||  in r600_gpu_init()
2189 		   ((rdev->family) == CHIP_RV630)) {  in r600_gpu_init()
2210 	if ((rdev->family) == CHIP_R600) {  in r600_gpu_init()
2224 	} else if (((rdev->family) == CHIP_RV610) ||  in r600_gpu_init()
2225 		   ((rdev->family) == CHIP_RV620) ||  in r600_gpu_init()
2226 		   ((rdev->family) == CHIP_RS780) ||  in r600_gpu_init()
2227 		   ((rdev->family) == CHIP_RS880)) {  in r600_gpu_init()
2244 	} else if (((rdev->family) == CHIP_RV630) ||  in r600_gpu_init()
2245 		   ((rdev->family) == CHIP_RV635)) {  in r600_gpu_init()
2259 	} else if ((rdev->family) == CHIP_RV670) {  in r600_gpu_init()
2282 	if (((rdev->family) == CHIP_RV610) ||  in r600_gpu_init()
2283 	    ((rdev->family) == CHIP_RV620) ||  in r600_gpu_init()
2284 	    ((rdev->family) == CHIP_RS780) ||  in r600_gpu_init()
2285 	    ((rdev->family) == CHIP_RS880)) {  in r600_gpu_init()
2308 	tmp = rdev->config.r600.max_pipes * 16;  in r600_gpu_init()
2309 	switch (rdev->family) {  in r600_gpu_init()
2352 	switch (rdev->family) {  in r600_gpu_init()
2390 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)  in r600_pciep_rreg()  argument
2395 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in r600_pciep_rreg()
2399 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in r600_pciep_rreg()
2403 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in r600_pciep_wreg()  argument
2407 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in r600_pciep_wreg()
2412 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in r600_pciep_wreg()
2418 void r600_cp_stop(struct radeon_device *rdev)  in r600_cp_stop()  argument
2420 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in r600_cp_stop()
2421 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in r600_cp_stop()
2424 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in r600_cp_stop()
2427 int r600_init_microcode(struct radeon_device *rdev)  in r600_init_microcode()  argument
2438 	switch (rdev->family) {  in r600_init_microcode()
2532 	if (rdev->family >= CHIP_CEDAR) {  in r600_init_microcode()
2536 	} else if (rdev->family >= CHIP_RV770) {  in r600_init_microcode()
2549 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in r600_init_microcode()
2552 	if (rdev->pfp_fw->size != pfp_req_size) {  in r600_init_microcode()
2554 		       rdev->pfp_fw->size, fw_name);  in r600_init_microcode()
2560 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in r600_init_microcode()
2563 	if (rdev->me_fw->size != me_req_size) {  in r600_init_microcode()
2565 		       rdev->me_fw->size, fw_name);  in r600_init_microcode()
2570 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in r600_init_microcode()
2573 	if (rdev->rlc_fw->size != rlc_req_size) {  in r600_init_microcode()
2575 		       rdev->rlc_fw->size, fw_name);  in r600_init_microcode()
2579 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {  in r600_init_microcode()
2581 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in r600_init_microcode()
2584 			release_firmware(rdev->smc_fw);  in r600_init_microcode()
2585 			rdev->smc_fw = NULL;  in r600_init_microcode()
2587 		} else if (rdev->smc_fw->size != smc_req_size) {  in r600_init_microcode()
2589 			       rdev->smc_fw->size, fw_name);  in r600_init_microcode()
2599 		release_firmware(rdev->pfp_fw);  in r600_init_microcode()
2600 		rdev->pfp_fw = NULL;  in r600_init_microcode()
2601 		release_firmware(rdev->me_fw);  in r600_init_microcode()
2602 		rdev->me_fw = NULL;  in r600_init_microcode()
2603 		release_firmware(rdev->rlc_fw);  in r600_init_microcode()
2604 		rdev->rlc_fw = NULL;  in r600_init_microcode()
2605 		release_firmware(rdev->smc_fw);  in r600_init_microcode()
2606 		rdev->smc_fw = NULL;  in r600_init_microcode()
2611 u32 r600_gfx_get_rptr(struct radeon_device *rdev,  in r600_gfx_get_rptr()  argument
2616 	if (rdev->wb.enabled)  in r600_gfx_get_rptr()
2617 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in r600_gfx_get_rptr()
2624 u32 r600_gfx_get_wptr(struct radeon_device *rdev,  in r600_gfx_get_wptr()  argument
2630 void r600_gfx_set_wptr(struct radeon_device *rdev,  in r600_gfx_set_wptr()  argument
2637 static int r600_cp_load_microcode(struct radeon_device *rdev)  in r600_cp_load_microcode()  argument
2642 	if (!rdev->me_fw || !rdev->pfp_fw)  in r600_cp_load_microcode()
2645 	r600_cp_stop(rdev);  in r600_cp_load_microcode()
2661 	fw_data = (const __be32 *)rdev->me_fw->data;  in r600_cp_load_microcode()
2667 	fw_data = (const __be32 *)rdev->pfp_fw->data;  in r600_cp_load_microcode()
2679 int r600_cp_start(struct radeon_device *rdev)  in r600_cp_start()  argument
2681 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in r600_cp_start()
2685 	r = radeon_ring_lock(rdev, ring, 7);  in r600_cp_start()
2692 	if (rdev->family >= CHIP_RV770) {  in r600_cp_start()
2694 		radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);  in r600_cp_start()
2697 		radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);  in r600_cp_start()
2702 	radeon_ring_unlock_commit(rdev, ring, false);  in r600_cp_start()
2709 int r600_cp_resume(struct radeon_device *rdev)  in r600_cp_resume()  argument
2711 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in r600_cp_resume()
2742 	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));  in r600_cp_resume()
2743 	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in r600_cp_resume()
2744 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in r600_cp_resume()
2746 	if (rdev->wb.enabled)  in r600_cp_resume()
2759 	r600_cp_start(rdev);  in r600_cp_resume()
2761 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);  in r600_cp_resume()
2767 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in r600_cp_resume()
2768 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in r600_cp_resume()
2773 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)  in r600_ring_init()  argument
2784 	if (radeon_ring_supports_scratch_reg(rdev, ring)) {  in r600_ring_init()
2785 		r = radeon_scratch_get(rdev, &ring->rptr_save_reg);  in r600_ring_init()
2793 void r600_cp_fini(struct radeon_device *rdev)  in r600_cp_fini()  argument
2795 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in r600_cp_fini()
2796 	r600_cp_stop(rdev);  in r600_cp_fini()
2797 	radeon_ring_fini(rdev, ring);  in r600_cp_fini()
2798 	radeon_scratch_free(rdev, ring->rptr_save_reg);  in r600_cp_fini()
2804 void r600_scratch_init(struct radeon_device *rdev)  in r600_scratch_init()  argument
2808 	rdev->scratch.num_reg = 7;  in r600_scratch_init()
2809 	rdev->scratch.reg_base = SCRATCH_REG0;  in r600_scratch_init()
2810 	for (i = 0; i < rdev->scratch.num_reg; i++) {  in r600_scratch_init()
2811 		rdev->scratch.free[i] = true;  in r600_scratch_init()
2812 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);  in r600_scratch_init()
2816 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)  in r600_ring_test()  argument
2823 	r = radeon_scratch_get(rdev, &scratch);  in r600_ring_test()
2829 	r = radeon_ring_lock(rdev, ring, 3);  in r600_ring_test()
2832 		radeon_scratch_free(rdev, scratch);  in r600_ring_test()
2838 	radeon_ring_unlock_commit(rdev, ring, false);  in r600_ring_test()
2839 	for (i = 0; i < rdev->usec_timeout; i++) {  in r600_ring_test()
2845 	if (i < rdev->usec_timeout) {  in r600_ring_test()
2852 	radeon_scratch_free(rdev, scratch);  in r600_ring_test()
2860 void r600_fence_ring_emit(struct radeon_device *rdev,  in r600_fence_ring_emit()  argument
2863 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in r600_fence_ring_emit()
2867 	if (rdev->family >= CHIP_RV770)  in r600_fence_ring_emit()
2870 	if (rdev->wb.use_event) {  in r600_fence_ring_emit()
2871 		u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in r600_fence_ring_emit()
2900 …radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET…  in r600_fence_ring_emit()
2919 bool r600_semaphore_ring_emit(struct radeon_device *rdev,  in r600_semaphore_ring_emit()  argument
2927 	if (rdev->family < CHIP_CAYMAN)  in r600_semaphore_ring_emit()
2935 	if (emit_wait && (rdev->family >= CHIP_CEDAR)) {  in r600_semaphore_ring_emit()
2957 struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev,  in r600_copy_cpdma()  argument
2964 	int ring_index = rdev->asic->copy.blit_ring_index;  in r600_copy_cpdma()
2965 	struct radeon_ring *ring = &rdev->ring[ring_index];  in r600_copy_cpdma()
2974 	r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24);  in r600_copy_cpdma()
2977 		radeon_sync_free(rdev, &sync, NULL);  in r600_copy_cpdma()
2981 	radeon_sync_resv(rdev, &sync, resv, false);  in r600_copy_cpdma()
2982 	radeon_sync_rings(rdev, &sync, ring->idx);  in r600_copy_cpdma()
3008 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in r600_copy_cpdma()
3010 		radeon_ring_unlock_undo(rdev, ring);  in r600_copy_cpdma()
3011 		radeon_sync_free(rdev, &sync, NULL);  in r600_copy_cpdma()
3015 	radeon_ring_unlock_commit(rdev, ring, false);  in r600_copy_cpdma()
3016 	radeon_sync_free(rdev, &sync, fence);  in r600_copy_cpdma()
3021 int r600_set_surface_reg(struct radeon_device *rdev, int reg,  in r600_set_surface_reg()  argument
3029 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)  in r600_clear_surface_reg()  argument
3034 static void r600_uvd_init(struct radeon_device *rdev)  in r600_uvd_init()  argument
3038 	if (!rdev->has_uvd)  in r600_uvd_init()
3041 	r = radeon_uvd_init(rdev);  in r600_uvd_init()
3043 		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);  in r600_uvd_init()
3050 		rdev->has_uvd = 0;  in r600_uvd_init()
3053 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;  in r600_uvd_init()
3054 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);  in r600_uvd_init()
3057 static void r600_uvd_start(struct radeon_device *rdev)  in r600_uvd_start()  argument
3061 	if (!rdev->has_uvd)  in r600_uvd_start()
3064 	r = uvd_v1_0_resume(rdev);  in r600_uvd_start()
3066 		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);  in r600_uvd_start()
3069 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);  in r600_uvd_start()
3071 		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);  in r600_uvd_start()
3077 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in r600_uvd_start()
3080 static void r600_uvd_resume(struct radeon_device *rdev)  in r600_uvd_resume()  argument
3085 	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)  in r600_uvd_resume()
3088 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in r600_uvd_resume()
3089 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));  in r600_uvd_resume()
3091 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);  in r600_uvd_resume()
3094 	r = uvd_v1_0_init(rdev);  in r600_uvd_resume()
3096 		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);  in r600_uvd_resume()
3101 static int r600_startup(struct radeon_device *rdev)  in r600_startup()  argument
3107 	r600_pcie_gen2_enable(rdev);  in r600_startup()
3110 	r = r600_vram_scratch_init(rdev);  in r600_startup()
3114 	r600_mc_program(rdev);  in r600_startup()
3116 	if (rdev->flags & RADEON_IS_AGP) {  in r600_startup()
3117 		r600_agp_enable(rdev);  in r600_startup()
3119 		r = r600_pcie_gart_enable(rdev);  in r600_startup()
3123 	r600_gpu_init(rdev);  in r600_startup()
3126 	r = radeon_wb_init(rdev);  in r600_startup()
3130 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in r600_startup()
3132 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in r600_startup()
3136 	r600_uvd_start(rdev);  in r600_startup()
3139 	if (!rdev->irq.installed) {  in r600_startup()
3140 		r = radeon_irq_kms_init(rdev);  in r600_startup()
3145 	r = r600_irq_init(rdev);  in r600_startup()
3148 		radeon_irq_kms_fini(rdev);  in r600_startup()
3151 	r600_irq_set(rdev);  in r600_startup()
3153 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in r600_startup()
3154 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in r600_startup()
3159 	r = r600_cp_load_microcode(rdev);  in r600_startup()
3162 	r = r600_cp_resume(rdev);  in r600_startup()
3166 	r600_uvd_resume(rdev);  in r600_startup()
3168 	r = radeon_ib_pool_init(rdev);  in r600_startup()
3170 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in r600_startup()
3174 	r = radeon_audio_init(rdev);  in r600_startup()
3183 void r600_vga_set_state(struct radeon_device *rdev, bool state)  in r600_vga_set_state()  argument
3197 int r600_resume(struct radeon_device *rdev)  in r600_resume()  argument
3206 	atom_asic_init(rdev->mode_info.atom_context);  in r600_resume()
3208 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in r600_resume()
3209 		radeon_pm_resume(rdev);  in r600_resume()
3211 	rdev->accel_working = true;  in r600_resume()
3212 	r = r600_startup(rdev);  in r600_resume()
3215 		rdev->accel_working = false;  in r600_resume()
3222 int r600_suspend(struct radeon_device *rdev)  in r600_suspend()  argument
3224 	radeon_pm_suspend(rdev);  in r600_suspend()
3225 	radeon_audio_fini(rdev);  in r600_suspend()
3226 	r600_cp_stop(rdev);  in r600_suspend()
3227 	if (rdev->has_uvd) {  in r600_suspend()
3228 		uvd_v1_0_fini(rdev);  in r600_suspend()
3229 		radeon_uvd_suspend(rdev);  in r600_suspend()
3231 	r600_irq_suspend(rdev);  in r600_suspend()
3232 	radeon_wb_disable(rdev);  in r600_suspend()
3233 	r600_pcie_gart_disable(rdev);  in r600_suspend()
3244 int r600_init(struct radeon_device *rdev)  in r600_init()  argument
3248 	if (r600_debugfs_mc_info_init(rdev)) {  in r600_init()
3252 	if (!radeon_get_bios(rdev)) {  in r600_init()
3253 		if (ASIC_IS_AVIVO(rdev))  in r600_init()
3257 	if (!rdev->is_atom_bios) {  in r600_init()
3258 		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");  in r600_init()
3261 	r = radeon_atombios_init(rdev);  in r600_init()
3265 	if (!radeon_card_posted(rdev)) {  in r600_init()
3266 		if (!rdev->bios) {  in r600_init()
3267 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in r600_init()
3271 		atom_asic_init(rdev->mode_info.atom_context);  in r600_init()
3274 	r600_scratch_init(rdev);  in r600_init()
3276 	radeon_surface_init(rdev);  in r600_init()
3278 	radeon_get_clock_info(rdev->ddev);  in r600_init()
3280 	r = radeon_fence_driver_init(rdev);  in r600_init()
3283 	if (rdev->flags & RADEON_IS_AGP) {  in r600_init()
3284 		r = radeon_agp_init(rdev);  in r600_init()
3286 			radeon_agp_disable(rdev);  in r600_init()
3288 	r = r600_mc_init(rdev);  in r600_init()
3292 	r = radeon_bo_init(rdev);  in r600_init()
3296 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {  in r600_init()
3297 		r = r600_init_microcode(rdev);  in r600_init()
3305 	radeon_pm_init(rdev);  in r600_init()
3307 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;  in r600_init()
3308 	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);  in r600_init()
3310 	r600_uvd_init(rdev);  in r600_init()
3312 	rdev->ih.ring_obj = NULL;  in r600_init()
3313 	r600_ih_ring_init(rdev, 64 * 1024);  in r600_init()
3315 	r = r600_pcie_gart_init(rdev);  in r600_init()
3319 	rdev->accel_working = true;  in r600_init()
3320 	r = r600_startup(rdev);  in r600_init()
3322 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in r600_init()
3323 		r600_cp_fini(rdev);  in r600_init()
3324 		r600_irq_fini(rdev);  in r600_init()
3325 		radeon_wb_fini(rdev);  in r600_init()
3326 		radeon_ib_pool_fini(rdev);  in r600_init()
3327 		radeon_irq_kms_fini(rdev);  in r600_init()
3328 		r600_pcie_gart_fini(rdev);  in r600_init()
3329 		rdev->accel_working = false;  in r600_init()
3335 void r600_fini(struct radeon_device *rdev)  in r600_fini()  argument
3337 	radeon_pm_fini(rdev);  in r600_fini()
3338 	radeon_audio_fini(rdev);  in r600_fini()
3339 	r600_cp_fini(rdev);  in r600_fini()
3340 	r600_irq_fini(rdev);  in r600_fini()
3341 	if (rdev->has_uvd) {  in r600_fini()
3342 		uvd_v1_0_fini(rdev);  in r600_fini()
3343 		radeon_uvd_fini(rdev);  in r600_fini()
3345 	radeon_wb_fini(rdev);  in r600_fini()
3346 	radeon_ib_pool_fini(rdev);  in r600_fini()
3347 	radeon_irq_kms_fini(rdev);  in r600_fini()
3348 	r600_pcie_gart_fini(rdev);  in r600_fini()
3349 	r600_vram_scratch_fini(rdev);  in r600_fini()
3350 	radeon_agp_fini(rdev);  in r600_fini()
3351 	radeon_gem_fini(rdev);  in r600_fini()
3352 	radeon_fence_driver_fini(rdev);  in r600_fini()
3353 	radeon_bo_fini(rdev);  in r600_fini()
3354 	radeon_atombios_fini(rdev);  in r600_fini()
3355 	kfree(rdev->bios);  in r600_fini()
3356 	rdev->bios = NULL;  in r600_fini()
3363 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in r600_ring_ib_execute()  argument
3365 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in r600_ring_ib_execute()
3374 	} else if (rdev->wb.enabled) {  in r600_ring_ib_execute()
3393 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)  in r600_ib_test()  argument
3401 	r = radeon_scratch_get(rdev, &scratch);  in r600_ib_test()
3407 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in r600_ib_test()
3416 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in r600_ib_test()
3432 	for (i = 0; i < rdev->usec_timeout; i++) {  in r600_ib_test()
3438 	if (i < rdev->usec_timeout) {  in r600_ib_test()
3446 	radeon_ib_free(rdev, &ib);  in r600_ib_test()
3448 	radeon_scratch_free(rdev, scratch);  in r600_ib_test()
3463 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)  in r600_ih_ring_init()  argument
3470 	rdev->ih.ring_size = ring_size;  in r600_ih_ring_init()
3471 	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;  in r600_ih_ring_init()
3472 	rdev->ih.rptr = 0;  in r600_ih_ring_init()
3475 int r600_ih_ring_alloc(struct radeon_device *rdev)  in r600_ih_ring_alloc()  argument
3480 	if (rdev->ih.ring_obj == NULL) {  in r600_ih_ring_alloc()
3481 		r = radeon_bo_create(rdev, rdev->ih.ring_size,  in r600_ih_ring_alloc()
3484 				     NULL, NULL, &rdev->ih.ring_obj);  in r600_ih_ring_alloc()
3489 		r = radeon_bo_reserve(rdev->ih.ring_obj, false);  in r600_ih_ring_alloc()
3492 		r = radeon_bo_pin(rdev->ih.ring_obj,  in r600_ih_ring_alloc()
3494 				  &rdev->ih.gpu_addr);  in r600_ih_ring_alloc()
3496 			radeon_bo_unreserve(rdev->ih.ring_obj);  in r600_ih_ring_alloc()
3500 		r = radeon_bo_kmap(rdev->ih.ring_obj,  in r600_ih_ring_alloc()
3501 				   (void **)&rdev->ih.ring);  in r600_ih_ring_alloc()
3502 		radeon_bo_unreserve(rdev->ih.ring_obj);  in r600_ih_ring_alloc()
3511 void r600_ih_ring_fini(struct radeon_device *rdev)  in r600_ih_ring_fini()  argument
3514 	if (rdev->ih.ring_obj) {  in r600_ih_ring_fini()
3515 		r = radeon_bo_reserve(rdev->ih.ring_obj, false);  in r600_ih_ring_fini()
3517 			radeon_bo_kunmap(rdev->ih.ring_obj);  in r600_ih_ring_fini()
3518 			radeon_bo_unpin(rdev->ih.ring_obj);  in r600_ih_ring_fini()
3519 			radeon_bo_unreserve(rdev->ih.ring_obj);  in r600_ih_ring_fini()
3521 		radeon_bo_unref(&rdev->ih.ring_obj);  in r600_ih_ring_fini()
3522 		rdev->ih.ring = NULL;  in r600_ih_ring_fini()
3523 		rdev->ih.ring_obj = NULL;  in r600_ih_ring_fini()
3527 void r600_rlc_stop(struct radeon_device *rdev)  in r600_rlc_stop()  argument
3530 	if ((rdev->family >= CHIP_RV770) &&  in r600_rlc_stop()
3531 	    (rdev->family <= CHIP_RV740)) {  in r600_rlc_stop()
3543 static void r600_rlc_start(struct radeon_device *rdev)  in r600_rlc_start()  argument
3548 static int r600_rlc_resume(struct radeon_device *rdev)  in r600_rlc_resume()  argument
3553 	if (!rdev->rlc_fw)  in r600_rlc_resume()
3556 	r600_rlc_stop(rdev);  in r600_rlc_resume()
3568 	fw_data = (const __be32 *)rdev->rlc_fw->data;  in r600_rlc_resume()
3569 	if (rdev->family >= CHIP_RV770) {  in r600_rlc_resume()
3582 	r600_rlc_start(rdev);  in r600_rlc_resume()
3587 static void r600_enable_interrupts(struct radeon_device *rdev)  in r600_enable_interrupts()  argument
3596 	rdev->ih.enabled = true;  in r600_enable_interrupts()
3599 void r600_disable_interrupts(struct radeon_device *rdev)  in r600_disable_interrupts()  argument
3611 	rdev->ih.enabled = false;  in r600_disable_interrupts()
3612 	rdev->ih.rptr = 0;  in r600_disable_interrupts()
3615 static void r600_disable_interrupt_state(struct radeon_device *rdev)  in r600_disable_interrupt_state()  argument
3626 	if (ASIC_IS_DCE3(rdev)) {  in r600_disable_interrupt_state()
3637 		if (ASIC_IS_DCE32(rdev)) {  in r600_disable_interrupt_state()
3668 int r600_irq_init(struct radeon_device *rdev)  in r600_irq_init()  argument
3675 	ret = r600_ih_ring_alloc(rdev);  in r600_irq_init()
3680 	r600_disable_interrupts(rdev);  in r600_irq_init()
3683 	if (rdev->family >= CHIP_CEDAR)  in r600_irq_init()
3684 		ret = evergreen_rlc_resume(rdev);  in r600_irq_init()
3686 		ret = r600_rlc_resume(rdev);  in r600_irq_init()
3688 		r600_ih_ring_fini(rdev);  in r600_irq_init()
3694 	WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);  in r600_irq_init()
3704 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);  in r600_irq_init()
3705 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);  in r600_irq_init()
3711 	if (rdev->wb.enabled)  in r600_irq_init()
3715 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);  in r600_irq_init()
3716 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);  in r600_irq_init()
3727 	if (rdev->msi_enabled)  in r600_irq_init()
3732 	if (rdev->family >= CHIP_CEDAR)  in r600_irq_init()
3733 		evergreen_disable_interrupt_state(rdev);  in r600_irq_init()
3735 		r600_disable_interrupt_state(rdev);  in r600_irq_init()
3738 	pci_set_master(rdev->pdev);  in r600_irq_init()
3741 	r600_enable_interrupts(rdev);  in r600_irq_init()
3746 void r600_irq_suspend(struct radeon_device *rdev)  in r600_irq_suspend()  argument
3748 	r600_irq_disable(rdev);  in r600_irq_suspend()
3749 	r600_rlc_stop(rdev);  in r600_irq_suspend()
3752 void r600_irq_fini(struct radeon_device *rdev)  in r600_irq_fini()  argument
3754 	r600_irq_suspend(rdev);  in r600_irq_fini()
3755 	r600_ih_ring_fini(rdev);  in r600_irq_fini()
3758 int r600_irq_set(struct radeon_device *rdev)  in r600_irq_set()  argument
3768 	if (!rdev->irq.installed) {  in r600_irq_set()
3773 	if (!rdev->ih.enabled) {  in r600_irq_set()
3774 		r600_disable_interrupts(rdev);  in r600_irq_set()
3776 		r600_disable_interrupt_state(rdev);  in r600_irq_set()
3780 	if (ASIC_IS_DCE3(rdev)) {  in r600_irq_set()
3785 		if (ASIC_IS_DCE32(rdev)) {  in r600_irq_set()
3804 	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {  in r600_irq_set()
3807 	} else if (rdev->family >= CHIP_RV770) {  in r600_irq_set()
3811 	if (rdev->irq.dpm_thermal) {  in r600_irq_set()
3816 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in r600_irq_set()
3822 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in r600_irq_set()
3827 	if (rdev->irq.crtc_vblank_int[0] ||  in r600_irq_set()
3828 	    atomic_read(&rdev->irq.pflip[0])) {  in r600_irq_set()
3832 	if (rdev->irq.crtc_vblank_int[1] ||  in r600_irq_set()
3833 	    atomic_read(&rdev->irq.pflip[1])) {  in r600_irq_set()
3837 	if (rdev->irq.hpd[0]) {  in r600_irq_set()
3841 	if (rdev->irq.hpd[1]) {  in r600_irq_set()
3845 	if (rdev->irq.hpd[2]) {  in r600_irq_set()
3849 	if (rdev->irq.hpd[3]) {  in r600_irq_set()
3853 	if (rdev->irq.hpd[4]) {  in r600_irq_set()
3857 	if (rdev->irq.hpd[5]) {  in r600_irq_set()
3861 	if (rdev->irq.afmt[0]) {  in r600_irq_set()
3865 	if (rdev->irq.afmt[1]) {  in r600_irq_set()
3876 	if (ASIC_IS_DCE3(rdev)) {  in r600_irq_set()
3881 		if (ASIC_IS_DCE32(rdev)) {  in r600_irq_set()
3897 	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {  in r600_irq_set()
3899 	} else if (rdev->family >= CHIP_RV770) {  in r600_irq_set()
3909 static void r600_irq_ack(struct radeon_device *rdev)  in r600_irq_ack()  argument
3913 	if (ASIC_IS_DCE3(rdev)) {  in r600_irq_ack()
3914 		rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);  in r600_irq_ack()
3915 		rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);  in r600_irq_ack()
3916 		rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);  in r600_irq_ack()
3917 		if (ASIC_IS_DCE32(rdev)) {  in r600_irq_ack()
3918 			rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);  in r600_irq_ack()
3919 			rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);  in r600_irq_ack()
3921 			rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);  in r600_irq_ack()
3922 			rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);  in r600_irq_ack()
3925 		rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);  in r600_irq_ack()
3926 		rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);  in r600_irq_ack()
3927 		rdev->irq.stat_regs.r600.disp_int_cont2 = 0;  in r600_irq_ack()
3928 		rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);  in r600_irq_ack()
3929 		rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);  in r600_irq_ack()
3931 	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);  in r600_irq_ack()
3932 	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);  in r600_irq_ack()
3934 	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)  in r600_irq_ack()
3936 	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)  in r600_irq_ack()
3938 	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)  in r600_irq_ack()
3940 	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)  in r600_irq_ack()
3942 	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)  in r600_irq_ack()
3944 	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)  in r600_irq_ack()
3946 	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {  in r600_irq_ack()
3947 		if (ASIC_IS_DCE3(rdev)) {  in r600_irq_ack()
3957 	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {  in r600_irq_ack()
3958 		if (ASIC_IS_DCE3(rdev)) {  in r600_irq_ack()
3968 	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {  in r600_irq_ack()
3969 		if (ASIC_IS_DCE3(rdev)) {  in r600_irq_ack()
3979 	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {  in r600_irq_ack()
3984 	if (ASIC_IS_DCE32(rdev)) {  in r600_irq_ack()
3985 		if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {  in r600_irq_ack()
3990 		if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {  in r600_irq_ack()
3995 		if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {  in r600_irq_ack()
4000 		if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {  in r600_irq_ack()
4006 		if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {  in r600_irq_ack()
4011 		if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {  in r600_irq_ack()
4012 			if (ASIC_IS_DCE3(rdev)) {  in r600_irq_ack()
4025 void r600_irq_disable(struct radeon_device *rdev)  in r600_irq_disable()  argument
4027 	r600_disable_interrupts(rdev);  in r600_irq_disable()
4030 	r600_irq_ack(rdev);  in r600_irq_disable()
4031 	r600_disable_interrupt_state(rdev);  in r600_irq_disable()
4034 static u32 r600_get_ih_wptr(struct radeon_device *rdev)  in r600_get_ih_wptr()  argument
4038 	if (rdev->wb.enabled)  in r600_get_ih_wptr()
4039 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in r600_get_ih_wptr()
4049 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in r600_get_ih_wptr()
4050 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in r600_get_ih_wptr()
4051 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in r600_get_ih_wptr()
4056 	return (wptr & rdev->ih.ptr_mask);  in r600_get_ih_wptr()
4089 int r600_irq_process(struct radeon_device *rdev)  in r600_irq_process()  argument
4099 	if (!rdev->ih.enabled || rdev->shutdown)  in r600_irq_process()
4103 	if (!rdev->msi_enabled)  in r600_irq_process()
4106 	wptr = r600_get_ih_wptr(rdev);  in r600_irq_process()
4110 	if (atomic_xchg(&rdev->ih.lock, 1))  in r600_irq_process()
4113 	rptr = rdev->ih.rptr;  in r600_irq_process()
4120 	r600_irq_ack(rdev);  in r600_irq_process()
4125 		src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in r600_irq_process()
4126 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in r600_irq_process()
4132 				if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT))  in r600_irq_process()
4135 				if (rdev->irq.crtc_vblank_int[0]) {  in r600_irq_process()
4136 					drm_handle_vblank(rdev->ddev, 0);  in r600_irq_process()
4137 					rdev->pm.vblank_sync = true;  in r600_irq_process()
4138 					wake_up(&rdev->irq.vblank_queue);  in r600_irq_process()
4140 				if (atomic_read(&rdev->irq.pflip[0]))  in r600_irq_process()
4141 					radeon_crtc_handle_vblank(rdev, 0);  in r600_irq_process()
4142 				rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;  in r600_irq_process()
4147 				if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT))  in r600_irq_process()
4150 				rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;  in r600_irq_process()
4162 				if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT))  in r600_irq_process()
4165 				if (rdev->irq.crtc_vblank_int[1]) {  in r600_irq_process()
4166 					drm_handle_vblank(rdev->ddev, 1);  in r600_irq_process()
4167 					rdev->pm.vblank_sync = true;  in r600_irq_process()
4168 					wake_up(&rdev->irq.vblank_queue);  in r600_irq_process()
4170 				if (atomic_read(&rdev->irq.pflip[1]))  in r600_irq_process()
4171 					radeon_crtc_handle_vblank(rdev, 1);  in r600_irq_process()
4172 				rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;  in r600_irq_process()
4177 				if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT))  in r600_irq_process()
4180 				rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;  in r600_irq_process()
4192 				radeon_crtc_handle_flip(rdev, 0);  in r600_irq_process()
4197 				radeon_crtc_handle_flip(rdev, 1);  in r600_irq_process()
4202 				if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT))  in r600_irq_process()
4205 				rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;  in r600_irq_process()
4210 				if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT))  in r600_irq_process()
4213 				rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;  in r600_irq_process()
4218 				if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT))  in r600_irq_process()
4221 				rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;  in r600_irq_process()
4226 				if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT))  in r600_irq_process()
4229 				rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;  in r600_irq_process()
4234 				if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT))  in r600_irq_process()
4237 				rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;  in r600_irq_process()
4242 				if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT))  in r600_irq_process()
4245 				rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;  in r600_irq_process()
4258 				if (!(rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG))  in r600_irq_process()
4261 				rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;  in r600_irq_process()
4267 				if (!(rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG))  in r600_irq_process()
4270 				rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;  in r600_irq_process()
4282 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);  in r600_irq_process()
4288 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in r600_irq_process()
4292 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in r600_irq_process()
4296 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);  in r600_irq_process()
4300 			rdev->pm.dpm.thermal.high_to_low = false;  in r600_irq_process()
4305 			rdev->pm.dpm.thermal.high_to_low = true;  in r600_irq_process()
4318 		rptr &= rdev->ih.ptr_mask;  in r600_irq_process()
4322 		schedule_delayed_work(&rdev->hotplug_work, 0);  in r600_irq_process()
4324 		schedule_work(&rdev->audio_work);  in r600_irq_process()
4325 	if (queue_thermal && rdev->pm.dpm_enabled)  in r600_irq_process()
4326 		schedule_work(&rdev->pm.dpm.thermal.work);  in r600_irq_process()
4327 	rdev->ih.rptr = rptr;  in r600_irq_process()
4328 	atomic_set(&rdev->ih.lock, 0);  in r600_irq_process()
4331 	wptr = r600_get_ih_wptr(rdev);  in r600_irq_process()
4347 	struct radeon_device *rdev = dev->dev_private;  in r600_debugfs_mc_info()  local
4349 	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);  in r600_debugfs_mc_info()
4350 	DREG32_SYS(m, rdev, VM_L2_STATUS);  in r600_debugfs_mc_info()
4359 int r600_debugfs_mc_info_init(struct radeon_device *rdev)  in r600_debugfs_mc_info_init()  argument
4362 	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));  in r600_debugfs_mc_info_init()
4377 void r600_mmio_hdp_flush(struct radeon_device *rdev)  in r600_mmio_hdp_flush()  argument
4384 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&  in r600_mmio_hdp_flush()
4385 	    rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {  in r600_mmio_hdp_flush()
4386 		void __iomem *ptr = (void *)rdev->vram_scratch.ptr;  in r600_mmio_hdp_flush()
4395 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)  in r600_set_pcie_lanes()  argument
4399 	if (rdev->flags & RADEON_IS_IGP)  in r600_set_pcie_lanes()
4402 	if (!(rdev->flags & RADEON_IS_PCIE))  in r600_set_pcie_lanes()
4406 	if (ASIC_IS_X2(rdev))  in r600_set_pcie_lanes()
4409 	radeon_gui_idle(rdev);  in r600_set_pcie_lanes()
4448 int r600_get_pcie_lanes(struct radeon_device *rdev)  in r600_get_pcie_lanes()  argument
4452 	if (rdev->flags & RADEON_IS_IGP)  in r600_get_pcie_lanes()
4455 	if (!(rdev->flags & RADEON_IS_PCIE))  in r600_get_pcie_lanes()
4459 	if (ASIC_IS_X2(rdev))  in r600_get_pcie_lanes()
4462 	radeon_gui_idle(rdev);  in r600_get_pcie_lanes()
4485 static void r600_pcie_gen2_enable(struct radeon_device *rdev)  in r600_pcie_gen2_enable()  argument
4493 	if (rdev->flags & RADEON_IS_IGP)  in r600_pcie_gen2_enable()
4496 	if (!(rdev->flags & RADEON_IS_PCIE))  in r600_pcie_gen2_enable()
4500 	if (ASIC_IS_X2(rdev))  in r600_pcie_gen2_enable()
4504 	if (rdev->family <= CHIP_R600)  in r600_pcie_gen2_enable()
4507 	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&  in r600_pcie_gen2_enable()
4508 		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))  in r600_pcie_gen2_enable()
4520 	if ((rdev->family == CHIP_RV670) ||  in r600_pcie_gen2_enable()
4521 	    (rdev->family == CHIP_RV620) ||  in r600_pcie_gen2_enable()
4522 	    (rdev->family == CHIP_RV635)) {  in r600_pcie_gen2_enable()
4545 		if ((rdev->family == CHIP_RV670) ||  in r600_pcie_gen2_enable()
4546 		    (rdev->family == CHIP_RV620) ||  in r600_pcie_gen2_enable()
4547 		    (rdev->family == CHIP_RV635)) {  in r600_pcie_gen2_enable()
4572 		if ((rdev->family == CHIP_RV670) ||  in r600_pcie_gen2_enable()
4573 		    (rdev->family == CHIP_RV620) ||  in r600_pcie_gen2_enable()
4574 		    (rdev->family == CHIP_RV635)) {  in r600_pcie_gen2_enable()
4607 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)  in r600_get_gpu_clock_counter()  argument
4611 	mutex_lock(&rdev->gpu_clock_mutex);  in r600_get_gpu_clock_counter()
4615 	mutex_unlock(&rdev->gpu_clock_mutex);  in r600_get_gpu_clock_counter()