57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)  in psb_mmu_clflush()  argument
59 	if (!driver->has_clflush)  in psb_mmu_clflush()
68 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)  in psb_mmu_clflush()  argument
74 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)  in psb_mmu_flush_pd_locked()  argument
76 	struct drm_device *dev = driver->dev;  in psb_mmu_flush_pd_locked()
79 	if (atomic_read(&driver->needs_tlbflush) || force) {  in psb_mmu_flush_pd_locked()
87 		if (driver->msvdx_mmu_invaldc)  in psb_mmu_flush_pd_locked()
88 			atomic_set(driver->msvdx_mmu_invaldc, 1);  in psb_mmu_flush_pd_locked()
90 	atomic_set(&driver->needs_tlbflush, 0);  in psb_mmu_flush_pd_locked()
94 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
96 	down_write(&driver->sem);
97 	psb_mmu_flush_pd_locked(driver, force);
98 	up_write(&driver->sem);
102 void psb_mmu_flush(struct psb_mmu_driver *driver)  in psb_mmu_flush()  argument
104 	struct drm_device *dev = driver->dev;  in psb_mmu_flush()
108 	down_write(&driver->sem);  in psb_mmu_flush()
110 	if (atomic_read(&driver->needs_tlbflush))  in psb_mmu_flush()
122 	atomic_set(&driver->needs_tlbflush, 0);  in psb_mmu_flush()
123 	if (driver->msvdx_mmu_invaldc)  in psb_mmu_flush()
124 		atomic_set(driver->msvdx_mmu_invaldc, 1);  in psb_mmu_flush()
125 	up_write(&driver->sem);  in psb_mmu_flush()
130 	struct drm_device *dev = pd->driver->dev;  in psb_mmu_set_pd_context()
135 	down_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
138 	psb_mmu_flush_pd_locked(pd->driver, 1);  in psb_mmu_set_pd_context()
140 	up_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
165 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,  in psb_mmu_alloc_pd()  argument
216 	pd->driver = driver;  in psb_mmu_alloc_pd()
239 	struct psb_mmu_driver *driver = pd->driver;  in psb_mmu_free_pagedir()  local
240 	struct drm_device *dev = driver->dev;  in psb_mmu_free_pagedir()
245 	down_write(&driver->sem);  in psb_mmu_free_pagedir()
248 		psb_mmu_flush_pd_locked(driver, 1);  in psb_mmu_free_pagedir()
265 	up_write(&driver->sem);  in psb_mmu_free_pagedir()
272 	uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;  in psb_mmu_alloc_pt()
274 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_alloc_pt()
297 	if (pd->driver->has_clflush && pd->hw_context != -1) {  in psb_mmu_alloc_pt()
322 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_alloc_map_lock()
348 			psb_mmu_clflush(pd->driver, (void *)&v[index]);  in psb_mmu_pt_alloc_map_lock()
349 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_alloc_map_lock()
361 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_map_lock()
385 			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);  in psb_mmu_pt_unmap_unlock()
386 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_unmap_unlock()
389 		spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
393 	spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
408 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)  in psb_mmu_get_default_pd()  argument
412 	down_read(&driver->sem);  in psb_mmu_get_default_pd()
413 	pd = driver->default_pd;  in psb_mmu_get_default_pd()
414 	up_read(&driver->sem);  in psb_mmu_get_default_pd()
420 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)  in psb_get_default_pd_addr()  argument
424 	pd = psb_mmu_get_default_pd(driver);  in psb_get_default_pd_addr()
428 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)  in psb_mmu_driver_takedown()  argument
430 	struct drm_device *dev = driver->dev;  in psb_mmu_driver_takedown()
433 	PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);  in psb_mmu_driver_takedown()
434 	psb_mmu_free_pagedir(driver->default_pd);  in psb_mmu_driver_takedown()
435 	kfree(driver);  in psb_mmu_driver_takedown()
443 	struct psb_mmu_driver *driver;  in psb_mmu_driver_init()  local
446 	driver = kmalloc(sizeof(*driver), GFP_KERNEL);  in psb_mmu_driver_init()
448 	if (!driver)  in psb_mmu_driver_init()
451 	driver->dev = dev;  in psb_mmu_driver_init()
452 	driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,  in psb_mmu_driver_init()
454 	if (!driver->default_pd)  in psb_mmu_driver_init()
457 	spin_lock_init(&driver->lock);  in psb_mmu_driver_init()
458 	init_rwsem(&driver->sem);  in psb_mmu_driver_init()
459 	down_write(&driver->sem);  in psb_mmu_driver_init()
460 	atomic_set(&driver->needs_tlbflush, 1);  in psb_mmu_driver_init()
461 	driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;  in psb_mmu_driver_init()
463 	driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);  in psb_mmu_driver_init()
464 	PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,  in psb_mmu_driver_init()
466 	PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,  in psb_mmu_driver_init()
469 	driver->has_clflush = 0;  in psb_mmu_driver_init()
482 		driver->has_clflush = 1;  in psb_mmu_driver_init()
483 		driver->clflush_add =  in psb_mmu_driver_init()
485 		driver->clflush_mask = driver->clflush_add - 1;  in psb_mmu_driver_init()
486 		driver->clflush_mask = ~driver->clflush_mask;  in psb_mmu_driver_init()
490 	up_write(&driver->sem);  in psb_mmu_driver_init()
491 	return driver;  in psb_mmu_driver_init()
494 	kfree(driver);  in psb_mmu_driver_init()
511 	unsigned long clflush_add = pd->driver->clflush_add;  in psb_mmu_flush_ptes()
512 	unsigned long clflush_mask = pd->driver->clflush_mask;  in psb_mmu_flush_ptes()
514 	if (!pd->driver->has_clflush)  in psb_mmu_flush_ptes()
564 	down_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
586 	up_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
589 		psb_mmu_flush(pd->driver);  in psb_mmu_remove_pfn_sequence()
616 	down_read(&pd->driver->sem);  in psb_mmu_remove_pages()
644 	up_read(&pd->driver->sem);  in psb_mmu_remove_pages()
647 		psb_mmu_flush(pd->driver);  in psb_mmu_remove_pages()
662 	down_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
688 	up_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
691 		psb_mmu_flush(pd->driver);  in psb_mmu_insert_pfn_sequence()
724 	down_read(&pd->driver->sem);  in psb_mmu_insert_pages()
755 	up_read(&pd->driver->sem);  in psb_mmu_insert_pages()
758 		psb_mmu_flush(pd->driver);  in psb_mmu_insert_pages()
769 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_virtual_to_pfn()
771 	down_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()
800 	up_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()