231 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)  in psb_mmu_free_pt()  argument
233 	__free_page(pt->p);  in psb_mmu_free_pt()
234 	kfree(pt);  in psb_mmu_free_pt()
242 	struct psb_mmu_pt *pt;  in psb_mmu_free_pagedir()  local
255 		pt = pd->tables[i];  in psb_mmu_free_pagedir()
256 		if (pt)  in psb_mmu_free_pagedir()
257 			psb_mmu_free_pt(pt);  in psb_mmu_free_pagedir()
270 	struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);  in psb_mmu_alloc_pt()  local
279 	if (!pt)  in psb_mmu_alloc_pt()
282 	pt->p = alloc_page(GFP_DMA32);  in psb_mmu_alloc_pt()
283 	if (!pt->p) {  in psb_mmu_alloc_pt()
284 		kfree(pt);  in psb_mmu_alloc_pt()
290 	v = kmap_atomic(pt->p);  in psb_mmu_alloc_pt()
309 	pt->count = 0;  in psb_mmu_alloc_pt()
310 	pt->pd = pd;  in psb_mmu_alloc_pt()
311 	pt->index = 0;  in psb_mmu_alloc_pt()
313 	return pt;  in psb_mmu_alloc_pt()
320 	struct psb_mmu_pt *pt;  in psb_mmu_pt_alloc_map_lock()  local
325 	pt = pd->tables[index];  in psb_mmu_pt_alloc_map_lock()
326 	while (!pt) {  in psb_mmu_pt_alloc_map_lock()
328 		pt = psb_mmu_alloc_pt(pd);  in psb_mmu_pt_alloc_map_lock()
329 		if (!pt)  in psb_mmu_pt_alloc_map_lock()
335 			psb_mmu_free_pt(pt);  in psb_mmu_pt_alloc_map_lock()
337 			pt = pd->tables[index];  in psb_mmu_pt_alloc_map_lock()
342 		pd->tables[index] = pt;  in psb_mmu_pt_alloc_map_lock()
343 		v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;  in psb_mmu_pt_alloc_map_lock()
344 		pt->index = index;  in psb_mmu_pt_alloc_map_lock()
352 	pt->v = kmap_atomic(pt->p);  in psb_mmu_pt_alloc_map_lock()
353 	return pt;  in psb_mmu_pt_alloc_map_lock()
360 	struct psb_mmu_pt *pt;  in psb_mmu_pt_map_lock()  local
364 	pt = pd->tables[index];  in psb_mmu_pt_map_lock()
365 	if (!pt) {  in psb_mmu_pt_map_lock()
369 	pt->v = kmap_atomic(pt->p);  in psb_mmu_pt_map_lock()
370 	return pt;  in psb_mmu_pt_map_lock()
373 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)  in psb_mmu_pt_unmap_unlock()  argument
375 	struct psb_mmu_pd *pd = pt->pd;  in psb_mmu_pt_unmap_unlock()
378 	kunmap_atomic(pt->v);  in psb_mmu_pt_unmap_unlock()
379 	if (pt->count == 0) {  in psb_mmu_pt_unmap_unlock()
381 		v[pt->index] = pd->invalid_pde;  in psb_mmu_pt_unmap_unlock()
382 		pd->tables[pt->index] = NULL;  in psb_mmu_pt_unmap_unlock()
385 			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);  in psb_mmu_pt_unmap_unlock()
390 		psb_mmu_free_pt(pt);  in psb_mmu_pt_unmap_unlock()
396 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,  in psb_mmu_set_pte()  argument
399 	pt->v[psb_mmu_pt_index(addr)] = pte;  in psb_mmu_set_pte()
402 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,  in psb_mmu_invalidate_pte()  argument
405 	pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;  in psb_mmu_invalidate_pte()
503 	struct psb_mmu_pt *pt;  in psb_mmu_flush_ptes()  local
532 			pt = psb_mmu_pt_map_lock(pd, addr);  in psb_mmu_flush_ptes()
533 			if (!pt)  in psb_mmu_flush_ptes()
536 				psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);  in psb_mmu_flush_ptes()
540 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_flush_ptes()
558 	struct psb_mmu_pt *pt;  in psb_mmu_remove_pfn_sequence()  local
571 		pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_remove_pfn_sequence()
572 		if (!pt)  in psb_mmu_remove_pfn_sequence()
575 			psb_mmu_invalidate_pte(pt, addr);  in psb_mmu_remove_pfn_sequence()
576 			--pt->count;  in psb_mmu_remove_pfn_sequence()
578 		psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_remove_pfn_sequence()
598 	struct psb_mmu_pt *pt;  in psb_mmu_remove_pages()  local
627 			pt = psb_mmu_pt_map_lock(pd, addr);  in psb_mmu_remove_pages()
628 			if (!pt)  in psb_mmu_remove_pages()
631 				psb_mmu_invalidate_pte(pt, addr);  in psb_mmu_remove_pages()
632 				--pt->count;  in psb_mmu_remove_pages()
635 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_remove_pages()
654 	struct psb_mmu_pt *pt;  in psb_mmu_insert_pfn_sequence()  local
669 		pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_insert_pfn_sequence()
670 		if (!pt) {  in psb_mmu_insert_pfn_sequence()
676 			psb_mmu_set_pte(pt, addr, pte);  in psb_mmu_insert_pfn_sequence()
677 			pt->count++;  in psb_mmu_insert_pfn_sequence()
679 		psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_insert_pfn_sequence()
701 	struct psb_mmu_pt *pt;  in psb_mmu_insert_pages()  local
733 			pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_insert_pages()
734 			if (!pt)  in psb_mmu_insert_pages()
739 				psb_mmu_set_pte(pt, addr, pte);  in psb_mmu_insert_pages()
740 				pt->count++;  in psb_mmu_insert_pages()
742 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_insert_pages()
767 	struct psb_mmu_pt *pt;  in psb_mmu_virtual_to_pfn()  local
772 	pt = psb_mmu_pt_map_lock(pd, virtual);  in psb_mmu_virtual_to_pfn()
773 	if (!pt) {  in psb_mmu_virtual_to_pfn()
791 	tmp = pt->v[psb_mmu_pt_index(virtual)];  in psb_mmu_virtual_to_pfn()
798 	psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_virtual_to_pfn()