1387 static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl,  in pnv_pci_p7ioc_tce_invalidate()  argument
1391 			&tbl->it_group_list, struct iommu_table_group_link,  in pnv_pci_p7ioc_tce_invalidate()
1398 	start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);  in pnv_pci_p7ioc_tce_invalidate()
1399 	end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +  in pnv_pci_p7ioc_tce_invalidate()
1424 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,  in pnv_ioda1_tce_build()  argument
1429 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,  in pnv_ioda1_tce_build()
1433 		pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);  in pnv_ioda1_tce_build()
1440 static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index,  in pnv_ioda_tce_xchg_no_kill()  argument
1444 	return pnv_tce_xchg(tbl, index, hpa, direction, !realmode);  in pnv_ioda_tce_xchg_no_kill()
1448 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,  in pnv_ioda1_tce_free()  argument
1451 	pnv_tce_free(tbl, index, npages);  in pnv_ioda1_tce_free()
1453 	pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);  in pnv_ioda1_tce_free()
1531 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,  in pnv_pci_ioda2_tce_invalidate()  argument
1536 	list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {  in pnv_pci_ioda2_tce_invalidate()
1540 		unsigned int shift = tbl->it_page_shift;  in pnv_pci_ioda2_tce_invalidate()
1575 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,  in pnv_ioda2_tce_build()  argument
1580 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,  in pnv_ioda2_tce_build()
1584 		pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);  in pnv_ioda2_tce_build()
1589 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,  in pnv_ioda2_tce_free()  argument
1592 	pnv_tce_free(tbl, index, npages);  in pnv_ioda2_tce_free()
1594 	pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);  in pnv_ioda2_tce_free()
1662 	struct iommu_table *tbl;  in pnv_pci_ioda1_setup_dma_pe()  local
1706 	tbl = pnv_pci_table_alloc(phb->hose->node);  in pnv_pci_ioda1_setup_dma_pe()
1707 	if (WARN_ON(!tbl))  in pnv_pci_ioda1_setup_dma_pe()
1712 	pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);  in pnv_pci_ioda1_setup_dma_pe()
1758 	pnv_pci_setup_iommu_table(tbl, addr, tce32_segsz * segs,  in pnv_pci_ioda1_setup_dma_pe()
1762 	tbl->it_ops = &pnv_ioda1_iommu_ops;  in pnv_pci_ioda1_setup_dma_pe()
1763 	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;  in pnv_pci_ioda1_setup_dma_pe()
1764 	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;  in pnv_pci_ioda1_setup_dma_pe()
1765 	iommu_init_table(tbl, phb->hose->node, 0, 0);  in pnv_pci_ioda1_setup_dma_pe()
1773 	if (tbl) {  in pnv_pci_ioda1_setup_dma_pe()
1774 		pnv_pci_unlink_table_and_group(tbl, &pe->table_group);  in pnv_pci_ioda1_setup_dma_pe()
1775 		iommu_tce_table_put(tbl);  in pnv_pci_ioda1_setup_dma_pe()
1780 		int num, struct iommu_table *tbl)  in pnv_pci_ioda2_set_window()  argument
1786 	const unsigned long size = tbl->it_indirect_levels ?  in pnv_pci_ioda2_set_window()
1787 			tbl->it_level_size : tbl->it_size;  in pnv_pci_ioda2_set_window()
1788 	const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;  in pnv_pci_ioda2_set_window()
1789 	const __u64 win_size = tbl->it_size << tbl->it_page_shift;  in pnv_pci_ioda2_set_window()
1793 		IOMMU_PAGE_SIZE(tbl));  in pnv_pci_ioda2_set_window()
1802 			tbl->it_indirect_levels + 1,  in pnv_pci_ioda2_set_window()
1803 			__pa(tbl->it_base),  in pnv_pci_ioda2_set_window()
1805 			IOMMU_PAGE_SIZE(tbl));  in pnv_pci_ioda2_set_window()
1812 			tbl, &pe->table_group);  in pnv_pci_ioda2_set_window()
1855 	struct iommu_table *tbl;  in pnv_pci_ioda2_create_table()  local
1857 	tbl = pnv_pci_table_alloc(nid);  in pnv_pci_ioda2_create_table()
1858 	if (!tbl)  in pnv_pci_ioda2_create_table()
1861 	tbl->it_ops = &pnv_ioda2_iommu_ops;  in pnv_pci_ioda2_create_table()
1865 			levels, alloc_userspace_copy, tbl);  in pnv_pci_ioda2_create_table()
1867 		iommu_tce_table_put(tbl);  in pnv_pci_ioda2_create_table()
1871 	*ptbl = tbl;  in pnv_pci_ioda2_create_table()
1878 	struct iommu_table *tbl = NULL;  in pnv_pci_ioda2_setup_default_config()  local
1919 			window_size, levels, false, &tbl);  in pnv_pci_ioda2_setup_default_config()
1930 		res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;  in pnv_pci_ioda2_setup_default_config()
1931 		res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;  in pnv_pci_ioda2_setup_default_config()
1933 	iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end);  in pnv_pci_ioda2_setup_default_config()
1935 	rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);  in pnv_pci_ioda2_setup_default_config()
1939 		iommu_tce_table_put(tbl);  in pnv_pci_ioda2_setup_default_config()
1952 		set_iommu_table_base(&pe->pdev->dev, tbl);  in pnv_pci_ioda2_setup_default_config()
2046 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_ioda2_take_ownership()  local
2054 	iommu_tce_table_put(tbl);  in pnv_ioda2_take_ownership()
2676 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_pci_ioda1_release_pe_dma()  local
2686 	pnv_pci_p7ioc_tce_invalidate(tbl, tbl->it_offset, tbl->it_size, false);  in pnv_pci_ioda1_release_pe_dma()
2692 	free_pages(tbl->it_base, get_order(tbl->it_size << 3));  in pnv_pci_ioda1_release_pe_dma()
2693 	iommu_tce_table_put(tbl);  in pnv_pci_ioda1_release_pe_dma()
2698 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_pci_ioda2_release_pe_dma()  local
2714 	iommu_tce_table_put(tbl);  in pnv_pci_ioda2_release_pe_dma()