53 	iommu_tce_table_put(stit->tbl);  in kvm_spapr_tce_iommu_table_free()
85 				if (table_group->tables[i] != stit->tbl)  in kvm_spapr_tce_release_iommu_group()
101 	struct iommu_table *tbl = NULL;  in kvm_spapr_tce_attach_iommu_group()  local
144 			tbl = iommu_tce_table_get(tbltmp);  in kvm_spapr_tce_attach_iommu_group()
148 	if (!tbl)  in kvm_spapr_tce_attach_iommu_group()
153 		if (tbl != stit->tbl)  in kvm_spapr_tce_attach_iommu_group()
158 			iommu_tce_table_put(tbl);  in kvm_spapr_tce_attach_iommu_group()
173 		iommu_tce_table_put(tbl);  in kvm_spapr_tce_attach_iommu_group()
177 	stit->tbl = tbl;  in kvm_spapr_tce_attach_iommu_group()
381 		long shift = stit->tbl->it_page_shift;  in kvmppc_tce_validate()
403 	u64 *tbl;  in kvmppc_tce_put()  local
419 	tbl = page_to_virt(page);  in kvmppc_tce_put()
421 	tbl[idx % TCES_PER_PAGE] = tce;  in kvmppc_tce_put()
425 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_clear_tce()  argument
428 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
429 	unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);  in kvmppc_clear_tce()
435 		iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir);  in kvmppc_clear_tce()
440 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_tce_iommu_mapped_dec()  argument
443 	const unsigned long pgsize = 1ULL << tbl->it_page_shift;  in kvmppc_tce_iommu_mapped_dec()
444 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);  in kvmppc_tce_iommu_mapped_dec()
461 		struct iommu_table *tbl, unsigned long entry)  in kvmppc_tce_iommu_do_unmap()  argument
467 	if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa,  in kvmppc_tce_iommu_do_unmap()
474 	ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);  in kvmppc_tce_iommu_do_unmap()
476 		iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);  in kvmppc_tce_iommu_do_unmap()
482 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_unmap()  argument
486 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_unmap()
490 		ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i);  in kvmppc_tce_iommu_unmap()
495 	iommu_tce_kill(tbl, io_entry, subpages);  in kvmppc_tce_iommu_unmap()
500 static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,  in kvmppc_tce_iommu_do_map()  argument
506 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);  in kvmppc_tce_iommu_do_map()
513 	mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);  in kvmppc_tce_iommu_do_map()
518 	if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa)))  in kvmppc_tce_iommu_do_map()
524 	ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);  in kvmppc_tce_iommu_do_map()
531 		kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);  in kvmppc_tce_iommu_do_map()
539 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_map()  argument
544 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_map()
548 			++i, pgoff += IOMMU_PAGE_SIZE(tbl)) {  in kvmppc_tce_iommu_map()
550 		ret = kvmppc_tce_iommu_do_map(kvm, tbl,  in kvmppc_tce_iommu_map()
556 	iommu_tce_kill(tbl, io_entry, subpages);  in kvmppc_tce_iommu_map()
599 					stit->tbl, entry);  in kvmppc_h_put_tce()
601 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,  in kvmppc_h_put_tce()
606 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);  in kvmppc_h_put_tce()
693 					stit->tbl, entry + i, ua,  in kvmppc_h_put_tce_indirect()
697 				kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,  in kvmppc_h_put_tce_indirect()
738 					stit->tbl, entry + i);  in kvmppc_h_stuff_tce()
747 			kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);  in kvmppc_h_stuff_tce()