96 	if (info->iommu->segment == data->segment &&  in search_pasid_table()
133 	info = dev->archdata.iommu;  in intel_pasid_alloc_table()
155 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
180 	info = dev->archdata.iommu;  in intel_pasid_free_table()
206 	info = dev->archdata.iommu;  in intel_pasid_get_table()
217 	info = dev->archdata.iommu;  in intel_pasid_get_dev_max_id()
238 	info = dev->archdata.iommu;  in intel_pasid_get_entry()
245 		entries = alloc_pgtable_page(info->iommu->node);  in intel_pasid_get_entry()
399 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
409 	qi_submit_sync(&desc, iommu);  in pasid_cache_invalidation_with_pasid()
413 iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid)  in iotlb_invalidation_with_pasid()  argument
423 	qi_submit_sync(&desc, iommu);  in iotlb_invalidation_with_pasid()
427 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
433 	info = dev->archdata.iommu;  in devtlb_invalidation_with_pasid()
441 	qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
444 void intel_pasid_tear_down_entry(struct intel_iommu *iommu,  in intel_pasid_tear_down_entry()  argument
457 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
460 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
461 	iotlb_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
464 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
465 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
472 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
478 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
480 		       iommu->name);  in intel_pasid_setup_first_level()
493 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_first_level()
495 			       iommu->name);  in intel_pasid_setup_first_level()
507 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
508 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
514 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_first_level()
517 	if (cap_caching_mode(iommu->cap)) {  in intel_pasid_setup_first_level()
518 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_first_level()
519 		iotlb_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_first_level()
521 		iommu_flush_write_buffer(iommu);  in intel_pasid_setup_first_level()
530 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
544 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
546 		       iommu->name);  in intel_pasid_setup_second_level()
555 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in intel_pasid_setup_second_level()
564 	did = domain->iommu_did[iommu->seq_id];  in intel_pasid_setup_second_level()
578 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
587 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_second_level()
590 	if (cap_caching_mode(iommu->cap)) {  in intel_pasid_setup_second_level()
591 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_second_level()
592 		iotlb_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_second_level()
594 		iommu_flush_write_buffer(iommu);  in intel_pasid_setup_second_level()
603 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
618 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
621 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
630 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_pass_through()
633 	if (cap_caching_mode(iommu->cap)) {  in intel_pasid_setup_pass_through()
634 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_pass_through()
635 		iotlb_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_pass_through()
637 		iommu_flush_write_buffer(iommu);  in intel_pasid_setup_pass_through()