862 	struct iova_cpu_rcache *cpu_rcache;  in init_iova_rcaches()  local
871 		rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());  in init_iova_rcaches()
875 			cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);  in init_iova_rcaches()
876 			spin_lock_init(&cpu_rcache->lock);  in init_iova_rcaches()
877 			cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);  in init_iova_rcaches()
878 			cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);  in init_iova_rcaches()
894 	struct iova_cpu_rcache *cpu_rcache;  in __iova_rcache_insert()  local
898 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);  in __iova_rcache_insert()
899 	spin_lock_irqsave(&cpu_rcache->lock, flags);  in __iova_rcache_insert()
901 	if (!iova_magazine_full(cpu_rcache->loaded)) {  in __iova_rcache_insert()
903 	} else if (!iova_magazine_full(cpu_rcache->prev)) {  in __iova_rcache_insert()
904 		swap(cpu_rcache->prev, cpu_rcache->loaded);  in __iova_rcache_insert()
913 						cpu_rcache->loaded;  in __iova_rcache_insert()
915 				mag_to_free = cpu_rcache->loaded;  in __iova_rcache_insert()
919 			cpu_rcache->loaded = new_mag;  in __iova_rcache_insert()
925 		iova_magazine_push(cpu_rcache->loaded, iova_pfn);  in __iova_rcache_insert()
927 	spin_unlock_irqrestore(&cpu_rcache->lock, flags);  in __iova_rcache_insert()
956 	struct iova_cpu_rcache *cpu_rcache;  in __iova_rcache_get()  local
961 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);  in __iova_rcache_get()
962 	spin_lock_irqsave(&cpu_rcache->lock, flags);  in __iova_rcache_get()
964 	if (!iova_magazine_empty(cpu_rcache->loaded)) {  in __iova_rcache_get()
966 	} else if (!iova_magazine_empty(cpu_rcache->prev)) {  in __iova_rcache_get()
967 		swap(cpu_rcache->prev, cpu_rcache->loaded);  in __iova_rcache_get()
972 			iova_magazine_free(cpu_rcache->loaded);  in __iova_rcache_get()
973 			cpu_rcache->loaded = rcache->depot[--rcache->depot_size];  in __iova_rcache_get()
980 		iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);  in __iova_rcache_get()
982 	spin_unlock_irqrestore(&cpu_rcache->lock, flags);  in __iova_rcache_get()
1010 	struct iova_cpu_rcache *cpu_rcache;  in free_iova_rcaches()  local
1017 			cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);  in free_iova_rcaches()
1018 			iova_magazine_free(cpu_rcache->loaded);  in free_iova_rcaches()
1019 			iova_magazine_free(cpu_rcache->prev);  in free_iova_rcaches()
1032 	struct iova_cpu_rcache *cpu_rcache;  in free_cpu_cached_iovas()  local
1039 		cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);  in free_cpu_cached_iovas()
1040 		spin_lock_irqsave(&cpu_rcache->lock, flags);  in free_cpu_cached_iovas()
1041 		iova_magazine_free_pfns(cpu_rcache->loaded, iovad);  in free_cpu_cached_iovas()
1042 		iova_magazine_free_pfns(cpu_rcache->prev, iovad);  in free_cpu_cached_iovas()
1043 		spin_unlock_irqrestore(&cpu_rcache->lock, flags);  in free_cpu_cached_iovas()