• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
4  *
5  * Rewrite, cleanup, new allocation schemes, virtual merging:
6  * Copyright (C) 2004 Olof Johansson, IBM Corporation
7  *               and  Ben. Herrenschmidt, IBM Corporation
8  *
9  * Dynamic DMA mapping support, bus-independent parts.
10  */
11 
12 
13 #include <linux/init.h>
14 #include <linux/types.h>
15 #include <linux/slab.h>
16 #include <linux/mm.h>
17 #include <linux/spinlock.h>
18 #include <linux/string.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/bitmap.h>
21 #include <linux/iommu-helper.h>
22 #include <linux/crash_dump.h>
23 #include <linux/hash.h>
24 #include <linux/fault-inject.h>
25 #include <linux/pci.h>
26 #include <linux/iommu.h>
27 #include <linux/sched.h>
28 #include <asm/io.h>
29 #include <asm/prom.h>
30 #include <asm/iommu.h>
31 #include <asm/pci-bridge.h>
32 #include <asm/machdep.h>
33 #include <asm/kdump.h>
34 #include <asm/fadump.h>
35 #include <asm/vio.h>
36 #include <asm/tce.h>
37 #include <asm/mmu_context.h>
38 
39 #define DBG(...)
40 
41 static int novmerge;
42 
43 static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int);
44 
setup_iommu(char * str)45 static int __init setup_iommu(char *str)
46 {
47 	if (!strcmp(str, "novmerge"))
48 		novmerge = 1;
49 	else if (!strcmp(str, "vmerge"))
50 		novmerge = 0;
51 	return 1;
52 }
53 
54 __setup("iommu=", setup_iommu);
55 
56 static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
57 
58 /*
59  * We precalculate the hash to avoid doing it on every allocation.
60  *
61  * The hash is important to spread CPUs across all the pools. For example,
62  * on a POWER7 with 4 way SMT we want interrupts on the primary threads and
63  * with 4 pools all primary threads would map to the same pool.
64  */
setup_iommu_pool_hash(void)65 static int __init setup_iommu_pool_hash(void)
66 {
67 	unsigned int i;
68 
69 	for_each_possible_cpu(i)
70 		per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
71 
72 	return 0;
73 }
74 subsys_initcall(setup_iommu_pool_hash);
75 
76 #ifdef CONFIG_FAIL_IOMMU
77 
78 static DECLARE_FAULT_ATTR(fail_iommu);
79 
setup_fail_iommu(char * str)80 static int __init setup_fail_iommu(char *str)
81 {
82 	return setup_fault_attr(&fail_iommu, str);
83 }
84 __setup("fail_iommu=", setup_fail_iommu);
85 
should_fail_iommu(struct device * dev)86 static bool should_fail_iommu(struct device *dev)
87 {
88 	return dev->archdata.fail_iommu && should_fail(&fail_iommu, 1);
89 }
90 
fail_iommu_debugfs(void)91 static int __init fail_iommu_debugfs(void)
92 {
93 	struct dentry *dir = fault_create_debugfs_attr("fail_iommu",
94 						       NULL, &fail_iommu);
95 
96 	return PTR_ERR_OR_ZERO(dir);
97 }
98 late_initcall(fail_iommu_debugfs);
99 
fail_iommu_show(struct device * dev,struct device_attribute * attr,char * buf)100 static ssize_t fail_iommu_show(struct device *dev,
101 			       struct device_attribute *attr, char *buf)
102 {
103 	return sprintf(buf, "%d\n", dev->archdata.fail_iommu);
104 }
105 
fail_iommu_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)106 static ssize_t fail_iommu_store(struct device *dev,
107 				struct device_attribute *attr, const char *buf,
108 				size_t count)
109 {
110 	int i;
111 
112 	if (count > 0 && sscanf(buf, "%d", &i) > 0)
113 		dev->archdata.fail_iommu = (i == 0) ? 0 : 1;
114 
115 	return count;
116 }
117 
118 static DEVICE_ATTR_RW(fail_iommu);
119 
fail_iommu_bus_notify(struct notifier_block * nb,unsigned long action,void * data)120 static int fail_iommu_bus_notify(struct notifier_block *nb,
121 				 unsigned long action, void *data)
122 {
123 	struct device *dev = data;
124 
125 	if (action == BUS_NOTIFY_ADD_DEVICE) {
126 		if (device_create_file(dev, &dev_attr_fail_iommu))
127 			pr_warn("Unable to create IOMMU fault injection sysfs "
128 				"entries\n");
129 	} else if (action == BUS_NOTIFY_DEL_DEVICE) {
130 		device_remove_file(dev, &dev_attr_fail_iommu);
131 	}
132 
133 	return 0;
134 }
135 
136 /*
137  * PCI and VIO buses need separate notifier_block structs, since they're linked
138  * list nodes.  Sharing a notifier_block would mean that any notifiers later
139  * registered for PCI buses would also get called by VIO buses and vice versa.
140  */
141 static struct notifier_block fail_iommu_pci_bus_notifier = {
142 	.notifier_call = fail_iommu_bus_notify
143 };
144 
145 #ifdef CONFIG_IBMVIO
146 static struct notifier_block fail_iommu_vio_bus_notifier = {
147 	.notifier_call = fail_iommu_bus_notify
148 };
149 #endif
150 
fail_iommu_setup(void)151 static int __init fail_iommu_setup(void)
152 {
153 #ifdef CONFIG_PCI
154 	bus_register_notifier(&pci_bus_type, &fail_iommu_pci_bus_notifier);
155 #endif
156 #ifdef CONFIG_IBMVIO
157 	bus_register_notifier(&vio_bus_type, &fail_iommu_vio_bus_notifier);
158 #endif
159 
160 	return 0;
161 }
162 /*
163  * Must execute after PCI and VIO subsystem have initialised but before
164  * devices are probed.
165  */
166 arch_initcall(fail_iommu_setup);
167 #else
should_fail_iommu(struct device * dev)168 static inline bool should_fail_iommu(struct device *dev)
169 {
170 	return false;
171 }
172 #endif
173 
iommu_range_alloc(struct device * dev,struct iommu_table * tbl,unsigned long npages,unsigned long * handle,unsigned long mask,unsigned int align_order)174 static unsigned long iommu_range_alloc(struct device *dev,
175 				       struct iommu_table *tbl,
176                                        unsigned long npages,
177                                        unsigned long *handle,
178                                        unsigned long mask,
179                                        unsigned int align_order)
180 {
181 	unsigned long n, end, start;
182 	unsigned long limit;
183 	int largealloc = npages > 15;
184 	int pass = 0;
185 	unsigned long align_mask;
186 	unsigned long flags;
187 	unsigned int pool_nr;
188 	struct iommu_pool *pool;
189 
190 	align_mask = (1ull << align_order) - 1;
191 
192 	/* This allocator was derived from x86_64's bit string search */
193 
194 	/* Sanity check */
195 	if (unlikely(npages == 0)) {
196 		if (printk_ratelimit())
197 			WARN_ON(1);
198 		return DMA_MAPPING_ERROR;
199 	}
200 
201 	if (should_fail_iommu(dev))
202 		return DMA_MAPPING_ERROR;
203 
204 	/*
205 	 * We don't need to disable preemption here because any CPU can
206 	 * safely use any IOMMU pool.
207 	 */
208 	pool_nr = raw_cpu_read(iommu_pool_hash) & (tbl->nr_pools - 1);
209 
210 	if (largealloc)
211 		pool = &(tbl->large_pool);
212 	else
213 		pool = &(tbl->pools[pool_nr]);
214 
215 	spin_lock_irqsave(&(pool->lock), flags);
216 
217 again:
218 	if ((pass == 0) && handle && *handle &&
219 	    (*handle >= pool->start) && (*handle < pool->end))
220 		start = *handle;
221 	else
222 		start = pool->hint;
223 
224 	limit = pool->end;
225 
226 	/* The case below can happen if we have a small segment appended
227 	 * to a large, or when the previous alloc was at the very end of
228 	 * the available space. If so, go back to the initial start.
229 	 */
230 	if (start >= limit)
231 		start = pool->start;
232 
233 	if (limit + tbl->it_offset > mask) {
234 		limit = mask - tbl->it_offset + 1;
235 		/* If we're constrained on address range, first try
236 		 * at the masked hint to avoid O(n) search complexity,
237 		 * but on second pass, start at 0 in pool 0.
238 		 */
239 		if ((start & mask) >= limit || pass > 0) {
240 			spin_unlock(&(pool->lock));
241 			pool = &(tbl->pools[0]);
242 			spin_lock(&(pool->lock));
243 			start = pool->start;
244 		} else {
245 			start &= mask;
246 		}
247 	}
248 
249 	n = iommu_area_alloc(tbl->it_map, limit, start, npages, tbl->it_offset,
250 			dma_get_seg_boundary_nr_pages(dev, tbl->it_page_shift),
251 			align_mask);
252 	if (n == -1) {
253 		if (likely(pass == 0)) {
254 			/* First try the pool from the start */
255 			pool->hint = pool->start;
256 			pass++;
257 			goto again;
258 
259 		} else if (pass <= tbl->nr_pools) {
260 			/* Now try scanning all the other pools */
261 			spin_unlock(&(pool->lock));
262 			pool_nr = (pool_nr + 1) & (tbl->nr_pools - 1);
263 			pool = &tbl->pools[pool_nr];
264 			spin_lock(&(pool->lock));
265 			pool->hint = pool->start;
266 			pass++;
267 			goto again;
268 
269 		} else {
270 			/* Give up */
271 			spin_unlock_irqrestore(&(pool->lock), flags);
272 			return DMA_MAPPING_ERROR;
273 		}
274 	}
275 
276 	end = n + npages;
277 
278 	/* Bump the hint to a new block for small allocs. */
279 	if (largealloc) {
280 		/* Don't bump to new block to avoid fragmentation */
281 		pool->hint = end;
282 	} else {
283 		/* Overflow will be taken care of at the next allocation */
284 		pool->hint = (end + tbl->it_blocksize - 1) &
285 		                ~(tbl->it_blocksize - 1);
286 	}
287 
288 	/* Update handle for SG allocations */
289 	if (handle)
290 		*handle = end;
291 
292 	spin_unlock_irqrestore(&(pool->lock), flags);
293 
294 	return n;
295 }
296 
iommu_alloc(struct device * dev,struct iommu_table * tbl,void * page,unsigned int npages,enum dma_data_direction direction,unsigned long mask,unsigned int align_order,unsigned long attrs)297 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
298 			      void *page, unsigned int npages,
299 			      enum dma_data_direction direction,
300 			      unsigned long mask, unsigned int align_order,
301 			      unsigned long attrs)
302 {
303 	unsigned long entry;
304 	dma_addr_t ret = DMA_MAPPING_ERROR;
305 	int build_fail;
306 
307 	entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
308 
309 	if (unlikely(entry == DMA_MAPPING_ERROR))
310 		return DMA_MAPPING_ERROR;
311 
312 	entry += tbl->it_offset;	/* Offset into real TCE table */
313 	ret = entry << tbl->it_page_shift;	/* Set the return dma address */
314 
315 	/* Put the TCEs in the HW table */
316 	build_fail = tbl->it_ops->set(tbl, entry, npages,
317 				      (unsigned long)page &
318 				      IOMMU_PAGE_MASK(tbl), direction, attrs);
319 
320 	/* tbl->it_ops->set() only returns non-zero for transient errors.
321 	 * Clean up the table bitmap in this case and return
322 	 * DMA_MAPPING_ERROR. For all other errors the functionality is
323 	 * not altered.
324 	 */
325 	if (unlikely(build_fail)) {
326 		__iommu_free(tbl, ret, npages);
327 		return DMA_MAPPING_ERROR;
328 	}
329 
330 	/* Flush/invalidate TLB caches if necessary */
331 	if (tbl->it_ops->flush)
332 		tbl->it_ops->flush(tbl);
333 
334 	/* Make sure updates are seen by hardware */
335 	mb();
336 
337 	return ret;
338 }
339 
iommu_free_check(struct iommu_table * tbl,dma_addr_t dma_addr,unsigned int npages)340 static bool iommu_free_check(struct iommu_table *tbl, dma_addr_t dma_addr,
341 			     unsigned int npages)
342 {
343 	unsigned long entry, free_entry;
344 
345 	entry = dma_addr >> tbl->it_page_shift;
346 	free_entry = entry - tbl->it_offset;
347 
348 	if (((free_entry + npages) > tbl->it_size) ||
349 	    (entry < tbl->it_offset)) {
350 		if (printk_ratelimit()) {
351 			printk(KERN_INFO "iommu_free: invalid entry\n");
352 			printk(KERN_INFO "\tentry     = 0x%lx\n", entry);
353 			printk(KERN_INFO "\tdma_addr  = 0x%llx\n", (u64)dma_addr);
354 			printk(KERN_INFO "\tTable     = 0x%llx\n", (u64)tbl);
355 			printk(KERN_INFO "\tbus#      = 0x%llx\n", (u64)tbl->it_busno);
356 			printk(KERN_INFO "\tsize      = 0x%llx\n", (u64)tbl->it_size);
357 			printk(KERN_INFO "\tstartOff  = 0x%llx\n", (u64)tbl->it_offset);
358 			printk(KERN_INFO "\tindex     = 0x%llx\n", (u64)tbl->it_index);
359 			WARN_ON(1);
360 		}
361 
362 		return false;
363 	}
364 
365 	return true;
366 }
367 
get_pool(struct iommu_table * tbl,unsigned long entry)368 static struct iommu_pool *get_pool(struct iommu_table *tbl,
369 				   unsigned long entry)
370 {
371 	struct iommu_pool *p;
372 	unsigned long largepool_start = tbl->large_pool.start;
373 
374 	/* The large pool is the last pool at the top of the table */
375 	if (entry >= largepool_start) {
376 		p = &tbl->large_pool;
377 	} else {
378 		unsigned int pool_nr = entry / tbl->poolsize;
379 
380 		BUG_ON(pool_nr > tbl->nr_pools);
381 		p = &tbl->pools[pool_nr];
382 	}
383 
384 	return p;
385 }
386 
__iommu_free(struct iommu_table * tbl,dma_addr_t dma_addr,unsigned int npages)387 static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
388 			 unsigned int npages)
389 {
390 	unsigned long entry, free_entry;
391 	unsigned long flags;
392 	struct iommu_pool *pool;
393 
394 	entry = dma_addr >> tbl->it_page_shift;
395 	free_entry = entry - tbl->it_offset;
396 
397 	pool = get_pool(tbl, free_entry);
398 
399 	if (!iommu_free_check(tbl, dma_addr, npages))
400 		return;
401 
402 	tbl->it_ops->clear(tbl, entry, npages);
403 
404 	spin_lock_irqsave(&(pool->lock), flags);
405 	bitmap_clear(tbl->it_map, free_entry, npages);
406 	spin_unlock_irqrestore(&(pool->lock), flags);
407 }
408 
iommu_free(struct iommu_table * tbl,dma_addr_t dma_addr,unsigned int npages)409 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
410 		unsigned int npages)
411 {
412 	__iommu_free(tbl, dma_addr, npages);
413 
414 	/* Make sure TLB cache is flushed if the HW needs it. We do
415 	 * not do an mb() here on purpose, it is not needed on any of
416 	 * the current platforms.
417 	 */
418 	if (tbl->it_ops->flush)
419 		tbl->it_ops->flush(tbl);
420 }
421 
ppc_iommu_map_sg(struct device * dev,struct iommu_table * tbl,struct scatterlist * sglist,int nelems,unsigned long mask,enum dma_data_direction direction,unsigned long attrs)422 int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl,
423 		     struct scatterlist *sglist, int nelems,
424 		     unsigned long mask, enum dma_data_direction direction,
425 		     unsigned long attrs)
426 {
427 	dma_addr_t dma_next = 0, dma_addr;
428 	struct scatterlist *s, *outs, *segstart;
429 	int outcount, incount, i, build_fail = 0;
430 	unsigned int align;
431 	unsigned long handle;
432 	unsigned int max_seg_size;
433 
434 	BUG_ON(direction == DMA_NONE);
435 
436 	if ((nelems == 0) || !tbl)
437 		return 0;
438 
439 	outs = s = segstart = &sglist[0];
440 	outcount = 1;
441 	incount = nelems;
442 	handle = 0;
443 
444 	/* Init first segment length for backout at failure */
445 	outs->dma_length = 0;
446 
447 	DBG("sg mapping %d elements:\n", nelems);
448 
449 	max_seg_size = dma_get_max_seg_size(dev);
450 	for_each_sg(sglist, s, nelems, i) {
451 		unsigned long vaddr, npages, entry, slen;
452 
453 		slen = s->length;
454 		/* Sanity check */
455 		if (slen == 0) {
456 			dma_next = 0;
457 			continue;
458 		}
459 		/* Allocate iommu entries for that segment */
460 		vaddr = (unsigned long) sg_virt(s);
461 		npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE(tbl));
462 		align = 0;
463 		if (tbl->it_page_shift < PAGE_SHIFT && slen >= PAGE_SIZE &&
464 		    (vaddr & ~PAGE_MASK) == 0)
465 			align = PAGE_SHIFT - tbl->it_page_shift;
466 		entry = iommu_range_alloc(dev, tbl, npages, &handle,
467 					  mask >> tbl->it_page_shift, align);
468 
469 		DBG("  - vaddr: %lx, size: %lx\n", vaddr, slen);
470 
471 		/* Handle failure */
472 		if (unlikely(entry == DMA_MAPPING_ERROR)) {
473 			if (!(attrs & DMA_ATTR_NO_WARN) &&
474 			    printk_ratelimit())
475 				dev_info(dev, "iommu_alloc failed, tbl %p "
476 					 "vaddr %lx npages %lu\n", tbl, vaddr,
477 					 npages);
478 			goto failure;
479 		}
480 
481 		/* Convert entry to a dma_addr_t */
482 		entry += tbl->it_offset;
483 		dma_addr = entry << tbl->it_page_shift;
484 		dma_addr |= (s->offset & ~IOMMU_PAGE_MASK(tbl));
485 
486 		DBG("  - %lu pages, entry: %lx, dma_addr: %lx\n",
487 			    npages, entry, dma_addr);
488 
489 		/* Insert into HW table */
490 		build_fail = tbl->it_ops->set(tbl, entry, npages,
491 					      vaddr & IOMMU_PAGE_MASK(tbl),
492 					      direction, attrs);
493 		if(unlikely(build_fail))
494 			goto failure;
495 
496 		/* If we are in an open segment, try merging */
497 		if (segstart != s) {
498 			DBG("  - trying merge...\n");
499 			/* We cannot merge if:
500 			 * - allocated dma_addr isn't contiguous to previous allocation
501 			 */
502 			if (novmerge || (dma_addr != dma_next) ||
503 			    (outs->dma_length + s->length > max_seg_size)) {
504 				/* Can't merge: create a new segment */
505 				segstart = s;
506 				outcount++;
507 				outs = sg_next(outs);
508 				DBG("    can't merge, new segment.\n");
509 			} else {
510 				outs->dma_length += s->length;
511 				DBG("    merged, new len: %ux\n", outs->dma_length);
512 			}
513 		}
514 
515 		if (segstart == s) {
516 			/* This is a new segment, fill entries */
517 			DBG("  - filling new segment.\n");
518 			outs->dma_address = dma_addr;
519 			outs->dma_length = slen;
520 		}
521 
522 		/* Calculate next page pointer for contiguous check */
523 		dma_next = dma_addr + slen;
524 
525 		DBG("  - dma next is: %lx\n", dma_next);
526 	}
527 
528 	/* Flush/invalidate TLB caches if necessary */
529 	if (tbl->it_ops->flush)
530 		tbl->it_ops->flush(tbl);
531 
532 	DBG("mapped %d elements:\n", outcount);
533 
534 	/* For the sake of ppc_iommu_unmap_sg, we clear out the length in the
535 	 * next entry of the sglist if we didn't fill the list completely
536 	 */
537 	if (outcount < incount) {
538 		outs = sg_next(outs);
539 		outs->dma_address = DMA_MAPPING_ERROR;
540 		outs->dma_length = 0;
541 	}
542 
543 	/* Make sure updates are seen by hardware */
544 	mb();
545 
546 	return outcount;
547 
548  failure:
549 	for_each_sg(sglist, s, nelems, i) {
550 		if (s->dma_length != 0) {
551 			unsigned long vaddr, npages;
552 
553 			vaddr = s->dma_address & IOMMU_PAGE_MASK(tbl);
554 			npages = iommu_num_pages(s->dma_address, s->dma_length,
555 						 IOMMU_PAGE_SIZE(tbl));
556 			__iommu_free(tbl, vaddr, npages);
557 			s->dma_address = DMA_MAPPING_ERROR;
558 			s->dma_length = 0;
559 		}
560 		if (s == outs)
561 			break;
562 	}
563 	return 0;
564 }
565 
566 
ppc_iommu_unmap_sg(struct iommu_table * tbl,struct scatterlist * sglist,int nelems,enum dma_data_direction direction,unsigned long attrs)567 void ppc_iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist,
568 			int nelems, enum dma_data_direction direction,
569 			unsigned long attrs)
570 {
571 	struct scatterlist *sg;
572 
573 	BUG_ON(direction == DMA_NONE);
574 
575 	if (!tbl)
576 		return;
577 
578 	sg = sglist;
579 	while (nelems--) {
580 		unsigned int npages;
581 		dma_addr_t dma_handle = sg->dma_address;
582 
583 		if (sg->dma_length == 0)
584 			break;
585 		npages = iommu_num_pages(dma_handle, sg->dma_length,
586 					 IOMMU_PAGE_SIZE(tbl));
587 		__iommu_free(tbl, dma_handle, npages);
588 		sg = sg_next(sg);
589 	}
590 
591 	/* Flush/invalidate TLBs if necessary. As for iommu_free(), we
592 	 * do not do an mb() here, the affected platforms do not need it
593 	 * when freeing.
594 	 */
595 	if (tbl->it_ops->flush)
596 		tbl->it_ops->flush(tbl);
597 }
598 
iommu_table_clear(struct iommu_table * tbl)599 static void iommu_table_clear(struct iommu_table *tbl)
600 {
601 	/*
602 	 * In case of firmware assisted dump system goes through clean
603 	 * reboot process at the time of system crash. Hence it's safe to
604 	 * clear the TCE entries if firmware assisted dump is active.
605 	 */
606 	if (!is_kdump_kernel() || is_fadump_active()) {
607 		/* Clear the table in case firmware left allocations in it */
608 		tbl->it_ops->clear(tbl, tbl->it_offset, tbl->it_size);
609 		return;
610 	}
611 
612 #ifdef CONFIG_CRASH_DUMP
613 	if (tbl->it_ops->get) {
614 		unsigned long index, tceval, tcecount = 0;
615 
616 		/* Reserve the existing mappings left by the first kernel. */
617 		for (index = 0; index < tbl->it_size; index++) {
618 			tceval = tbl->it_ops->get(tbl, index + tbl->it_offset);
619 			/*
620 			 * Freed TCE entry contains 0x7fffffffffffffff on JS20
621 			 */
622 			if (tceval && (tceval != 0x7fffffffffffffffUL)) {
623 				__set_bit(index, tbl->it_map);
624 				tcecount++;
625 			}
626 		}
627 
628 		if ((tbl->it_size - tcecount) < KDUMP_MIN_TCE_ENTRIES) {
629 			printk(KERN_WARNING "TCE table is full; freeing ");
630 			printk(KERN_WARNING "%d entries for the kdump boot\n",
631 				KDUMP_MIN_TCE_ENTRIES);
632 			for (index = tbl->it_size - KDUMP_MIN_TCE_ENTRIES;
633 				index < tbl->it_size; index++)
634 				__clear_bit(index, tbl->it_map);
635 		}
636 	}
637 #endif
638 }
639 
iommu_table_reserve_pages(struct iommu_table * tbl,unsigned long res_start,unsigned long res_end)640 static void iommu_table_reserve_pages(struct iommu_table *tbl,
641 		unsigned long res_start, unsigned long res_end)
642 {
643 	int i;
644 
645 	WARN_ON_ONCE(res_end < res_start);
646 	/*
647 	 * Reserve page 0 so it will not be used for any mappings.
648 	 * This avoids buggy drivers that consider page 0 to be invalid
649 	 * to crash the machine or even lose data.
650 	 */
651 	if (tbl->it_offset == 0)
652 		set_bit(0, tbl->it_map);
653 
654 	tbl->it_reserved_start = res_start;
655 	tbl->it_reserved_end = res_end;
656 
657 	/* Check if res_start..res_end isn't empty and overlaps the table */
658 	if (res_start && res_end &&
659 			(tbl->it_offset + tbl->it_size < res_start ||
660 			 res_end < tbl->it_offset))
661 		return;
662 
663 	for (i = tbl->it_reserved_start; i < tbl->it_reserved_end; ++i)
664 		set_bit(i - tbl->it_offset, tbl->it_map);
665 }
666 
iommu_table_release_pages(struct iommu_table * tbl)667 static void iommu_table_release_pages(struct iommu_table *tbl)
668 {
669 	int i;
670 
671 	/*
672 	 * In case we have reserved the first bit, we should not emit
673 	 * the warning below.
674 	 */
675 	if (tbl->it_offset == 0)
676 		clear_bit(0, tbl->it_map);
677 
678 	for (i = tbl->it_reserved_start; i < tbl->it_reserved_end; ++i)
679 		clear_bit(i - tbl->it_offset, tbl->it_map);
680 }
681 
682 /*
683  * Build a iommu_table structure.  This contains a bit map which
684  * is used to manage allocation of the tce space.
685  */
iommu_init_table(struct iommu_table * tbl,int nid,unsigned long res_start,unsigned long res_end)686 struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid,
687 		unsigned long res_start, unsigned long res_end)
688 {
689 	unsigned long sz;
690 	static int welcomed = 0;
691 	struct page *page;
692 	unsigned int i;
693 	struct iommu_pool *p;
694 
695 	BUG_ON(!tbl->it_ops);
696 
697 	/* number of bytes needed for the bitmap */
698 	sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
699 
700 	page = alloc_pages_node(nid, GFP_KERNEL, get_order(sz));
701 	if (!page)
702 		panic("iommu_init_table: Can't allocate %ld bytes\n", sz);
703 	tbl->it_map = page_address(page);
704 	memset(tbl->it_map, 0, sz);
705 
706 	iommu_table_reserve_pages(tbl, res_start, res_end);
707 
708 	/* We only split the IOMMU table if we have 1GB or more of space */
709 	if ((tbl->it_size << tbl->it_page_shift) >= (1UL * 1024 * 1024 * 1024))
710 		tbl->nr_pools = IOMMU_NR_POOLS;
711 	else
712 		tbl->nr_pools = 1;
713 
714 	/* We reserve the top 1/4 of the table for large allocations */
715 	tbl->poolsize = (tbl->it_size * 3 / 4) / tbl->nr_pools;
716 
717 	for (i = 0; i < tbl->nr_pools; i++) {
718 		p = &tbl->pools[i];
719 		spin_lock_init(&(p->lock));
720 		p->start = tbl->poolsize * i;
721 		p->hint = p->start;
722 		p->end = p->start + tbl->poolsize;
723 	}
724 
725 	p = &tbl->large_pool;
726 	spin_lock_init(&(p->lock));
727 	p->start = tbl->poolsize * i;
728 	p->hint = p->start;
729 	p->end = tbl->it_size;
730 
731 	iommu_table_clear(tbl);
732 
733 	if (!welcomed) {
734 		printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n",
735 		       novmerge ? "disabled" : "enabled");
736 		welcomed = 1;
737 	}
738 
739 	return tbl;
740 }
741 
iommu_table_free(struct kref * kref)742 static void iommu_table_free(struct kref *kref)
743 {
744 	unsigned long bitmap_sz;
745 	unsigned int order;
746 	struct iommu_table *tbl;
747 
748 	tbl = container_of(kref, struct iommu_table, it_kref);
749 
750 	if (tbl->it_ops->free)
751 		tbl->it_ops->free(tbl);
752 
753 	if (!tbl->it_map) {
754 		kfree(tbl);
755 		return;
756 	}
757 
758 	iommu_table_release_pages(tbl);
759 
760 	/* verify that table contains no entries */
761 	if (!bitmap_empty(tbl->it_map, tbl->it_size))
762 		pr_warn("%s: Unexpected TCEs\n", __func__);
763 
764 	/* calculate bitmap size in bytes */
765 	bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
766 
767 	/* free bitmap */
768 	order = get_order(bitmap_sz);
769 	free_pages((unsigned long) tbl->it_map, order);
770 
771 	/* free table */
772 	kfree(tbl);
773 }
774 
iommu_tce_table_get(struct iommu_table * tbl)775 struct iommu_table *iommu_tce_table_get(struct iommu_table *tbl)
776 {
777 	if (kref_get_unless_zero(&tbl->it_kref))
778 		return tbl;
779 
780 	return NULL;
781 }
782 EXPORT_SYMBOL_GPL(iommu_tce_table_get);
783 
iommu_tce_table_put(struct iommu_table * tbl)784 int iommu_tce_table_put(struct iommu_table *tbl)
785 {
786 	if (WARN_ON(!tbl))
787 		return 0;
788 
789 	return kref_put(&tbl->it_kref, iommu_table_free);
790 }
791 EXPORT_SYMBOL_GPL(iommu_tce_table_put);
792 
793 /* Creates TCEs for a user provided buffer.  The user buffer must be
794  * contiguous real kernel storage (not vmalloc).  The address passed here
795  * comprises a page address and offset into that page. The dma_addr_t
796  * returned will point to the same byte within the page as was passed in.
797  */
iommu_map_page(struct device * dev,struct iommu_table * tbl,struct page * page,unsigned long offset,size_t size,unsigned long mask,enum dma_data_direction direction,unsigned long attrs)798 dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl,
799 			  struct page *page, unsigned long offset, size_t size,
800 			  unsigned long mask, enum dma_data_direction direction,
801 			  unsigned long attrs)
802 {
803 	dma_addr_t dma_handle = DMA_MAPPING_ERROR;
804 	void *vaddr;
805 	unsigned long uaddr;
806 	unsigned int npages, align;
807 
808 	BUG_ON(direction == DMA_NONE);
809 
810 	vaddr = page_address(page) + offset;
811 	uaddr = (unsigned long)vaddr;
812 
813 	if (tbl) {
814 		npages = iommu_num_pages(uaddr, size, IOMMU_PAGE_SIZE(tbl));
815 		align = 0;
816 		if (tbl->it_page_shift < PAGE_SHIFT && size >= PAGE_SIZE &&
817 		    ((unsigned long)vaddr & ~PAGE_MASK) == 0)
818 			align = PAGE_SHIFT - tbl->it_page_shift;
819 
820 		dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction,
821 					 mask >> tbl->it_page_shift, align,
822 					 attrs);
823 		if (dma_handle == DMA_MAPPING_ERROR) {
824 			if (!(attrs & DMA_ATTR_NO_WARN) &&
825 			    printk_ratelimit())  {
826 				dev_info(dev, "iommu_alloc failed, tbl %p "
827 					 "vaddr %p npages %d\n", tbl, vaddr,
828 					 npages);
829 			}
830 		} else
831 			dma_handle |= (uaddr & ~IOMMU_PAGE_MASK(tbl));
832 	}
833 
834 	return dma_handle;
835 }
836 
iommu_unmap_page(struct iommu_table * tbl,dma_addr_t dma_handle,size_t size,enum dma_data_direction direction,unsigned long attrs)837 void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle,
838 		      size_t size, enum dma_data_direction direction,
839 		      unsigned long attrs)
840 {
841 	unsigned int npages;
842 
843 	BUG_ON(direction == DMA_NONE);
844 
845 	if (tbl) {
846 		npages = iommu_num_pages(dma_handle, size,
847 					 IOMMU_PAGE_SIZE(tbl));
848 		iommu_free(tbl, dma_handle, npages);
849 	}
850 }
851 
852 /* Allocates a contiguous real buffer and creates mappings over it.
853  * Returns the virtual address of the buffer and sets dma_handle
854  * to the dma address (mapping) of the first page.
855  */
iommu_alloc_coherent(struct device * dev,struct iommu_table * tbl,size_t size,dma_addr_t * dma_handle,unsigned long mask,gfp_t flag,int node)856 void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl,
857 			   size_t size,	dma_addr_t *dma_handle,
858 			   unsigned long mask, gfp_t flag, int node)
859 {
860 	void *ret = NULL;
861 	dma_addr_t mapping;
862 	unsigned int order;
863 	unsigned int nio_pages, io_order;
864 	struct page *page;
865 
866 	size = PAGE_ALIGN(size);
867 	order = get_order(size);
868 
869  	/*
870 	 * Client asked for way too much space.  This is checked later
871 	 * anyway.  It is easier to debug here for the drivers than in
872 	 * the tce tables.
873 	 */
874 	if (order >= IOMAP_MAX_ORDER) {
875 		dev_info(dev, "iommu_alloc_consistent size too large: 0x%lx\n",
876 			 size);
877 		return NULL;
878 	}
879 
880 	if (!tbl)
881 		return NULL;
882 
883 	/* Alloc enough pages (and possibly more) */
884 	page = alloc_pages_node(node, flag, order);
885 	if (!page)
886 		return NULL;
887 	ret = page_address(page);
888 	memset(ret, 0, size);
889 
890 	/* Set up tces to cover the allocated range */
891 	nio_pages = size >> tbl->it_page_shift;
892 	io_order = get_iommu_order(size, tbl);
893 	mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
894 			      mask >> tbl->it_page_shift, io_order, 0);
895 	if (mapping == DMA_MAPPING_ERROR) {
896 		free_pages((unsigned long)ret, order);
897 		return NULL;
898 	}
899 	*dma_handle = mapping;
900 	return ret;
901 }
902 
iommu_free_coherent(struct iommu_table * tbl,size_t size,void * vaddr,dma_addr_t dma_handle)903 void iommu_free_coherent(struct iommu_table *tbl, size_t size,
904 			 void *vaddr, dma_addr_t dma_handle)
905 {
906 	if (tbl) {
907 		unsigned int nio_pages;
908 
909 		size = PAGE_ALIGN(size);
910 		nio_pages = size >> tbl->it_page_shift;
911 		iommu_free(tbl, dma_handle, nio_pages);
912 		size = PAGE_ALIGN(size);
913 		free_pages((unsigned long)vaddr, get_order(size));
914 	}
915 }
916 
iommu_direction_to_tce_perm(enum dma_data_direction dir)917 unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir)
918 {
919 	switch (dir) {
920 	case DMA_BIDIRECTIONAL:
921 		return TCE_PCI_READ | TCE_PCI_WRITE;
922 	case DMA_FROM_DEVICE:
923 		return TCE_PCI_WRITE;
924 	case DMA_TO_DEVICE:
925 		return TCE_PCI_READ;
926 	default:
927 		return 0;
928 	}
929 }
930 EXPORT_SYMBOL_GPL(iommu_direction_to_tce_perm);
931 
932 #ifdef CONFIG_IOMMU_API
933 /*
934  * SPAPR TCE API
935  */
group_release(void * iommu_data)936 static void group_release(void *iommu_data)
937 {
938 	struct iommu_table_group *table_group = iommu_data;
939 
940 	table_group->group = NULL;
941 }
942 
iommu_register_group(struct iommu_table_group * table_group,int pci_domain_number,unsigned long pe_num)943 void iommu_register_group(struct iommu_table_group *table_group,
944 		int pci_domain_number, unsigned long pe_num)
945 {
946 	struct iommu_group *grp;
947 	char *name;
948 
949 	grp = iommu_group_alloc();
950 	if (IS_ERR(grp)) {
951 		pr_warn("powerpc iommu api: cannot create new group, err=%ld\n",
952 				PTR_ERR(grp));
953 		return;
954 	}
955 	table_group->group = grp;
956 	iommu_group_set_iommudata(grp, table_group, group_release);
957 	name = kasprintf(GFP_KERNEL, "domain%d-pe%lx",
958 			pci_domain_number, pe_num);
959 	if (!name)
960 		return;
961 	iommu_group_set_name(grp, name);
962 	kfree(name);
963 }
964 
iommu_tce_direction(unsigned long tce)965 enum dma_data_direction iommu_tce_direction(unsigned long tce)
966 {
967 	if ((tce & TCE_PCI_READ) && (tce & TCE_PCI_WRITE))
968 		return DMA_BIDIRECTIONAL;
969 	else if (tce & TCE_PCI_READ)
970 		return DMA_TO_DEVICE;
971 	else if (tce & TCE_PCI_WRITE)
972 		return DMA_FROM_DEVICE;
973 	else
974 		return DMA_NONE;
975 }
976 EXPORT_SYMBOL_GPL(iommu_tce_direction);
977 
iommu_flush_tce(struct iommu_table * tbl)978 void iommu_flush_tce(struct iommu_table *tbl)
979 {
980 	/* Flush/invalidate TLB caches if necessary */
981 	if (tbl->it_ops->flush)
982 		tbl->it_ops->flush(tbl);
983 
984 	/* Make sure updates are seen by hardware */
985 	mb();
986 }
987 EXPORT_SYMBOL_GPL(iommu_flush_tce);
988 
iommu_tce_check_ioba(unsigned long page_shift,unsigned long offset,unsigned long size,unsigned long ioba,unsigned long npages)989 int iommu_tce_check_ioba(unsigned long page_shift,
990 		unsigned long offset, unsigned long size,
991 		unsigned long ioba, unsigned long npages)
992 {
993 	unsigned long mask = (1UL << page_shift) - 1;
994 
995 	if (ioba & mask)
996 		return -EINVAL;
997 
998 	ioba >>= page_shift;
999 	if (ioba < offset)
1000 		return -EINVAL;
1001 
1002 	if ((ioba + 1) > (offset + size))
1003 		return -EINVAL;
1004 
1005 	return 0;
1006 }
1007 EXPORT_SYMBOL_GPL(iommu_tce_check_ioba);
1008 
iommu_tce_check_gpa(unsigned long page_shift,unsigned long gpa)1009 int iommu_tce_check_gpa(unsigned long page_shift, unsigned long gpa)
1010 {
1011 	unsigned long mask = (1UL << page_shift) - 1;
1012 
1013 	if (gpa & mask)
1014 		return -EINVAL;
1015 
1016 	return 0;
1017 }
1018 EXPORT_SYMBOL_GPL(iommu_tce_check_gpa);
1019 
iommu_tce_xchg_no_kill(struct mm_struct * mm,struct iommu_table * tbl,unsigned long entry,unsigned long * hpa,enum dma_data_direction * direction)1020 extern long iommu_tce_xchg_no_kill(struct mm_struct *mm,
1021 		struct iommu_table *tbl,
1022 		unsigned long entry, unsigned long *hpa,
1023 		enum dma_data_direction *direction)
1024 {
1025 	long ret;
1026 	unsigned long size = 0;
1027 
1028 	ret = tbl->it_ops->xchg_no_kill(tbl, entry, hpa, direction, false);
1029 	if (!ret && ((*direction == DMA_FROM_DEVICE) ||
1030 			(*direction == DMA_BIDIRECTIONAL)) &&
1031 			!mm_iommu_is_devmem(mm, *hpa, tbl->it_page_shift,
1032 					&size))
1033 		SetPageDirty(pfn_to_page(*hpa >> PAGE_SHIFT));
1034 
1035 	return ret;
1036 }
1037 EXPORT_SYMBOL_GPL(iommu_tce_xchg_no_kill);
1038 
iommu_tce_kill(struct iommu_table * tbl,unsigned long entry,unsigned long pages)1039 void iommu_tce_kill(struct iommu_table *tbl,
1040 		unsigned long entry, unsigned long pages)
1041 {
1042 	if (tbl->it_ops->tce_kill)
1043 		tbl->it_ops->tce_kill(tbl, entry, pages, false);
1044 }
1045 EXPORT_SYMBOL_GPL(iommu_tce_kill);
1046 
iommu_take_ownership(struct iommu_table * tbl)1047 int iommu_take_ownership(struct iommu_table *tbl)
1048 {
1049 	unsigned long flags, i, sz = (tbl->it_size + 7) >> 3;
1050 	int ret = 0;
1051 
1052 	/*
1053 	 * VFIO does not control TCE entries allocation and the guest
1054 	 * can write new TCEs on top of existing ones so iommu_tce_build()
1055 	 * must be able to release old pages. This functionality
1056 	 * requires exchange() callback defined so if it is not
1057 	 * implemented, we disallow taking ownership over the table.
1058 	 */
1059 	if (!tbl->it_ops->xchg_no_kill)
1060 		return -EINVAL;
1061 
1062 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
1063 	for (i = 0; i < tbl->nr_pools; i++)
1064 		spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
1065 
1066 	iommu_table_release_pages(tbl);
1067 
1068 	if (!bitmap_empty(tbl->it_map, tbl->it_size)) {
1069 		pr_err("iommu_tce: it_map is not empty");
1070 		ret = -EBUSY;
1071 		/* Undo iommu_table_release_pages, i.e. restore bit#0, etc */
1072 		iommu_table_reserve_pages(tbl, tbl->it_reserved_start,
1073 				tbl->it_reserved_end);
1074 	} else {
1075 		memset(tbl->it_map, 0xff, sz);
1076 	}
1077 
1078 	for (i = 0; i < tbl->nr_pools; i++)
1079 		spin_unlock(&tbl->pools[i].lock);
1080 	spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
1081 
1082 	return ret;
1083 }
1084 EXPORT_SYMBOL_GPL(iommu_take_ownership);
1085 
iommu_release_ownership(struct iommu_table * tbl)1086 void iommu_release_ownership(struct iommu_table *tbl)
1087 {
1088 	unsigned long flags, i, sz = (tbl->it_size + 7) >> 3;
1089 
1090 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
1091 	for (i = 0; i < tbl->nr_pools; i++)
1092 		spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
1093 
1094 	memset(tbl->it_map, 0, sz);
1095 
1096 	iommu_table_reserve_pages(tbl, tbl->it_reserved_start,
1097 			tbl->it_reserved_end);
1098 
1099 	for (i = 0; i < tbl->nr_pools; i++)
1100 		spin_unlock(&tbl->pools[i].lock);
1101 	spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
1102 }
1103 EXPORT_SYMBOL_GPL(iommu_release_ownership);
1104 
iommu_add_device(struct iommu_table_group * table_group,struct device * dev)1105 int iommu_add_device(struct iommu_table_group *table_group, struct device *dev)
1106 {
1107 	/*
1108 	 * The sysfs entries should be populated before
1109 	 * binding IOMMU group. If sysfs entries isn't
1110 	 * ready, we simply bail.
1111 	 */
1112 	if (!device_is_registered(dev))
1113 		return -ENOENT;
1114 
1115 	if (device_iommu_mapped(dev)) {
1116 		pr_debug("%s: Skipping device %s with iommu group %d\n",
1117 			 __func__, dev_name(dev),
1118 			 iommu_group_id(dev->iommu_group));
1119 		return -EBUSY;
1120 	}
1121 
1122 	pr_debug("%s: Adding %s to iommu group %d\n",
1123 		 __func__, dev_name(dev),  iommu_group_id(table_group->group));
1124 
1125 	return iommu_group_add_device(table_group->group, dev);
1126 }
1127 EXPORT_SYMBOL_GPL(iommu_add_device);
1128 
iommu_del_device(struct device * dev)1129 void iommu_del_device(struct device *dev)
1130 {
1131 	/*
1132 	 * Some devices might not have IOMMU table and group
1133 	 * and we needn't detach them from the associated
1134 	 * IOMMU groups
1135 	 */
1136 	if (!device_iommu_mapped(dev)) {
1137 		pr_debug("iommu_tce: skipping device %s with no tbl\n",
1138 			 dev_name(dev));
1139 		return;
1140 	}
1141 
1142 	iommu_group_remove_device(dev);
1143 }
1144 EXPORT_SYMBOL_GPL(iommu_del_device);
1145 #endif /* CONFIG_IOMMU_API */
1146