1 /*
2 ** IA64 System Bus Adapter (SBA) I/O MMU manager
3 **
4 ** (c) Copyright 2002-2005 Alex Williamson
5 ** (c) Copyright 2002-2003 Grant Grundler
6 ** (c) Copyright 2002-2005 Hewlett-Packard Company
7 **
8 ** Portions (c) 2000 Grant Grundler (from parisc I/O MMU code)
9 ** Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
10 **
11 ** This program is free software; you can redistribute it and/or modify
12 ** it under the terms of the GNU General Public License as published by
13 ** the Free Software Foundation; either version 2 of the License, or
14 ** (at your option) any later version.
15 **
16 **
17 ** This module initializes the IOC (I/O Controller) found on HP
18 ** McKinley machines and their successors.
19 **
20 */
21
22 #include <linux/types.h>
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/init.h>
28 #include <linux/mm.h>
29 #include <linux/string.h>
30 #include <linux/pci.h>
31 #include <linux/proc_fs.h>
32 #include <linux/seq_file.h>
33 #include <linux/acpi.h>
34 #include <linux/efi.h>
35 #include <linux/nodemask.h>
36 #include <linux/bitops.h> /* hweight64() */
37 #include <linux/crash_dump.h>
38 #include <linux/iommu-helper.h>
39
40 #include <asm/delay.h> /* ia64_get_itc() */
41 #include <asm/io.h>
42 #include <asm/page.h> /* PAGE_OFFSET */
43 #include <asm/dma.h>
44 #include <asm/system.h> /* wmb() */
45
46 #include <asm/acpi-ext.h>
47
48 extern int swiotlb_late_init_with_default_size (size_t size);
49
50 #define PFX "IOC: "
51
52 /*
53 ** Enabling timing search of the pdir resource map. Output in /proc.
54 ** Disabled by default to optimize performance.
55 */
56 #undef PDIR_SEARCH_TIMING
57
58 /*
59 ** This option allows cards capable of 64bit DMA to bypass the IOMMU. If
60 ** not defined, all DMA will be 32bit and go through the TLB.
61 ** There's potentially a conflict in the bio merge code with us
62 ** advertising an iommu, but then bypassing it. Since I/O MMU bypassing
63 ** appears to give more performance than bio-level virtual merging, we'll
64 ** do the former for now. NOTE: BYPASS_SG also needs to be undef'd to
65 ** completely restrict DMA to the IOMMU.
66 */
67 #define ALLOW_IOV_BYPASS
68
69 /*
70 ** This option specifically allows/disallows bypassing scatterlists with
71 ** multiple entries. Coalescing these entries can allow better DMA streaming
72 ** and in some cases shows better performance than entirely bypassing the
73 ** IOMMU. Performance increase on the order of 1-2% sequential output/input
74 ** using bonnie++ on a RAID0 MD device (sym2 & mpt).
75 */
76 #undef ALLOW_IOV_BYPASS_SG
77
78 /*
79 ** If a device prefetches beyond the end of a valid pdir entry, it will cause
80 ** a hard failure, ie. MCA. Version 3.0 and later of the zx1 LBA should
81 ** disconnect on 4k boundaries and prevent such issues. If the device is
82 ** particularly aggressive, this option will keep the entire pdir valid such
83 ** that prefetching will hit a valid address. This could severely impact
84 ** error containment, and is therefore off by default. The page that is
85 ** used for spill-over is poisoned, so that should help debugging somewhat.
86 */
87 #undef FULL_VALID_PDIR
88
89 #define ENABLE_MARK_CLEAN
90
91 /*
92 ** The number of debug flags is a clue - this code is fragile. NOTE: since
93 ** tightening the use of res_lock the resource bitmap and actual pdir are no
94 ** longer guaranteed to stay in sync. The sanity checking code isn't going to
95 ** like that.
96 */
97 #undef DEBUG_SBA_INIT
98 #undef DEBUG_SBA_RUN
99 #undef DEBUG_SBA_RUN_SG
100 #undef DEBUG_SBA_RESOURCE
101 #undef ASSERT_PDIR_SANITY
102 #undef DEBUG_LARGE_SG_ENTRIES
103 #undef DEBUG_BYPASS
104
105 #if defined(FULL_VALID_PDIR) && defined(ASSERT_PDIR_SANITY)
106 #error FULL_VALID_PDIR and ASSERT_PDIR_SANITY are mutually exclusive
107 #endif
108
109 #define SBA_INLINE __inline__
110 /* #define SBA_INLINE */
111
112 #ifdef DEBUG_SBA_INIT
113 #define DBG_INIT(x...) printk(x)
114 #else
115 #define DBG_INIT(x...)
116 #endif
117
118 #ifdef DEBUG_SBA_RUN
119 #define DBG_RUN(x...) printk(x)
120 #else
121 #define DBG_RUN(x...)
122 #endif
123
124 #ifdef DEBUG_SBA_RUN_SG
125 #define DBG_RUN_SG(x...) printk(x)
126 #else
127 #define DBG_RUN_SG(x...)
128 #endif
129
130
131 #ifdef DEBUG_SBA_RESOURCE
132 #define DBG_RES(x...) printk(x)
133 #else
134 #define DBG_RES(x...)
135 #endif
136
137 #ifdef DEBUG_BYPASS
138 #define DBG_BYPASS(x...) printk(x)
139 #else
140 #define DBG_BYPASS(x...)
141 #endif
142
143 #ifdef ASSERT_PDIR_SANITY
144 #define ASSERT(expr) \
145 if(!(expr)) { \
146 printk( "\n" __FILE__ ":%d: Assertion " #expr " failed!\n",__LINE__); \
147 panic(#expr); \
148 }
149 #else
150 #define ASSERT(expr)
151 #endif
152
153 /*
154 ** The number of pdir entries to "free" before issuing
155 ** a read to PCOM register to flush out PCOM writes.
156 ** Interacts with allocation granularity (ie 4 or 8 entries
157 ** allocated and free'd/purged at a time might make this
158 ** less interesting).
159 */
160 #define DELAYED_RESOURCE_CNT 64
161
162 #define PCI_DEVICE_ID_HP_SX2000_IOC 0x12ec
163
164 #define ZX1_IOC_ID ((PCI_DEVICE_ID_HP_ZX1_IOC << 16) | PCI_VENDOR_ID_HP)
165 #define ZX2_IOC_ID ((PCI_DEVICE_ID_HP_ZX2_IOC << 16) | PCI_VENDOR_ID_HP)
166 #define REO_IOC_ID ((PCI_DEVICE_ID_HP_REO_IOC << 16) | PCI_VENDOR_ID_HP)
167 #define SX1000_IOC_ID ((PCI_DEVICE_ID_HP_SX1000_IOC << 16) | PCI_VENDOR_ID_HP)
168 #define SX2000_IOC_ID ((PCI_DEVICE_ID_HP_SX2000_IOC << 16) | PCI_VENDOR_ID_HP)
169
170 #define ZX1_IOC_OFFSET 0x1000 /* ACPI reports SBA, we want IOC */
171
172 #define IOC_FUNC_ID 0x000
173 #define IOC_FCLASS 0x008 /* function class, bist, header, rev... */
174 #define IOC_IBASE 0x300 /* IO TLB */
175 #define IOC_IMASK 0x308
176 #define IOC_PCOM 0x310
177 #define IOC_TCNFG 0x318
178 #define IOC_PDIR_BASE 0x320
179
180 #define IOC_ROPE0_CFG 0x500
181 #define IOC_ROPE_AO 0x10 /* Allow "Relaxed Ordering" */
182
183
184 /* AGP GART driver looks for this */
185 #define ZX1_SBA_IOMMU_COOKIE 0x0000badbadc0ffeeUL
186
187 /*
188 ** The zx1 IOC supports 4/8/16/64KB page sizes (see TCNFG register)
189 **
190 ** Some IOCs (sx1000) can run at the above pages sizes, but are
191 ** really only supported using the IOC at a 4k page size.
192 **
193 ** iovp_size could only be greater than PAGE_SIZE if we are
194 ** confident the drivers really only touch the next physical
195 ** page iff that driver instance owns it.
196 */
197 static unsigned long iovp_size;
198 static unsigned long iovp_shift;
199 static unsigned long iovp_mask;
200
201 struct ioc {
202 void __iomem *ioc_hpa; /* I/O MMU base address */
203 char *res_map; /* resource map, bit == pdir entry */
204 u64 *pdir_base; /* physical base address */
205 unsigned long ibase; /* pdir IOV Space base */
206 unsigned long imask; /* pdir IOV Space mask */
207
208 unsigned long *res_hint; /* next avail IOVP - circular search */
209 unsigned long dma_mask;
210 spinlock_t res_lock; /* protects the resource bitmap, but must be held when */
211 /* clearing pdir to prevent races with allocations. */
212 unsigned int res_bitshift; /* from the RIGHT! */
213 unsigned int res_size; /* size of resource map in bytes */
214 #ifdef CONFIG_NUMA
215 unsigned int node; /* node where this IOC lives */
216 #endif
217 #if DELAYED_RESOURCE_CNT > 0
218 spinlock_t saved_lock; /* may want to try to get this on a separate cacheline */
219 /* than res_lock for bigger systems. */
220 int saved_cnt;
221 struct sba_dma_pair {
222 dma_addr_t iova;
223 size_t size;
224 } saved[DELAYED_RESOURCE_CNT];
225 #endif
226
227 #ifdef PDIR_SEARCH_TIMING
228 #define SBA_SEARCH_SAMPLE 0x100
229 unsigned long avg_search[SBA_SEARCH_SAMPLE];
230 unsigned long avg_idx; /* current index into avg_search */
231 #endif
232
233 /* Stuff we don't need in performance path */
234 struct ioc *next; /* list of IOC's in system */
235 acpi_handle handle; /* for multiple IOC's */
236 const char *name;
237 unsigned int func_id;
238 unsigned int rev; /* HW revision of chip */
239 u32 iov_size;
240 unsigned int pdir_size; /* in bytes, determined by IOV Space size */
241 struct pci_dev *sac_only_dev;
242 };
243
244 static struct ioc *ioc_list;
245 static int reserve_sba_gart = 1;
246
247 static SBA_INLINE void sba_mark_invalid(struct ioc *, dma_addr_t, size_t);
248 static SBA_INLINE void sba_free_range(struct ioc *, dma_addr_t, size_t);
249
250 #define sba_sg_address(sg) sg_virt((sg))
251
252 #ifdef FULL_VALID_PDIR
253 static u64 prefetch_spill_page;
254 #endif
255
256 #ifdef CONFIG_PCI
257 # define GET_IOC(dev) (((dev)->bus == &pci_bus_type) \
258 ? ((struct ioc *) PCI_CONTROLLER(to_pci_dev(dev))->iommu) : NULL)
259 #else
260 # define GET_IOC(dev) NULL
261 #endif
262
263 /*
264 ** DMA_CHUNK_SIZE is used by the SCSI mid-layer to break up
265 ** (or rather not merge) DMAs into manageable chunks.
266 ** On parisc, this is more of the software/tuning constraint
267 ** rather than the HW. I/O MMU allocation algorithms can be
268 ** faster with smaller sizes (to some degree).
269 */
270 #define DMA_CHUNK_SIZE (BITS_PER_LONG*iovp_size)
271
272 #define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
273
274 /************************************
275 ** SBA register read and write support
276 **
277 ** BE WARNED: register writes are posted.
278 ** (ie follow writes which must reach HW with a read)
279 **
280 */
281 #define READ_REG(addr) __raw_readq(addr)
282 #define WRITE_REG(val, addr) __raw_writeq(val, addr)
283
284 #ifdef DEBUG_SBA_INIT
285
286 /**
287 * sba_dump_tlb - debugging only - print IOMMU operating parameters
288 * @hpa: base address of the IOMMU
289 *
290 * Print the size/location of the IO MMU PDIR.
291 */
292 static void
sba_dump_tlb(char * hpa)293 sba_dump_tlb(char *hpa)
294 {
295 DBG_INIT("IO TLB at 0x%p\n", (void *)hpa);
296 DBG_INIT("IOC_IBASE : %016lx\n", READ_REG(hpa+IOC_IBASE));
297 DBG_INIT("IOC_IMASK : %016lx\n", READ_REG(hpa+IOC_IMASK));
298 DBG_INIT("IOC_TCNFG : %016lx\n", READ_REG(hpa+IOC_TCNFG));
299 DBG_INIT("IOC_PDIR_BASE: %016lx\n", READ_REG(hpa+IOC_PDIR_BASE));
300 DBG_INIT("\n");
301 }
302 #endif
303
304
305 #ifdef ASSERT_PDIR_SANITY
306
307 /**
308 * sba_dump_pdir_entry - debugging only - print one IOMMU PDIR entry
309 * @ioc: IO MMU structure which owns the pdir we are interested in.
310 * @msg: text to print ont the output line.
311 * @pide: pdir index.
312 *
313 * Print one entry of the IO MMU PDIR in human readable form.
314 */
315 static void
sba_dump_pdir_entry(struct ioc * ioc,char * msg,uint pide)316 sba_dump_pdir_entry(struct ioc *ioc, char *msg, uint pide)
317 {
318 /* start printing from lowest pde in rval */
319 u64 *ptr = &ioc->pdir_base[pide & ~(BITS_PER_LONG - 1)];
320 unsigned long *rptr = (unsigned long *) &ioc->res_map[(pide >>3) & -sizeof(unsigned long)];
321 uint rcnt;
322
323 printk(KERN_DEBUG "SBA: %s rp %p bit %d rval 0x%lx\n",
324 msg, rptr, pide & (BITS_PER_LONG - 1), *rptr);
325
326 rcnt = 0;
327 while (rcnt < BITS_PER_LONG) {
328 printk(KERN_DEBUG "%s %2d %p %016Lx\n",
329 (rcnt == (pide & (BITS_PER_LONG - 1)))
330 ? " -->" : " ",
331 rcnt, ptr, (unsigned long long) *ptr );
332 rcnt++;
333 ptr++;
334 }
335 printk(KERN_DEBUG "%s", msg);
336 }
337
338
339 /**
340 * sba_check_pdir - debugging only - consistency checker
341 * @ioc: IO MMU structure which owns the pdir we are interested in.
342 * @msg: text to print ont the output line.
343 *
344 * Verify the resource map and pdir state is consistent
345 */
346 static int
sba_check_pdir(struct ioc * ioc,char * msg)347 sba_check_pdir(struct ioc *ioc, char *msg)
348 {
349 u64 *rptr_end = (u64 *) &(ioc->res_map[ioc->res_size]);
350 u64 *rptr = (u64 *) ioc->res_map; /* resource map ptr */
351 u64 *pptr = ioc->pdir_base; /* pdir ptr */
352 uint pide = 0;
353
354 while (rptr < rptr_end) {
355 u64 rval;
356 int rcnt; /* number of bits we might check */
357
358 rval = *rptr;
359 rcnt = 64;
360
361 while (rcnt) {
362 /* Get last byte and highest bit from that */
363 u32 pde = ((u32)((*pptr >> (63)) & 0x1));
364 if ((rval & 0x1) ^ pde)
365 {
366 /*
367 ** BUMMER! -- res_map != pdir --
368 ** Dump rval and matching pdir entries
369 */
370 sba_dump_pdir_entry(ioc, msg, pide);
371 return(1);
372 }
373 rcnt--;
374 rval >>= 1; /* try the next bit */
375 pptr++;
376 pide++;
377 }
378 rptr++; /* look at next word of res_map */
379 }
380 /* It'd be nice if we always got here :^) */
381 return 0;
382 }
383
384
385 /**
386 * sba_dump_sg - debugging only - print Scatter-Gather list
387 * @ioc: IO MMU structure which owns the pdir we are interested in.
388 * @startsg: head of the SG list
389 * @nents: number of entries in SG list
390 *
391 * print the SG list so we can verify it's correct by hand.
392 */
393 static void
sba_dump_sg(struct ioc * ioc,struct scatterlist * startsg,int nents)394 sba_dump_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
395 {
396 while (nents-- > 0) {
397 printk(KERN_DEBUG " %d : DMA %08lx/%05x CPU %p\n", nents,
398 startsg->dma_address, startsg->dma_length,
399 sba_sg_address(startsg));
400 startsg = sg_next(startsg);
401 }
402 }
403
404 static void
sba_check_sg(struct ioc * ioc,struct scatterlist * startsg,int nents)405 sba_check_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
406 {
407 struct scatterlist *the_sg = startsg;
408 int the_nents = nents;
409
410 while (the_nents-- > 0) {
411 if (sba_sg_address(the_sg) == 0x0UL)
412 sba_dump_sg(NULL, startsg, nents);
413 the_sg = sg_next(the_sg);
414 }
415 }
416
417 #endif /* ASSERT_PDIR_SANITY */
418
419
420
421
422 /**************************************************************
423 *
424 * I/O Pdir Resource Management
425 *
426 * Bits set in the resource map are in use.
427 * Each bit can represent a number of pages.
428 * LSbs represent lower addresses (IOVA's).
429 *
430 ***************************************************************/
431 #define PAGES_PER_RANGE 1 /* could increase this to 4 or 8 if needed */
432
433 /* Convert from IOVP to IOVA and vice versa. */
434 #define SBA_IOVA(ioc,iovp,offset) ((ioc->ibase) | (iovp) | (offset))
435 #define SBA_IOVP(ioc,iova) ((iova) & ~(ioc->ibase))
436
437 #define PDIR_ENTRY_SIZE sizeof(u64)
438
439 #define PDIR_INDEX(iovp) ((iovp)>>iovp_shift)
440
441 #define RESMAP_MASK(n) ~(~0UL << (n))
442 #define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
443
444
445 /**
446 * For most cases the normal get_order is sufficient, however it limits us
447 * to PAGE_SIZE being the minimum mapping alignment and TC flush granularity.
448 * It only incurs about 1 clock cycle to use this one with the static variable
449 * and makes the code more intuitive.
450 */
451 static SBA_INLINE int
get_iovp_order(unsigned long size)452 get_iovp_order (unsigned long size)
453 {
454 long double d = size - 1;
455 long order;
456
457 order = ia64_getf_exp(d);
458 order = order - iovp_shift - 0xffff + 1;
459 if (order < 0)
460 order = 0;
461 return order;
462 }
463
ptr_to_pide(struct ioc * ioc,unsigned long * res_ptr,unsigned int bitshiftcnt)464 static unsigned long ptr_to_pide(struct ioc *ioc, unsigned long *res_ptr,
465 unsigned int bitshiftcnt)
466 {
467 return (((unsigned long)res_ptr - (unsigned long)ioc->res_map) << 3)
468 + bitshiftcnt;
469 }
470
471 /**
472 * sba_search_bitmap - find free space in IO PDIR resource bitmap
473 * @ioc: IO MMU structure which owns the pdir we are interested in.
474 * @bits_wanted: number of entries we need.
475 * @use_hint: use res_hint to indicate where to start looking
476 *
477 * Find consecutive free bits in resource bitmap.
478 * Each bit represents one entry in the IO Pdir.
479 * Cool perf optimization: search for log2(size) bits at a time.
480 */
481 static SBA_INLINE unsigned long
sba_search_bitmap(struct ioc * ioc,struct device * dev,unsigned long bits_wanted,int use_hint)482 sba_search_bitmap(struct ioc *ioc, struct device *dev,
483 unsigned long bits_wanted, int use_hint)
484 {
485 unsigned long *res_ptr;
486 unsigned long *res_end = (unsigned long *) &(ioc->res_map[ioc->res_size]);
487 unsigned long flags, pide = ~0UL, tpide;
488 unsigned long boundary_size;
489 unsigned long shift;
490 int ret;
491
492 ASSERT(((unsigned long) ioc->res_hint & (sizeof(unsigned long) - 1UL)) == 0);
493 ASSERT(res_ptr < res_end);
494
495 boundary_size = (unsigned long long)dma_get_seg_boundary(dev) + 1;
496 boundary_size = ALIGN(boundary_size, 1ULL << iovp_shift) >> iovp_shift;
497
498 BUG_ON(ioc->ibase & ~iovp_mask);
499 shift = ioc->ibase >> iovp_shift;
500
501 spin_lock_irqsave(&ioc->res_lock, flags);
502
503 /* Allow caller to force a search through the entire resource space */
504 if (likely(use_hint)) {
505 res_ptr = ioc->res_hint;
506 } else {
507 res_ptr = (ulong *)ioc->res_map;
508 ioc->res_bitshift = 0;
509 }
510
511 /*
512 * N.B. REO/Grande defect AR2305 can cause TLB fetch timeouts
513 * if a TLB entry is purged while in use. sba_mark_invalid()
514 * purges IOTLB entries in power-of-two sizes, so we also
515 * allocate IOVA space in power-of-two sizes.
516 */
517 bits_wanted = 1UL << get_iovp_order(bits_wanted << iovp_shift);
518
519 if (likely(bits_wanted == 1)) {
520 unsigned int bitshiftcnt;
521 for(; res_ptr < res_end ; res_ptr++) {
522 if (likely(*res_ptr != ~0UL)) {
523 bitshiftcnt = ffz(*res_ptr);
524 *res_ptr |= (1UL << bitshiftcnt);
525 pide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
526 ioc->res_bitshift = bitshiftcnt + bits_wanted;
527 goto found_it;
528 }
529 }
530 goto not_found;
531
532 }
533
534 if (likely(bits_wanted <= BITS_PER_LONG/2)) {
535 /*
536 ** Search the resource bit map on well-aligned values.
537 ** "o" is the alignment.
538 ** We need the alignment to invalidate I/O TLB using
539 ** SBA HW features in the unmap path.
540 */
541 unsigned long o = 1 << get_iovp_order(bits_wanted << iovp_shift);
542 uint bitshiftcnt = ROUNDUP(ioc->res_bitshift, o);
543 unsigned long mask, base_mask;
544
545 base_mask = RESMAP_MASK(bits_wanted);
546 mask = base_mask << bitshiftcnt;
547
548 DBG_RES("%s() o %ld %p", __func__, o, res_ptr);
549 for(; res_ptr < res_end ; res_ptr++)
550 {
551 DBG_RES(" %p %lx %lx\n", res_ptr, mask, *res_ptr);
552 ASSERT(0 != mask);
553 for (; mask ; mask <<= o, bitshiftcnt += o) {
554 tpide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
555 ret = iommu_is_span_boundary(tpide, bits_wanted,
556 shift,
557 boundary_size);
558 if ((0 == ((*res_ptr) & mask)) && !ret) {
559 *res_ptr |= mask; /* mark resources busy! */
560 pide = tpide;
561 ioc->res_bitshift = bitshiftcnt + bits_wanted;
562 goto found_it;
563 }
564 }
565
566 bitshiftcnt = 0;
567 mask = base_mask;
568
569 }
570
571 } else {
572 int qwords, bits, i;
573 unsigned long *end;
574
575 qwords = bits_wanted >> 6; /* /64 */
576 bits = bits_wanted - (qwords * BITS_PER_LONG);
577
578 end = res_end - qwords;
579
580 for (; res_ptr < end; res_ptr++) {
581 tpide = ptr_to_pide(ioc, res_ptr, 0);
582 ret = iommu_is_span_boundary(tpide, bits_wanted,
583 shift, boundary_size);
584 if (ret)
585 goto next_ptr;
586 for (i = 0 ; i < qwords ; i++) {
587 if (res_ptr[i] != 0)
588 goto next_ptr;
589 }
590 if (bits && res_ptr[i] && (__ffs(res_ptr[i]) < bits))
591 continue;
592
593 /* Found it, mark it */
594 for (i = 0 ; i < qwords ; i++)
595 res_ptr[i] = ~0UL;
596 res_ptr[i] |= RESMAP_MASK(bits);
597
598 pide = tpide;
599 res_ptr += qwords;
600 ioc->res_bitshift = bits;
601 goto found_it;
602 next_ptr:
603 ;
604 }
605 }
606
607 not_found:
608 prefetch(ioc->res_map);
609 ioc->res_hint = (unsigned long *) ioc->res_map;
610 ioc->res_bitshift = 0;
611 spin_unlock_irqrestore(&ioc->res_lock, flags);
612 return (pide);
613
614 found_it:
615 ioc->res_hint = res_ptr;
616 spin_unlock_irqrestore(&ioc->res_lock, flags);
617 return (pide);
618 }
619
620
621 /**
622 * sba_alloc_range - find free bits and mark them in IO PDIR resource bitmap
623 * @ioc: IO MMU structure which owns the pdir we are interested in.
624 * @size: number of bytes to create a mapping for
625 *
626 * Given a size, find consecutive unmarked and then mark those bits in the
627 * resource bit map.
628 */
629 static int
sba_alloc_range(struct ioc * ioc,struct device * dev,size_t size)630 sba_alloc_range(struct ioc *ioc, struct device *dev, size_t size)
631 {
632 unsigned int pages_needed = size >> iovp_shift;
633 #ifdef PDIR_SEARCH_TIMING
634 unsigned long itc_start;
635 #endif
636 unsigned long pide;
637
638 ASSERT(pages_needed);
639 ASSERT(0 == (size & ~iovp_mask));
640
641 #ifdef PDIR_SEARCH_TIMING
642 itc_start = ia64_get_itc();
643 #endif
644 /*
645 ** "seek and ye shall find"...praying never hurts either...
646 */
647 pide = sba_search_bitmap(ioc, dev, pages_needed, 1);
648 if (unlikely(pide >= (ioc->res_size << 3))) {
649 pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
650 if (unlikely(pide >= (ioc->res_size << 3))) {
651 #if DELAYED_RESOURCE_CNT > 0
652 unsigned long flags;
653
654 /*
655 ** With delayed resource freeing, we can give this one more shot. We're
656 ** getting close to being in trouble here, so do what we can to make this
657 ** one count.
658 */
659 spin_lock_irqsave(&ioc->saved_lock, flags);
660 if (ioc->saved_cnt > 0) {
661 struct sba_dma_pair *d;
662 int cnt = ioc->saved_cnt;
663
664 d = &(ioc->saved[ioc->saved_cnt - 1]);
665
666 spin_lock(&ioc->res_lock);
667 while (cnt--) {
668 sba_mark_invalid(ioc, d->iova, d->size);
669 sba_free_range(ioc, d->iova, d->size);
670 d--;
671 }
672 ioc->saved_cnt = 0;
673 READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
674 spin_unlock(&ioc->res_lock);
675 }
676 spin_unlock_irqrestore(&ioc->saved_lock, flags);
677
678 pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
679 if (unlikely(pide >= (ioc->res_size << 3)))
680 panic(__FILE__ ": I/O MMU @ %p is out of mapping resources\n",
681 ioc->ioc_hpa);
682 #else
683 panic(__FILE__ ": I/O MMU @ %p is out of mapping resources\n",
684 ioc->ioc_hpa);
685 #endif
686 }
687 }
688
689 #ifdef PDIR_SEARCH_TIMING
690 ioc->avg_search[ioc->avg_idx++] = (ia64_get_itc() - itc_start) / pages_needed;
691 ioc->avg_idx &= SBA_SEARCH_SAMPLE - 1;
692 #endif
693
694 prefetchw(&(ioc->pdir_base[pide]));
695
696 #ifdef ASSERT_PDIR_SANITY
697 /* verify the first enable bit is clear */
698 if(0x00 != ((u8 *) ioc->pdir_base)[pide*PDIR_ENTRY_SIZE + 7]) {
699 sba_dump_pdir_entry(ioc, "sba_search_bitmap() botched it?", pide);
700 }
701 #endif
702
703 DBG_RES("%s(%x) %d -> %lx hint %x/%x\n",
704 __func__, size, pages_needed, pide,
705 (uint) ((unsigned long) ioc->res_hint - (unsigned long) ioc->res_map),
706 ioc->res_bitshift );
707
708 return (pide);
709 }
710
711
712 /**
713 * sba_free_range - unmark bits in IO PDIR resource bitmap
714 * @ioc: IO MMU structure which owns the pdir we are interested in.
715 * @iova: IO virtual address which was previously allocated.
716 * @size: number of bytes to create a mapping for
717 *
718 * clear bits in the ioc's resource map
719 */
720 static SBA_INLINE void
sba_free_range(struct ioc * ioc,dma_addr_t iova,size_t size)721 sba_free_range(struct ioc *ioc, dma_addr_t iova, size_t size)
722 {
723 unsigned long iovp = SBA_IOVP(ioc, iova);
724 unsigned int pide = PDIR_INDEX(iovp);
725 unsigned int ridx = pide >> 3; /* convert bit to byte address */
726 unsigned long *res_ptr = (unsigned long *) &((ioc)->res_map[ridx & ~RESMAP_IDX_MASK]);
727 int bits_not_wanted = size >> iovp_shift;
728 unsigned long m;
729
730 /* Round up to power-of-two size: see AR2305 note above */
731 bits_not_wanted = 1UL << get_iovp_order(bits_not_wanted << iovp_shift);
732 for (; bits_not_wanted > 0 ; res_ptr++) {
733
734 if (unlikely(bits_not_wanted > BITS_PER_LONG)) {
735
736 /* these mappings start 64bit aligned */
737 *res_ptr = 0UL;
738 bits_not_wanted -= BITS_PER_LONG;
739 pide += BITS_PER_LONG;
740
741 } else {
742
743 /* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
744 m = RESMAP_MASK(bits_not_wanted) << (pide & (BITS_PER_LONG - 1));
745 bits_not_wanted = 0;
746
747 DBG_RES("%s( ,%x,%x) %x/%lx %x %p %lx\n", __func__, (uint) iova, size,
748 bits_not_wanted, m, pide, res_ptr, *res_ptr);
749
750 ASSERT(m != 0);
751 ASSERT(bits_not_wanted);
752 ASSERT((*res_ptr & m) == m); /* verify same bits are set */
753 *res_ptr &= ~m;
754 }
755 }
756 }
757
758
759 /**************************************************************
760 *
761 * "Dynamic DMA Mapping" support (aka "Coherent I/O")
762 *
763 ***************************************************************/
764
765 /**
766 * sba_io_pdir_entry - fill in one IO PDIR entry
767 * @pdir_ptr: pointer to IO PDIR entry
768 * @vba: Virtual CPU address of buffer to map
769 *
770 * SBA Mapping Routine
771 *
772 * Given a virtual address (vba, arg1) sba_io_pdir_entry()
773 * loads the I/O PDIR entry pointed to by pdir_ptr (arg0).
774 * Each IO Pdir entry consists of 8 bytes as shown below
775 * (LSB == bit 0):
776 *
777 * 63 40 11 7 0
778 * +-+---------------------+----------------------------------+----+--------+
779 * |V| U | PPN[39:12] | U | FF |
780 * +-+---------------------+----------------------------------+----+--------+
781 *
782 * V == Valid Bit
783 * U == Unused
784 * PPN == Physical Page Number
785 *
786 * The physical address fields are filled with the results of virt_to_phys()
787 * on the vba.
788 */
789
790 #if 1
791 #define sba_io_pdir_entry(pdir_ptr, vba) *pdir_ptr = ((vba & ~0xE000000000000FFFULL) \
792 | 0x8000000000000000ULL)
793 #else
794 void SBA_INLINE
sba_io_pdir_entry(u64 * pdir_ptr,unsigned long vba)795 sba_io_pdir_entry(u64 *pdir_ptr, unsigned long vba)
796 {
797 *pdir_ptr = ((vba & ~0xE000000000000FFFULL) | 0x80000000000000FFULL);
798 }
799 #endif
800
801 #ifdef ENABLE_MARK_CLEAN
802 /**
803 * Since DMA is i-cache coherent, any (complete) pages that were written via
804 * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to
805 * flush them when they get mapped into an executable vm-area.
806 */
807 static void
mark_clean(void * addr,size_t size)808 mark_clean (void *addr, size_t size)
809 {
810 unsigned long pg_addr, end;
811
812 pg_addr = PAGE_ALIGN((unsigned long) addr);
813 end = (unsigned long) addr + size;
814 while (pg_addr + PAGE_SIZE <= end) {
815 struct page *page = virt_to_page((void *)pg_addr);
816 set_bit(PG_arch_1, &page->flags);
817 pg_addr += PAGE_SIZE;
818 }
819 }
820 #endif
821
822 /**
823 * sba_mark_invalid - invalidate one or more IO PDIR entries
824 * @ioc: IO MMU structure which owns the pdir we are interested in.
825 * @iova: IO Virtual Address mapped earlier
826 * @byte_cnt: number of bytes this mapping covers.
827 *
828 * Marking the IO PDIR entry(ies) as Invalid and invalidate
829 * corresponding IO TLB entry. The PCOM (Purge Command Register)
830 * is to purge stale entries in the IO TLB when unmapping entries.
831 *
832 * The PCOM register supports purging of multiple pages, with a minium
833 * of 1 page and a maximum of 2GB. Hardware requires the address be
834 * aligned to the size of the range being purged. The size of the range
835 * must be a power of 2. The "Cool perf optimization" in the
836 * allocation routine helps keep that true.
837 */
838 static SBA_INLINE void
sba_mark_invalid(struct ioc * ioc,dma_addr_t iova,size_t byte_cnt)839 sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)
840 {
841 u32 iovp = (u32) SBA_IOVP(ioc,iova);
842
843 int off = PDIR_INDEX(iovp);
844
845 /* Must be non-zero and rounded up */
846 ASSERT(byte_cnt > 0);
847 ASSERT(0 == (byte_cnt & ~iovp_mask));
848
849 #ifdef ASSERT_PDIR_SANITY
850 /* Assert first pdir entry is set */
851 if (!(ioc->pdir_base[off] >> 60)) {
852 sba_dump_pdir_entry(ioc,"sba_mark_invalid()", PDIR_INDEX(iovp));
853 }
854 #endif
855
856 if (byte_cnt <= iovp_size)
857 {
858 ASSERT(off < ioc->pdir_size);
859
860 iovp |= iovp_shift; /* set "size" field for PCOM */
861
862 #ifndef FULL_VALID_PDIR
863 /*
864 ** clear I/O PDIR entry "valid" bit
865 ** Do NOT clear the rest - save it for debugging.
866 ** We should only clear bits that have previously
867 ** been enabled.
868 */
869 ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
870 #else
871 /*
872 ** If we want to maintain the PDIR as valid, put in
873 ** the spill page so devices prefetching won't
874 ** cause a hard fail.
875 */
876 ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
877 #endif
878 } else {
879 u32 t = get_iovp_order(byte_cnt) + iovp_shift;
880
881 iovp |= t;
882 ASSERT(t <= 31); /* 2GB! Max value of "size" field */
883
884 do {
885 /* verify this pdir entry is enabled */
886 ASSERT(ioc->pdir_base[off] >> 63);
887 #ifndef FULL_VALID_PDIR
888 /* clear I/O Pdir entry "valid" bit first */
889 ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
890 #else
891 ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
892 #endif
893 off++;
894 byte_cnt -= iovp_size;
895 } while (byte_cnt > 0);
896 }
897
898 WRITE_REG(iovp | ioc->ibase, ioc->ioc_hpa+IOC_PCOM);
899 }
900
901 /**
902 * sba_map_single_attrs - map one buffer and return IOVA for DMA
903 * @dev: instance of PCI owned by the driver that's asking.
904 * @addr: driver buffer to map.
905 * @size: number of bytes to map in driver buffer.
906 * @dir: R/W or both.
907 * @attrs: optional dma attributes
908 *
909 * See Documentation/PCI/PCI-DMA-mapping.txt
910 */
911 dma_addr_t
sba_map_single_attrs(struct device * dev,void * addr,size_t size,int dir,struct dma_attrs * attrs)912 sba_map_single_attrs(struct device *dev, void *addr, size_t size, int dir,
913 struct dma_attrs *attrs)
914 {
915 struct ioc *ioc;
916 dma_addr_t iovp;
917 dma_addr_t offset;
918 u64 *pdir_start;
919 int pide;
920 #ifdef ASSERT_PDIR_SANITY
921 unsigned long flags;
922 #endif
923 #ifdef ALLOW_IOV_BYPASS
924 unsigned long pci_addr = virt_to_phys(addr);
925 #endif
926
927 #ifdef ALLOW_IOV_BYPASS
928 ASSERT(to_pci_dev(dev)->dma_mask);
929 /*
930 ** Check if the PCI device can DMA to ptr... if so, just return ptr
931 */
932 if (likely((pci_addr & ~to_pci_dev(dev)->dma_mask) == 0)) {
933 /*
934 ** Device is bit capable of DMA'ing to the buffer...
935 ** just return the PCI address of ptr
936 */
937 DBG_BYPASS("sba_map_single_attrs() bypass mask/addr: "
938 "0x%lx/0x%lx\n",
939 to_pci_dev(dev)->dma_mask, pci_addr);
940 return pci_addr;
941 }
942 #endif
943 ioc = GET_IOC(dev);
944 ASSERT(ioc);
945
946 prefetch(ioc->res_hint);
947
948 ASSERT(size > 0);
949 ASSERT(size <= DMA_CHUNK_SIZE);
950
951 /* save offset bits */
952 offset = ((dma_addr_t) (long) addr) & ~iovp_mask;
953
954 /* round up to nearest iovp_size */
955 size = (size + offset + ~iovp_mask) & iovp_mask;
956
957 #ifdef ASSERT_PDIR_SANITY
958 spin_lock_irqsave(&ioc->res_lock, flags);
959 if (sba_check_pdir(ioc,"Check before sba_map_single_attrs()"))
960 panic("Sanity check failed");
961 spin_unlock_irqrestore(&ioc->res_lock, flags);
962 #endif
963
964 pide = sba_alloc_range(ioc, dev, size);
965
966 iovp = (dma_addr_t) pide << iovp_shift;
967
968 DBG_RUN("%s() 0x%p -> 0x%lx\n", __func__, addr, (long) iovp | offset);
969
970 pdir_start = &(ioc->pdir_base[pide]);
971
972 while (size > 0) {
973 ASSERT(((u8 *)pdir_start)[7] == 0); /* verify availability */
974 sba_io_pdir_entry(pdir_start, (unsigned long) addr);
975
976 DBG_RUN(" pdir 0x%p %lx\n", pdir_start, *pdir_start);
977
978 addr += iovp_size;
979 size -= iovp_size;
980 pdir_start++;
981 }
982 /* force pdir update */
983 wmb();
984
985 /* form complete address */
986 #ifdef ASSERT_PDIR_SANITY
987 spin_lock_irqsave(&ioc->res_lock, flags);
988 sba_check_pdir(ioc,"Check after sba_map_single_attrs()");
989 spin_unlock_irqrestore(&ioc->res_lock, flags);
990 #endif
991 return SBA_IOVA(ioc, iovp, offset);
992 }
993 EXPORT_SYMBOL(sba_map_single_attrs);
994
995 #ifdef ENABLE_MARK_CLEAN
996 static SBA_INLINE void
sba_mark_clean(struct ioc * ioc,dma_addr_t iova,size_t size)997 sba_mark_clean(struct ioc *ioc, dma_addr_t iova, size_t size)
998 {
999 u32 iovp = (u32) SBA_IOVP(ioc,iova);
1000 int off = PDIR_INDEX(iovp);
1001 void *addr;
1002
1003 if (size <= iovp_size) {
1004 addr = phys_to_virt(ioc->pdir_base[off] &
1005 ~0xE000000000000FFFULL);
1006 mark_clean(addr, size);
1007 } else {
1008 do {
1009 addr = phys_to_virt(ioc->pdir_base[off] &
1010 ~0xE000000000000FFFULL);
1011 mark_clean(addr, min(size, iovp_size));
1012 off++;
1013 size -= iovp_size;
1014 } while (size > 0);
1015 }
1016 }
1017 #endif
1018
1019 /**
1020 * sba_unmap_single_attrs - unmap one IOVA and free resources
1021 * @dev: instance of PCI owned by the driver that's asking.
1022 * @iova: IOVA of driver buffer previously mapped.
1023 * @size: number of bytes mapped in driver buffer.
1024 * @dir: R/W or both.
1025 * @attrs: optional dma attributes
1026 *
1027 * See Documentation/PCI/PCI-DMA-mapping.txt
1028 */
sba_unmap_single_attrs(struct device * dev,dma_addr_t iova,size_t size,int dir,struct dma_attrs * attrs)1029 void sba_unmap_single_attrs(struct device *dev, dma_addr_t iova, size_t size,
1030 int dir, struct dma_attrs *attrs)
1031 {
1032 struct ioc *ioc;
1033 #if DELAYED_RESOURCE_CNT > 0
1034 struct sba_dma_pair *d;
1035 #endif
1036 unsigned long flags;
1037 dma_addr_t offset;
1038
1039 ioc = GET_IOC(dev);
1040 ASSERT(ioc);
1041
1042 #ifdef ALLOW_IOV_BYPASS
1043 if (likely((iova & ioc->imask) != ioc->ibase)) {
1044 /*
1045 ** Address does not fall w/in IOVA, must be bypassing
1046 */
1047 DBG_BYPASS("sba_unmap_single_atttrs() bypass addr: 0x%lx\n",
1048 iova);
1049
1050 #ifdef ENABLE_MARK_CLEAN
1051 if (dir == DMA_FROM_DEVICE) {
1052 mark_clean(phys_to_virt(iova), size);
1053 }
1054 #endif
1055 return;
1056 }
1057 #endif
1058 offset = iova & ~iovp_mask;
1059
1060 DBG_RUN("%s() iovp 0x%lx/%x\n", __func__, (long) iova, size);
1061
1062 iova ^= offset; /* clear offset bits */
1063 size += offset;
1064 size = ROUNDUP(size, iovp_size);
1065
1066 #ifdef ENABLE_MARK_CLEAN
1067 if (dir == DMA_FROM_DEVICE)
1068 sba_mark_clean(ioc, iova, size);
1069 #endif
1070
1071 #if DELAYED_RESOURCE_CNT > 0
1072 spin_lock_irqsave(&ioc->saved_lock, flags);
1073 d = &(ioc->saved[ioc->saved_cnt]);
1074 d->iova = iova;
1075 d->size = size;
1076 if (unlikely(++(ioc->saved_cnt) >= DELAYED_RESOURCE_CNT)) {
1077 int cnt = ioc->saved_cnt;
1078 spin_lock(&ioc->res_lock);
1079 while (cnt--) {
1080 sba_mark_invalid(ioc, d->iova, d->size);
1081 sba_free_range(ioc, d->iova, d->size);
1082 d--;
1083 }
1084 ioc->saved_cnt = 0;
1085 READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
1086 spin_unlock(&ioc->res_lock);
1087 }
1088 spin_unlock_irqrestore(&ioc->saved_lock, flags);
1089 #else /* DELAYED_RESOURCE_CNT == 0 */
1090 spin_lock_irqsave(&ioc->res_lock, flags);
1091 sba_mark_invalid(ioc, iova, size);
1092 sba_free_range(ioc, iova, size);
1093 READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
1094 spin_unlock_irqrestore(&ioc->res_lock, flags);
1095 #endif /* DELAYED_RESOURCE_CNT == 0 */
1096 }
1097 EXPORT_SYMBOL(sba_unmap_single_attrs);
1098
1099 /**
1100 * sba_alloc_coherent - allocate/map shared mem for DMA
1101 * @dev: instance of PCI owned by the driver that's asking.
1102 * @size: number of bytes mapped in driver buffer.
1103 * @dma_handle: IOVA of new buffer.
1104 *
1105 * See Documentation/PCI/PCI-DMA-mapping.txt
1106 */
1107 void *
sba_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flags)1108 sba_alloc_coherent (struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags)
1109 {
1110 struct ioc *ioc;
1111 void *addr;
1112
1113 ioc = GET_IOC(dev);
1114 ASSERT(ioc);
1115
1116 #ifdef CONFIG_NUMA
1117 {
1118 struct page *page;
1119 page = alloc_pages_node(ioc->node == MAX_NUMNODES ?
1120 numa_node_id() : ioc->node, flags,
1121 get_order(size));
1122
1123 if (unlikely(!page))
1124 return NULL;
1125
1126 addr = page_address(page);
1127 }
1128 #else
1129 addr = (void *) __get_free_pages(flags, get_order(size));
1130 #endif
1131 if (unlikely(!addr))
1132 return NULL;
1133
1134 memset(addr, 0, size);
1135 *dma_handle = virt_to_phys(addr);
1136
1137 #ifdef ALLOW_IOV_BYPASS
1138 ASSERT(dev->coherent_dma_mask);
1139 /*
1140 ** Check if the PCI device can DMA to ptr... if so, just return ptr
1141 */
1142 if (likely((*dma_handle & ~dev->coherent_dma_mask) == 0)) {
1143 DBG_BYPASS("sba_alloc_coherent() bypass mask/addr: 0x%lx/0x%lx\n",
1144 dev->coherent_dma_mask, *dma_handle);
1145
1146 return addr;
1147 }
1148 #endif
1149
1150 /*
1151 * If device can't bypass or bypass is disabled, pass the 32bit fake
1152 * device to map single to get an iova mapping.
1153 */
1154 *dma_handle = sba_map_single_attrs(&ioc->sac_only_dev->dev, addr,
1155 size, 0, NULL);
1156
1157 return addr;
1158 }
1159
1160
1161 /**
1162 * sba_free_coherent - free/unmap shared mem for DMA
1163 * @dev: instance of PCI owned by the driver that's asking.
1164 * @size: number of bytes mapped in driver buffer.
1165 * @vaddr: virtual address IOVA of "consistent" buffer.
1166 * @dma_handler: IO virtual address of "consistent" buffer.
1167 *
1168 * See Documentation/PCI/PCI-DMA-mapping.txt
1169 */
sba_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle)1170 void sba_free_coherent (struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle)
1171 {
1172 sba_unmap_single_attrs(dev, dma_handle, size, 0, NULL);
1173 free_pages((unsigned long) vaddr, get_order(size));
1174 }
1175
1176
1177 /*
1178 ** Since 0 is a valid pdir_base index value, can't use that
1179 ** to determine if a value is valid or not. Use a flag to indicate
1180 ** the SG list entry contains a valid pdir index.
1181 */
1182 #define PIDE_FLAG 0x1UL
1183
1184 #ifdef DEBUG_LARGE_SG_ENTRIES
1185 int dump_run_sg = 0;
1186 #endif
1187
1188
1189 /**
1190 * sba_fill_pdir - write allocated SG entries into IO PDIR
1191 * @ioc: IO MMU structure which owns the pdir we are interested in.
1192 * @startsg: list of IOVA/size pairs
1193 * @nents: number of entries in startsg list
1194 *
1195 * Take preprocessed SG list and write corresponding entries
1196 * in the IO PDIR.
1197 */
1198
1199 static SBA_INLINE int
sba_fill_pdir(struct ioc * ioc,struct scatterlist * startsg,int nents)1200 sba_fill_pdir(
1201 struct ioc *ioc,
1202 struct scatterlist *startsg,
1203 int nents)
1204 {
1205 struct scatterlist *dma_sg = startsg; /* pointer to current DMA */
1206 int n_mappings = 0;
1207 u64 *pdirp = NULL;
1208 unsigned long dma_offset = 0;
1209
1210 while (nents-- > 0) {
1211 int cnt = startsg->dma_length;
1212 startsg->dma_length = 0;
1213
1214 #ifdef DEBUG_LARGE_SG_ENTRIES
1215 if (dump_run_sg)
1216 printk(" %2d : %08lx/%05x %p\n",
1217 nents, startsg->dma_address, cnt,
1218 sba_sg_address(startsg));
1219 #else
1220 DBG_RUN_SG(" %d : %08lx/%05x %p\n",
1221 nents, startsg->dma_address, cnt,
1222 sba_sg_address(startsg));
1223 #endif
1224 /*
1225 ** Look for the start of a new DMA stream
1226 */
1227 if (startsg->dma_address & PIDE_FLAG) {
1228 u32 pide = startsg->dma_address & ~PIDE_FLAG;
1229 dma_offset = (unsigned long) pide & ~iovp_mask;
1230 startsg->dma_address = 0;
1231 if (n_mappings)
1232 dma_sg = sg_next(dma_sg);
1233 dma_sg->dma_address = pide | ioc->ibase;
1234 pdirp = &(ioc->pdir_base[pide >> iovp_shift]);
1235 n_mappings++;
1236 }
1237
1238 /*
1239 ** Look for a VCONTIG chunk
1240 */
1241 if (cnt) {
1242 unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
1243 ASSERT(pdirp);
1244
1245 /* Since multiple Vcontig blocks could make up
1246 ** one DMA stream, *add* cnt to dma_len.
1247 */
1248 dma_sg->dma_length += cnt;
1249 cnt += dma_offset;
1250 dma_offset=0; /* only want offset on first chunk */
1251 cnt = ROUNDUP(cnt, iovp_size);
1252 do {
1253 sba_io_pdir_entry(pdirp, vaddr);
1254 vaddr += iovp_size;
1255 cnt -= iovp_size;
1256 pdirp++;
1257 } while (cnt > 0);
1258 }
1259 startsg = sg_next(startsg);
1260 }
1261 /* force pdir update */
1262 wmb();
1263
1264 #ifdef DEBUG_LARGE_SG_ENTRIES
1265 dump_run_sg = 0;
1266 #endif
1267 return(n_mappings);
1268 }
1269
1270
1271 /*
1272 ** Two address ranges are DMA contiguous *iff* "end of prev" and
1273 ** "start of next" are both on an IOV page boundary.
1274 **
1275 ** (shift left is a quick trick to mask off upper bits)
1276 */
1277 #define DMA_CONTIG(__X, __Y) \
1278 (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - iovp_shift)) == 0UL)
1279
1280
1281 /**
1282 * sba_coalesce_chunks - preprocess the SG list
1283 * @ioc: IO MMU structure which owns the pdir we are interested in.
1284 * @startsg: list of IOVA/size pairs
1285 * @nents: number of entries in startsg list
1286 *
1287 * First pass is to walk the SG list and determine where the breaks are
1288 * in the DMA stream. Allocates PDIR entries but does not fill them.
1289 * Returns the number of DMA chunks.
1290 *
1291 * Doing the fill separate from the coalescing/allocation keeps the
1292 * code simpler. Future enhancement could make one pass through
1293 * the sglist do both.
1294 */
1295 static SBA_INLINE int
sba_coalesce_chunks(struct ioc * ioc,struct device * dev,struct scatterlist * startsg,int nents)1296 sba_coalesce_chunks(struct ioc *ioc, struct device *dev,
1297 struct scatterlist *startsg,
1298 int nents)
1299 {
1300 struct scatterlist *vcontig_sg; /* VCONTIG chunk head */
1301 unsigned long vcontig_len; /* len of VCONTIG chunk */
1302 unsigned long vcontig_end;
1303 struct scatterlist *dma_sg; /* next DMA stream head */
1304 unsigned long dma_offset, dma_len; /* start/len of DMA stream */
1305 int n_mappings = 0;
1306 unsigned int max_seg_size = dma_get_max_seg_size(dev);
1307
1308 while (nents > 0) {
1309 unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
1310
1311 /*
1312 ** Prepare for first/next DMA stream
1313 */
1314 dma_sg = vcontig_sg = startsg;
1315 dma_len = vcontig_len = vcontig_end = startsg->length;
1316 vcontig_end += vaddr;
1317 dma_offset = vaddr & ~iovp_mask;
1318
1319 /* PARANOID: clear entries */
1320 startsg->dma_address = startsg->dma_length = 0;
1321
1322 /*
1323 ** This loop terminates one iteration "early" since
1324 ** it's always looking one "ahead".
1325 */
1326 while (--nents > 0) {
1327 unsigned long vaddr; /* tmp */
1328
1329 startsg = sg_next(startsg);
1330
1331 /* PARANOID */
1332 startsg->dma_address = startsg->dma_length = 0;
1333
1334 /* catch brokenness in SCSI layer */
1335 ASSERT(startsg->length <= DMA_CHUNK_SIZE);
1336
1337 /*
1338 ** First make sure current dma stream won't
1339 ** exceed DMA_CHUNK_SIZE if we coalesce the
1340 ** next entry.
1341 */
1342 if (((dma_len + dma_offset + startsg->length + ~iovp_mask) & iovp_mask)
1343 > DMA_CHUNK_SIZE)
1344 break;
1345
1346 if (dma_len + startsg->length > max_seg_size)
1347 break;
1348
1349 /*
1350 ** Then look for virtually contiguous blocks.
1351 **
1352 ** append the next transaction?
1353 */
1354 vaddr = (unsigned long) sba_sg_address(startsg);
1355 if (vcontig_end == vaddr)
1356 {
1357 vcontig_len += startsg->length;
1358 vcontig_end += startsg->length;
1359 dma_len += startsg->length;
1360 continue;
1361 }
1362
1363 #ifdef DEBUG_LARGE_SG_ENTRIES
1364 dump_run_sg = (vcontig_len > iovp_size);
1365 #endif
1366
1367 /*
1368 ** Not virtually contigous.
1369 ** Terminate prev chunk.
1370 ** Start a new chunk.
1371 **
1372 ** Once we start a new VCONTIG chunk, dma_offset
1373 ** can't change. And we need the offset from the first
1374 ** chunk - not the last one. Ergo Successive chunks
1375 ** must start on page boundaries and dove tail
1376 ** with it's predecessor.
1377 */
1378 vcontig_sg->dma_length = vcontig_len;
1379
1380 vcontig_sg = startsg;
1381 vcontig_len = startsg->length;
1382
1383 /*
1384 ** 3) do the entries end/start on page boundaries?
1385 ** Don't update vcontig_end until we've checked.
1386 */
1387 if (DMA_CONTIG(vcontig_end, vaddr))
1388 {
1389 vcontig_end = vcontig_len + vaddr;
1390 dma_len += vcontig_len;
1391 continue;
1392 } else {
1393 break;
1394 }
1395 }
1396
1397 /*
1398 ** End of DMA Stream
1399 ** Terminate last VCONTIG block.
1400 ** Allocate space for DMA stream.
1401 */
1402 vcontig_sg->dma_length = vcontig_len;
1403 dma_len = (dma_len + dma_offset + ~iovp_mask) & iovp_mask;
1404 ASSERT(dma_len <= DMA_CHUNK_SIZE);
1405 dma_sg->dma_address = (dma_addr_t) (PIDE_FLAG
1406 | (sba_alloc_range(ioc, dev, dma_len) << iovp_shift)
1407 | dma_offset);
1408 n_mappings++;
1409 }
1410
1411 return n_mappings;
1412 }
1413
1414
1415 /**
1416 * sba_map_sg - map Scatter/Gather list
1417 * @dev: instance of PCI owned by the driver that's asking.
1418 * @sglist: array of buffer/length pairs
1419 * @nents: number of entries in list
1420 * @dir: R/W or both.
1421 * @attrs: optional dma attributes
1422 *
1423 * See Documentation/PCI/PCI-DMA-mapping.txt
1424 */
sba_map_sg_attrs(struct device * dev,struct scatterlist * sglist,int nents,int dir,struct dma_attrs * attrs)1425 int sba_map_sg_attrs(struct device *dev, struct scatterlist *sglist, int nents,
1426 int dir, struct dma_attrs *attrs)
1427 {
1428 struct ioc *ioc;
1429 int coalesced, filled = 0;
1430 #ifdef ASSERT_PDIR_SANITY
1431 unsigned long flags;
1432 #endif
1433 #ifdef ALLOW_IOV_BYPASS_SG
1434 struct scatterlist *sg;
1435 #endif
1436
1437 DBG_RUN_SG("%s() START %d entries\n", __func__, nents);
1438 ioc = GET_IOC(dev);
1439 ASSERT(ioc);
1440
1441 #ifdef ALLOW_IOV_BYPASS_SG
1442 ASSERT(to_pci_dev(dev)->dma_mask);
1443 if (likely((ioc->dma_mask & ~to_pci_dev(dev)->dma_mask) == 0)) {
1444 for_each_sg(sglist, sg, nents, filled) {
1445 sg->dma_length = sg->length;
1446 sg->dma_address = virt_to_phys(sba_sg_address(sg));
1447 }
1448 return filled;
1449 }
1450 #endif
1451 /* Fast path single entry scatterlists. */
1452 if (nents == 1) {
1453 sglist->dma_length = sglist->length;
1454 sglist->dma_address = sba_map_single_attrs(dev, sba_sg_address(sglist), sglist->length, dir, attrs);
1455 return 1;
1456 }
1457
1458 #ifdef ASSERT_PDIR_SANITY
1459 spin_lock_irqsave(&ioc->res_lock, flags);
1460 if (sba_check_pdir(ioc,"Check before sba_map_sg_attrs()"))
1461 {
1462 sba_dump_sg(ioc, sglist, nents);
1463 panic("Check before sba_map_sg_attrs()");
1464 }
1465 spin_unlock_irqrestore(&ioc->res_lock, flags);
1466 #endif
1467
1468 prefetch(ioc->res_hint);
1469
1470 /*
1471 ** First coalesce the chunks and allocate I/O pdir space
1472 **
1473 ** If this is one DMA stream, we can properly map using the
1474 ** correct virtual address associated with each DMA page.
1475 ** w/o this association, we wouldn't have coherent DMA!
1476 ** Access to the virtual address is what forces a two pass algorithm.
1477 */
1478 coalesced = sba_coalesce_chunks(ioc, dev, sglist, nents);
1479
1480 /*
1481 ** Program the I/O Pdir
1482 **
1483 ** map the virtual addresses to the I/O Pdir
1484 ** o dma_address will contain the pdir index
1485 ** o dma_len will contain the number of bytes to map
1486 ** o address contains the virtual address.
1487 */
1488 filled = sba_fill_pdir(ioc, sglist, nents);
1489
1490 #ifdef ASSERT_PDIR_SANITY
1491 spin_lock_irqsave(&ioc->res_lock, flags);
1492 if (sba_check_pdir(ioc,"Check after sba_map_sg_attrs()"))
1493 {
1494 sba_dump_sg(ioc, sglist, nents);
1495 panic("Check after sba_map_sg_attrs()\n");
1496 }
1497 spin_unlock_irqrestore(&ioc->res_lock, flags);
1498 #endif
1499
1500 ASSERT(coalesced == filled);
1501 DBG_RUN_SG("%s() DONE %d mappings\n", __func__, filled);
1502
1503 return filled;
1504 }
1505 EXPORT_SYMBOL(sba_map_sg_attrs);
1506
1507 /**
1508 * sba_unmap_sg_attrs - unmap Scatter/Gather list
1509 * @dev: instance of PCI owned by the driver that's asking.
1510 * @sglist: array of buffer/length pairs
1511 * @nents: number of entries in list
1512 * @dir: R/W or both.
1513 * @attrs: optional dma attributes
1514 *
1515 * See Documentation/PCI/PCI-DMA-mapping.txt
1516 */
sba_unmap_sg_attrs(struct device * dev,struct scatterlist * sglist,int nents,int dir,struct dma_attrs * attrs)1517 void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist,
1518 int nents, int dir, struct dma_attrs *attrs)
1519 {
1520 #ifdef ASSERT_PDIR_SANITY
1521 struct ioc *ioc;
1522 unsigned long flags;
1523 #endif
1524
1525 DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1526 __func__, nents, sba_sg_address(sglist), sglist->length);
1527
1528 #ifdef ASSERT_PDIR_SANITY
1529 ioc = GET_IOC(dev);
1530 ASSERT(ioc);
1531
1532 spin_lock_irqsave(&ioc->res_lock, flags);
1533 sba_check_pdir(ioc,"Check before sba_unmap_sg_attrs()");
1534 spin_unlock_irqrestore(&ioc->res_lock, flags);
1535 #endif
1536
1537 while (nents && sglist->dma_length) {
1538
1539 sba_unmap_single_attrs(dev, sglist->dma_address,
1540 sglist->dma_length, dir, attrs);
1541 sglist = sg_next(sglist);
1542 nents--;
1543 }
1544
1545 DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
1546
1547 #ifdef ASSERT_PDIR_SANITY
1548 spin_lock_irqsave(&ioc->res_lock, flags);
1549 sba_check_pdir(ioc,"Check after sba_unmap_sg_attrs()");
1550 spin_unlock_irqrestore(&ioc->res_lock, flags);
1551 #endif
1552
1553 }
1554 EXPORT_SYMBOL(sba_unmap_sg_attrs);
1555
1556 /**************************************************************
1557 *
1558 * Initialization and claim
1559 *
1560 ***************************************************************/
1561
1562 static void __init
ioc_iova_init(struct ioc * ioc)1563 ioc_iova_init(struct ioc *ioc)
1564 {
1565 int tcnfg;
1566 int agp_found = 0;
1567 struct pci_dev *device = NULL;
1568 #ifdef FULL_VALID_PDIR
1569 unsigned long index;
1570 #endif
1571
1572 /*
1573 ** Firmware programs the base and size of a "safe IOVA space"
1574 ** (one that doesn't overlap memory or LMMIO space) in the
1575 ** IBASE and IMASK registers.
1576 */
1577 ioc->ibase = READ_REG(ioc->ioc_hpa + IOC_IBASE) & ~0x1UL;
1578 ioc->imask = READ_REG(ioc->ioc_hpa + IOC_IMASK) | 0xFFFFFFFF00000000UL;
1579
1580 ioc->iov_size = ~ioc->imask + 1;
1581
1582 DBG_INIT("%s() hpa %p IOV base 0x%lx mask 0x%lx (%dMB)\n",
1583 __func__, ioc->ioc_hpa, ioc->ibase, ioc->imask,
1584 ioc->iov_size >> 20);
1585
1586 switch (iovp_size) {
1587 case 4*1024: tcnfg = 0; break;
1588 case 8*1024: tcnfg = 1; break;
1589 case 16*1024: tcnfg = 2; break;
1590 case 64*1024: tcnfg = 3; break;
1591 default:
1592 panic(PFX "Unsupported IOTLB page size %ldK",
1593 iovp_size >> 10);
1594 break;
1595 }
1596 WRITE_REG(tcnfg, ioc->ioc_hpa + IOC_TCNFG);
1597
1598 ioc->pdir_size = (ioc->iov_size / iovp_size) * PDIR_ENTRY_SIZE;
1599 ioc->pdir_base = (void *) __get_free_pages(GFP_KERNEL,
1600 get_order(ioc->pdir_size));
1601 if (!ioc->pdir_base)
1602 panic(PFX "Couldn't allocate I/O Page Table\n");
1603
1604 memset(ioc->pdir_base, 0, ioc->pdir_size);
1605
1606 DBG_INIT("%s() IOV page size %ldK pdir %p size %x\n", __func__,
1607 iovp_size >> 10, ioc->pdir_base, ioc->pdir_size);
1608
1609 ASSERT(ALIGN((unsigned long) ioc->pdir_base, 4*1024) == (unsigned long) ioc->pdir_base);
1610 WRITE_REG(virt_to_phys(ioc->pdir_base), ioc->ioc_hpa + IOC_PDIR_BASE);
1611
1612 /*
1613 ** If an AGP device is present, only use half of the IOV space
1614 ** for PCI DMA. Unfortunately we can't know ahead of time
1615 ** whether GART support will actually be used, for now we
1616 ** can just key on an AGP device found in the system.
1617 ** We program the next pdir index after we stop w/ a key for
1618 ** the GART code to handshake on.
1619 */
1620 for_each_pci_dev(device)
1621 agp_found |= pci_find_capability(device, PCI_CAP_ID_AGP);
1622
1623 if (agp_found && reserve_sba_gart) {
1624 printk(KERN_INFO PFX "reserving %dMb of IOVA space at 0x%lx for agpgart\n",
1625 ioc->iov_size/2 >> 20, ioc->ibase + ioc->iov_size/2);
1626 ioc->pdir_size /= 2;
1627 ((u64 *)ioc->pdir_base)[PDIR_INDEX(ioc->iov_size/2)] = ZX1_SBA_IOMMU_COOKIE;
1628 }
1629 #ifdef FULL_VALID_PDIR
1630 /*
1631 ** Check to see if the spill page has been allocated, we don't need more than
1632 ** one across multiple SBAs.
1633 */
1634 if (!prefetch_spill_page) {
1635 char *spill_poison = "SBAIOMMU POISON";
1636 int poison_size = 16;
1637 void *poison_addr, *addr;
1638
1639 addr = (void *)__get_free_pages(GFP_KERNEL, get_order(iovp_size));
1640 if (!addr)
1641 panic(PFX "Couldn't allocate PDIR spill page\n");
1642
1643 poison_addr = addr;
1644 for ( ; (u64) poison_addr < addr + iovp_size; poison_addr += poison_size)
1645 memcpy(poison_addr, spill_poison, poison_size);
1646
1647 prefetch_spill_page = virt_to_phys(addr);
1648
1649 DBG_INIT("%s() prefetch spill addr: 0x%lx\n", __func__, prefetch_spill_page);
1650 }
1651 /*
1652 ** Set all the PDIR entries valid w/ the spill page as the target
1653 */
1654 for (index = 0 ; index < (ioc->pdir_size / PDIR_ENTRY_SIZE) ; index++)
1655 ((u64 *)ioc->pdir_base)[index] = (0x80000000000000FF | prefetch_spill_page);
1656 #endif
1657
1658 /* Clear I/O TLB of any possible entries */
1659 WRITE_REG(ioc->ibase | (get_iovp_order(ioc->iov_size) + iovp_shift), ioc->ioc_hpa + IOC_PCOM);
1660 READ_REG(ioc->ioc_hpa + IOC_PCOM);
1661
1662 /* Enable IOVA translation */
1663 WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa + IOC_IBASE);
1664 READ_REG(ioc->ioc_hpa + IOC_IBASE);
1665 }
1666
1667 static void __init
ioc_resource_init(struct ioc * ioc)1668 ioc_resource_init(struct ioc *ioc)
1669 {
1670 spin_lock_init(&ioc->res_lock);
1671 #if DELAYED_RESOURCE_CNT > 0
1672 spin_lock_init(&ioc->saved_lock);
1673 #endif
1674
1675 /* resource map size dictated by pdir_size */
1676 ioc->res_size = ioc->pdir_size / PDIR_ENTRY_SIZE; /* entries */
1677 ioc->res_size >>= 3; /* convert bit count to byte count */
1678 DBG_INIT("%s() res_size 0x%x\n", __func__, ioc->res_size);
1679
1680 ioc->res_map = (char *) __get_free_pages(GFP_KERNEL,
1681 get_order(ioc->res_size));
1682 if (!ioc->res_map)
1683 panic(PFX "Couldn't allocate resource map\n");
1684
1685 memset(ioc->res_map, 0, ioc->res_size);
1686 /* next available IOVP - circular search */
1687 ioc->res_hint = (unsigned long *) ioc->res_map;
1688
1689 #ifdef ASSERT_PDIR_SANITY
1690 /* Mark first bit busy - ie no IOVA 0 */
1691 ioc->res_map[0] = 0x1;
1692 ioc->pdir_base[0] = 0x8000000000000000ULL | ZX1_SBA_IOMMU_COOKIE;
1693 #endif
1694 #ifdef FULL_VALID_PDIR
1695 /* Mark the last resource used so we don't prefetch beyond IOVA space */
1696 ioc->res_map[ioc->res_size - 1] |= 0x80UL; /* res_map is chars */
1697 ioc->pdir_base[(ioc->pdir_size / PDIR_ENTRY_SIZE) - 1] = (0x80000000000000FF
1698 | prefetch_spill_page);
1699 #endif
1700
1701 DBG_INIT("%s() res_map %x %p\n", __func__,
1702 ioc->res_size, (void *) ioc->res_map);
1703 }
1704
1705 static void __init
ioc_sac_init(struct ioc * ioc)1706 ioc_sac_init(struct ioc *ioc)
1707 {
1708 struct pci_dev *sac = NULL;
1709 struct pci_controller *controller = NULL;
1710
1711 /*
1712 * pci_alloc_coherent() must return a DMA address which is
1713 * SAC (single address cycle) addressable, so allocate a
1714 * pseudo-device to enforce that.
1715 */
1716 sac = kzalloc(sizeof(*sac), GFP_KERNEL);
1717 if (!sac)
1718 panic(PFX "Couldn't allocate struct pci_dev");
1719
1720 controller = kzalloc(sizeof(*controller), GFP_KERNEL);
1721 if (!controller)
1722 panic(PFX "Couldn't allocate struct pci_controller");
1723
1724 controller->iommu = ioc;
1725 sac->sysdata = controller;
1726 sac->dma_mask = 0xFFFFFFFFUL;
1727 #ifdef CONFIG_PCI
1728 sac->dev.bus = &pci_bus_type;
1729 #endif
1730 ioc->sac_only_dev = sac;
1731 }
1732
1733 static void __init
ioc_zx1_init(struct ioc * ioc)1734 ioc_zx1_init(struct ioc *ioc)
1735 {
1736 unsigned long rope_config;
1737 unsigned int i;
1738
1739 if (ioc->rev < 0x20)
1740 panic(PFX "IOC 2.0 or later required for IOMMU support\n");
1741
1742 /* 38 bit memory controller + extra bit for range displaced by MMIO */
1743 ioc->dma_mask = (0x1UL << 39) - 1;
1744
1745 /*
1746 ** Clear ROPE(N)_CONFIG AO bit.
1747 ** Disables "NT Ordering" (~= !"Relaxed Ordering")
1748 ** Overrides bit 1 in DMA Hint Sets.
1749 ** Improves netperf UDP_STREAM by ~10% for tg3 on bcm5701.
1750 */
1751 for (i=0; i<(8*8); i+=8) {
1752 rope_config = READ_REG(ioc->ioc_hpa + IOC_ROPE0_CFG + i);
1753 rope_config &= ~IOC_ROPE_AO;
1754 WRITE_REG(rope_config, ioc->ioc_hpa + IOC_ROPE0_CFG + i);
1755 }
1756 }
1757
1758 typedef void (initfunc)(struct ioc *);
1759
1760 struct ioc_iommu {
1761 u32 func_id;
1762 char *name;
1763 initfunc *init;
1764 };
1765
1766 static struct ioc_iommu ioc_iommu_info[] __initdata = {
1767 { ZX1_IOC_ID, "zx1", ioc_zx1_init },
1768 { ZX2_IOC_ID, "zx2", NULL },
1769 { SX1000_IOC_ID, "sx1000", NULL },
1770 { SX2000_IOC_ID, "sx2000", NULL },
1771 };
1772
1773 static struct ioc * __init
ioc_init(u64 hpa,void * handle)1774 ioc_init(u64 hpa, void *handle)
1775 {
1776 struct ioc *ioc;
1777 struct ioc_iommu *info;
1778
1779 ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
1780 if (!ioc)
1781 return NULL;
1782
1783 ioc->next = ioc_list;
1784 ioc_list = ioc;
1785
1786 ioc->handle = handle;
1787 ioc->ioc_hpa = ioremap(hpa, 0x1000);
1788
1789 ioc->func_id = READ_REG(ioc->ioc_hpa + IOC_FUNC_ID);
1790 ioc->rev = READ_REG(ioc->ioc_hpa + IOC_FCLASS) & 0xFFUL;
1791 ioc->dma_mask = 0xFFFFFFFFFFFFFFFFUL; /* conservative */
1792
1793 for (info = ioc_iommu_info; info < ioc_iommu_info + ARRAY_SIZE(ioc_iommu_info); info++) {
1794 if (ioc->func_id == info->func_id) {
1795 ioc->name = info->name;
1796 if (info->init)
1797 (info->init)(ioc);
1798 }
1799 }
1800
1801 iovp_size = (1 << iovp_shift);
1802 iovp_mask = ~(iovp_size - 1);
1803
1804 DBG_INIT("%s: PAGE_SIZE %ldK, iovp_size %ldK\n", __func__,
1805 PAGE_SIZE >> 10, iovp_size >> 10);
1806
1807 if (!ioc->name) {
1808 ioc->name = kmalloc(24, GFP_KERNEL);
1809 if (ioc->name)
1810 sprintf((char *) ioc->name, "Unknown (%04x:%04x)",
1811 ioc->func_id & 0xFFFF, (ioc->func_id >> 16) & 0xFFFF);
1812 else
1813 ioc->name = "Unknown";
1814 }
1815
1816 ioc_iova_init(ioc);
1817 ioc_resource_init(ioc);
1818 ioc_sac_init(ioc);
1819
1820 if ((long) ~iovp_mask > (long) ia64_max_iommu_merge_mask)
1821 ia64_max_iommu_merge_mask = ~iovp_mask;
1822
1823 printk(KERN_INFO PFX
1824 "%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
1825 ioc->name, (ioc->rev >> 4) & 0xF, ioc->rev & 0xF,
1826 hpa, ioc->iov_size >> 20, ioc->ibase);
1827
1828 return ioc;
1829 }
1830
1831
1832
1833 /**************************************************************************
1834 **
1835 ** SBA initialization code (HW and SW)
1836 **
1837 ** o identify SBA chip itself
1838 ** o FIXME: initialize DMA hints for reasonable defaults
1839 **
1840 **************************************************************************/
1841
1842 #ifdef CONFIG_PROC_FS
1843 static void *
ioc_start(struct seq_file * s,loff_t * pos)1844 ioc_start(struct seq_file *s, loff_t *pos)
1845 {
1846 struct ioc *ioc;
1847 loff_t n = *pos;
1848
1849 for (ioc = ioc_list; ioc; ioc = ioc->next)
1850 if (!n--)
1851 return ioc;
1852
1853 return NULL;
1854 }
1855
1856 static void *
ioc_next(struct seq_file * s,void * v,loff_t * pos)1857 ioc_next(struct seq_file *s, void *v, loff_t *pos)
1858 {
1859 struct ioc *ioc = v;
1860
1861 ++*pos;
1862 return ioc->next;
1863 }
1864
1865 static void
ioc_stop(struct seq_file * s,void * v)1866 ioc_stop(struct seq_file *s, void *v)
1867 {
1868 }
1869
1870 static int
ioc_show(struct seq_file * s,void * v)1871 ioc_show(struct seq_file *s, void *v)
1872 {
1873 struct ioc *ioc = v;
1874 unsigned long *res_ptr = (unsigned long *)ioc->res_map;
1875 int i, used = 0;
1876
1877 seq_printf(s, "Hewlett Packard %s IOC rev %d.%d\n",
1878 ioc->name, ((ioc->rev >> 4) & 0xF), (ioc->rev & 0xF));
1879 #ifdef CONFIG_NUMA
1880 if (ioc->node != MAX_NUMNODES)
1881 seq_printf(s, "NUMA node : %d\n", ioc->node);
1882 #endif
1883 seq_printf(s, "IOVA size : %ld MB\n", ((ioc->pdir_size >> 3) * iovp_size)/(1024*1024));
1884 seq_printf(s, "IOVA page size : %ld kb\n", iovp_size/1024);
1885
1886 for (i = 0; i < (ioc->res_size / sizeof(unsigned long)); ++i, ++res_ptr)
1887 used += hweight64(*res_ptr);
1888
1889 seq_printf(s, "PDIR size : %d entries\n", ioc->pdir_size >> 3);
1890 seq_printf(s, "PDIR used : %d entries\n", used);
1891
1892 #ifdef PDIR_SEARCH_TIMING
1893 {
1894 unsigned long i = 0, avg = 0, min, max;
1895 min = max = ioc->avg_search[0];
1896 for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
1897 avg += ioc->avg_search[i];
1898 if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
1899 if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
1900 }
1901 avg /= SBA_SEARCH_SAMPLE;
1902 seq_printf(s, "Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles/IOVA page)\n",
1903 min, avg, max);
1904 }
1905 #endif
1906 #ifndef ALLOW_IOV_BYPASS
1907 seq_printf(s, "IOVA bypass disabled\n");
1908 #endif
1909 return 0;
1910 }
1911
1912 static const struct seq_operations ioc_seq_ops = {
1913 .start = ioc_start,
1914 .next = ioc_next,
1915 .stop = ioc_stop,
1916 .show = ioc_show
1917 };
1918
1919 static int
ioc_open(struct inode * inode,struct file * file)1920 ioc_open(struct inode *inode, struct file *file)
1921 {
1922 return seq_open(file, &ioc_seq_ops);
1923 }
1924
1925 static const struct file_operations ioc_fops = {
1926 .open = ioc_open,
1927 .read = seq_read,
1928 .llseek = seq_lseek,
1929 .release = seq_release
1930 };
1931
1932 static void __init
ioc_proc_init(void)1933 ioc_proc_init(void)
1934 {
1935 struct proc_dir_entry *dir;
1936
1937 dir = proc_mkdir("bus/mckinley", NULL);
1938 if (!dir)
1939 return;
1940
1941 proc_create(ioc_list->name, 0, dir, &ioc_fops);
1942 }
1943 #endif
1944
1945 static void
sba_connect_bus(struct pci_bus * bus)1946 sba_connect_bus(struct pci_bus *bus)
1947 {
1948 acpi_handle handle, parent;
1949 acpi_status status;
1950 struct ioc *ioc;
1951
1952 if (!PCI_CONTROLLER(bus))
1953 panic(PFX "no sysdata on bus %d!\n", bus->number);
1954
1955 if (PCI_CONTROLLER(bus)->iommu)
1956 return;
1957
1958 handle = PCI_CONTROLLER(bus)->acpi_handle;
1959 if (!handle)
1960 return;
1961
1962 /*
1963 * The IOC scope encloses PCI root bridges in the ACPI
1964 * namespace, so work our way out until we find an IOC we
1965 * claimed previously.
1966 */
1967 do {
1968 for (ioc = ioc_list; ioc; ioc = ioc->next)
1969 if (ioc->handle == handle) {
1970 PCI_CONTROLLER(bus)->iommu = ioc;
1971 return;
1972 }
1973
1974 status = acpi_get_parent(handle, &parent);
1975 handle = parent;
1976 } while (ACPI_SUCCESS(status));
1977
1978 printk(KERN_WARNING "No IOC for PCI Bus %04x:%02x in ACPI\n", pci_domain_nr(bus), bus->number);
1979 }
1980
1981 #ifdef CONFIG_NUMA
1982 static void __init
sba_map_ioc_to_node(struct ioc * ioc,acpi_handle handle)1983 sba_map_ioc_to_node(struct ioc *ioc, acpi_handle handle)
1984 {
1985 unsigned int node;
1986 int pxm;
1987
1988 ioc->node = MAX_NUMNODES;
1989
1990 pxm = acpi_get_pxm(handle);
1991
1992 if (pxm < 0)
1993 return;
1994
1995 node = pxm_to_node(pxm);
1996
1997 if (node >= MAX_NUMNODES || !node_online(node))
1998 return;
1999
2000 ioc->node = node;
2001 return;
2002 }
2003 #else
2004 #define sba_map_ioc_to_node(ioc, handle)
2005 #endif
2006
2007 static int __init
acpi_sba_ioc_add(struct acpi_device * device)2008 acpi_sba_ioc_add(struct acpi_device *device)
2009 {
2010 struct ioc *ioc;
2011 acpi_status status;
2012 u64 hpa, length;
2013 struct acpi_buffer buffer;
2014 struct acpi_device_info *dev_info;
2015
2016 status = hp_acpi_csr_space(device->handle, &hpa, &length);
2017 if (ACPI_FAILURE(status))
2018 return 1;
2019
2020 buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER;
2021 status = acpi_get_object_info(device->handle, &buffer);
2022 if (ACPI_FAILURE(status))
2023 return 1;
2024 dev_info = buffer.pointer;
2025
2026 /*
2027 * For HWP0001, only SBA appears in ACPI namespace. It encloses the PCI
2028 * root bridges, and its CSR space includes the IOC function.
2029 */
2030 if (strncmp("HWP0001", dev_info->hardware_id.value, 7) == 0) {
2031 hpa += ZX1_IOC_OFFSET;
2032 /* zx1 based systems default to kernel page size iommu pages */
2033 if (!iovp_shift)
2034 iovp_shift = min(PAGE_SHIFT, 16);
2035 }
2036 kfree(dev_info);
2037
2038 /*
2039 * default anything not caught above or specified on cmdline to 4k
2040 * iommu page size
2041 */
2042 if (!iovp_shift)
2043 iovp_shift = 12;
2044
2045 ioc = ioc_init(hpa, device->handle);
2046 if (!ioc)
2047 return 1;
2048
2049 /* setup NUMA node association */
2050 sba_map_ioc_to_node(ioc, device->handle);
2051 return 0;
2052 }
2053
2054 static const struct acpi_device_id hp_ioc_iommu_device_ids[] = {
2055 {"HWP0001", 0},
2056 {"HWP0004", 0},
2057 {"", 0},
2058 };
2059 static struct acpi_driver acpi_sba_ioc_driver = {
2060 .name = "IOC IOMMU Driver",
2061 .ids = hp_ioc_iommu_device_ids,
2062 .ops = {
2063 .add = acpi_sba_ioc_add,
2064 },
2065 };
2066
2067 static int __init
sba_init(void)2068 sba_init(void)
2069 {
2070 if (!ia64_platform_is("hpzx1") && !ia64_platform_is("hpzx1_swiotlb"))
2071 return 0;
2072
2073 #if defined(CONFIG_IA64_GENERIC)
2074 /* If we are booting a kdump kernel, the sba_iommu will
2075 * cause devices that were not shutdown properly to MCA
2076 * as soon as they are turned back on. Our only option for
2077 * a successful kdump kernel boot is to use the swiotlb.
2078 */
2079 if (is_kdump_kernel()) {
2080 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2081 panic("Unable to initialize software I/O TLB:"
2082 " Try machvec=dig boot option");
2083 machvec_init("dig");
2084 return 0;
2085 }
2086 #endif
2087
2088 acpi_bus_register_driver(&acpi_sba_ioc_driver);
2089 if (!ioc_list) {
2090 #ifdef CONFIG_IA64_GENERIC
2091 /*
2092 * If we didn't find something sba_iommu can claim, we
2093 * need to setup the swiotlb and switch to the dig machvec.
2094 */
2095 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2096 panic("Unable to find SBA IOMMU or initialize "
2097 "software I/O TLB: Try machvec=dig boot option");
2098 machvec_init("dig");
2099 #else
2100 panic("Unable to find SBA IOMMU: Try a generic or DIG kernel");
2101 #endif
2102 return 0;
2103 }
2104
2105 #if defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_HP_ZX1_SWIOTLB)
2106 /*
2107 * hpzx1_swiotlb needs to have a fairly small swiotlb bounce
2108 * buffer setup to support devices with smaller DMA masks than
2109 * sba_iommu can handle.
2110 */
2111 if (ia64_platform_is("hpzx1_swiotlb")) {
2112 extern void hwsw_init(void);
2113
2114 hwsw_init();
2115 }
2116 #endif
2117
2118 #ifdef CONFIG_PCI
2119 {
2120 struct pci_bus *b = NULL;
2121 while ((b = pci_find_next_bus(b)) != NULL)
2122 sba_connect_bus(b);
2123 }
2124 #endif
2125
2126 #ifdef CONFIG_PROC_FS
2127 ioc_proc_init();
2128 #endif
2129 return 0;
2130 }
2131
2132 subsys_initcall(sba_init); /* must be initialized after ACPI etc., but before any drivers... */
2133
2134 static int __init
nosbagart(char * str)2135 nosbagart(char *str)
2136 {
2137 reserve_sba_gart = 0;
2138 return 1;
2139 }
2140
2141 int
sba_dma_supported(struct device * dev,u64 mask)2142 sba_dma_supported (struct device *dev, u64 mask)
2143 {
2144 /* make sure it's at least 32bit capable */
2145 return ((mask & 0xFFFFFFFFUL) == 0xFFFFFFFFUL);
2146 }
2147
2148 int
sba_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)2149 sba_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
2150 {
2151 return 0;
2152 }
2153
2154 __setup("nosbagart", nosbagart);
2155
2156 static int __init
sba_page_override(char * str)2157 sba_page_override(char *str)
2158 {
2159 unsigned long page_size;
2160
2161 page_size = memparse(str, &str);
2162 switch (page_size) {
2163 case 4096:
2164 case 8192:
2165 case 16384:
2166 case 65536:
2167 iovp_shift = ffs(page_size) - 1;
2168 break;
2169 default:
2170 printk("%s: unknown/unsupported iommu page size %ld\n",
2171 __func__, page_size);
2172 }
2173
2174 return 1;
2175 }
2176
2177 __setup("sbapagesize=",sba_page_override);
2178
2179 EXPORT_SYMBOL(sba_dma_mapping_error);
2180 EXPORT_SYMBOL(sba_dma_supported);
2181 EXPORT_SYMBOL(sba_alloc_coherent);
2182 EXPORT_SYMBOL(sba_free_coherent);
2183