1 /*
2 * Firmware replacement code.
3 *
4 * Work around broken BIOSes that don't set an aperture, only set the
5 * aperture in the AGP bridge, or set too small aperture.
6 *
7 * If all fails map the aperture over some low memory. This is cheaper than
8 * doing bounce buffering. The memory is lost. This is done at early boot
9 * because only the bootmem allocator can allocate 32+MB.
10 *
11 * Copyright 2002 Andi Kleen, SuSE Labs.
12 */
13 #include <linux/kernel.h>
14 #include <linux/types.h>
15 #include <linux/init.h>
16 #include <linux/bootmem.h>
17 #include <linux/mmzone.h>
18 #include <linux/pci_ids.h>
19 #include <linux/pci.h>
20 #include <linux/bitops.h>
21 #include <linux/ioport.h>
22 #include <linux/suspend.h>
23 #include <asm/e820.h>
24 #include <asm/io.h>
25 #include <asm/iommu.h>
26 #include <asm/gart.h>
27 #include <asm/pci-direct.h>
28 #include <asm/dma.h>
29 #include <asm/k8.h>
30
31 int gart_iommu_aperture;
32 int gart_iommu_aperture_disabled __initdata;
33 int gart_iommu_aperture_allowed __initdata;
34
35 int fallback_aper_order __initdata = 1; /* 64MB */
36 int fallback_aper_force __initdata;
37
38 int fix_aperture __initdata = 1;
39
40 struct bus_dev_range {
41 int bus;
42 int dev_base;
43 int dev_limit;
44 };
45
46 static struct bus_dev_range bus_dev_ranges[] __initdata = {
47 { 0x00, 0x18, 0x20},
48 { 0xff, 0x00, 0x20},
49 { 0xfe, 0x00, 0x20}
50 };
51
52 static struct resource gart_resource = {
53 .name = "GART",
54 .flags = IORESOURCE_MEM,
55 };
56
insert_aperture_resource(u32 aper_base,u32 aper_size)57 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
58 {
59 gart_resource.start = aper_base;
60 gart_resource.end = aper_base + aper_size - 1;
61 insert_resource(&iomem_resource, &gart_resource);
62 }
63
64 /* This code runs before the PCI subsystem is initialized, so just
65 access the northbridge directly. */
66
allocate_aperture(void)67 static u32 __init allocate_aperture(void)
68 {
69 u32 aper_size;
70 void *p;
71
72 /* aper_size should <= 1G */
73 if (fallback_aper_order > 5)
74 fallback_aper_order = 5;
75 aper_size = (32 * 1024 * 1024) << fallback_aper_order;
76
77 /*
78 * Aperture has to be naturally aligned. This means a 2GB aperture
79 * won't have much chance of finding a place in the lower 4GB of
80 * memory. Unfortunately we cannot move it up because that would
81 * make the IOMMU useless.
82 */
83 /*
84 * using 512M as goal, in case kexec will load kernel_big
85 * that will do the on position decompress, and could overlap with
86 * that positon with gart that is used.
87 * sequende:
88 * kernel_small
89 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
90 * ==> kernel_small(gart area become e820_reserved)
91 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
92 * ==> kerne_big (uncompressed size will be big than 64M or 128M)
93 * so don't use 512M below as gart iommu, leave the space for kernel
94 * code for safe
95 */
96 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
97 if (!p || __pa(p)+aper_size > 0xffffffff) {
98 printk(KERN_ERR
99 "Cannot allocate aperture memory hole (%p,%uK)\n",
100 p, aper_size>>10);
101 if (p)
102 free_bootmem(__pa(p), aper_size);
103 return 0;
104 }
105 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
106 aper_size >> 10, __pa(p));
107 insert_aperture_resource((u32)__pa(p), aper_size);
108 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT,
109 (u32)__pa(p+aper_size) >> PAGE_SHIFT);
110
111 return (u32)__pa(p);
112 }
113
114
115 /* Find a PCI capability */
find_cap(int bus,int slot,int func,int cap)116 static u32 __init find_cap(int bus, int slot, int func, int cap)
117 {
118 int bytes;
119 u8 pos;
120
121 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
122 PCI_STATUS_CAP_LIST))
123 return 0;
124
125 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
126 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
127 u8 id;
128
129 pos &= ~3;
130 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
131 if (id == 0xff)
132 break;
133 if (id == cap)
134 return pos;
135 pos = read_pci_config_byte(bus, slot, func,
136 pos+PCI_CAP_LIST_NEXT);
137 }
138 return 0;
139 }
140
141 /* Read a standard AGPv3 bridge header */
read_agp(int bus,int slot,int func,int cap,u32 * order)142 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
143 {
144 u32 apsize;
145 u32 apsizereg;
146 int nbits;
147 u32 aper_low, aper_hi;
148 u64 aper;
149 u32 old_order;
150
151 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
152 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
153 if (apsizereg == 0xffffffff) {
154 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
155 return 0;
156 }
157
158 /* old_order could be the value from NB gart setting */
159 old_order = *order;
160
161 apsize = apsizereg & 0xfff;
162 /* Some BIOS use weird encodings not in the AGPv3 table. */
163 if (apsize & 0xff)
164 apsize |= 0xf00;
165 nbits = hweight16(apsize);
166 *order = 7 - nbits;
167 if ((int)*order < 0) /* < 32MB */
168 *order = 0;
169
170 aper_low = read_pci_config(bus, slot, func, 0x10);
171 aper_hi = read_pci_config(bus, slot, func, 0x14);
172 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);
173
174 /*
175 * On some sick chips, APSIZE is 0. It means it wants 4G
176 * so let double check that order, and lets trust AMD NB settings:
177 */
178 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
179 aper, 32 << old_order);
180 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
181 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
182 32 << *order, apsizereg);
183 *order = old_order;
184 }
185
186 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
187 aper, 32 << *order, apsizereg);
188
189 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
190 return 0;
191 return (u32)aper;
192 }
193
194 /*
195 * Look for an AGP bridge. Windows only expects the aperture in the
196 * AGP bridge and some BIOS forget to initialize the Northbridge too.
197 * Work around this here.
198 *
199 * Do an PCI bus scan by hand because we're running before the PCI
200 * subsystem.
201 *
202 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan
203 * generically. It's probably overkill to always scan all slots because
204 * the AGP bridges should be always an own bus on the HT hierarchy,
205 * but do it here for future safety.
206 */
search_agp_bridge(u32 * order,int * valid_agp)207 static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
208 {
209 int bus, slot, func;
210
211 /* Poor man's PCI discovery */
212 for (bus = 0; bus < 256; bus++) {
213 for (slot = 0; slot < 32; slot++) {
214 for (func = 0; func < 8; func++) {
215 u32 class, cap;
216 u8 type;
217 class = read_pci_config(bus, slot, func,
218 PCI_CLASS_REVISION);
219 if (class == 0xffffffff)
220 break;
221
222 switch (class >> 16) {
223 case PCI_CLASS_BRIDGE_HOST:
224 case PCI_CLASS_BRIDGE_OTHER: /* needed? */
225 /* AGP bridge? */
226 cap = find_cap(bus, slot, func,
227 PCI_CAP_ID_AGP);
228 if (!cap)
229 break;
230 *valid_agp = 1;
231 return read_agp(bus, slot, func, cap,
232 order);
233 }
234
235 /* No multi-function device? */
236 type = read_pci_config_byte(bus, slot, func,
237 PCI_HEADER_TYPE);
238 if (!(type & 0x80))
239 break;
240 }
241 }
242 }
243 printk(KERN_INFO "No AGP bridge found\n");
244
245 return 0;
246 }
247
248 static int gart_fix_e820 __initdata = 1;
249
parse_gart_mem(char * p)250 static int __init parse_gart_mem(char *p)
251 {
252 if (!p)
253 return -EINVAL;
254
255 if (!strncmp(p, "off", 3))
256 gart_fix_e820 = 0;
257 else if (!strncmp(p, "on", 2))
258 gart_fix_e820 = 1;
259
260 return 0;
261 }
262 early_param("gart_fix_e820", parse_gart_mem);
263
early_gart_iommu_check(void)264 void __init early_gart_iommu_check(void)
265 {
266 /*
267 * in case it is enabled before, esp for kexec/kdump,
268 * previous kernel already enable that. memset called
269 * by allocate_aperture/__alloc_bootmem_nopanic cause restart.
270 * or second kernel have different position for GART hole. and new
271 * kernel could use hole as RAM that is still used by GART set by
272 * first kernel
273 * or BIOS forget to put that in reserved.
274 * try to update e820 to make that region as reserved.
275 */
276 int i, fix, slot;
277 u32 ctl;
278 u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
279 u64 aper_base = 0, last_aper_base = 0;
280 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0;
281
282 if (!early_pci_allowed())
283 return;
284
285 /* This is mostly duplicate of iommu_hole_init */
286 fix = 0;
287 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
288 int bus;
289 int dev_base, dev_limit;
290
291 bus = bus_dev_ranges[i].bus;
292 dev_base = bus_dev_ranges[i].dev_base;
293 dev_limit = bus_dev_ranges[i].dev_limit;
294
295 for (slot = dev_base; slot < dev_limit; slot++) {
296 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
297 continue;
298
299 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
300 aper_enabled = ctl & AMD64_GARTEN;
301 aper_order = (ctl >> 1) & 7;
302 aper_size = (32 * 1024 * 1024) << aper_order;
303 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
304 aper_base <<= 25;
305
306 if (last_valid) {
307 if ((aper_order != last_aper_order) ||
308 (aper_base != last_aper_base) ||
309 (aper_enabled != last_aper_enabled)) {
310 fix = 1;
311 break;
312 }
313 }
314
315 last_aper_order = aper_order;
316 last_aper_base = aper_base;
317 last_aper_enabled = aper_enabled;
318 last_valid = 1;
319 }
320 }
321
322 if (!fix && !aper_enabled)
323 return;
324
325 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL)
326 fix = 1;
327
328 if (gart_fix_e820 && !fix && aper_enabled) {
329 if (e820_any_mapped(aper_base, aper_base + aper_size,
330 E820_RAM)) {
331 /* reserve it, so we can reuse it in second kernel */
332 printk(KERN_INFO "update e820 for GART\n");
333 e820_add_region(aper_base, aper_size, E820_RESERVED);
334 update_e820();
335 }
336 }
337
338 if (!fix)
339 return;
340
341 /* different nodes have different setting, disable them all at first*/
342 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
343 int bus;
344 int dev_base, dev_limit;
345
346 bus = bus_dev_ranges[i].bus;
347 dev_base = bus_dev_ranges[i].dev_base;
348 dev_limit = bus_dev_ranges[i].dev_limit;
349
350 for (slot = dev_base; slot < dev_limit; slot++) {
351 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
352 continue;
353
354 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
355 ctl &= ~AMD64_GARTEN;
356 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
357 }
358 }
359
360 }
361
362 static int __initdata printed_gart_size_msg;
363
gart_iommu_hole_init(void)364 void __init gart_iommu_hole_init(void)
365 {
366 u32 agp_aper_base = 0, agp_aper_order = 0;
367 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
368 u64 aper_base, last_aper_base = 0;
369 int fix, slot, valid_agp = 0;
370 int i, node;
371
372 if (gart_iommu_aperture_disabled || !fix_aperture ||
373 !early_pci_allowed())
374 return;
375
376 printk(KERN_INFO "Checking aperture...\n");
377
378 if (!fallback_aper_force)
379 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);
380
381 fix = 0;
382 node = 0;
383 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
384 int bus;
385 int dev_base, dev_limit;
386
387 bus = bus_dev_ranges[i].bus;
388 dev_base = bus_dev_ranges[i].dev_base;
389 dev_limit = bus_dev_ranges[i].dev_limit;
390
391 for (slot = dev_base; slot < dev_limit; slot++) {
392 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
393 continue;
394
395 iommu_detected = 1;
396 gart_iommu_aperture = 1;
397
398 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7;
399 aper_size = (32 * 1024 * 1024) << aper_order;
400 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
401 aper_base <<= 25;
402
403 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
404 node, aper_base, aper_size >> 20);
405 node++;
406
407 if (!aperture_valid(aper_base, aper_size, 64<<20)) {
408 if (valid_agp && agp_aper_base &&
409 agp_aper_base == aper_base &&
410 agp_aper_order == aper_order) {
411 /* the same between two setting from NB and agp */
412 if (!no_iommu &&
413 max_pfn > MAX_DMA32_PFN &&
414 !printed_gart_size_msg) {
415 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
416 printk(KERN_ERR "please increase GART size in your BIOS setup\n");
417 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
418 printed_gart_size_msg = 1;
419 }
420 } else {
421 fix = 1;
422 goto out;
423 }
424 }
425
426 if ((last_aper_order && aper_order != last_aper_order) ||
427 (last_aper_base && aper_base != last_aper_base)) {
428 fix = 1;
429 goto out;
430 }
431 last_aper_order = aper_order;
432 last_aper_base = aper_base;
433 }
434 }
435
436 out:
437 if (!fix && !fallback_aper_force) {
438 if (last_aper_base) {
439 unsigned long n = (32 * 1024 * 1024) << last_aper_order;
440
441 insert_aperture_resource((u32)last_aper_base, n);
442 }
443 return;
444 }
445
446 if (!fallback_aper_force) {
447 aper_alloc = agp_aper_base;
448 aper_order = agp_aper_order;
449 }
450
451 if (aper_alloc) {
452 /* Got the aperture from the AGP bridge */
453 } else if (swiotlb && !valid_agp) {
454 /* Do nothing */
455 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
456 force_iommu ||
457 valid_agp ||
458 fallback_aper_force) {
459 printk(KERN_INFO
460 "Your BIOS doesn't leave a aperture memory hole\n");
461 printk(KERN_INFO
462 "Please enable the IOMMU option in the BIOS setup\n");
463 printk(KERN_INFO
464 "This costs you %d MB of RAM\n",
465 32 << fallback_aper_order);
466
467 aper_order = fallback_aper_order;
468 aper_alloc = allocate_aperture();
469 if (!aper_alloc) {
470 /*
471 * Could disable AGP and IOMMU here, but it's
472 * probably not worth it. But the later users
473 * cannot deal with bad apertures and turning
474 * on the aperture over memory causes very
475 * strange problems, so it's better to panic
476 * early.
477 */
478 panic("Not enough memory for aperture");
479 }
480 } else {
481 return;
482 }
483
484 /* Fix up the north bridges */
485 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
486 int bus;
487 int dev_base, dev_limit;
488
489 bus = bus_dev_ranges[i].bus;
490 dev_base = bus_dev_ranges[i].dev_base;
491 dev_limit = bus_dev_ranges[i].dev_limit;
492 for (slot = dev_base; slot < dev_limit; slot++) {
493 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
494 continue;
495
496 /* Don't enable translation yet. That is done later.
497 Assume this BIOS didn't initialise the GART so
498 just overwrite all previous bits */
499 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
500 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
501 }
502 }
503
504 set_up_gart_resume(aper_order, aper_alloc);
505 }
506