1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #define pr_fmt(fmt) "AMD-Vi: " fmt
9 #define dev_fmt(fmt) pr_fmt(fmt)
10
11 #include <linux/pci.h>
12 #include <linux/acpi.h>
13 #include <linux/list.h>
14 #include <linux/bitmap.h>
15 #include <linux/slab.h>
16 #include <linux/syscore_ops.h>
17 #include <linux/interrupt.h>
18 #include <linux/msi.h>
19 #include <linux/irq.h>
20 #include <linux/amd-iommu.h>
21 #include <linux/export.h>
22 #include <linux/kmemleak.h>
23 #include <linux/cc_platform.h>
24 #include <linux/iopoll.h>
25 #include <asm/pci-direct.h>
26 #include <asm/iommu.h>
27 #include <asm/apic.h>
28 #include <asm/gart.h>
29 #include <asm/x86_init.h>
30 #include <asm/io_apic.h>
31 #include <asm/irq_remapping.h>
32 #include <asm/set_memory.h>
33
34 #include <linux/crash_dump.h>
35
36 #include "amd_iommu.h"
37 #include "../irq_remapping.h"
38
39 /*
40 * definitions for the ACPI scanning code
41 */
42 #define IVRS_HEADER_LENGTH 48
43
44 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40
45 #define ACPI_IVMD_TYPE_ALL 0x20
46 #define ACPI_IVMD_TYPE 0x21
47 #define ACPI_IVMD_TYPE_RANGE 0x22
48
49 #define IVHD_DEV_ALL 0x01
50 #define IVHD_DEV_SELECT 0x02
51 #define IVHD_DEV_SELECT_RANGE_START 0x03
52 #define IVHD_DEV_RANGE_END 0x04
53 #define IVHD_DEV_ALIAS 0x42
54 #define IVHD_DEV_ALIAS_RANGE 0x43
55 #define IVHD_DEV_EXT_SELECT 0x46
56 #define IVHD_DEV_EXT_SELECT_RANGE 0x47
57 #define IVHD_DEV_SPECIAL 0x48
58 #define IVHD_DEV_ACPI_HID 0xf0
59
60 #define UID_NOT_PRESENT 0
61 #define UID_IS_INTEGER 1
62 #define UID_IS_CHARACTER 2
63
64 #define IVHD_SPECIAL_IOAPIC 1
65 #define IVHD_SPECIAL_HPET 2
66
67 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01
68 #define IVHD_FLAG_PASSPW_EN_MASK 0x02
69 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
70 #define IVHD_FLAG_ISOC_EN_MASK 0x08
71
72 #define IVMD_FLAG_EXCL_RANGE 0x08
73 #define IVMD_FLAG_IW 0x04
74 #define IVMD_FLAG_IR 0x02
75 #define IVMD_FLAG_UNITY_MAP 0x01
76
77 #define ACPI_DEVFLAG_INITPASS 0x01
78 #define ACPI_DEVFLAG_EXTINT 0x02
79 #define ACPI_DEVFLAG_NMI 0x04
80 #define ACPI_DEVFLAG_SYSMGT1 0x10
81 #define ACPI_DEVFLAG_SYSMGT2 0x20
82 #define ACPI_DEVFLAG_LINT0 0x40
83 #define ACPI_DEVFLAG_LINT1 0x80
84 #define ACPI_DEVFLAG_ATSDIS 0x10000000
85
86 #define LOOP_TIMEOUT 2000000
87
88 #define IVRS_GET_SBDF_ID(seg, bus, dev, fd) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
89 | ((dev & 0x1f) << 3) | (fn & 0x7))
90
91 /*
92 * ACPI table definitions
93 *
94 * These data structures are laid over the table to parse the important values
95 * out of it.
96 */
97
98 /*
99 * structure describing one IOMMU in the ACPI table. Typically followed by one
100 * or more ivhd_entrys.
101 */
102 struct ivhd_header {
103 u8 type;
104 u8 flags;
105 u16 length;
106 u16 devid;
107 u16 cap_ptr;
108 u64 mmio_phys;
109 u16 pci_seg;
110 u16 info;
111 u32 efr_attr;
112
113 /* Following only valid on IVHD type 11h and 40h */
114 u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */
115 u64 efr_reg2;
116 } __attribute__((packed));
117
118 /*
119 * A device entry describing which devices a specific IOMMU translates and
120 * which requestor ids they use.
121 */
122 struct ivhd_entry {
123 u8 type;
124 u16 devid;
125 u8 flags;
126 struct_group(ext_hid,
127 u32 ext;
128 u32 hidh;
129 );
130 u64 cid;
131 u8 uidf;
132 u8 uidl;
133 u8 uid;
134 } __attribute__((packed));
135
136 /*
137 * An AMD IOMMU memory definition structure. It defines things like exclusion
138 * ranges for devices and regions that should be unity mapped.
139 */
140 struct ivmd_header {
141 u8 type;
142 u8 flags;
143 u16 length;
144 u16 devid;
145 u16 aux;
146 u16 pci_seg;
147 u8 resv[6];
148 u64 range_start;
149 u64 range_length;
150 } __attribute__((packed));
151
152 bool amd_iommu_dump;
153 bool amd_iommu_irq_remap __read_mostly;
154
155 enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1;
156
157 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
158 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
159
160 static bool amd_iommu_detected;
161 static bool amd_iommu_disabled __initdata;
162 static bool amd_iommu_force_enable __initdata;
163 static bool amd_iommu_irtcachedis;
164 static int amd_iommu_target_ivhd_type;
165
166 /* Global EFR and EFR2 registers */
167 u64 amd_iommu_efr;
168 u64 amd_iommu_efr2;
169
170 /* SNP is enabled on the system? */
171 bool amd_iommu_snp_en;
172 EXPORT_SYMBOL(amd_iommu_snp_en);
173
174 LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */
175 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
176 system */
177
178 /* Array to assign indices to IOMMUs*/
179 struct amd_iommu *amd_iommus[MAX_IOMMUS];
180
181 /* Number of IOMMUs present in the system */
182 static int amd_iommus_present;
183
184 /* IOMMUs have a non-present cache? */
185 bool amd_iommu_np_cache __read_mostly;
186 bool amd_iommu_iotlb_sup __read_mostly = true;
187
188 u32 amd_iommu_max_pasid __read_mostly = ~0;
189
190 bool amd_iommu_v2_present __read_mostly;
191 static bool amd_iommu_pc_present __read_mostly;
192 bool amdr_ivrs_remap_support __read_mostly;
193
194 bool amd_iommu_force_isolation __read_mostly;
195
196 /*
197 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
198 * to know which ones are already in use.
199 */
200 unsigned long *amd_iommu_pd_alloc_bitmap;
201
202 enum iommu_init_state {
203 IOMMU_START_STATE,
204 IOMMU_IVRS_DETECTED,
205 IOMMU_ACPI_FINISHED,
206 IOMMU_ENABLED,
207 IOMMU_PCI_INIT,
208 IOMMU_INTERRUPTS_EN,
209 IOMMU_INITIALIZED,
210 IOMMU_NOT_FOUND,
211 IOMMU_INIT_ERROR,
212 IOMMU_CMDLINE_DISABLED,
213 };
214
215 /* Early ioapic and hpet maps from kernel command line */
216 #define EARLY_MAP_SIZE 4
217 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
218 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
219 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
220
221 static int __initdata early_ioapic_map_size;
222 static int __initdata early_hpet_map_size;
223 static int __initdata early_acpihid_map_size;
224
225 static bool __initdata cmdline_maps;
226
227 static enum iommu_init_state init_state = IOMMU_START_STATE;
228
229 static int amd_iommu_enable_interrupts(void);
230 static int __init iommu_go_to_state(enum iommu_init_state state);
231 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
232
233 static bool amd_iommu_pre_enabled = true;
234
235 static u32 amd_iommu_ivinfo __initdata;
236
translation_pre_enabled(struct amd_iommu * iommu)237 bool translation_pre_enabled(struct amd_iommu *iommu)
238 {
239 return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
240 }
241
clear_translation_pre_enabled(struct amd_iommu * iommu)242 static void clear_translation_pre_enabled(struct amd_iommu *iommu)
243 {
244 iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
245 }
246
init_translation_status(struct amd_iommu * iommu)247 static void init_translation_status(struct amd_iommu *iommu)
248 {
249 u64 ctrl;
250
251 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
252 if (ctrl & (1<<CONTROL_IOMMU_EN))
253 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
254 }
255
tbl_size(int entry_size,int last_bdf)256 static inline unsigned long tbl_size(int entry_size, int last_bdf)
257 {
258 unsigned shift = PAGE_SHIFT +
259 get_order((last_bdf + 1) * entry_size);
260
261 return 1UL << shift;
262 }
263
amd_iommu_get_num_iommus(void)264 int amd_iommu_get_num_iommus(void)
265 {
266 return amd_iommus_present;
267 }
268
269 /*
270 * Iterate through all the IOMMUs to get common EFR
271 * masks among all IOMMUs and warn if found inconsistency.
272 */
get_global_efr(void)273 static void get_global_efr(void)
274 {
275 struct amd_iommu *iommu;
276
277 for_each_iommu(iommu) {
278 u64 tmp = iommu->features;
279 u64 tmp2 = iommu->features2;
280
281 if (list_is_first(&iommu->list, &amd_iommu_list)) {
282 amd_iommu_efr = tmp;
283 amd_iommu_efr2 = tmp2;
284 continue;
285 }
286
287 if (amd_iommu_efr == tmp &&
288 amd_iommu_efr2 == tmp2)
289 continue;
290
291 pr_err(FW_BUG
292 "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n",
293 tmp, tmp2, amd_iommu_efr, amd_iommu_efr2,
294 iommu->index, iommu->pci_seg->id,
295 PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),
296 PCI_FUNC(iommu->devid));
297
298 amd_iommu_efr &= tmp;
299 amd_iommu_efr2 &= tmp2;
300 }
301
302 pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
303 }
304
check_feature_on_all_iommus(u64 mask)305 static bool check_feature_on_all_iommus(u64 mask)
306 {
307 return !!(amd_iommu_efr & mask);
308 }
309
310 /*
311 * For IVHD type 0x11/0x40, EFR is also available via IVHD.
312 * Default to IVHD EFR since it is available sooner
313 * (i.e. before PCI init).
314 */
early_iommu_features_init(struct amd_iommu * iommu,struct ivhd_header * h)315 static void __init early_iommu_features_init(struct amd_iommu *iommu,
316 struct ivhd_header *h)
317 {
318 if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) {
319 iommu->features = h->efr_reg;
320 iommu->features2 = h->efr_reg2;
321 }
322 if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP)
323 amdr_ivrs_remap_support = true;
324 }
325
326 /* Access to l1 and l2 indexed register spaces */
327
iommu_read_l1(struct amd_iommu * iommu,u16 l1,u8 address)328 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
329 {
330 u32 val;
331
332 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
333 pci_read_config_dword(iommu->dev, 0xfc, &val);
334 return val;
335 }
336
iommu_write_l1(struct amd_iommu * iommu,u16 l1,u8 address,u32 val)337 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
338 {
339 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
340 pci_write_config_dword(iommu->dev, 0xfc, val);
341 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
342 }
343
iommu_read_l2(struct amd_iommu * iommu,u8 address)344 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
345 {
346 u32 val;
347
348 pci_write_config_dword(iommu->dev, 0xf0, address);
349 pci_read_config_dword(iommu->dev, 0xf4, &val);
350 return val;
351 }
352
iommu_write_l2(struct amd_iommu * iommu,u8 address,u32 val)353 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
354 {
355 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
356 pci_write_config_dword(iommu->dev, 0xf4, val);
357 }
358
359 /****************************************************************************
360 *
361 * AMD IOMMU MMIO register space handling functions
362 *
363 * These functions are used to program the IOMMU device registers in
364 * MMIO space required for that driver.
365 *
366 ****************************************************************************/
367
368 /*
369 * This function set the exclusion range in the IOMMU. DMA accesses to the
370 * exclusion range are passed through untranslated
371 */
iommu_set_exclusion_range(struct amd_iommu * iommu)372 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
373 {
374 u64 start = iommu->exclusion_start & PAGE_MASK;
375 u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
376 u64 entry;
377
378 if (!iommu->exclusion_start)
379 return;
380
381 entry = start | MMIO_EXCL_ENABLE_MASK;
382 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
383 &entry, sizeof(entry));
384
385 entry = limit;
386 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
387 &entry, sizeof(entry));
388 }
389
iommu_set_cwwb_range(struct amd_iommu * iommu)390 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
391 {
392 u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
393 u64 entry = start & PM_ADDR_MASK;
394
395 if (!check_feature_on_all_iommus(FEATURE_SNP))
396 return;
397
398 /* Note:
399 * Re-purpose Exclusion base/limit registers for Completion wait
400 * write-back base/limit.
401 */
402 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
403 &entry, sizeof(entry));
404
405 /* Note:
406 * Default to 4 Kbytes, which can be specified by setting base
407 * address equal to the limit address.
408 */
409 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
410 &entry, sizeof(entry));
411 }
412
413 /* Programs the physical address of the device table into the IOMMU hardware */
iommu_set_device_table(struct amd_iommu * iommu)414 static void iommu_set_device_table(struct amd_iommu *iommu)
415 {
416 u64 entry;
417 u32 dev_table_size = iommu->pci_seg->dev_table_size;
418 void *dev_table = (void *)get_dev_table(iommu);
419
420 BUG_ON(iommu->mmio_base == NULL);
421
422 entry = iommu_virt_to_phys(dev_table);
423 entry |= (dev_table_size >> 12) - 1;
424 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
425 &entry, sizeof(entry));
426 }
427
428 /* Generic functions to enable/disable certain features of the IOMMU. */
iommu_feature_enable(struct amd_iommu * iommu,u8 bit)429 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
430 {
431 u64 ctrl;
432
433 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
434 ctrl |= (1ULL << bit);
435 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
436 }
437
iommu_feature_disable(struct amd_iommu * iommu,u8 bit)438 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
439 {
440 u64 ctrl;
441
442 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
443 ctrl &= ~(1ULL << bit);
444 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
445 }
446
iommu_set_inv_tlb_timeout(struct amd_iommu * iommu,int timeout)447 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
448 {
449 u64 ctrl;
450
451 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
452 ctrl &= ~CTRL_INV_TO_MASK;
453 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
454 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
455 }
456
457 /* Function to enable the hardware */
iommu_enable(struct amd_iommu * iommu)458 static void iommu_enable(struct amd_iommu *iommu)
459 {
460 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
461 }
462
iommu_disable(struct amd_iommu * iommu)463 static void iommu_disable(struct amd_iommu *iommu)
464 {
465 if (!iommu->mmio_base)
466 return;
467
468 /* Disable command buffer */
469 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
470
471 /* Disable event logging and event interrupts */
472 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
473 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
474
475 /* Disable IOMMU GA_LOG */
476 iommu_feature_disable(iommu, CONTROL_GALOG_EN);
477 iommu_feature_disable(iommu, CONTROL_GAINT_EN);
478
479 /* Disable IOMMU hardware itself */
480 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
481
482 /* Clear IRTE cache disabling bit */
483 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
484 }
485
486 /*
487 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
488 * the system has one.
489 */
iommu_map_mmio_space(u64 address,u64 end)490 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
491 {
492 if (!request_mem_region(address, end, "amd_iommu")) {
493 pr_err("Can not reserve memory region %llx-%llx for mmio\n",
494 address, end);
495 pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
496 return NULL;
497 }
498
499 return (u8 __iomem *)ioremap(address, end);
500 }
501
iommu_unmap_mmio_space(struct amd_iommu * iommu)502 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
503 {
504 if (iommu->mmio_base)
505 iounmap(iommu->mmio_base);
506 release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
507 }
508
get_ivhd_header_size(struct ivhd_header * h)509 static inline u32 get_ivhd_header_size(struct ivhd_header *h)
510 {
511 u32 size = 0;
512
513 switch (h->type) {
514 case 0x10:
515 size = 24;
516 break;
517 case 0x11:
518 case 0x40:
519 size = 40;
520 break;
521 }
522 return size;
523 }
524
525 /****************************************************************************
526 *
527 * The functions below belong to the first pass of AMD IOMMU ACPI table
528 * parsing. In this pass we try to find out the highest device id this
529 * code has to handle. Upon this information the size of the shared data
530 * structures is determined later.
531 *
532 ****************************************************************************/
533
534 /*
535 * This function calculates the length of a given IVHD entry
536 */
ivhd_entry_length(u8 * ivhd)537 static inline int ivhd_entry_length(u8 *ivhd)
538 {
539 u32 type = ((struct ivhd_entry *)ivhd)->type;
540
541 if (type < 0x80) {
542 return 0x04 << (*ivhd >> 6);
543 } else if (type == IVHD_DEV_ACPI_HID) {
544 /* For ACPI_HID, offset 21 is uid len */
545 return *((u8 *)ivhd + 21) + 22;
546 }
547 return 0;
548 }
549
550 /*
551 * After reading the highest device id from the IOMMU PCI capability header
552 * this function looks if there is a higher device id defined in the ACPI table
553 */
find_last_devid_from_ivhd(struct ivhd_header * h)554 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
555 {
556 u8 *p = (void *)h, *end = (void *)h;
557 struct ivhd_entry *dev;
558 int last_devid = -EINVAL;
559
560 u32 ivhd_size = get_ivhd_header_size(h);
561
562 if (!ivhd_size) {
563 pr_err("Unsupported IVHD type %#x\n", h->type);
564 return -EINVAL;
565 }
566
567 p += ivhd_size;
568 end += h->length;
569
570 while (p < end) {
571 dev = (struct ivhd_entry *)p;
572 switch (dev->type) {
573 case IVHD_DEV_ALL:
574 /* Use maximum BDF value for DEV_ALL */
575 return 0xffff;
576 case IVHD_DEV_SELECT:
577 case IVHD_DEV_RANGE_END:
578 case IVHD_DEV_ALIAS:
579 case IVHD_DEV_EXT_SELECT:
580 /* all the above subfield types refer to device ids */
581 if (dev->devid > last_devid)
582 last_devid = dev->devid;
583 break;
584 default:
585 break;
586 }
587 p += ivhd_entry_length(p);
588 }
589
590 WARN_ON(p != end);
591
592 return last_devid;
593 }
594
check_ivrs_checksum(struct acpi_table_header * table)595 static int __init check_ivrs_checksum(struct acpi_table_header *table)
596 {
597 int i;
598 u8 checksum = 0, *p = (u8 *)table;
599
600 for (i = 0; i < table->length; ++i)
601 checksum += p[i];
602 if (checksum != 0) {
603 /* ACPI table corrupt */
604 pr_err(FW_BUG "IVRS invalid checksum\n");
605 return -ENODEV;
606 }
607
608 return 0;
609 }
610
611 /*
612 * Iterate over all IVHD entries in the ACPI table and find the highest device
613 * id which we need to handle. This is the first of three functions which parse
614 * the ACPI table. So we check the checksum here.
615 */
find_last_devid_acpi(struct acpi_table_header * table,u16 pci_seg)616 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
617 {
618 u8 *p = (u8 *)table, *end = (u8 *)table;
619 struct ivhd_header *h;
620 int last_devid, last_bdf = 0;
621
622 p += IVRS_HEADER_LENGTH;
623
624 end += table->length;
625 while (p < end) {
626 h = (struct ivhd_header *)p;
627 if (h->pci_seg == pci_seg &&
628 h->type == amd_iommu_target_ivhd_type) {
629 last_devid = find_last_devid_from_ivhd(h);
630
631 if (last_devid < 0)
632 return -EINVAL;
633 if (last_devid > last_bdf)
634 last_bdf = last_devid;
635 }
636 p += h->length;
637 }
638 WARN_ON(p != end);
639
640 return last_bdf;
641 }
642
643 /****************************************************************************
644 *
645 * The following functions belong to the code path which parses the ACPI table
646 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
647 * data structures, initialize the per PCI segment device/alias/rlookup table
648 * and also basically initialize the hardware.
649 *
650 ****************************************************************************/
651
652 /* Allocate per PCI segment device table */
alloc_dev_table(struct amd_iommu_pci_seg * pci_seg)653 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
654 {
655 pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
656 get_order(pci_seg->dev_table_size));
657 if (!pci_seg->dev_table)
658 return -ENOMEM;
659
660 return 0;
661 }
662
free_dev_table(struct amd_iommu_pci_seg * pci_seg)663 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
664 {
665 free_pages((unsigned long)pci_seg->dev_table,
666 get_order(pci_seg->dev_table_size));
667 pci_seg->dev_table = NULL;
668 }
669
670 /* Allocate per PCI segment IOMMU rlookup table. */
alloc_rlookup_table(struct amd_iommu_pci_seg * pci_seg)671 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
672 {
673 pci_seg->rlookup_table = (void *)__get_free_pages(
674 GFP_KERNEL | __GFP_ZERO,
675 get_order(pci_seg->rlookup_table_size));
676 if (pci_seg->rlookup_table == NULL)
677 return -ENOMEM;
678
679 return 0;
680 }
681
free_rlookup_table(struct amd_iommu_pci_seg * pci_seg)682 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
683 {
684 free_pages((unsigned long)pci_seg->rlookup_table,
685 get_order(pci_seg->rlookup_table_size));
686 pci_seg->rlookup_table = NULL;
687 }
688
alloc_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)689 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
690 {
691 pci_seg->irq_lookup_table = (void *)__get_free_pages(
692 GFP_KERNEL | __GFP_ZERO,
693 get_order(pci_seg->rlookup_table_size));
694 kmemleak_alloc(pci_seg->irq_lookup_table,
695 pci_seg->rlookup_table_size, 1, GFP_KERNEL);
696 if (pci_seg->irq_lookup_table == NULL)
697 return -ENOMEM;
698
699 return 0;
700 }
701
free_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)702 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
703 {
704 kmemleak_free(pci_seg->irq_lookup_table);
705 free_pages((unsigned long)pci_seg->irq_lookup_table,
706 get_order(pci_seg->rlookup_table_size));
707 pci_seg->irq_lookup_table = NULL;
708 }
709
alloc_alias_table(struct amd_iommu_pci_seg * pci_seg)710 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
711 {
712 int i;
713
714 pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL,
715 get_order(pci_seg->alias_table_size));
716 if (!pci_seg->alias_table)
717 return -ENOMEM;
718
719 /*
720 * let all alias entries point to itself
721 */
722 for (i = 0; i <= pci_seg->last_bdf; ++i)
723 pci_seg->alias_table[i] = i;
724
725 return 0;
726 }
727
free_alias_table(struct amd_iommu_pci_seg * pci_seg)728 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
729 {
730 free_pages((unsigned long)pci_seg->alias_table,
731 get_order(pci_seg->alias_table_size));
732 pci_seg->alias_table = NULL;
733 }
734
735 /*
736 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
737 * write commands to that buffer later and the IOMMU will execute them
738 * asynchronously
739 */
alloc_command_buffer(struct amd_iommu * iommu)740 static int __init alloc_command_buffer(struct amd_iommu *iommu)
741 {
742 iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
743 get_order(CMD_BUFFER_SIZE));
744
745 return iommu->cmd_buf ? 0 : -ENOMEM;
746 }
747
748 /*
749 * This function restarts event logging in case the IOMMU experienced
750 * an event log buffer overflow.
751 */
amd_iommu_restart_event_logging(struct amd_iommu * iommu)752 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
753 {
754 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
755 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
756 }
757
758 /*
759 * This function restarts event logging in case the IOMMU experienced
760 * an GA log overflow.
761 */
amd_iommu_restart_ga_log(struct amd_iommu * iommu)762 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
763 {
764 u32 status;
765
766 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
767 if (status & MMIO_STATUS_GALOG_RUN_MASK)
768 return;
769
770 pr_info_ratelimited("IOMMU GA Log restarting\n");
771
772 iommu_feature_disable(iommu, CONTROL_GALOG_EN);
773 iommu_feature_disable(iommu, CONTROL_GAINT_EN);
774
775 writel(MMIO_STATUS_GALOG_OVERFLOW_MASK,
776 iommu->mmio_base + MMIO_STATUS_OFFSET);
777
778 iommu_feature_enable(iommu, CONTROL_GAINT_EN);
779 iommu_feature_enable(iommu, CONTROL_GALOG_EN);
780 }
781
782 /*
783 * This function resets the command buffer if the IOMMU stopped fetching
784 * commands from it.
785 */
amd_iommu_reset_cmd_buffer(struct amd_iommu * iommu)786 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
787 {
788 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
789
790 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
791 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
792 iommu->cmd_buf_head = 0;
793 iommu->cmd_buf_tail = 0;
794
795 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
796 }
797
798 /*
799 * This function writes the command buffer address to the hardware and
800 * enables it.
801 */
iommu_enable_command_buffer(struct amd_iommu * iommu)802 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
803 {
804 u64 entry;
805
806 BUG_ON(iommu->cmd_buf == NULL);
807
808 entry = iommu_virt_to_phys(iommu->cmd_buf);
809 entry |= MMIO_CMD_SIZE_512;
810
811 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
812 &entry, sizeof(entry));
813
814 amd_iommu_reset_cmd_buffer(iommu);
815 }
816
817 /*
818 * This function disables the command buffer
819 */
iommu_disable_command_buffer(struct amd_iommu * iommu)820 static void iommu_disable_command_buffer(struct amd_iommu *iommu)
821 {
822 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
823 }
824
free_command_buffer(struct amd_iommu * iommu)825 static void __init free_command_buffer(struct amd_iommu *iommu)
826 {
827 free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
828 }
829
iommu_alloc_4k_pages(struct amd_iommu * iommu,gfp_t gfp,size_t size)830 static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,
831 gfp_t gfp, size_t size)
832 {
833 int order = get_order(size);
834 void *buf = (void *)__get_free_pages(gfp, order);
835
836 if (buf &&
837 check_feature_on_all_iommus(FEATURE_SNP) &&
838 set_memory_4k((unsigned long)buf, (1 << order))) {
839 free_pages((unsigned long)buf, order);
840 buf = NULL;
841 }
842
843 return buf;
844 }
845
846 /* allocates the memory where the IOMMU will log its events to */
alloc_event_buffer(struct amd_iommu * iommu)847 static int __init alloc_event_buffer(struct amd_iommu *iommu)
848 {
849 iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
850 EVT_BUFFER_SIZE);
851
852 return iommu->evt_buf ? 0 : -ENOMEM;
853 }
854
iommu_enable_event_buffer(struct amd_iommu * iommu)855 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
856 {
857 u64 entry;
858
859 BUG_ON(iommu->evt_buf == NULL);
860
861 entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
862
863 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
864 &entry, sizeof(entry));
865
866 /* set head and tail to zero manually */
867 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
868 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
869
870 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
871 }
872
873 /*
874 * This function disables the event log buffer
875 */
iommu_disable_event_buffer(struct amd_iommu * iommu)876 static void iommu_disable_event_buffer(struct amd_iommu *iommu)
877 {
878 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
879 }
880
free_event_buffer(struct amd_iommu * iommu)881 static void __init free_event_buffer(struct amd_iommu *iommu)
882 {
883 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
884 }
885
886 /* allocates the memory where the IOMMU will log its events to */
alloc_ppr_log(struct amd_iommu * iommu)887 static int __init alloc_ppr_log(struct amd_iommu *iommu)
888 {
889 iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
890 PPR_LOG_SIZE);
891
892 return iommu->ppr_log ? 0 : -ENOMEM;
893 }
894
iommu_enable_ppr_log(struct amd_iommu * iommu)895 static void iommu_enable_ppr_log(struct amd_iommu *iommu)
896 {
897 u64 entry;
898
899 if (iommu->ppr_log == NULL)
900 return;
901
902 entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
903
904 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
905 &entry, sizeof(entry));
906
907 /* set head and tail to zero manually */
908 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
909 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
910
911 iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
912 iommu_feature_enable(iommu, CONTROL_PPR_EN);
913 }
914
free_ppr_log(struct amd_iommu * iommu)915 static void __init free_ppr_log(struct amd_iommu *iommu)
916 {
917 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
918 }
919
free_ga_log(struct amd_iommu * iommu)920 static void free_ga_log(struct amd_iommu *iommu)
921 {
922 #ifdef CONFIG_IRQ_REMAP
923 free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE));
924 free_pages((unsigned long)iommu->ga_log_tail, get_order(8));
925 #endif
926 }
927
928 #ifdef CONFIG_IRQ_REMAP
iommu_ga_log_enable(struct amd_iommu * iommu)929 static int iommu_ga_log_enable(struct amd_iommu *iommu)
930 {
931 u32 status, i;
932 u64 entry;
933
934 if (!iommu->ga_log)
935 return -EINVAL;
936
937 entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
938 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
939 &entry, sizeof(entry));
940 entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
941 (BIT_ULL(52)-1)) & ~7ULL;
942 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
943 &entry, sizeof(entry));
944 writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
945 writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
946
947
948 iommu_feature_enable(iommu, CONTROL_GAINT_EN);
949 iommu_feature_enable(iommu, CONTROL_GALOG_EN);
950
951 for (i = 0; i < LOOP_TIMEOUT; ++i) {
952 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
953 if (status & (MMIO_STATUS_GALOG_RUN_MASK))
954 break;
955 udelay(10);
956 }
957
958 if (WARN_ON(i >= LOOP_TIMEOUT))
959 return -EINVAL;
960
961 return 0;
962 }
963
iommu_init_ga_log(struct amd_iommu * iommu)964 static int iommu_init_ga_log(struct amd_iommu *iommu)
965 {
966 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
967 return 0;
968
969 iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
970 get_order(GA_LOG_SIZE));
971 if (!iommu->ga_log)
972 goto err_out;
973
974 iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
975 get_order(8));
976 if (!iommu->ga_log_tail)
977 goto err_out;
978
979 return 0;
980 err_out:
981 free_ga_log(iommu);
982 return -EINVAL;
983 }
984 #endif /* CONFIG_IRQ_REMAP */
985
alloc_cwwb_sem(struct amd_iommu * iommu)986 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
987 {
988 iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1);
989
990 return iommu->cmd_sem ? 0 : -ENOMEM;
991 }
992
free_cwwb_sem(struct amd_iommu * iommu)993 static void __init free_cwwb_sem(struct amd_iommu *iommu)
994 {
995 if (iommu->cmd_sem)
996 free_page((unsigned long)iommu->cmd_sem);
997 }
998
iommu_enable_xt(struct amd_iommu * iommu)999 static void iommu_enable_xt(struct amd_iommu *iommu)
1000 {
1001 #ifdef CONFIG_IRQ_REMAP
1002 /*
1003 * XT mode (32-bit APIC destination ID) requires
1004 * GA mode (128-bit IRTE support) as a prerequisite.
1005 */
1006 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
1007 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
1008 iommu_feature_enable(iommu, CONTROL_XT_EN);
1009 #endif /* CONFIG_IRQ_REMAP */
1010 }
1011
iommu_enable_gt(struct amd_iommu * iommu)1012 static void iommu_enable_gt(struct amd_iommu *iommu)
1013 {
1014 if (!iommu_feature(iommu, FEATURE_GT))
1015 return;
1016
1017 iommu_feature_enable(iommu, CONTROL_GT_EN);
1018 }
1019
1020 /* sets a specific bit in the device table entry. */
__set_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)1021 static void __set_dev_entry_bit(struct dev_table_entry *dev_table,
1022 u16 devid, u8 bit)
1023 {
1024 int i = (bit >> 6) & 0x03;
1025 int _bit = bit & 0x3f;
1026
1027 dev_table[devid].data[i] |= (1UL << _bit);
1028 }
1029
set_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1030 static void set_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1031 {
1032 struct dev_table_entry *dev_table = get_dev_table(iommu);
1033
1034 return __set_dev_entry_bit(dev_table, devid, bit);
1035 }
1036
__get_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)1037 static int __get_dev_entry_bit(struct dev_table_entry *dev_table,
1038 u16 devid, u8 bit)
1039 {
1040 int i = (bit >> 6) & 0x03;
1041 int _bit = bit & 0x3f;
1042
1043 return (dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
1044 }
1045
get_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1046 static int get_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1047 {
1048 struct dev_table_entry *dev_table = get_dev_table(iommu);
1049
1050 return __get_dev_entry_bit(dev_table, devid, bit);
1051 }
1052
__copy_device_table(struct amd_iommu * iommu)1053 static bool __copy_device_table(struct amd_iommu *iommu)
1054 {
1055 u64 int_ctl, int_tab_len, entry = 0;
1056 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1057 struct dev_table_entry *old_devtb = NULL;
1058 u32 lo, hi, devid, old_devtb_size;
1059 phys_addr_t old_devtb_phys;
1060 u16 dom_id, dte_v, irq_v;
1061 gfp_t gfp_flag;
1062 u64 tmp;
1063
1064 /* Each IOMMU use separate device table with the same size */
1065 lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
1066 hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
1067 entry = (((u64) hi) << 32) + lo;
1068
1069 old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
1070 if (old_devtb_size != pci_seg->dev_table_size) {
1071 pr_err("The device table size of IOMMU:%d is not expected!\n",
1072 iommu->index);
1073 return false;
1074 }
1075
1076 /*
1077 * When SME is enabled in the first kernel, the entry includes the
1078 * memory encryption mask(sme_me_mask), we must remove the memory
1079 * encryption mask to obtain the true physical address in kdump kernel.
1080 */
1081 old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
1082
1083 if (old_devtb_phys >= 0x100000000ULL) {
1084 pr_err("The address of old device table is above 4G, not trustworthy!\n");
1085 return false;
1086 }
1087 old_devtb = (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && is_kdump_kernel())
1088 ? (__force void *)ioremap_encrypted(old_devtb_phys,
1089 pci_seg->dev_table_size)
1090 : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB);
1091
1092 if (!old_devtb)
1093 return false;
1094
1095 gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
1096 pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
1097 get_order(pci_seg->dev_table_size));
1098 if (pci_seg->old_dev_tbl_cpy == NULL) {
1099 pr_err("Failed to allocate memory for copying old device table!\n");
1100 memunmap(old_devtb);
1101 return false;
1102 }
1103
1104 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
1105 pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid];
1106 dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
1107 dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
1108
1109 if (dte_v && dom_id) {
1110 pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
1111 pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
1112 __set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
1113 /* If gcr3 table existed, mask it out */
1114 if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
1115 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1116 tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1117 pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;
1118 tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
1119 tmp |= DTE_FLAG_GV;
1120 pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;
1121 }
1122 }
1123
1124 irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
1125 int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
1126 int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;
1127 if (irq_v && (int_ctl || int_tab_len)) {
1128 if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
1129 (int_tab_len != DTE_INTTABLEN)) {
1130 pr_err("Wrong old irq remapping flag: %#x\n", devid);
1131 memunmap(old_devtb);
1132 return false;
1133 }
1134
1135 pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
1136 }
1137 }
1138 memunmap(old_devtb);
1139
1140 return true;
1141 }
1142
copy_device_table(void)1143 static bool copy_device_table(void)
1144 {
1145 struct amd_iommu *iommu;
1146 struct amd_iommu_pci_seg *pci_seg;
1147
1148 if (!amd_iommu_pre_enabled)
1149 return false;
1150
1151 pr_warn("Translation is already enabled - trying to copy translation structures\n");
1152
1153 /*
1154 * All IOMMUs within PCI segment shares common device table.
1155 * Hence copy device table only once per PCI segment.
1156 */
1157 for_each_pci_segment(pci_seg) {
1158 for_each_iommu(iommu) {
1159 if (pci_seg->id != iommu->pci_seg->id)
1160 continue;
1161 if (!__copy_device_table(iommu))
1162 return false;
1163 break;
1164 }
1165 }
1166
1167 return true;
1168 }
1169
amd_iommu_apply_erratum_63(struct amd_iommu * iommu,u16 devid)1170 void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid)
1171 {
1172 int sysmgt;
1173
1174 sysmgt = get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1) |
1175 (get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2) << 1);
1176
1177 if (sysmgt == 0x01)
1178 set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW);
1179 }
1180
1181 /*
1182 * This function takes the device specific flags read from the ACPI
1183 * table and sets up the device table entry with that information
1184 */
set_dev_entry_from_acpi(struct amd_iommu * iommu,u16 devid,u32 flags,u32 ext_flags)1185 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
1186 u16 devid, u32 flags, u32 ext_flags)
1187 {
1188 if (flags & ACPI_DEVFLAG_INITPASS)
1189 set_dev_entry_bit(iommu, devid, DEV_ENTRY_INIT_PASS);
1190 if (flags & ACPI_DEVFLAG_EXTINT)
1191 set_dev_entry_bit(iommu, devid, DEV_ENTRY_EINT_PASS);
1192 if (flags & ACPI_DEVFLAG_NMI)
1193 set_dev_entry_bit(iommu, devid, DEV_ENTRY_NMI_PASS);
1194 if (flags & ACPI_DEVFLAG_SYSMGT1)
1195 set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1);
1196 if (flags & ACPI_DEVFLAG_SYSMGT2)
1197 set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2);
1198 if (flags & ACPI_DEVFLAG_LINT0)
1199 set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT0_PASS);
1200 if (flags & ACPI_DEVFLAG_LINT1)
1201 set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT1_PASS);
1202
1203 amd_iommu_apply_erratum_63(iommu, devid);
1204
1205 amd_iommu_set_rlookup_table(iommu, devid);
1206 }
1207
add_special_device(u8 type,u8 id,u32 * devid,bool cmd_line)1208 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line)
1209 {
1210 struct devid_map *entry;
1211 struct list_head *list;
1212
1213 if (type == IVHD_SPECIAL_IOAPIC)
1214 list = &ioapic_map;
1215 else if (type == IVHD_SPECIAL_HPET)
1216 list = &hpet_map;
1217 else
1218 return -EINVAL;
1219
1220 list_for_each_entry(entry, list, list) {
1221 if (!(entry->id == id && entry->cmd_line))
1222 continue;
1223
1224 pr_info("Command-line override present for %s id %d - ignoring\n",
1225 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
1226
1227 *devid = entry->devid;
1228
1229 return 0;
1230 }
1231
1232 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1233 if (!entry)
1234 return -ENOMEM;
1235
1236 entry->id = id;
1237 entry->devid = *devid;
1238 entry->cmd_line = cmd_line;
1239
1240 list_add_tail(&entry->list, list);
1241
1242 return 0;
1243 }
1244
add_acpi_hid_device(u8 * hid,u8 * uid,u32 * devid,bool cmd_line)1245 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid,
1246 bool cmd_line)
1247 {
1248 struct acpihid_map_entry *entry;
1249 struct list_head *list = &acpihid_map;
1250
1251 list_for_each_entry(entry, list, list) {
1252 if (strcmp(entry->hid, hid) ||
1253 (*uid && *entry->uid && strcmp(entry->uid, uid)) ||
1254 !entry->cmd_line)
1255 continue;
1256
1257 pr_info("Command-line override for hid:%s uid:%s\n",
1258 hid, uid);
1259 *devid = entry->devid;
1260 return 0;
1261 }
1262
1263 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1264 if (!entry)
1265 return -ENOMEM;
1266
1267 memcpy(entry->uid, uid, strlen(uid));
1268 memcpy(entry->hid, hid, strlen(hid));
1269 entry->devid = *devid;
1270 entry->cmd_line = cmd_line;
1271 entry->root_devid = (entry->devid & (~0x7));
1272
1273 pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n",
1274 entry->cmd_line ? "cmd" : "ivrs",
1275 entry->hid, entry->uid, entry->root_devid);
1276
1277 list_add_tail(&entry->list, list);
1278 return 0;
1279 }
1280
add_early_maps(void)1281 static int __init add_early_maps(void)
1282 {
1283 int i, ret;
1284
1285 for (i = 0; i < early_ioapic_map_size; ++i) {
1286 ret = add_special_device(IVHD_SPECIAL_IOAPIC,
1287 early_ioapic_map[i].id,
1288 &early_ioapic_map[i].devid,
1289 early_ioapic_map[i].cmd_line);
1290 if (ret)
1291 return ret;
1292 }
1293
1294 for (i = 0; i < early_hpet_map_size; ++i) {
1295 ret = add_special_device(IVHD_SPECIAL_HPET,
1296 early_hpet_map[i].id,
1297 &early_hpet_map[i].devid,
1298 early_hpet_map[i].cmd_line);
1299 if (ret)
1300 return ret;
1301 }
1302
1303 for (i = 0; i < early_acpihid_map_size; ++i) {
1304 ret = add_acpi_hid_device(early_acpihid_map[i].hid,
1305 early_acpihid_map[i].uid,
1306 &early_acpihid_map[i].devid,
1307 early_acpihid_map[i].cmd_line);
1308 if (ret)
1309 return ret;
1310 }
1311
1312 return 0;
1313 }
1314
1315 /*
1316 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
1317 * initializes the hardware and our data structures with it.
1318 */
init_iommu_from_acpi(struct amd_iommu * iommu,struct ivhd_header * h)1319 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
1320 struct ivhd_header *h)
1321 {
1322 u8 *p = (u8 *)h;
1323 u8 *end = p, flags = 0;
1324 u16 devid = 0, devid_start = 0, devid_to = 0, seg_id;
1325 u32 dev_i, ext_flags = 0;
1326 bool alias = false;
1327 struct ivhd_entry *e;
1328 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1329 u32 ivhd_size;
1330 int ret;
1331
1332
1333 ret = add_early_maps();
1334 if (ret)
1335 return ret;
1336
1337 amd_iommu_apply_ivrs_quirks();
1338
1339 /*
1340 * First save the recommended feature enable bits from ACPI
1341 */
1342 iommu->acpi_flags = h->flags;
1343
1344 /*
1345 * Done. Now parse the device entries
1346 */
1347 ivhd_size = get_ivhd_header_size(h);
1348 if (!ivhd_size) {
1349 pr_err("Unsupported IVHD type %#x\n", h->type);
1350 return -EINVAL;
1351 }
1352
1353 p += ivhd_size;
1354
1355 end += h->length;
1356
1357
1358 while (p < end) {
1359 e = (struct ivhd_entry *)p;
1360 seg_id = pci_seg->id;
1361
1362 switch (e->type) {
1363 case IVHD_DEV_ALL:
1364
1365 DUMP_printk(" DEV_ALL\t\t\tflags: %02x\n", e->flags);
1366
1367 for (dev_i = 0; dev_i <= pci_seg->last_bdf; ++dev_i)
1368 set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0);
1369 break;
1370 case IVHD_DEV_SELECT:
1371
1372 DUMP_printk(" DEV_SELECT\t\t\t devid: %04x:%02x:%02x.%x "
1373 "flags: %02x\n",
1374 seg_id, PCI_BUS_NUM(e->devid),
1375 PCI_SLOT(e->devid),
1376 PCI_FUNC(e->devid),
1377 e->flags);
1378
1379 devid = e->devid;
1380 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1381 break;
1382 case IVHD_DEV_SELECT_RANGE_START:
1383
1384 DUMP_printk(" DEV_SELECT_RANGE_START\t "
1385 "devid: %04x:%02x:%02x.%x flags: %02x\n",
1386 seg_id, PCI_BUS_NUM(e->devid),
1387 PCI_SLOT(e->devid),
1388 PCI_FUNC(e->devid),
1389 e->flags);
1390
1391 devid_start = e->devid;
1392 flags = e->flags;
1393 ext_flags = 0;
1394 alias = false;
1395 break;
1396 case IVHD_DEV_ALIAS:
1397
1398 DUMP_printk(" DEV_ALIAS\t\t\t devid: %04x:%02x:%02x.%x "
1399 "flags: %02x devid_to: %02x:%02x.%x\n",
1400 seg_id, PCI_BUS_NUM(e->devid),
1401 PCI_SLOT(e->devid),
1402 PCI_FUNC(e->devid),
1403 e->flags,
1404 PCI_BUS_NUM(e->ext >> 8),
1405 PCI_SLOT(e->ext >> 8),
1406 PCI_FUNC(e->ext >> 8));
1407
1408 devid = e->devid;
1409 devid_to = e->ext >> 8;
1410 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
1411 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
1412 pci_seg->alias_table[devid] = devid_to;
1413 break;
1414 case IVHD_DEV_ALIAS_RANGE:
1415
1416 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
1417 "devid: %04x:%02x:%02x.%x flags: %02x "
1418 "devid_to: %04x:%02x:%02x.%x\n",
1419 seg_id, PCI_BUS_NUM(e->devid),
1420 PCI_SLOT(e->devid),
1421 PCI_FUNC(e->devid),
1422 e->flags,
1423 seg_id, PCI_BUS_NUM(e->ext >> 8),
1424 PCI_SLOT(e->ext >> 8),
1425 PCI_FUNC(e->ext >> 8));
1426
1427 devid_start = e->devid;
1428 flags = e->flags;
1429 devid_to = e->ext >> 8;
1430 ext_flags = 0;
1431 alias = true;
1432 break;
1433 case IVHD_DEV_EXT_SELECT:
1434
1435 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %04x:%02x:%02x.%x "
1436 "flags: %02x ext: %08x\n",
1437 seg_id, PCI_BUS_NUM(e->devid),
1438 PCI_SLOT(e->devid),
1439 PCI_FUNC(e->devid),
1440 e->flags, e->ext);
1441
1442 devid = e->devid;
1443 set_dev_entry_from_acpi(iommu, devid, e->flags,
1444 e->ext);
1445 break;
1446 case IVHD_DEV_EXT_SELECT_RANGE:
1447
1448 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
1449 "%04x:%02x:%02x.%x flags: %02x ext: %08x\n",
1450 seg_id, PCI_BUS_NUM(e->devid),
1451 PCI_SLOT(e->devid),
1452 PCI_FUNC(e->devid),
1453 e->flags, e->ext);
1454
1455 devid_start = e->devid;
1456 flags = e->flags;
1457 ext_flags = e->ext;
1458 alias = false;
1459 break;
1460 case IVHD_DEV_RANGE_END:
1461
1462 DUMP_printk(" DEV_RANGE_END\t\t devid: %04x:%02x:%02x.%x\n",
1463 seg_id, PCI_BUS_NUM(e->devid),
1464 PCI_SLOT(e->devid),
1465 PCI_FUNC(e->devid));
1466
1467 devid = e->devid;
1468 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
1469 if (alias) {
1470 pci_seg->alias_table[dev_i] = devid_to;
1471 set_dev_entry_from_acpi(iommu,
1472 devid_to, flags, ext_flags);
1473 }
1474 set_dev_entry_from_acpi(iommu, dev_i,
1475 flags, ext_flags);
1476 }
1477 break;
1478 case IVHD_DEV_SPECIAL: {
1479 u8 handle, type;
1480 const char *var;
1481 u32 devid;
1482 int ret;
1483
1484 handle = e->ext & 0xff;
1485 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8));
1486 type = (e->ext >> 24) & 0xff;
1487
1488 if (type == IVHD_SPECIAL_IOAPIC)
1489 var = "IOAPIC";
1490 else if (type == IVHD_SPECIAL_HPET)
1491 var = "HPET";
1492 else
1493 var = "UNKNOWN";
1494
1495 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x\n",
1496 var, (int)handle,
1497 seg_id, PCI_BUS_NUM(devid),
1498 PCI_SLOT(devid),
1499 PCI_FUNC(devid));
1500
1501 ret = add_special_device(type, handle, &devid, false);
1502 if (ret)
1503 return ret;
1504
1505 /*
1506 * add_special_device might update the devid in case a
1507 * command-line override is present. So call
1508 * set_dev_entry_from_acpi after add_special_device.
1509 */
1510 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1511
1512 break;
1513 }
1514 case IVHD_DEV_ACPI_HID: {
1515 u32 devid;
1516 u8 hid[ACPIHID_HID_LEN];
1517 u8 uid[ACPIHID_UID_LEN];
1518 int ret;
1519
1520 if (h->type != 0x40) {
1521 pr_err(FW_BUG "Invalid IVHD device type %#x\n",
1522 e->type);
1523 break;
1524 }
1525
1526 BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1);
1527 memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1);
1528 hid[ACPIHID_HID_LEN - 1] = '\0';
1529
1530 if (!(*hid)) {
1531 pr_err(FW_BUG "Invalid HID.\n");
1532 break;
1533 }
1534
1535 uid[0] = '\0';
1536 switch (e->uidf) {
1537 case UID_NOT_PRESENT:
1538
1539 if (e->uidl != 0)
1540 pr_warn(FW_BUG "Invalid UID length.\n");
1541
1542 break;
1543 case UID_IS_INTEGER:
1544
1545 sprintf(uid, "%d", e->uid);
1546
1547 break;
1548 case UID_IS_CHARACTER:
1549
1550 memcpy(uid, &e->uid, e->uidl);
1551 uid[e->uidl] = '\0';
1552
1553 break;
1554 default:
1555 break;
1556 }
1557
1558 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid);
1559 DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x\n",
1560 hid, uid, seg_id,
1561 PCI_BUS_NUM(devid),
1562 PCI_SLOT(devid),
1563 PCI_FUNC(devid));
1564
1565 flags = e->flags;
1566
1567 ret = add_acpi_hid_device(hid, uid, &devid, false);
1568 if (ret)
1569 return ret;
1570
1571 /*
1572 * add_special_device might update the devid in case a
1573 * command-line override is present. So call
1574 * set_dev_entry_from_acpi after add_special_device.
1575 */
1576 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1577
1578 break;
1579 }
1580 default:
1581 break;
1582 }
1583
1584 p += ivhd_entry_length(p);
1585 }
1586
1587 return 0;
1588 }
1589
1590 /* Allocate PCI segment data structure */
alloc_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1591 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1592 struct acpi_table_header *ivrs_base)
1593 {
1594 struct amd_iommu_pci_seg *pci_seg;
1595 int last_bdf;
1596
1597 /*
1598 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1599 * handle in this PCI segment. Upon this information the shared data
1600 * structures for the PCI segments in the system will be allocated.
1601 */
1602 last_bdf = find_last_devid_acpi(ivrs_base, id);
1603 if (last_bdf < 0)
1604 return NULL;
1605
1606 pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1607 if (pci_seg == NULL)
1608 return NULL;
1609
1610 pci_seg->last_bdf = last_bdf;
1611 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1612 pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf);
1613 pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf);
1614 pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf);
1615
1616 pci_seg->id = id;
1617 init_llist_head(&pci_seg->dev_data_list);
1618 INIT_LIST_HEAD(&pci_seg->unity_map);
1619 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1620
1621 if (alloc_dev_table(pci_seg))
1622 return NULL;
1623 if (alloc_alias_table(pci_seg))
1624 return NULL;
1625 if (alloc_rlookup_table(pci_seg))
1626 return NULL;
1627
1628 return pci_seg;
1629 }
1630
get_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1631 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1632 struct acpi_table_header *ivrs_base)
1633 {
1634 struct amd_iommu_pci_seg *pci_seg;
1635
1636 for_each_pci_segment(pci_seg) {
1637 if (pci_seg->id == id)
1638 return pci_seg;
1639 }
1640
1641 return alloc_pci_segment(id, ivrs_base);
1642 }
1643
free_pci_segments(void)1644 static void __init free_pci_segments(void)
1645 {
1646 struct amd_iommu_pci_seg *pci_seg, *next;
1647
1648 for_each_pci_segment_safe(pci_seg, next) {
1649 list_del(&pci_seg->list);
1650 free_irq_lookup_table(pci_seg);
1651 free_rlookup_table(pci_seg);
1652 free_alias_table(pci_seg);
1653 free_dev_table(pci_seg);
1654 kfree(pci_seg);
1655 }
1656 }
1657
free_iommu_one(struct amd_iommu * iommu)1658 static void __init free_iommu_one(struct amd_iommu *iommu)
1659 {
1660 free_cwwb_sem(iommu);
1661 free_command_buffer(iommu);
1662 free_event_buffer(iommu);
1663 free_ppr_log(iommu);
1664 free_ga_log(iommu);
1665 iommu_unmap_mmio_space(iommu);
1666 }
1667
free_iommu_all(void)1668 static void __init free_iommu_all(void)
1669 {
1670 struct amd_iommu *iommu, *next;
1671
1672 for_each_iommu_safe(iommu, next) {
1673 list_del(&iommu->list);
1674 free_iommu_one(iommu);
1675 kfree(iommu);
1676 }
1677 }
1678
1679 /*
1680 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1681 * Workaround:
1682 * BIOS should disable L2B micellaneous clock gating by setting
1683 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1684 */
amd_iommu_erratum_746_workaround(struct amd_iommu * iommu)1685 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1686 {
1687 u32 value;
1688
1689 if ((boot_cpu_data.x86 != 0x15) ||
1690 (boot_cpu_data.x86_model < 0x10) ||
1691 (boot_cpu_data.x86_model > 0x1f))
1692 return;
1693
1694 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1695 pci_read_config_dword(iommu->dev, 0xf4, &value);
1696
1697 if (value & BIT(2))
1698 return;
1699
1700 /* Select NB indirect register 0x90 and enable writing */
1701 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1702
1703 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1704 pci_info(iommu->dev, "Applying erratum 746 workaround\n");
1705
1706 /* Clear the enable writing bit */
1707 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1708 }
1709
1710 /*
1711 * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission)
1712 * Workaround:
1713 * BIOS should enable ATS write permission check by setting
1714 * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b
1715 */
amd_iommu_ats_write_check_workaround(struct amd_iommu * iommu)1716 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
1717 {
1718 u32 value;
1719
1720 if ((boot_cpu_data.x86 != 0x15) ||
1721 (boot_cpu_data.x86_model < 0x30) ||
1722 (boot_cpu_data.x86_model > 0x3f))
1723 return;
1724
1725 /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */
1726 value = iommu_read_l2(iommu, 0x47);
1727
1728 if (value & BIT(0))
1729 return;
1730
1731 /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
1732 iommu_write_l2(iommu, 0x47, value | BIT(0));
1733
1734 pci_info(iommu->dev, "Applying ATS write check workaround\n");
1735 }
1736
1737 /*
1738 * This function glues the initialization function for one IOMMU
1739 * together and also allocates the command buffer and programs the
1740 * hardware. It does NOT enable the IOMMU. This is done afterwards.
1741 */
init_iommu_one(struct amd_iommu * iommu,struct ivhd_header * h,struct acpi_table_header * ivrs_base)1742 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1743 struct acpi_table_header *ivrs_base)
1744 {
1745 struct amd_iommu_pci_seg *pci_seg;
1746
1747 pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1748 if (pci_seg == NULL)
1749 return -ENOMEM;
1750 iommu->pci_seg = pci_seg;
1751
1752 raw_spin_lock_init(&iommu->lock);
1753 iommu->cmd_sem_val = 0;
1754
1755 /* Add IOMMU to internal data structures */
1756 list_add_tail(&iommu->list, &amd_iommu_list);
1757 iommu->index = amd_iommus_present++;
1758
1759 if (unlikely(iommu->index >= MAX_IOMMUS)) {
1760 WARN(1, "System has more IOMMUs than supported by this driver\n");
1761 return -ENOSYS;
1762 }
1763
1764 /* Index is fine - add IOMMU to the array */
1765 amd_iommus[iommu->index] = iommu;
1766
1767 /*
1768 * Copy data from ACPI table entry to the iommu struct
1769 */
1770 iommu->devid = h->devid;
1771 iommu->cap_ptr = h->cap_ptr;
1772 iommu->mmio_phys = h->mmio_phys;
1773
1774 switch (h->type) {
1775 case 0x10:
1776 /* Check if IVHD EFR contains proper max banks/counters */
1777 if ((h->efr_attr != 0) &&
1778 ((h->efr_attr & (0xF << 13)) != 0) &&
1779 ((h->efr_attr & (0x3F << 17)) != 0))
1780 iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1781 else
1782 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1783
1784 /*
1785 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1786 * GAM also requires GA mode. Therefore, we need to
1787 * check cmpxchg16b support before enabling it.
1788 */
1789 if (!boot_cpu_has(X86_FEATURE_CX16) ||
1790 ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
1791 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1792 break;
1793 case 0x11:
1794 case 0x40:
1795 if (h->efr_reg & (1 << 9))
1796 iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1797 else
1798 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1799
1800 /*
1801 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1802 * XT, GAM also requires GA mode. Therefore, we need to
1803 * check cmpxchg16b support before enabling them.
1804 */
1805 if (!boot_cpu_has(X86_FEATURE_CX16) ||
1806 ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) {
1807 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1808 break;
1809 }
1810
1811 if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
1812 amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
1813
1814 early_iommu_features_init(iommu, h);
1815
1816 break;
1817 default:
1818 return -EINVAL;
1819 }
1820
1821 iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
1822 iommu->mmio_phys_end);
1823 if (!iommu->mmio_base)
1824 return -ENOMEM;
1825
1826 return init_iommu_from_acpi(iommu, h);
1827 }
1828
init_iommu_one_late(struct amd_iommu * iommu)1829 static int __init init_iommu_one_late(struct amd_iommu *iommu)
1830 {
1831 int ret;
1832
1833 if (alloc_cwwb_sem(iommu))
1834 return -ENOMEM;
1835
1836 if (alloc_command_buffer(iommu))
1837 return -ENOMEM;
1838
1839 if (alloc_event_buffer(iommu))
1840 return -ENOMEM;
1841
1842 iommu->int_enabled = false;
1843
1844 init_translation_status(iommu);
1845 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
1846 iommu_disable(iommu);
1847 clear_translation_pre_enabled(iommu);
1848 pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
1849 iommu->index);
1850 }
1851 if (amd_iommu_pre_enabled)
1852 amd_iommu_pre_enabled = translation_pre_enabled(iommu);
1853
1854 if (amd_iommu_irq_remap) {
1855 ret = amd_iommu_create_irq_domain(iommu);
1856 if (ret)
1857 return ret;
1858 }
1859
1860 /*
1861 * Make sure IOMMU is not considered to translate itself. The IVRS
1862 * table tells us so, but this is a lie!
1863 */
1864 iommu->pci_seg->rlookup_table[iommu->devid] = NULL;
1865
1866 return 0;
1867 }
1868
1869 /**
1870 * get_highest_supported_ivhd_type - Look up the appropriate IVHD type
1871 * @ivrs: Pointer to the IVRS header
1872 *
1873 * This function search through all IVDB of the maximum supported IVHD
1874 */
get_highest_supported_ivhd_type(struct acpi_table_header * ivrs)1875 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs)
1876 {
1877 u8 *base = (u8 *)ivrs;
1878 struct ivhd_header *ivhd = (struct ivhd_header *)
1879 (base + IVRS_HEADER_LENGTH);
1880 u8 last_type = ivhd->type;
1881 u16 devid = ivhd->devid;
1882
1883 while (((u8 *)ivhd - base < ivrs->length) &&
1884 (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) {
1885 u8 *p = (u8 *) ivhd;
1886
1887 if (ivhd->devid == devid)
1888 last_type = ivhd->type;
1889 ivhd = (struct ivhd_header *)(p + ivhd->length);
1890 }
1891
1892 return last_type;
1893 }
1894
1895 /*
1896 * Iterates over all IOMMU entries in the ACPI table, allocates the
1897 * IOMMU structure and initializes it with init_iommu_one()
1898 */
init_iommu_all(struct acpi_table_header * table)1899 static int __init init_iommu_all(struct acpi_table_header *table)
1900 {
1901 u8 *p = (u8 *)table, *end = (u8 *)table;
1902 struct ivhd_header *h;
1903 struct amd_iommu *iommu;
1904 int ret;
1905
1906 end += table->length;
1907 p += IVRS_HEADER_LENGTH;
1908
1909 /* Phase 1: Process all IVHD blocks */
1910 while (p < end) {
1911 h = (struct ivhd_header *)p;
1912 if (*p == amd_iommu_target_ivhd_type) {
1913
1914 DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x "
1915 "flags: %01x info %04x\n",
1916 h->pci_seg, PCI_BUS_NUM(h->devid),
1917 PCI_SLOT(h->devid), PCI_FUNC(h->devid),
1918 h->cap_ptr, h->flags, h->info);
1919 DUMP_printk(" mmio-addr: %016llx\n",
1920 h->mmio_phys);
1921
1922 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1923 if (iommu == NULL)
1924 return -ENOMEM;
1925
1926 ret = init_iommu_one(iommu, h, table);
1927 if (ret)
1928 return ret;
1929 }
1930 p += h->length;
1931
1932 }
1933 WARN_ON(p != end);
1934
1935 /* Phase 2 : Early feature support check */
1936 get_global_efr();
1937
1938 /* Phase 3 : Enabling IOMMU features */
1939 for_each_iommu(iommu) {
1940 ret = init_iommu_one_late(iommu);
1941 if (ret)
1942 return ret;
1943 }
1944
1945 return 0;
1946 }
1947
init_iommu_perf_ctr(struct amd_iommu * iommu)1948 static void init_iommu_perf_ctr(struct amd_iommu *iommu)
1949 {
1950 u64 val;
1951 struct pci_dev *pdev = iommu->dev;
1952
1953 if (!iommu_feature(iommu, FEATURE_PC))
1954 return;
1955
1956 amd_iommu_pc_present = true;
1957
1958 pci_info(pdev, "IOMMU performance counters supported\n");
1959
1960 val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
1961 iommu->max_banks = (u8) ((val >> 12) & 0x3f);
1962 iommu->max_counters = (u8) ((val >> 7) & 0xf);
1963
1964 return;
1965 }
1966
amd_iommu_show_cap(struct device * dev,struct device_attribute * attr,char * buf)1967 static ssize_t amd_iommu_show_cap(struct device *dev,
1968 struct device_attribute *attr,
1969 char *buf)
1970 {
1971 struct amd_iommu *iommu = dev_to_amd_iommu(dev);
1972 return sprintf(buf, "%x\n", iommu->cap);
1973 }
1974 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
1975
amd_iommu_show_features(struct device * dev,struct device_attribute * attr,char * buf)1976 static ssize_t amd_iommu_show_features(struct device *dev,
1977 struct device_attribute *attr,
1978 char *buf)
1979 {
1980 struct amd_iommu *iommu = dev_to_amd_iommu(dev);
1981 return sprintf(buf, "%llx:%llx\n", iommu->features2, iommu->features);
1982 }
1983 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
1984
1985 static struct attribute *amd_iommu_attrs[] = {
1986 &dev_attr_cap.attr,
1987 &dev_attr_features.attr,
1988 NULL,
1989 };
1990
1991 static struct attribute_group amd_iommu_group = {
1992 .name = "amd-iommu",
1993 .attrs = amd_iommu_attrs,
1994 };
1995
1996 static const struct attribute_group *amd_iommu_groups[] = {
1997 &amd_iommu_group,
1998 NULL,
1999 };
2000
2001 /*
2002 * Note: IVHD 0x11 and 0x40 also contains exact copy
2003 * of the IOMMU Extended Feature Register [MMIO Offset 0030h].
2004 * Default to EFR in IVHD since it is available sooner (i.e. before PCI init).
2005 */
late_iommu_features_init(struct amd_iommu * iommu)2006 static void __init late_iommu_features_init(struct amd_iommu *iommu)
2007 {
2008 u64 features, features2;
2009
2010 if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
2011 return;
2012
2013 /* read extended feature bits */
2014 features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
2015 features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
2016
2017 if (!iommu->features) {
2018 iommu->features = features;
2019 iommu->features2 = features2;
2020 return;
2021 }
2022
2023 /*
2024 * Sanity check and warn if EFR values from
2025 * IVHD and MMIO conflict.
2026 */
2027 if (features != iommu->features ||
2028 features2 != iommu->features2) {
2029 pr_warn(FW_WARN
2030 "EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
2031 features, iommu->features,
2032 features2, iommu->features2);
2033 }
2034 }
2035
iommu_init_pci(struct amd_iommu * iommu)2036 static int __init iommu_init_pci(struct amd_iommu *iommu)
2037 {
2038 int cap_ptr = iommu->cap_ptr;
2039 int ret;
2040
2041 iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2042 PCI_BUS_NUM(iommu->devid),
2043 iommu->devid & 0xff);
2044 if (!iommu->dev)
2045 return -ENODEV;
2046
2047 /* Prevent binding other PCI device drivers to IOMMU devices */
2048 iommu->dev->match_driver = false;
2049
2050 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
2051 &iommu->cap);
2052
2053 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
2054 amd_iommu_iotlb_sup = false;
2055
2056 late_iommu_features_init(iommu);
2057
2058 if (iommu_feature(iommu, FEATURE_GT)) {
2059 int glxval;
2060 u32 max_pasid;
2061 u64 pasmax;
2062
2063 pasmax = iommu->features & FEATURE_PASID_MASK;
2064 pasmax >>= FEATURE_PASID_SHIFT;
2065 max_pasid = (1 << (pasmax + 1)) - 1;
2066
2067 amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
2068
2069 BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
2070
2071 glxval = iommu->features & FEATURE_GLXVAL_MASK;
2072 glxval >>= FEATURE_GLXVAL_SHIFT;
2073
2074 if (amd_iommu_max_glx_val == -1)
2075 amd_iommu_max_glx_val = glxval;
2076 else
2077 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
2078 }
2079
2080 if (iommu_feature(iommu, FEATURE_GT) &&
2081 iommu_feature(iommu, FEATURE_PPR)) {
2082 iommu->is_iommu_v2 = true;
2083 amd_iommu_v2_present = true;
2084 }
2085
2086 if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu))
2087 return -ENOMEM;
2088
2089 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
2090 pr_info("Using strict mode due to virtualization\n");
2091 iommu_set_dma_strict();
2092 amd_iommu_np_cache = true;
2093 }
2094
2095 init_iommu_perf_ctr(iommu);
2096
2097 if (amd_iommu_pgtable == AMD_IOMMU_V2) {
2098 if (!iommu_feature(iommu, FEATURE_GIOSUP) ||
2099 !iommu_feature(iommu, FEATURE_GT)) {
2100 pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
2101 amd_iommu_pgtable = AMD_IOMMU_V1;
2102 } else if (iommu_default_passthrough()) {
2103 pr_warn("V2 page table doesn't support passthrough mode. Fallback to v1.\n");
2104 amd_iommu_pgtable = AMD_IOMMU_V1;
2105 }
2106 }
2107
2108 if (is_rd890_iommu(iommu->dev)) {
2109 int i, j;
2110
2111 iommu->root_pdev =
2112 pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2113 iommu->dev->bus->number,
2114 PCI_DEVFN(0, 0));
2115
2116 /*
2117 * Some rd890 systems may not be fully reconfigured by the
2118 * BIOS, so it's necessary for us to store this information so
2119 * it can be reprogrammed on resume
2120 */
2121 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
2122 &iommu->stored_addr_lo);
2123 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
2124 &iommu->stored_addr_hi);
2125
2126 /* Low bit locks writes to configuration space */
2127 iommu->stored_addr_lo &= ~1;
2128
2129 for (i = 0; i < 6; i++)
2130 for (j = 0; j < 0x12; j++)
2131 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
2132
2133 for (i = 0; i < 0x83; i++)
2134 iommu->stored_l2[i] = iommu_read_l2(iommu, i);
2135 }
2136
2137 amd_iommu_erratum_746_workaround(iommu);
2138 amd_iommu_ats_write_check_workaround(iommu);
2139
2140 ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,
2141 amd_iommu_groups, "ivhd%d", iommu->index);
2142 if (ret)
2143 return ret;
2144
2145 iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);
2146
2147 return pci_enable_device(iommu->dev);
2148 }
2149
print_iommu_info(void)2150 static void print_iommu_info(void)
2151 {
2152 static const char * const feat_str[] = {
2153 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
2154 "IA", "GA", "HE", "PC"
2155 };
2156 struct amd_iommu *iommu;
2157
2158 for_each_iommu(iommu) {
2159 struct pci_dev *pdev = iommu->dev;
2160 int i;
2161
2162 pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr);
2163
2164 if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
2165 pr_info("Extended features (%#llx, %#llx):", iommu->features, iommu->features2);
2166
2167 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
2168 if (iommu_feature(iommu, (1ULL << i)))
2169 pr_cont(" %s", feat_str[i]);
2170 }
2171
2172 if (iommu->features & FEATURE_GAM_VAPIC)
2173 pr_cont(" GA_vAPIC");
2174
2175 if (iommu->features & FEATURE_SNP)
2176 pr_cont(" SNP");
2177
2178 pr_cont("\n");
2179 }
2180 }
2181 if (irq_remapping_enabled) {
2182 pr_info("Interrupt remapping enabled\n");
2183 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2184 pr_info("X2APIC enabled\n");
2185 }
2186 if (amd_iommu_pgtable == AMD_IOMMU_V2)
2187 pr_info("V2 page table enabled\n");
2188 }
2189
amd_iommu_init_pci(void)2190 static int __init amd_iommu_init_pci(void)
2191 {
2192 struct amd_iommu *iommu;
2193 struct amd_iommu_pci_seg *pci_seg;
2194 int ret;
2195
2196 for_each_iommu(iommu) {
2197 ret = iommu_init_pci(iommu);
2198 if (ret) {
2199 pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
2200 iommu->index, ret);
2201 goto out;
2202 }
2203 /* Need to setup range after PCI init */
2204 iommu_set_cwwb_range(iommu);
2205 }
2206
2207 /*
2208 * Order is important here to make sure any unity map requirements are
2209 * fulfilled. The unity mappings are created and written to the device
2210 * table during the iommu_init_pci() call.
2211 *
2212 * After that we call init_device_table_dma() to make sure any
2213 * uninitialized DTE will block DMA, and in the end we flush the caches
2214 * of all IOMMUs to make sure the changes to the device table are
2215 * active.
2216 */
2217 for_each_pci_segment(pci_seg)
2218 init_device_table_dma(pci_seg);
2219
2220 for_each_iommu(iommu)
2221 iommu_flush_all_caches(iommu);
2222
2223 print_iommu_info();
2224
2225 out:
2226 return ret;
2227 }
2228
2229 /****************************************************************************
2230 *
2231 * The following functions initialize the MSI interrupts for all IOMMUs
2232 * in the system. It's a bit challenging because there could be multiple
2233 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
2234 * pci_dev.
2235 *
2236 ****************************************************************************/
2237
iommu_setup_msi(struct amd_iommu * iommu)2238 static int iommu_setup_msi(struct amd_iommu *iommu)
2239 {
2240 int r;
2241
2242 r = pci_enable_msi(iommu->dev);
2243 if (r)
2244 return r;
2245
2246 r = request_threaded_irq(iommu->dev->irq,
2247 amd_iommu_int_handler,
2248 amd_iommu_int_thread,
2249 0, "AMD-Vi",
2250 iommu);
2251
2252 if (r) {
2253 pci_disable_msi(iommu->dev);
2254 return r;
2255 }
2256
2257 return 0;
2258 }
2259
2260 union intcapxt {
2261 u64 capxt;
2262 struct {
2263 u64 reserved_0 : 2,
2264 dest_mode_logical : 1,
2265 reserved_1 : 5,
2266 destid_0_23 : 24,
2267 vector : 8,
2268 reserved_2 : 16,
2269 destid_24_31 : 8;
2270 };
2271 } __attribute__ ((packed));
2272
2273
2274 static struct irq_chip intcapxt_controller;
2275
intcapxt_irqdomain_activate(struct irq_domain * domain,struct irq_data * irqd,bool reserve)2276 static int intcapxt_irqdomain_activate(struct irq_domain *domain,
2277 struct irq_data *irqd, bool reserve)
2278 {
2279 return 0;
2280 }
2281
intcapxt_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irqd)2282 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
2283 struct irq_data *irqd)
2284 {
2285 }
2286
2287
intcapxt_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)2288 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2289 unsigned int nr_irqs, void *arg)
2290 {
2291 struct irq_alloc_info *info = arg;
2292 int i, ret;
2293
2294 if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI)
2295 return -EINVAL;
2296
2297 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
2298 if (ret < 0)
2299 return ret;
2300
2301 for (i = virq; i < virq + nr_irqs; i++) {
2302 struct irq_data *irqd = irq_domain_get_irq_data(domain, i);
2303
2304 irqd->chip = &intcapxt_controller;
2305 irqd->chip_data = info->data;
2306 __irq_set_handler(i, handle_edge_irq, 0, "edge");
2307 }
2308
2309 return ret;
2310 }
2311
intcapxt_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)2312 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2313 unsigned int nr_irqs)
2314 {
2315 irq_domain_free_irqs_top(domain, virq, nr_irqs);
2316 }
2317
2318
intcapxt_unmask_irq(struct irq_data * irqd)2319 static void intcapxt_unmask_irq(struct irq_data *irqd)
2320 {
2321 struct amd_iommu *iommu = irqd->chip_data;
2322 struct irq_cfg *cfg = irqd_cfg(irqd);
2323 union intcapxt xt;
2324
2325 xt.capxt = 0ULL;
2326 xt.dest_mode_logical = apic->dest_mode_logical;
2327 xt.vector = cfg->vector;
2328 xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
2329 xt.destid_24_31 = cfg->dest_apicid >> 24;
2330
2331 /**
2332 * Current IOMMU implementation uses the same IRQ for all
2333 * 3 IOMMU interrupts.
2334 */
2335 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
2336 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
2337 writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
2338 }
2339
intcapxt_mask_irq(struct irq_data * irqd)2340 static void intcapxt_mask_irq(struct irq_data *irqd)
2341 {
2342 struct amd_iommu *iommu = irqd->chip_data;
2343
2344 writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
2345 writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
2346 writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
2347 }
2348
2349
intcapxt_set_affinity(struct irq_data * irqd,const struct cpumask * mask,bool force)2350 static int intcapxt_set_affinity(struct irq_data *irqd,
2351 const struct cpumask *mask, bool force)
2352 {
2353 struct irq_data *parent = irqd->parent_data;
2354 int ret;
2355
2356 ret = parent->chip->irq_set_affinity(parent, mask, force);
2357 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
2358 return ret;
2359 return 0;
2360 }
2361
intcapxt_set_wake(struct irq_data * irqd,unsigned int on)2362 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
2363 {
2364 return on ? -EOPNOTSUPP : 0;
2365 }
2366
2367 static struct irq_chip intcapxt_controller = {
2368 .name = "IOMMU-MSI",
2369 .irq_unmask = intcapxt_unmask_irq,
2370 .irq_mask = intcapxt_mask_irq,
2371 .irq_ack = irq_chip_ack_parent,
2372 .irq_retrigger = irq_chip_retrigger_hierarchy,
2373 .irq_set_affinity = intcapxt_set_affinity,
2374 .irq_set_wake = intcapxt_set_wake,
2375 .flags = IRQCHIP_MASK_ON_SUSPEND,
2376 };
2377
2378 static const struct irq_domain_ops intcapxt_domain_ops = {
2379 .alloc = intcapxt_irqdomain_alloc,
2380 .free = intcapxt_irqdomain_free,
2381 .activate = intcapxt_irqdomain_activate,
2382 .deactivate = intcapxt_irqdomain_deactivate,
2383 };
2384
2385
2386 static struct irq_domain *iommu_irqdomain;
2387
iommu_get_irqdomain(void)2388 static struct irq_domain *iommu_get_irqdomain(void)
2389 {
2390 struct fwnode_handle *fn;
2391
2392 /* No need for locking here (yet) as the init is single-threaded */
2393 if (iommu_irqdomain)
2394 return iommu_irqdomain;
2395
2396 fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI");
2397 if (!fn)
2398 return NULL;
2399
2400 iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0,
2401 fn, &intcapxt_domain_ops,
2402 NULL);
2403 if (!iommu_irqdomain)
2404 irq_domain_free_fwnode(fn);
2405
2406 return iommu_irqdomain;
2407 }
2408
iommu_setup_intcapxt(struct amd_iommu * iommu)2409 static int iommu_setup_intcapxt(struct amd_iommu *iommu)
2410 {
2411 struct irq_domain *domain;
2412 struct irq_alloc_info info;
2413 int irq, ret;
2414
2415 domain = iommu_get_irqdomain();
2416 if (!domain)
2417 return -ENXIO;
2418
2419 init_irq_alloc_info(&info, NULL);
2420 info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
2421 info.data = iommu;
2422
2423 irq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info);
2424 if (irq < 0) {
2425 irq_domain_remove(domain);
2426 return irq;
2427 }
2428
2429 ret = request_threaded_irq(irq, amd_iommu_int_handler,
2430 amd_iommu_int_thread, 0, "AMD-Vi", iommu);
2431 if (ret) {
2432 irq_domain_free_irqs(irq, 1);
2433 irq_domain_remove(domain);
2434 return ret;
2435 }
2436
2437 return 0;
2438 }
2439
iommu_init_irq(struct amd_iommu * iommu)2440 static int iommu_init_irq(struct amd_iommu *iommu)
2441 {
2442 int ret;
2443
2444 if (iommu->int_enabled)
2445 goto enable_faults;
2446
2447 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2448 ret = iommu_setup_intcapxt(iommu);
2449 else if (iommu->dev->msi_cap)
2450 ret = iommu_setup_msi(iommu);
2451 else
2452 ret = -ENODEV;
2453
2454 if (ret)
2455 return ret;
2456
2457 iommu->int_enabled = true;
2458 enable_faults:
2459
2460 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2461 iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
2462
2463 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2464
2465 if (iommu->ppr_log != NULL)
2466 iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
2467 return 0;
2468 }
2469
2470 /****************************************************************************
2471 *
2472 * The next functions belong to the third pass of parsing the ACPI
2473 * table. In this last pass the memory mapping requirements are
2474 * gathered (like exclusion and unity mapping ranges).
2475 *
2476 ****************************************************************************/
2477
free_unity_maps(void)2478 static void __init free_unity_maps(void)
2479 {
2480 struct unity_map_entry *entry, *next;
2481 struct amd_iommu_pci_seg *p, *pci_seg;
2482
2483 for_each_pci_segment_safe(pci_seg, p) {
2484 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2485 list_del(&entry->list);
2486 kfree(entry);
2487 }
2488 }
2489 }
2490
2491 /* called for unity map ACPI definition */
init_unity_map_range(struct ivmd_header * m,struct acpi_table_header * ivrs_base)2492 static int __init init_unity_map_range(struct ivmd_header *m,
2493 struct acpi_table_header *ivrs_base)
2494 {
2495 struct unity_map_entry *e = NULL;
2496 struct amd_iommu_pci_seg *pci_seg;
2497 char *s;
2498
2499 pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2500 if (pci_seg == NULL)
2501 return -ENOMEM;
2502
2503 e = kzalloc(sizeof(*e), GFP_KERNEL);
2504 if (e == NULL)
2505 return -ENOMEM;
2506
2507 switch (m->type) {
2508 default:
2509 kfree(e);
2510 return 0;
2511 case ACPI_IVMD_TYPE:
2512 s = "IVMD_TYPEi\t\t\t";
2513 e->devid_start = e->devid_end = m->devid;
2514 break;
2515 case ACPI_IVMD_TYPE_ALL:
2516 s = "IVMD_TYPE_ALL\t\t";
2517 e->devid_start = 0;
2518 e->devid_end = pci_seg->last_bdf;
2519 break;
2520 case ACPI_IVMD_TYPE_RANGE:
2521 s = "IVMD_TYPE_RANGE\t\t";
2522 e->devid_start = m->devid;
2523 e->devid_end = m->aux;
2524 break;
2525 }
2526 e->address_start = PAGE_ALIGN(m->range_start);
2527 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
2528 e->prot = m->flags >> 1;
2529
2530 /*
2531 * Treat per-device exclusion ranges as r/w unity-mapped regions
2532 * since some buggy BIOSes might lead to the overwritten exclusion
2533 * range (exclusion_start and exclusion_length members). This
2534 * happens when there are multiple exclusion ranges (IVMD entries)
2535 * defined in ACPI table.
2536 */
2537 if (m->flags & IVMD_FLAG_EXCL_RANGE)
2538 e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
2539
2540 DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
2541 "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
2542 " flags: %x\n", s, m->pci_seg,
2543 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
2544 PCI_FUNC(e->devid_start), m->pci_seg,
2545 PCI_BUS_NUM(e->devid_end),
2546 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
2547 e->address_start, e->address_end, m->flags);
2548
2549 list_add_tail(&e->list, &pci_seg->unity_map);
2550
2551 return 0;
2552 }
2553
2554 /* iterates over all memory definitions we find in the ACPI table */
init_memory_definitions(struct acpi_table_header * table)2555 static int __init init_memory_definitions(struct acpi_table_header *table)
2556 {
2557 u8 *p = (u8 *)table, *end = (u8 *)table;
2558 struct ivmd_header *m;
2559
2560 end += table->length;
2561 p += IVRS_HEADER_LENGTH;
2562
2563 while (p < end) {
2564 m = (struct ivmd_header *)p;
2565 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2566 init_unity_map_range(m, table);
2567
2568 p += m->length;
2569 }
2570
2571 return 0;
2572 }
2573
2574 /*
2575 * Init the device table to not allow DMA access for devices
2576 */
init_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2577 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2578 {
2579 u32 devid;
2580 struct dev_table_entry *dev_table = pci_seg->dev_table;
2581
2582 if (dev_table == NULL)
2583 return;
2584
2585 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2586 __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_VALID);
2587 if (!amd_iommu_snp_en)
2588 __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_TRANSLATION);
2589 }
2590 }
2591
uninit_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2592 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2593 {
2594 u32 devid;
2595 struct dev_table_entry *dev_table = pci_seg->dev_table;
2596
2597 if (dev_table == NULL)
2598 return;
2599
2600 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2601 dev_table[devid].data[0] = 0ULL;
2602 dev_table[devid].data[1] = 0ULL;
2603 }
2604 }
2605
init_device_table(void)2606 static void init_device_table(void)
2607 {
2608 struct amd_iommu_pci_seg *pci_seg;
2609 u32 devid;
2610
2611 if (!amd_iommu_irq_remap)
2612 return;
2613
2614 for_each_pci_segment(pci_seg) {
2615 for (devid = 0; devid <= pci_seg->last_bdf; ++devid)
2616 __set_dev_entry_bit(pci_seg->dev_table,
2617 devid, DEV_ENTRY_IRQ_TBL_EN);
2618 }
2619 }
2620
iommu_init_flags(struct amd_iommu * iommu)2621 static void iommu_init_flags(struct amd_iommu *iommu)
2622 {
2623 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
2624 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
2625 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
2626
2627 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
2628 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
2629 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
2630
2631 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
2632 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
2633 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
2634
2635 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
2636 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
2637 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
2638
2639 /*
2640 * make IOMMU memory accesses cache coherent
2641 */
2642 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
2643
2644 /* Set IOTLB invalidation timeout to 1s */
2645 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
2646 }
2647
iommu_apply_resume_quirks(struct amd_iommu * iommu)2648 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
2649 {
2650 int i, j;
2651 u32 ioc_feature_control;
2652 struct pci_dev *pdev = iommu->root_pdev;
2653
2654 /* RD890 BIOSes may not have completely reconfigured the iommu */
2655 if (!is_rd890_iommu(iommu->dev) || !pdev)
2656 return;
2657
2658 /*
2659 * First, we need to ensure that the iommu is enabled. This is
2660 * controlled by a register in the northbridge
2661 */
2662
2663 /* Select Northbridge indirect register 0x75 and enable writing */
2664 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
2665 pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
2666
2667 /* Enable the iommu */
2668 if (!(ioc_feature_control & 0x1))
2669 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
2670
2671 /* Restore the iommu BAR */
2672 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2673 iommu->stored_addr_lo);
2674 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
2675 iommu->stored_addr_hi);
2676
2677 /* Restore the l1 indirect regs for each of the 6 l1s */
2678 for (i = 0; i < 6; i++)
2679 for (j = 0; j < 0x12; j++)
2680 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
2681
2682 /* Restore the l2 indirect regs */
2683 for (i = 0; i < 0x83; i++)
2684 iommu_write_l2(iommu, i, iommu->stored_l2[i]);
2685
2686 /* Lock PCI setup registers */
2687 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2688 iommu->stored_addr_lo | 1);
2689 }
2690
iommu_enable_ga(struct amd_iommu * iommu)2691 static void iommu_enable_ga(struct amd_iommu *iommu)
2692 {
2693 #ifdef CONFIG_IRQ_REMAP
2694 switch (amd_iommu_guest_ir) {
2695 case AMD_IOMMU_GUEST_IR_VAPIC:
2696 case AMD_IOMMU_GUEST_IR_LEGACY_GA:
2697 iommu_feature_enable(iommu, CONTROL_GA_EN);
2698 iommu->irte_ops = &irte_128_ops;
2699 break;
2700 default:
2701 iommu->irte_ops = &irte_32_ops;
2702 break;
2703 }
2704 #endif
2705 }
2706
iommu_disable_irtcachedis(struct amd_iommu * iommu)2707 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)
2708 {
2709 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
2710 }
2711
iommu_enable_irtcachedis(struct amd_iommu * iommu)2712 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
2713 {
2714 u64 ctrl;
2715
2716 if (!amd_iommu_irtcachedis)
2717 return;
2718
2719 /*
2720 * Note:
2721 * The support for IRTCacheDis feature is dertermined by
2722 * checking if the bit is writable.
2723 */
2724 iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);
2725 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
2726 ctrl &= (1ULL << CONTROL_IRTCACHEDIS);
2727 if (ctrl)
2728 iommu->irtcachedis_enabled = true;
2729 pr_info("iommu%d (%#06x) : IRT cache is %s\n",
2730 iommu->index, iommu->devid,
2731 iommu->irtcachedis_enabled ? "disabled" : "enabled");
2732 }
2733
early_enable_iommu(struct amd_iommu * iommu)2734 static void early_enable_iommu(struct amd_iommu *iommu)
2735 {
2736 iommu_disable(iommu);
2737 iommu_init_flags(iommu);
2738 iommu_set_device_table(iommu);
2739 iommu_enable_command_buffer(iommu);
2740 iommu_enable_event_buffer(iommu);
2741 iommu_set_exclusion_range(iommu);
2742 iommu_enable_ga(iommu);
2743 iommu_enable_xt(iommu);
2744 iommu_enable_irtcachedis(iommu);
2745 iommu_enable(iommu);
2746 iommu_flush_all_caches(iommu);
2747 }
2748
2749 /*
2750 * This function finally enables all IOMMUs found in the system after
2751 * they have been initialized.
2752 *
2753 * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
2754 * the old content of device table entries. Not this case or copy failed,
2755 * just continue as normal kernel does.
2756 */
early_enable_iommus(void)2757 static void early_enable_iommus(void)
2758 {
2759 struct amd_iommu *iommu;
2760 struct amd_iommu_pci_seg *pci_seg;
2761
2762 if (!copy_device_table()) {
2763 /*
2764 * If come here because of failure in copying device table from old
2765 * kernel with all IOMMUs enabled, print error message and try to
2766 * free allocated old_dev_tbl_cpy.
2767 */
2768 if (amd_iommu_pre_enabled)
2769 pr_err("Failed to copy DEV table from previous kernel.\n");
2770
2771 for_each_pci_segment(pci_seg) {
2772 if (pci_seg->old_dev_tbl_cpy != NULL) {
2773 free_pages((unsigned long)pci_seg->old_dev_tbl_cpy,
2774 get_order(pci_seg->dev_table_size));
2775 pci_seg->old_dev_tbl_cpy = NULL;
2776 }
2777 }
2778
2779 for_each_iommu(iommu) {
2780 clear_translation_pre_enabled(iommu);
2781 early_enable_iommu(iommu);
2782 }
2783 } else {
2784 pr_info("Copied DEV table from previous kernel.\n");
2785
2786 for_each_pci_segment(pci_seg) {
2787 free_pages((unsigned long)pci_seg->dev_table,
2788 get_order(pci_seg->dev_table_size));
2789 pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
2790 }
2791
2792 for_each_iommu(iommu) {
2793 iommu_disable_command_buffer(iommu);
2794 iommu_disable_event_buffer(iommu);
2795 iommu_disable_irtcachedis(iommu);
2796 iommu_enable_command_buffer(iommu);
2797 iommu_enable_event_buffer(iommu);
2798 iommu_enable_ga(iommu);
2799 iommu_enable_xt(iommu);
2800 iommu_enable_irtcachedis(iommu);
2801 iommu_set_device_table(iommu);
2802 iommu_flush_all_caches(iommu);
2803 }
2804 }
2805 }
2806
enable_iommus_v2(void)2807 static void enable_iommus_v2(void)
2808 {
2809 struct amd_iommu *iommu;
2810
2811 for_each_iommu(iommu) {
2812 iommu_enable_ppr_log(iommu);
2813 iommu_enable_gt(iommu);
2814 }
2815 }
2816
enable_iommus_vapic(void)2817 static void enable_iommus_vapic(void)
2818 {
2819 #ifdef CONFIG_IRQ_REMAP
2820 u32 status, i;
2821 struct amd_iommu *iommu;
2822
2823 for_each_iommu(iommu) {
2824 /*
2825 * Disable GALog if already running. It could have been enabled
2826 * in the previous boot before kdump.
2827 */
2828 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2829 if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2830 continue;
2831
2832 iommu_feature_disable(iommu, CONTROL_GALOG_EN);
2833 iommu_feature_disable(iommu, CONTROL_GAINT_EN);
2834
2835 /*
2836 * Need to set and poll check the GALOGRun bit to zero before
2837 * we can set/ modify GA Log registers safely.
2838 */
2839 for (i = 0; i < LOOP_TIMEOUT; ++i) {
2840 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2841 if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2842 break;
2843 udelay(10);
2844 }
2845
2846 if (WARN_ON(i >= LOOP_TIMEOUT))
2847 return;
2848 }
2849
2850 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
2851 !check_feature_on_all_iommus(FEATURE_GAM_VAPIC)) {
2852 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2853 return;
2854 }
2855
2856 if (amd_iommu_snp_en &&
2857 !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
2858 pr_warn("Force to disable Virtual APIC due to SNP\n");
2859 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2860 return;
2861 }
2862
2863 /* Enabling GAM and SNPAVIC support */
2864 for_each_iommu(iommu) {
2865 if (iommu_init_ga_log(iommu) ||
2866 iommu_ga_log_enable(iommu))
2867 return;
2868
2869 iommu_feature_enable(iommu, CONTROL_GAM_EN);
2870 if (amd_iommu_snp_en)
2871 iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);
2872 }
2873
2874 amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
2875 pr_info("Virtual APIC enabled\n");
2876 #endif
2877 }
2878
enable_iommus(void)2879 static void enable_iommus(void)
2880 {
2881 early_enable_iommus();
2882 enable_iommus_vapic();
2883 enable_iommus_v2();
2884 }
2885
disable_iommus(void)2886 static void disable_iommus(void)
2887 {
2888 struct amd_iommu *iommu;
2889
2890 for_each_iommu(iommu)
2891 iommu_disable(iommu);
2892
2893 #ifdef CONFIG_IRQ_REMAP
2894 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
2895 amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
2896 #endif
2897 }
2898
2899 /*
2900 * Suspend/Resume support
2901 * disable suspend until real resume implemented
2902 */
2903
amd_iommu_resume(void)2904 static void amd_iommu_resume(void)
2905 {
2906 struct amd_iommu *iommu;
2907
2908 for_each_iommu(iommu)
2909 iommu_apply_resume_quirks(iommu);
2910
2911 /* re-load the hardware */
2912 enable_iommus();
2913
2914 amd_iommu_enable_interrupts();
2915 }
2916
amd_iommu_suspend(void)2917 static int amd_iommu_suspend(void)
2918 {
2919 /* disable IOMMUs to go out of the way for BIOS */
2920 disable_iommus();
2921
2922 return 0;
2923 }
2924
2925 static struct syscore_ops amd_iommu_syscore_ops = {
2926 .suspend = amd_iommu_suspend,
2927 .resume = amd_iommu_resume,
2928 };
2929
free_iommu_resources(void)2930 static void __init free_iommu_resources(void)
2931 {
2932 kmem_cache_destroy(amd_iommu_irq_cache);
2933 amd_iommu_irq_cache = NULL;
2934
2935 free_iommu_all();
2936 free_pci_segments();
2937 }
2938
2939 /* SB IOAPIC is always on this device in AMD systems */
2940 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
2941
check_ioapic_information(void)2942 static bool __init check_ioapic_information(void)
2943 {
2944 const char *fw_bug = FW_BUG;
2945 bool ret, has_sb_ioapic;
2946 int idx;
2947
2948 has_sb_ioapic = false;
2949 ret = false;
2950
2951 /*
2952 * If we have map overrides on the kernel command line the
2953 * messages in this function might not describe firmware bugs
2954 * anymore - so be careful
2955 */
2956 if (cmdline_maps)
2957 fw_bug = "";
2958
2959 for (idx = 0; idx < nr_ioapics; idx++) {
2960 int devid, id = mpc_ioapic_id(idx);
2961
2962 devid = get_ioapic_devid(id);
2963 if (devid < 0) {
2964 pr_err("%s: IOAPIC[%d] not in IVRS table\n",
2965 fw_bug, id);
2966 ret = false;
2967 } else if (devid == IOAPIC_SB_DEVID) {
2968 has_sb_ioapic = true;
2969 ret = true;
2970 }
2971 }
2972
2973 if (!has_sb_ioapic) {
2974 /*
2975 * We expect the SB IOAPIC to be listed in the IVRS
2976 * table. The system timer is connected to the SB IOAPIC
2977 * and if we don't have it in the list the system will
2978 * panic at boot time. This situation usually happens
2979 * when the BIOS is buggy and provides us the wrong
2980 * device id for the IOAPIC in the system.
2981 */
2982 pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
2983 }
2984
2985 if (!ret)
2986 pr_err("Disabling interrupt remapping\n");
2987
2988 return ret;
2989 }
2990
free_dma_resources(void)2991 static void __init free_dma_resources(void)
2992 {
2993 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
2994 get_order(MAX_DOMAIN_ID/8));
2995 amd_iommu_pd_alloc_bitmap = NULL;
2996
2997 free_unity_maps();
2998 }
2999
ivinfo_init(void * ivrs)3000 static void __init ivinfo_init(void *ivrs)
3001 {
3002 amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET));
3003 }
3004
3005 /*
3006 * This is the hardware init function for AMD IOMMU in the system.
3007 * This function is called either from amd_iommu_init or from the interrupt
3008 * remapping setup code.
3009 *
3010 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
3011 * four times:
3012 *
3013 * 1 pass) Discover the most comprehensive IVHD type to use.
3014 *
3015 * 2 pass) Find the highest PCI device id the driver has to handle.
3016 * Upon this information the size of the data structures is
3017 * determined that needs to be allocated.
3018 *
3019 * 3 pass) Initialize the data structures just allocated with the
3020 * information in the ACPI table about available AMD IOMMUs
3021 * in the system. It also maps the PCI devices in the
3022 * system to specific IOMMUs
3023 *
3024 * 4 pass) After the basic data structures are allocated and
3025 * initialized we update them with information about memory
3026 * remapping requirements parsed out of the ACPI table in
3027 * this last pass.
3028 *
3029 * After everything is set up the IOMMUs are enabled and the necessary
3030 * hotplug and suspend notifiers are registered.
3031 */
early_amd_iommu_init(void)3032 static int __init early_amd_iommu_init(void)
3033 {
3034 struct acpi_table_header *ivrs_base;
3035 int remap_cache_sz, ret;
3036 acpi_status status;
3037
3038 if (!amd_iommu_detected)
3039 return -ENODEV;
3040
3041 status = acpi_get_table("IVRS", 0, &ivrs_base);
3042 if (status == AE_NOT_FOUND)
3043 return -ENODEV;
3044 else if (ACPI_FAILURE(status)) {
3045 const char *err = acpi_format_exception(status);
3046 pr_err("IVRS table error: %s\n", err);
3047 return -EINVAL;
3048 }
3049
3050 /*
3051 * Validate checksum here so we don't need to do it when
3052 * we actually parse the table
3053 */
3054 ret = check_ivrs_checksum(ivrs_base);
3055 if (ret)
3056 goto out;
3057
3058 ivinfo_init(ivrs_base);
3059
3060 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
3061 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
3062
3063 /* Device table - directly used by all IOMMUs */
3064 ret = -ENOMEM;
3065
3066 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
3067 GFP_KERNEL | __GFP_ZERO,
3068 get_order(MAX_DOMAIN_ID/8));
3069 if (amd_iommu_pd_alloc_bitmap == NULL)
3070 goto out;
3071
3072 /*
3073 * never allocate domain 0 because its used as the non-allocated and
3074 * error value placeholder
3075 */
3076 __set_bit(0, amd_iommu_pd_alloc_bitmap);
3077
3078 /*
3079 * now the data structures are allocated and basically initialized
3080 * start the real acpi table scan
3081 */
3082 ret = init_iommu_all(ivrs_base);
3083 if (ret)
3084 goto out;
3085
3086 /* Disable any previously enabled IOMMUs */
3087 if (!is_kdump_kernel() || amd_iommu_disabled)
3088 disable_iommus();
3089
3090 if (amd_iommu_irq_remap)
3091 amd_iommu_irq_remap = check_ioapic_information();
3092
3093 if (amd_iommu_irq_remap) {
3094 struct amd_iommu_pci_seg *pci_seg;
3095 /*
3096 * Interrupt remapping enabled, create kmem_cache for the
3097 * remapping tables.
3098 */
3099 ret = -ENOMEM;
3100 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3101 remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32);
3102 else
3103 remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2);
3104 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
3105 remap_cache_sz,
3106 DTE_INTTAB_ALIGNMENT,
3107 0, NULL);
3108 if (!amd_iommu_irq_cache)
3109 goto out;
3110
3111 for_each_pci_segment(pci_seg) {
3112 if (alloc_irq_lookup_table(pci_seg))
3113 goto out;
3114 }
3115 }
3116
3117 ret = init_memory_definitions(ivrs_base);
3118 if (ret)
3119 goto out;
3120
3121 /* init the device table */
3122 init_device_table();
3123
3124 out:
3125 /* Don't leak any ACPI memory */
3126 acpi_put_table(ivrs_base);
3127
3128 return ret;
3129 }
3130
amd_iommu_enable_interrupts(void)3131 static int amd_iommu_enable_interrupts(void)
3132 {
3133 struct amd_iommu *iommu;
3134 int ret = 0;
3135
3136 for_each_iommu(iommu) {
3137 ret = iommu_init_irq(iommu);
3138 if (ret)
3139 goto out;
3140 }
3141
3142 out:
3143 return ret;
3144 }
3145
detect_ivrs(void)3146 static bool __init detect_ivrs(void)
3147 {
3148 struct acpi_table_header *ivrs_base;
3149 acpi_status status;
3150 int i;
3151
3152 status = acpi_get_table("IVRS", 0, &ivrs_base);
3153 if (status == AE_NOT_FOUND)
3154 return false;
3155 else if (ACPI_FAILURE(status)) {
3156 const char *err = acpi_format_exception(status);
3157 pr_err("IVRS table error: %s\n", err);
3158 return false;
3159 }
3160
3161 acpi_put_table(ivrs_base);
3162
3163 if (amd_iommu_force_enable)
3164 goto out;
3165
3166 /* Don't use IOMMU if there is Stoney Ridge graphics */
3167 for (i = 0; i < 32; i++) {
3168 u32 pci_id;
3169
3170 pci_id = read_pci_config(0, i, 0, 0);
3171 if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) {
3172 pr_info("Disable IOMMU on Stoney Ridge\n");
3173 return false;
3174 }
3175 }
3176
3177 out:
3178 /* Make sure ACS will be enabled during PCI probe */
3179 pci_request_acs();
3180
3181 return true;
3182 }
3183
3184 /****************************************************************************
3185 *
3186 * AMD IOMMU Initialization State Machine
3187 *
3188 ****************************************************************************/
3189
state_next(void)3190 static int __init state_next(void)
3191 {
3192 int ret = 0;
3193
3194 switch (init_state) {
3195 case IOMMU_START_STATE:
3196 if (!detect_ivrs()) {
3197 init_state = IOMMU_NOT_FOUND;
3198 ret = -ENODEV;
3199 } else {
3200 init_state = IOMMU_IVRS_DETECTED;
3201 }
3202 break;
3203 case IOMMU_IVRS_DETECTED:
3204 if (amd_iommu_disabled) {
3205 init_state = IOMMU_CMDLINE_DISABLED;
3206 ret = -EINVAL;
3207 } else {
3208 ret = early_amd_iommu_init();
3209 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
3210 }
3211 break;
3212 case IOMMU_ACPI_FINISHED:
3213 early_enable_iommus();
3214 x86_platform.iommu_shutdown = disable_iommus;
3215 init_state = IOMMU_ENABLED;
3216 break;
3217 case IOMMU_ENABLED:
3218 register_syscore_ops(&amd_iommu_syscore_ops);
3219 ret = amd_iommu_init_pci();
3220 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
3221 enable_iommus_vapic();
3222 enable_iommus_v2();
3223 break;
3224 case IOMMU_PCI_INIT:
3225 ret = amd_iommu_enable_interrupts();
3226 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
3227 break;
3228 case IOMMU_INTERRUPTS_EN:
3229 init_state = IOMMU_INITIALIZED;
3230 break;
3231 case IOMMU_INITIALIZED:
3232 /* Nothing to do */
3233 break;
3234 case IOMMU_NOT_FOUND:
3235 case IOMMU_INIT_ERROR:
3236 case IOMMU_CMDLINE_DISABLED:
3237 /* Error states => do nothing */
3238 ret = -EINVAL;
3239 break;
3240 default:
3241 /* Unknown state */
3242 BUG();
3243 }
3244
3245 if (ret) {
3246 free_dma_resources();
3247 if (!irq_remapping_enabled) {
3248 disable_iommus();
3249 free_iommu_resources();
3250 } else {
3251 struct amd_iommu *iommu;
3252 struct amd_iommu_pci_seg *pci_seg;
3253
3254 for_each_pci_segment(pci_seg)
3255 uninit_device_table_dma(pci_seg);
3256
3257 for_each_iommu(iommu)
3258 iommu_flush_all_caches(iommu);
3259 }
3260 }
3261 return ret;
3262 }
3263
iommu_go_to_state(enum iommu_init_state state)3264 static int __init iommu_go_to_state(enum iommu_init_state state)
3265 {
3266 int ret = -EINVAL;
3267
3268 while (init_state != state) {
3269 if (init_state == IOMMU_NOT_FOUND ||
3270 init_state == IOMMU_INIT_ERROR ||
3271 init_state == IOMMU_CMDLINE_DISABLED)
3272 break;
3273 ret = state_next();
3274 }
3275
3276 return ret;
3277 }
3278
3279 #ifdef CONFIG_IRQ_REMAP
amd_iommu_prepare(void)3280 int __init amd_iommu_prepare(void)
3281 {
3282 int ret;
3283
3284 amd_iommu_irq_remap = true;
3285
3286 ret = iommu_go_to_state(IOMMU_ACPI_FINISHED);
3287 if (ret) {
3288 amd_iommu_irq_remap = false;
3289 return ret;
3290 }
3291
3292 return amd_iommu_irq_remap ? 0 : -ENODEV;
3293 }
3294
amd_iommu_enable(void)3295 int __init amd_iommu_enable(void)
3296 {
3297 int ret;
3298
3299 ret = iommu_go_to_state(IOMMU_ENABLED);
3300 if (ret)
3301 return ret;
3302
3303 irq_remapping_enabled = 1;
3304 return amd_iommu_xt_mode;
3305 }
3306
amd_iommu_disable(void)3307 void amd_iommu_disable(void)
3308 {
3309 amd_iommu_suspend();
3310 }
3311
amd_iommu_reenable(int mode)3312 int amd_iommu_reenable(int mode)
3313 {
3314 amd_iommu_resume();
3315
3316 return 0;
3317 }
3318
amd_iommu_enable_faulting(void)3319 int __init amd_iommu_enable_faulting(void)
3320 {
3321 /* We enable MSI later when PCI is initialized */
3322 return 0;
3323 }
3324 #endif
3325
3326 /*
3327 * This is the core init function for AMD IOMMU hardware in the system.
3328 * This function is called from the generic x86 DMA layer initialization
3329 * code.
3330 */
amd_iommu_init(void)3331 static int __init amd_iommu_init(void)
3332 {
3333 struct amd_iommu *iommu;
3334 int ret;
3335
3336 ret = iommu_go_to_state(IOMMU_INITIALIZED);
3337 #ifdef CONFIG_GART_IOMMU
3338 if (ret && list_empty(&amd_iommu_list)) {
3339 /*
3340 * We failed to initialize the AMD IOMMU - try fallback
3341 * to GART if possible.
3342 */
3343 gart_iommu_init();
3344 }
3345 #endif
3346
3347 for_each_iommu(iommu)
3348 amd_iommu_debugfs_setup(iommu);
3349
3350 return ret;
3351 }
3352
amd_iommu_sme_check(void)3353 static bool amd_iommu_sme_check(void)
3354 {
3355 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) ||
3356 (boot_cpu_data.x86 != 0x17))
3357 return true;
3358
3359 /* For Fam17h, a specific level of support is required */
3360 if (boot_cpu_data.microcode >= 0x08001205)
3361 return true;
3362
3363 if ((boot_cpu_data.microcode >= 0x08001126) &&
3364 (boot_cpu_data.microcode <= 0x080011ff))
3365 return true;
3366
3367 pr_notice("IOMMU not currently supported when SME is active\n");
3368
3369 return false;
3370 }
3371
3372 /****************************************************************************
3373 *
3374 * Early detect code. This code runs at IOMMU detection time in the DMA
3375 * layer. It just looks if there is an IVRS ACPI table to detect AMD
3376 * IOMMUs
3377 *
3378 ****************************************************************************/
amd_iommu_detect(void)3379 int __init amd_iommu_detect(void)
3380 {
3381 int ret;
3382
3383 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
3384 return -ENODEV;
3385
3386 if (!amd_iommu_sme_check())
3387 return -ENODEV;
3388
3389 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
3390 if (ret)
3391 return ret;
3392
3393 amd_iommu_detected = true;
3394 iommu_detected = 1;
3395 x86_init.iommu.iommu_init = amd_iommu_init;
3396
3397 return 1;
3398 }
3399
3400 /****************************************************************************
3401 *
3402 * Parsing functions for the AMD IOMMU specific kernel command line
3403 * options.
3404 *
3405 ****************************************************************************/
3406
parse_amd_iommu_dump(char * str)3407 static int __init parse_amd_iommu_dump(char *str)
3408 {
3409 amd_iommu_dump = true;
3410
3411 return 1;
3412 }
3413
parse_amd_iommu_intr(char * str)3414 static int __init parse_amd_iommu_intr(char *str)
3415 {
3416 for (; *str; ++str) {
3417 if (strncmp(str, "legacy", 6) == 0) {
3418 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
3419 break;
3420 }
3421 if (strncmp(str, "vapic", 5) == 0) {
3422 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
3423 break;
3424 }
3425 }
3426 return 1;
3427 }
3428
parse_amd_iommu_options(char * str)3429 static int __init parse_amd_iommu_options(char *str)
3430 {
3431 if (!str)
3432 return -EINVAL;
3433
3434 while (*str) {
3435 if (strncmp(str, "fullflush", 9) == 0) {
3436 pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n");
3437 iommu_set_dma_strict();
3438 } else if (strncmp(str, "force_enable", 12) == 0) {
3439 amd_iommu_force_enable = true;
3440 } else if (strncmp(str, "off", 3) == 0) {
3441 amd_iommu_disabled = true;
3442 } else if (strncmp(str, "force_isolation", 15) == 0) {
3443 amd_iommu_force_isolation = true;
3444 } else if (strncmp(str, "pgtbl_v1", 8) == 0) {
3445 amd_iommu_pgtable = AMD_IOMMU_V1;
3446 } else if (strncmp(str, "pgtbl_v2", 8) == 0) {
3447 amd_iommu_pgtable = AMD_IOMMU_V2;
3448 } else if (strncmp(str, "irtcachedis", 11) == 0) {
3449 amd_iommu_irtcachedis = true;
3450 } else {
3451 pr_notice("Unknown option - '%s'\n", str);
3452 }
3453
3454 str += strcspn(str, ",");
3455 while (*str == ',')
3456 str++;
3457 }
3458
3459 return 1;
3460 }
3461
parse_ivrs_ioapic(char * str)3462 static int __init parse_ivrs_ioapic(char *str)
3463 {
3464 u32 seg = 0, bus, dev, fn;
3465 int id, i;
3466 u32 devid;
3467
3468 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3469 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3470 goto found;
3471
3472 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3473 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3474 pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
3475 str, id, seg, bus, dev, fn);
3476 goto found;
3477 }
3478
3479 pr_err("Invalid command line: ivrs_ioapic%s\n", str);
3480 return 1;
3481
3482 found:
3483 if (early_ioapic_map_size == EARLY_MAP_SIZE) {
3484 pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
3485 str);
3486 return 1;
3487 }
3488
3489 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3490
3491 cmdline_maps = true;
3492 i = early_ioapic_map_size++;
3493 early_ioapic_map[i].id = id;
3494 early_ioapic_map[i].devid = devid;
3495 early_ioapic_map[i].cmd_line = true;
3496
3497 return 1;
3498 }
3499
parse_ivrs_hpet(char * str)3500 static int __init parse_ivrs_hpet(char *str)
3501 {
3502 u32 seg = 0, bus, dev, fn;
3503 int id, i;
3504 u32 devid;
3505
3506 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3507 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3508 goto found;
3509
3510 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3511 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3512 pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
3513 str, id, seg, bus, dev, fn);
3514 goto found;
3515 }
3516
3517 pr_err("Invalid command line: ivrs_hpet%s\n", str);
3518 return 1;
3519
3520 found:
3521 if (early_hpet_map_size == EARLY_MAP_SIZE) {
3522 pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
3523 str);
3524 return 1;
3525 }
3526
3527 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3528
3529 cmdline_maps = true;
3530 i = early_hpet_map_size++;
3531 early_hpet_map[i].id = id;
3532 early_hpet_map[i].devid = devid;
3533 early_hpet_map[i].cmd_line = true;
3534
3535 return 1;
3536 }
3537
3538 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN)
3539
parse_ivrs_acpihid(char * str)3540 static int __init parse_ivrs_acpihid(char *str)
3541 {
3542 u32 seg = 0, bus, dev, fn;
3543 char *hid, *uid, *p, *addr;
3544 char acpiid[ACPIID_LEN] = {0};
3545 int i;
3546
3547 addr = strchr(str, '@');
3548 if (!addr) {
3549 addr = strchr(str, '=');
3550 if (!addr)
3551 goto not_found;
3552
3553 ++addr;
3554
3555 if (strlen(addr) > ACPIID_LEN)
3556 goto not_found;
3557
3558 if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
3559 sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
3560 pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
3561 str, acpiid, seg, bus, dev, fn);
3562 goto found;
3563 }
3564 goto not_found;
3565 }
3566
3567 /* We have the '@', make it the terminator to get just the acpiid */
3568 *addr++ = 0;
3569
3570 if (strlen(str) > ACPIID_LEN + 1)
3571 goto not_found;
3572
3573 if (sscanf(str, "=%s", acpiid) != 1)
3574 goto not_found;
3575
3576 if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
3577 sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
3578 goto found;
3579
3580 not_found:
3581 pr_err("Invalid command line: ivrs_acpihid%s\n", str);
3582 return 1;
3583
3584 found:
3585 p = acpiid;
3586 hid = strsep(&p, ":");
3587 uid = p;
3588
3589 if (!hid || !(*hid) || !uid) {
3590 pr_err("Invalid command line: hid or uid\n");
3591 return 1;
3592 }
3593
3594 /*
3595 * Ignore leading zeroes after ':', so e.g., AMDI0095:00
3596 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
3597 */
3598 while (*uid == '0' && *(uid + 1))
3599 uid++;
3600
3601 i = early_acpihid_map_size++;
3602 memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
3603 memcpy(early_acpihid_map[i].uid, uid, strlen(uid));
3604 early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3605 early_acpihid_map[i].cmd_line = true;
3606
3607 return 1;
3608 }
3609
3610 __setup("amd_iommu_dump", parse_amd_iommu_dump);
3611 __setup("amd_iommu=", parse_amd_iommu_options);
3612 __setup("amd_iommu_intr=", parse_amd_iommu_intr);
3613 __setup("ivrs_ioapic", parse_ivrs_ioapic);
3614 __setup("ivrs_hpet", parse_ivrs_hpet);
3615 __setup("ivrs_acpihid", parse_ivrs_acpihid);
3616
amd_iommu_v2_supported(void)3617 bool amd_iommu_v2_supported(void)
3618 {
3619 /*
3620 * Since DTE[Mode]=0 is prohibited on SNP-enabled system
3621 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
3622 * setting up IOMMUv1 page table.
3623 */
3624 return amd_iommu_v2_present && !amd_iommu_snp_en;
3625 }
3626 EXPORT_SYMBOL(amd_iommu_v2_supported);
3627
get_amd_iommu(unsigned int idx)3628 struct amd_iommu *get_amd_iommu(unsigned int idx)
3629 {
3630 unsigned int i = 0;
3631 struct amd_iommu *iommu;
3632
3633 for_each_iommu(iommu)
3634 if (i++ == idx)
3635 return iommu;
3636 return NULL;
3637 }
3638
3639 /****************************************************************************
3640 *
3641 * IOMMU EFR Performance Counter support functionality. This code allows
3642 * access to the IOMMU PC functionality.
3643 *
3644 ****************************************************************************/
3645
amd_iommu_pc_get_max_banks(unsigned int idx)3646 u8 amd_iommu_pc_get_max_banks(unsigned int idx)
3647 {
3648 struct amd_iommu *iommu = get_amd_iommu(idx);
3649
3650 if (iommu)
3651 return iommu->max_banks;
3652
3653 return 0;
3654 }
3655 EXPORT_SYMBOL(amd_iommu_pc_get_max_banks);
3656
amd_iommu_pc_supported(void)3657 bool amd_iommu_pc_supported(void)
3658 {
3659 return amd_iommu_pc_present;
3660 }
3661 EXPORT_SYMBOL(amd_iommu_pc_supported);
3662
amd_iommu_pc_get_max_counters(unsigned int idx)3663 u8 amd_iommu_pc_get_max_counters(unsigned int idx)
3664 {
3665 struct amd_iommu *iommu = get_amd_iommu(idx);
3666
3667 if (iommu)
3668 return iommu->max_counters;
3669
3670 return 0;
3671 }
3672 EXPORT_SYMBOL(amd_iommu_pc_get_max_counters);
3673
iommu_pc_get_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value,bool is_write)3674 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
3675 u8 fxn, u64 *value, bool is_write)
3676 {
3677 u32 offset;
3678 u32 max_offset_lim;
3679
3680 /* Make sure the IOMMU PC resource is available */
3681 if (!amd_iommu_pc_present)
3682 return -ENODEV;
3683
3684 /* Check for valid iommu and pc register indexing */
3685 if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))
3686 return -ENODEV;
3687
3688 offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn);
3689
3690 /* Limit the offset to the hw defined mmio region aperture */
3691 max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |
3692 (iommu->max_counters << 8) | 0x28);
3693 if ((offset < MMIO_CNTR_REG_OFFSET) ||
3694 (offset > max_offset_lim))
3695 return -EINVAL;
3696
3697 if (is_write) {
3698 u64 val = *value & GENMASK_ULL(47, 0);
3699
3700 writel((u32)val, iommu->mmio_base + offset);
3701 writel((val >> 32), iommu->mmio_base + offset + 4);
3702 } else {
3703 *value = readl(iommu->mmio_base + offset + 4);
3704 *value <<= 32;
3705 *value |= readl(iommu->mmio_base + offset);
3706 *value &= GENMASK_ULL(47, 0);
3707 }
3708
3709 return 0;
3710 }
3711
amd_iommu_pc_get_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3712 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3713 {
3714 if (!iommu)
3715 return -EINVAL;
3716
3717 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);
3718 }
3719
amd_iommu_pc_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3720 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3721 {
3722 if (!iommu)
3723 return -EINVAL;
3724
3725 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);
3726 }
3727
3728 #ifdef CONFIG_AMD_MEM_ENCRYPT
amd_iommu_snp_enable(void)3729 int amd_iommu_snp_enable(void)
3730 {
3731 /*
3732 * The SNP support requires that IOMMU must be enabled, and is
3733 * not configured in the passthrough mode.
3734 */
3735 if (no_iommu || iommu_default_passthrough()) {
3736 pr_err("SNP: IOMMU is disabled or configured in passthrough mode, SNP cannot be supported");
3737 return -EINVAL;
3738 }
3739
3740 /*
3741 * Prevent enabling SNP after IOMMU_ENABLED state because this process
3742 * affect how IOMMU driver sets up data structures and configures
3743 * IOMMU hardware.
3744 */
3745 if (init_state > IOMMU_ENABLED) {
3746 pr_err("SNP: Too late to enable SNP for IOMMU.\n");
3747 return -EINVAL;
3748 }
3749
3750 amd_iommu_snp_en = check_feature_on_all_iommus(FEATURE_SNP);
3751 if (!amd_iommu_snp_en)
3752 return -EINVAL;
3753
3754 pr_info("SNP enabled\n");
3755
3756 /* Enforce IOMMU v1 pagetable when SNP is enabled. */
3757 if (amd_iommu_pgtable != AMD_IOMMU_V1) {
3758 pr_warn("Force to using AMD IOMMU v1 page table due to SNP\n");
3759 amd_iommu_pgtable = AMD_IOMMU_V1;
3760 }
3761
3762 return 0;
3763 }
3764 #endif
3765