1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7 #define pr_fmt(fmt) "riscv-imsic: " fmt
8 #include <linux/acpi.h>
9 #include <linux/cpu.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/irq.h>
13 #include <linux/irqchip.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqchip/riscv-imsic.h>
16 #include <linux/module.h>
17 #include <linux/pci.h>
18 #include <linux/spinlock.h>
19 #include <linux/smp.h>
20
21 #include "irq-riscv-imsic-state.h"
22
23 static int imsic_parent_irq;
24
25 #ifdef CONFIG_SMP
imsic_ipi_send(unsigned int cpu)26 static void imsic_ipi_send(unsigned int cpu)
27 {
28 struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu);
29
30 writel_relaxed(IMSIC_IPI_ID, local->msi_va);
31 }
32
imsic_ipi_starting_cpu(void)33 static void imsic_ipi_starting_cpu(void)
34 {
35 /* Enable IPIs for current CPU. */
36 __imsic_id_set_enable(IMSIC_IPI_ID);
37 }
38
imsic_ipi_dying_cpu(void)39 static void imsic_ipi_dying_cpu(void)
40 {
41 /* Disable IPIs for current CPU. */
42 __imsic_id_clear_enable(IMSIC_IPI_ID);
43 }
44
imsic_ipi_domain_init(void)45 static int __init imsic_ipi_domain_init(void)
46 {
47 int virq;
48
49 /* Create IMSIC IPI multiplexing */
50 virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
51 if (virq <= 0)
52 return virq < 0 ? virq : -ENOMEM;
53
54 /* Set vIRQ range */
55 riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
56
57 /* Announce that IMSIC is providing IPIs */
58 pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
59
60 return 0;
61 }
62 #else
imsic_ipi_starting_cpu(void)63 static void imsic_ipi_starting_cpu(void) { }
imsic_ipi_dying_cpu(void)64 static void imsic_ipi_dying_cpu(void) { }
imsic_ipi_domain_init(void)65 static int __init imsic_ipi_domain_init(void) { return 0; }
66 #endif
67
68 /*
69 * To handle an interrupt, we read the TOPEI CSR and write zero in one
70 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to
71 * Linux interrupt number and let Linux IRQ subsystem handle it.
72 */
imsic_handle_irq(struct irq_desc * desc)73 static void imsic_handle_irq(struct irq_desc *desc)
74 {
75 struct irq_chip *chip = irq_desc_get_chip(desc);
76 int err, cpu = smp_processor_id();
77 struct imsic_vector *vec;
78 unsigned long local_id;
79
80 /*
81 * Process pending local synchronization instead of waiting
82 * for per-CPU local timer to expire.
83 */
84 imsic_local_sync_all(false);
85
86 chained_irq_enter(chip, desc);
87
88 while ((local_id = csr_swap(CSR_TOPEI, 0))) {
89 local_id >>= TOPEI_ID_SHIFT;
90
91 if (local_id == IMSIC_IPI_ID) {
92 if (IS_ENABLED(CONFIG_SMP))
93 ipi_mux_process();
94 continue;
95 }
96
97 if (unlikely(!imsic->base_domain))
98 continue;
99
100 vec = imsic_vector_from_local_id(cpu, local_id);
101 if (!vec) {
102 pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id);
103 continue;
104 }
105
106 err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq);
107 if (unlikely(err))
108 pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq);
109 }
110
111 chained_irq_exit(chip, desc);
112 }
113
imsic_starting_cpu(unsigned int cpu)114 static int imsic_starting_cpu(unsigned int cpu)
115 {
116 /* Mark per-CPU IMSIC state as online */
117 imsic_state_online();
118
119 /* Enable per-CPU parent interrupt */
120 enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
121
122 /* Setup IPIs */
123 imsic_ipi_starting_cpu();
124
125 /*
126 * Interrupts identities might have been enabled/disabled while
127 * this CPU was not running so sync-up local enable/disable state.
128 */
129 imsic_local_sync_all(true);
130
131 /* Enable local interrupt delivery */
132 imsic_local_delivery(true);
133
134 return 0;
135 }
136
imsic_dying_cpu(unsigned int cpu)137 static int imsic_dying_cpu(unsigned int cpu)
138 {
139 /* Cleanup IPIs */
140 imsic_ipi_dying_cpu();
141
142 /* Mark per-CPU IMSIC state as offline */
143 imsic_state_offline();
144
145 return 0;
146 }
147
imsic_early_probe(struct fwnode_handle * fwnode)148 static int __init imsic_early_probe(struct fwnode_handle *fwnode)
149 {
150 struct irq_domain *domain;
151 int rc;
152
153 /* Find parent domain and register chained handler */
154 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
155 if (!domain) {
156 pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
157 return -ENOENT;
158 }
159 imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
160 if (!imsic_parent_irq) {
161 pr_err("%pfwP: Failed to create INTC mapping\n", fwnode);
162 return -ENOENT;
163 }
164
165 /* Initialize IPI domain */
166 rc = imsic_ipi_domain_init();
167 if (rc) {
168 pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode);
169 return rc;
170 }
171
172 /* Setup chained handler to the parent domain interrupt */
173 irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq);
174
175 /*
176 * Setup cpuhp state (must be done after setting imsic_parent_irq)
177 *
178 * Don't disable per-CPU IMSIC file when CPU goes offline
179 * because this affects IPI and the masking/unmasking of
180 * virtual IPIs is done via generic IPI-Mux
181 */
182 cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
183 imsic_starting_cpu, imsic_dying_cpu);
184
185 return 0;
186 }
187
imsic_early_dt_init(struct device_node * node,struct device_node * parent)188 static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
189 {
190 struct fwnode_handle *fwnode = &node->fwnode;
191 int rc;
192
193 /* Setup IMSIC state */
194 rc = imsic_setup_state(fwnode, NULL);
195 if (rc) {
196 pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc);
197 return rc;
198 }
199
200 /* Do early setup of IPIs */
201 rc = imsic_early_probe(fwnode);
202 if (rc)
203 return rc;
204
205 /* Ensure that OF platform device gets probed */
206 of_node_clear_flag(node, OF_POPULATED);
207 return 0;
208 }
209
210 IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init);
211
212 #ifdef CONFIG_ACPI
213
214 static struct fwnode_handle *imsic_acpi_fwnode;
215
imsic_acpi_get_fwnode(struct device * dev)216 struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev)
217 {
218 return imsic_acpi_fwnode;
219 }
220
imsic_early_acpi_init(union acpi_subtable_headers * header,const unsigned long end)221 static int __init imsic_early_acpi_init(union acpi_subtable_headers *header,
222 const unsigned long end)
223 {
224 struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header;
225 int rc;
226
227 imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic");
228 if (!imsic_acpi_fwnode) {
229 pr_err("unable to allocate IMSIC FW node\n");
230 return -ENOMEM;
231 }
232
233 /* Setup IMSIC state */
234 rc = imsic_setup_state(imsic_acpi_fwnode, imsic);
235 if (rc) {
236 pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc);
237 return rc;
238 }
239
240 /* Do early setup of IMSIC state and IPIs */
241 rc = imsic_early_probe(imsic_acpi_fwnode);
242 if (rc) {
243 irq_domain_free_fwnode(imsic_acpi_fwnode);
244 imsic_acpi_fwnode = NULL;
245 return rc;
246 }
247
248 rc = imsic_platform_acpi_probe(imsic_acpi_fwnode);
249
250 #ifdef CONFIG_PCI
251 if (!rc)
252 pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode);
253 #endif
254
255 if (rc)
256 pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n",
257 imsic_acpi_fwnode, rc);
258
259 /*
260 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can
261 * continue to work. So, no need to return failure. This is similar to
262 * DT where IPI works but MSI probe fails for some reason.
263 */
264 return 0;
265 }
266
267 IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL,
268 1, imsic_early_acpi_init);
269 #endif
270