1 /*
2 * Copyright (c) 2015-2021, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8
9 #include <arch.h>
10 #include <arch_helpers.h>
11 #include <common/debug.h>
12 #include <common/interrupt_props.h>
13 #include <drivers/arm/gicv3.h>
14 #include <lib/spinlock.h>
15
16 #include "gicv3_private.h"
17
18 const gicv3_driver_data_t *gicv3_driver_data;
19
20 /*
21 * Spinlock to guard registers needing read-modify-write. APIs protected by this
22 * spinlock are used either at boot time (when only a single CPU is active), or
23 * when the system is fully coherent.
24 */
25 static spinlock_t gic_lock;
26
27 /*
28 * Redistributor power operations are weakly bound so that they can be
29 * overridden
30 */
31 #pragma weak gicv3_rdistif_off
32 #pragma weak gicv3_rdistif_on
33
34 /* Check interrupt ID for SGI/(E)PPI and (E)SPIs */
35 static bool is_sgi_ppi(unsigned int id);
36
37 /*
38 * Helper macros to save and restore GICR and GICD registers
39 * corresponding to their numbers to and from the context
40 */
41 #define RESTORE_GICR_REG(base, ctx, name, i) \
42 gicr_write_##name((base), (i), (ctx)->gicr_##name[(i)])
43
44 #define SAVE_GICR_REG(base, ctx, name, i) \
45 (ctx)->gicr_##name[(i)] = gicr_read_##name((base), (i))
46
47 /* Helper macros to save and restore GICD registers to and from the context */
48 #define RESTORE_GICD_REGS(base, ctx, intr_num, reg, REG) \
49 do { \
50 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
51 int_id += (1U << REG##R_SHIFT)) { \
52 gicd_write_##reg((base), int_id, \
53 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
54 REG##R_SHIFT]); \
55 } \
56 } while (false)
57
58 #define SAVE_GICD_REGS(base, ctx, intr_num, reg, REG) \
59 do { \
60 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
61 int_id += (1U << REG##R_SHIFT)) { \
62 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
63 REG##R_SHIFT] = gicd_read_##reg((base), int_id); \
64 } \
65 } while (false)
66
67 #if GIC_EXT_INTID
68 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
69 do { \
70 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
71 int_id += (1U << REG##R_SHIFT)) { \
72 gicd_write_##reg((base), int_id, \
73 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - \
74 round_up(TOTAL_SPI_INTR_NUM, 1U << REG##R_SHIFT)))\
75 >> REG##R_SHIFT]); \
76 } \
77 } while (false)
78
79 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
80 do { \
81 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
82 int_id += (1U << REG##R_SHIFT)) { \
83 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - \
84 round_up(TOTAL_SPI_INTR_NUM, 1U << REG##R_SHIFT)))\
85 >> REG##R_SHIFT] = gicd_read_##reg((base), int_id);\
86 } \
87 } while (false)
88 #else
89 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG)
90 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG)
91 #endif /* GIC_EXT_INTID */
92
93 /*******************************************************************************
94 * This function initialises the ARM GICv3 driver in EL3 with provided platform
95 * inputs.
96 ******************************************************************************/
gicv3_driver_init(const gicv3_driver_data_t * plat_driver_data)97 void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
98 {
99 unsigned int gic_version;
100 unsigned int gicv2_compat;
101
102 assert(plat_driver_data != NULL);
103 assert(plat_driver_data->gicd_base != 0U);
104 assert(plat_driver_data->rdistif_num != 0U);
105 assert(plat_driver_data->rdistif_base_addrs != NULL);
106
107 assert(IS_IN_EL3());
108
109 assert((plat_driver_data->interrupt_props_num != 0U) ?
110 (plat_driver_data->interrupt_props != NULL) : 1);
111
112 /* Check for system register support */
113 #ifndef __aarch64__
114 assert((read_id_pfr1() &
115 (ID_PFR1_GIC_MASK << ID_PFR1_GIC_SHIFT)) != 0U);
116 #else
117 assert((read_id_aa64pfr0_el1() &
118 (ID_AA64PFR0_GIC_MASK << ID_AA64PFR0_GIC_SHIFT)) != 0U);
119 #endif /* !__aarch64__ */
120
121 gic_version = gicd_read_pidr2(plat_driver_data->gicd_base);
122 gic_version >>= PIDR2_ARCH_REV_SHIFT;
123 gic_version &= PIDR2_ARCH_REV_MASK;
124
125 /* Check GIC version */
126 #if GIC_ENABLE_V4_EXTN
127 assert(gic_version == ARCH_REV_GICV4);
128
129 /* GICv4 supports Direct Virtual LPI injection */
130 assert((gicd_read_typer(plat_driver_data->gicd_base)
131 & TYPER_DVIS) != 0);
132 #else
133 assert(gic_version == ARCH_REV_GICV3);
134 #endif
135 /*
136 * Find out whether the GIC supports the GICv2 compatibility mode.
137 * The ARE_S bit resets to 0 if supported
138 */
139 gicv2_compat = gicd_read_ctlr(plat_driver_data->gicd_base);
140 gicv2_compat >>= CTLR_ARE_S_SHIFT;
141 gicv2_compat = gicv2_compat & CTLR_ARE_S_MASK;
142
143 if (plat_driver_data->gicr_base != 0U) {
144 /*
145 * Find the base address of each implemented Redistributor interface.
146 * The number of interfaces should be equal to the number of CPUs in the
147 * system. The memory for saving these addresses has to be allocated by
148 * the platform port
149 */
150 gicv3_rdistif_base_addrs_probe(plat_driver_data->rdistif_base_addrs,
151 plat_driver_data->rdistif_num,
152 plat_driver_data->gicr_base,
153 plat_driver_data->mpidr_to_core_pos);
154 #if !HW_ASSISTED_COHERENCY
155 /*
156 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
157 */
158 flush_dcache_range((uintptr_t)(plat_driver_data->rdistif_base_addrs),
159 plat_driver_data->rdistif_num *
160 sizeof(*(plat_driver_data->rdistif_base_addrs)));
161 #endif
162 }
163 gicv3_driver_data = plat_driver_data;
164
165 /*
166 * The GIC driver data is initialized by the primary CPU with caches
167 * enabled. When the secondary CPU boots up, it initializes the
168 * GICC/GICR interface with the caches disabled. Hence flush the
169 * driver data to ensure coherency. This is not required if the
170 * platform has HW_ASSISTED_COHERENCY enabled.
171 */
172 #if !HW_ASSISTED_COHERENCY
173 flush_dcache_range((uintptr_t)&gicv3_driver_data,
174 sizeof(gicv3_driver_data));
175 flush_dcache_range((uintptr_t)gicv3_driver_data,
176 sizeof(*gicv3_driver_data));
177 #endif
178 INFO("GICv%u with%s legacy support detected.\n", gic_version,
179 (gicv2_compat == 0U) ? "" : "out");
180 INFO("ARM GICv%u driver initialized in EL3\n", gic_version);
181 }
182
183 /*******************************************************************************
184 * This function initialises the GIC distributor interface based upon the data
185 * provided by the platform while initialising the driver.
186 ******************************************************************************/
gicv3_distif_init(void)187 void __init gicv3_distif_init(void)
188 {
189 unsigned int bitmap;
190
191 assert(gicv3_driver_data != NULL);
192 assert(gicv3_driver_data->gicd_base != 0U);
193
194 assert(IS_IN_EL3());
195
196 /*
197 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
198 * the ARE_S bit. The Distributor might generate a system error
199 * otherwise.
200 */
201 gicd_clr_ctlr(gicv3_driver_data->gicd_base,
202 CTLR_ENABLE_G0_BIT |
203 CTLR_ENABLE_G1S_BIT |
204 CTLR_ENABLE_G1NS_BIT,
205 RWP_TRUE);
206
207 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
208 gicd_set_ctlr(gicv3_driver_data->gicd_base,
209 CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
210
211 /* Set the default attribute of all (E)SPIs */
212 gicv3_spis_config_defaults(gicv3_driver_data->gicd_base);
213
214 bitmap = gicv3_secure_spis_config_props(
215 gicv3_driver_data->gicd_base,
216 gicv3_driver_data->interrupt_props,
217 gicv3_driver_data->interrupt_props_num);
218
219 /* Enable the secure (E)SPIs now that they have been configured */
220 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
221 }
222
223 /*******************************************************************************
224 * This function initialises the GIC Redistributor interface of the calling CPU
225 * (identified by the 'proc_num' parameter) based upon the data provided by the
226 * platform while initialising the driver.
227 ******************************************************************************/
gicv3_rdistif_init(unsigned int proc_num)228 void gicv3_rdistif_init(unsigned int proc_num)
229 {
230 uintptr_t gicr_base;
231 unsigned int bitmap;
232 uint32_t ctlr;
233
234 assert(gicv3_driver_data != NULL);
235 assert(proc_num < gicv3_driver_data->rdistif_num);
236 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
237 assert(gicv3_driver_data->gicd_base != 0U);
238
239 ctlr = gicd_read_ctlr(gicv3_driver_data->gicd_base);
240 assert((ctlr & CTLR_ARE_S_BIT) != 0U);
241
242 assert(IS_IN_EL3());
243
244 /* Power on redistributor */
245 gicv3_rdistif_on(proc_num);
246
247 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
248 assert(gicr_base != 0U);
249
250 /* Set the default attribute of all SGIs and (E)PPIs */
251 gicv3_ppi_sgi_config_defaults(gicr_base);
252
253 bitmap = gicv3_secure_ppi_sgi_config_props(gicr_base,
254 gicv3_driver_data->interrupt_props,
255 gicv3_driver_data->interrupt_props_num);
256
257 /* Enable interrupt groups as required, if not already */
258 if ((ctlr & bitmap) != bitmap) {
259 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
260 }
261 }
262
263 /*******************************************************************************
264 * Functions to perform power operations on GIC Redistributor
265 ******************************************************************************/
gicv3_rdistif_off(unsigned int proc_num)266 void gicv3_rdistif_off(unsigned int proc_num)
267 {
268 }
269
gicv3_rdistif_on(unsigned int proc_num)270 void gicv3_rdistif_on(unsigned int proc_num)
271 {
272 }
273
274 /*******************************************************************************
275 * This function enables the GIC CPU interface of the calling CPU using only
276 * system register accesses.
277 ******************************************************************************/
gicv3_cpuif_enable(unsigned int proc_num)278 void gicv3_cpuif_enable(unsigned int proc_num)
279 {
280 uintptr_t gicr_base;
281 u_register_t scr_el3;
282 unsigned int icc_sre_el3;
283
284 assert(gicv3_driver_data != NULL);
285 assert(proc_num < gicv3_driver_data->rdistif_num);
286 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
287 assert(IS_IN_EL3());
288
289 /* Mark the connected core as awake */
290 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
291 gicv3_rdistif_mark_core_awake(gicr_base);
292
293 /* Disable the legacy interrupt bypass */
294 icc_sre_el3 = ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT;
295
296 /*
297 * Enable system register access for EL3 and allow lower exception
298 * levels to configure the same for themselves. If the legacy mode is
299 * not supported, the SRE bit is RAO/WI
300 */
301 icc_sre_el3 |= (ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT);
302 write_icc_sre_el3(read_icc_sre_el3() | icc_sre_el3);
303
304 scr_el3 = read_scr_el3();
305
306 /*
307 * Switch to NS state to write Non secure ICC_SRE_EL1 and
308 * ICC_SRE_EL2 registers.
309 */
310 write_scr_el3(scr_el3 | SCR_NS_BIT);
311 isb();
312
313 write_icc_sre_el2(read_icc_sre_el2() | icc_sre_el3);
314 write_icc_sre_el1(ICC_SRE_SRE_BIT);
315 isb();
316
317 /* Switch to secure state. */
318 write_scr_el3(scr_el3 & (~SCR_NS_BIT));
319 isb();
320
321 /* Write the secure ICC_SRE_EL1 register */
322 write_icc_sre_el1(ICC_SRE_SRE_BIT);
323 isb();
324
325 /* Program the idle priority in the PMR */
326 write_icc_pmr_el1(GIC_PRI_MASK);
327
328 /* Enable Group0 interrupts */
329 write_icc_igrpen0_el1(IGRPEN1_EL1_ENABLE_G0_BIT);
330
331 /* Enable Group1 Secure interrupts */
332 write_icc_igrpen1_el3(read_icc_igrpen1_el3() |
333 IGRPEN1_EL3_ENABLE_G1S_BIT);
334 isb();
335 /* Add DSB to ensure visibility of System register writes */
336 dsb();
337 }
338
339 /*******************************************************************************
340 * This function disables the GIC CPU interface of the calling CPU using
341 * only system register accesses.
342 ******************************************************************************/
gicv3_cpuif_disable(unsigned int proc_num)343 void gicv3_cpuif_disable(unsigned int proc_num)
344 {
345 uintptr_t gicr_base;
346
347 assert(gicv3_driver_data != NULL);
348 assert(proc_num < gicv3_driver_data->rdistif_num);
349 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
350
351 assert(IS_IN_EL3());
352
353 /* Disable legacy interrupt bypass */
354 write_icc_sre_el3(read_icc_sre_el3() |
355 (ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT));
356
357 /* Disable Group0 interrupts */
358 write_icc_igrpen0_el1(read_icc_igrpen0_el1() &
359 ~IGRPEN1_EL1_ENABLE_G0_BIT);
360
361 /* Disable Group1 Secure and Non-Secure interrupts */
362 write_icc_igrpen1_el3(read_icc_igrpen1_el3() &
363 ~(IGRPEN1_EL3_ENABLE_G1NS_BIT |
364 IGRPEN1_EL3_ENABLE_G1S_BIT));
365
366 /* Synchronise accesses to group enable registers */
367 isb();
368 /* Add DSB to ensure visibility of System register writes */
369 dsb();
370
371 /* Mark the connected core as asleep */
372 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
373 assert(gicr_base != 0U);
374 gicv3_rdistif_mark_core_asleep(gicr_base);
375 }
376
377 /*******************************************************************************
378 * This function returns the id of the highest priority pending interrupt at
379 * the GIC cpu interface.
380 ******************************************************************************/
gicv3_get_pending_interrupt_id(void)381 unsigned int gicv3_get_pending_interrupt_id(void)
382 {
383 unsigned int id;
384
385 assert(IS_IN_EL3());
386 id = (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
387
388 /*
389 * If the ID is special identifier corresponding to G1S or G1NS
390 * interrupt, then read the highest pending group 1 interrupt.
391 */
392 if ((id == PENDING_G1S_INTID) || (id == PENDING_G1NS_INTID)) {
393 return (uint32_t)read_icc_hppir1_el1() & HPPIR1_EL1_INTID_MASK;
394 }
395
396 return id;
397 }
398
399 /*******************************************************************************
400 * This function returns the type of the highest priority pending interrupt at
401 * the GIC cpu interface. The return values can be one of the following :
402 * PENDING_G1S_INTID : The interrupt type is secure Group 1.
403 * PENDING_G1NS_INTID : The interrupt type is non secure Group 1.
404 * 0 - 1019 : The interrupt type is secure Group 0.
405 * GIC_SPURIOUS_INTERRUPT : there is no pending interrupt with
406 * sufficient priority to be signaled
407 ******************************************************************************/
gicv3_get_pending_interrupt_type(void)408 unsigned int gicv3_get_pending_interrupt_type(void)
409 {
410 assert(IS_IN_EL3());
411 return (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
412 }
413
414 /*******************************************************************************
415 * This function returns the type of the interrupt id depending upon the group
416 * this interrupt has been configured under by the interrupt controller i.e.
417 * group0 or group1 Secure / Non Secure. The return value can be one of the
418 * following :
419 * INTR_GROUP0 : The interrupt type is a Secure Group 0 interrupt
420 * INTR_GROUP1S : The interrupt type is a Secure Group 1 secure interrupt
421 * INTR_GROUP1NS: The interrupt type is a Secure Group 1 non secure
422 * interrupt.
423 ******************************************************************************/
gicv3_get_interrupt_type(unsigned int id,unsigned int proc_num)424 unsigned int gicv3_get_interrupt_type(unsigned int id, unsigned int proc_num)
425 {
426 unsigned int igroup, grpmodr;
427 uintptr_t gicr_base;
428
429 assert(IS_IN_EL3());
430 assert(gicv3_driver_data != NULL);
431
432 /* Ensure the parameters are valid */
433 assert((id < PENDING_G1S_INTID) || (id >= MIN_LPI_ID));
434 assert(proc_num < gicv3_driver_data->rdistif_num);
435
436 /* All LPI interrupts are Group 1 non secure */
437 if (id >= MIN_LPI_ID) {
438 return INTR_GROUP1NS;
439 }
440
441 /* Check interrupt ID */
442 if (is_sgi_ppi(id)) {
443 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
444 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
445 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
446 igroup = gicr_get_igroupr(gicr_base, id);
447 grpmodr = gicr_get_igrpmodr(gicr_base, id);
448 } else {
449 /* SPIs: 32-1019, ESPIs: 4096-5119 */
450 assert(gicv3_driver_data->gicd_base != 0U);
451 igroup = gicd_get_igroupr(gicv3_driver_data->gicd_base, id);
452 grpmodr = gicd_get_igrpmodr(gicv3_driver_data->gicd_base, id);
453 }
454
455 /*
456 * If the IGROUP bit is set, then it is a Group 1 Non secure
457 * interrupt
458 */
459 if (igroup != 0U) {
460 return INTR_GROUP1NS;
461 }
462
463 /* If the GRPMOD bit is set, then it is a Group 1 Secure interrupt */
464 if (grpmodr != 0U) {
465 return INTR_GROUP1S;
466 }
467
468 /* Else it is a Group 0 Secure interrupt */
469 return INTR_GROUP0;
470 }
471
472 /*****************************************************************************
473 * Function to save and disable the GIC ITS register context. The power
474 * management of GIC ITS is implementation-defined and this function doesn't
475 * save any memory structures required to support ITS. As the sequence to save
476 * this state is implementation defined, it should be executed in platform
477 * specific code. Calling this function alone and then powering down the GIC and
478 * ITS without implementing the aforementioned platform specific code will
479 * corrupt the ITS state.
480 *
481 * This function must be invoked after the GIC CPU interface is disabled.
482 *****************************************************************************/
gicv3_its_save_disable(uintptr_t gits_base,gicv3_its_ctx_t * const its_ctx)483 void gicv3_its_save_disable(uintptr_t gits_base,
484 gicv3_its_ctx_t * const its_ctx)
485 {
486 unsigned int i;
487
488 assert(gicv3_driver_data != NULL);
489 assert(IS_IN_EL3());
490 assert(its_ctx != NULL);
491 assert(gits_base != 0U);
492
493 its_ctx->gits_ctlr = gits_read_ctlr(gits_base);
494
495 /* Disable the ITS */
496 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
497
498 /* Wait for quiescent state */
499 gits_wait_for_quiescent_bit(gits_base);
500
501 its_ctx->gits_cbaser = gits_read_cbaser(gits_base);
502 its_ctx->gits_cwriter = gits_read_cwriter(gits_base);
503
504 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
505 its_ctx->gits_baser[i] = gits_read_baser(gits_base, i);
506 }
507 }
508
509 /*****************************************************************************
510 * Function to restore the GIC ITS register context. The power
511 * management of GIC ITS is implementation defined and this function doesn't
512 * restore any memory structures required to support ITS. The assumption is
513 * that these structures are in memory and are retained during system suspend.
514 *
515 * This must be invoked before the GIC CPU interface is enabled.
516 *****************************************************************************/
gicv3_its_restore(uintptr_t gits_base,const gicv3_its_ctx_t * const its_ctx)517 void gicv3_its_restore(uintptr_t gits_base,
518 const gicv3_its_ctx_t * const its_ctx)
519 {
520 unsigned int i;
521
522 assert(gicv3_driver_data != NULL);
523 assert(IS_IN_EL3());
524 assert(its_ctx != NULL);
525 assert(gits_base != 0U);
526
527 /* Assert that the GITS is disabled and quiescent */
528 assert((gits_read_ctlr(gits_base) & GITS_CTLR_ENABLED_BIT) == 0U);
529 assert((gits_read_ctlr(gits_base) & GITS_CTLR_QUIESCENT_BIT) != 0U);
530
531 gits_write_cbaser(gits_base, its_ctx->gits_cbaser);
532 gits_write_cwriter(gits_base, its_ctx->gits_cwriter);
533
534 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
535 gits_write_baser(gits_base, i, its_ctx->gits_baser[i]);
536 }
537
538 /* Restore the ITS CTLR but leave the ITS disabled */
539 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
540 }
541
542 /*****************************************************************************
543 * Function to save the GIC Redistributor register context. This function
544 * must be invoked after CPU interface disable and prior to Distributor save.
545 *****************************************************************************/
gicv3_rdistif_save(unsigned int proc_num,gicv3_redist_ctx_t * const rdist_ctx)546 void gicv3_rdistif_save(unsigned int proc_num,
547 gicv3_redist_ctx_t * const rdist_ctx)
548 {
549 uintptr_t gicr_base;
550 unsigned int i, ppi_regs_num, regs_num;
551
552 assert(gicv3_driver_data != NULL);
553 assert(proc_num < gicv3_driver_data->rdistif_num);
554 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
555 assert(IS_IN_EL3());
556 assert(rdist_ctx != NULL);
557
558 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
559
560 #if GIC_EXT_INTID
561 /* Calculate number of PPI registers */
562 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
563 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
564 /* All other values except PPInum [0-2] are reserved */
565 if (ppi_regs_num > 3U) {
566 ppi_regs_num = 1U;
567 }
568 #else
569 ppi_regs_num = 1U;
570 #endif
571 /*
572 * Wait for any write to GICR_CTLR to complete before trying to save any
573 * state.
574 */
575 gicr_wait_for_pending_write(gicr_base);
576
577 rdist_ctx->gicr_ctlr = gicr_read_ctlr(gicr_base);
578
579 rdist_ctx->gicr_propbaser = gicr_read_propbaser(gicr_base);
580 rdist_ctx->gicr_pendbaser = gicr_read_pendbaser(gicr_base);
581
582 /* 32 interrupt IDs per register */
583 for (i = 0U; i < ppi_regs_num; ++i) {
584 SAVE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
585 SAVE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
586 SAVE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
587 SAVE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
588 SAVE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
589 }
590
591 /* 16 interrupt IDs per GICR_ICFGR register */
592 regs_num = ppi_regs_num << 1;
593 for (i = 0U; i < regs_num; ++i) {
594 SAVE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
595 }
596
597 rdist_ctx->gicr_nsacr = gicr_read_nsacr(gicr_base);
598
599 /* 4 interrupt IDs per GICR_IPRIORITYR register */
600 regs_num = ppi_regs_num << 3;
601 for (i = 0U; i < regs_num; ++i) {
602 rdist_ctx->gicr_ipriorityr[i] =
603 gicr_ipriorityr_read(gicr_base, i);
604 }
605
606 /*
607 * Call the pre-save hook that implements the IMP DEF sequence that may
608 * be required on some GIC implementations. As this may need to access
609 * the Redistributor registers, we pass it proc_num.
610 */
611 gicv3_distif_pre_save(proc_num);
612 }
613
614 /*****************************************************************************
615 * Function to restore the GIC Redistributor register context. We disable
616 * LPI and per-cpu interrupts before we start restore of the Redistributor.
617 * This function must be invoked after Distributor restore but prior to
618 * CPU interface enable. The pending and active interrupts are restored
619 * after the interrupts are fully configured and enabled.
620 *****************************************************************************/
gicv3_rdistif_init_restore(unsigned int proc_num,const gicv3_redist_ctx_t * const rdist_ctx)621 void gicv3_rdistif_init_restore(unsigned int proc_num,
622 const gicv3_redist_ctx_t * const rdist_ctx)
623 {
624 uintptr_t gicr_base;
625 unsigned int i, ppi_regs_num, regs_num;
626
627 assert(gicv3_driver_data != NULL);
628 assert(proc_num < gicv3_driver_data->rdistif_num);
629 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
630 assert(IS_IN_EL3());
631 assert(rdist_ctx != NULL);
632
633 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
634
635 #if GIC_EXT_INTID
636 /* Calculate number of PPI registers */
637 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
638 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
639 /* All other values except PPInum [0-2] are reserved */
640 if (ppi_regs_num > 3U) {
641 ppi_regs_num = 1U;
642 }
643 #else
644 ppi_regs_num = 1U;
645 #endif
646 /* Power on redistributor */
647 gicv3_rdistif_on(proc_num);
648
649 /*
650 * Call the post-restore hook that implements the IMP DEF sequence that
651 * may be required on some GIC implementations. As this may need to
652 * access the Redistributor registers, we pass it proc_num.
653 */
654 gicv3_distif_post_restore(proc_num);
655
656 /*
657 * Disable all SGIs (imp. def.)/(E)PPIs before configuring them.
658 * This is a more scalable approach as it avoids clearing the enable
659 * bits in the GICD_CTLR.
660 */
661 for (i = 0U; i < ppi_regs_num; ++i) {
662 gicr_write_icenabler(gicr_base, i, ~0U);
663 }
664
665 /* Wait for pending writes to GICR_ICENABLER */
666 gicr_wait_for_pending_write(gicr_base);
667
668 /*
669 * Disable the LPIs to avoid unpredictable behavior when writing to
670 * GICR_PROPBASER and GICR_PENDBASER.
671 */
672 gicr_write_ctlr(gicr_base,
673 rdist_ctx->gicr_ctlr & ~(GICR_CTLR_EN_LPIS_BIT));
674
675 /* Restore registers' content */
676 gicr_write_propbaser(gicr_base, rdist_ctx->gicr_propbaser);
677 gicr_write_pendbaser(gicr_base, rdist_ctx->gicr_pendbaser);
678
679 /* 32 interrupt IDs per register */
680 for (i = 0U; i < ppi_regs_num; ++i) {
681 RESTORE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
682 RESTORE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
683 }
684
685 /* 4 interrupt IDs per GICR_IPRIORITYR register */
686 regs_num = ppi_regs_num << 3;
687 for (i = 0U; i < regs_num; ++i) {
688 gicr_ipriorityr_write(gicr_base, i,
689 rdist_ctx->gicr_ipriorityr[i]);
690 }
691
692 /* 16 interrupt IDs per GICR_ICFGR register */
693 regs_num = ppi_regs_num << 1;
694 for (i = 0U; i < regs_num; ++i) {
695 RESTORE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
696 }
697
698 gicr_write_nsacr(gicr_base, rdist_ctx->gicr_nsacr);
699
700 /* Restore after group and priorities are set.
701 * 32 interrupt IDs per register
702 */
703 for (i = 0U; i < ppi_regs_num; ++i) {
704 RESTORE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
705 RESTORE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
706 }
707
708 /*
709 * Wait for all writes to the Distributor to complete before enabling
710 * the SGI and (E)PPIs.
711 */
712 gicr_wait_for_upstream_pending_write(gicr_base);
713
714 /* 32 interrupt IDs per GICR_ISENABLER register */
715 for (i = 0U; i < ppi_regs_num; ++i) {
716 RESTORE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
717 }
718
719 /*
720 * Restore GICR_CTLR.Enable_LPIs bit and wait for pending writes in case
721 * the first write to GICR_CTLR was still in flight (this write only
722 * restores GICR_CTLR.Enable_LPIs and no waiting is required for this
723 * bit).
724 */
725 gicr_write_ctlr(gicr_base, rdist_ctx->gicr_ctlr);
726 gicr_wait_for_pending_write(gicr_base);
727 }
728
729 /*****************************************************************************
730 * Function to save the GIC Distributor register context. This function
731 * must be invoked after CPU interface disable and Redistributor save.
732 *****************************************************************************/
gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)733 void gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)
734 {
735 assert(gicv3_driver_data != NULL);
736 assert(gicv3_driver_data->gicd_base != 0U);
737 assert(IS_IN_EL3());
738 assert(dist_ctx != NULL);
739
740 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
741 unsigned int num_ints = gicv3_get_spi_limit(gicd_base);
742 #if GIC_EXT_INTID
743 unsigned int num_eints = gicv3_get_espi_limit(gicd_base);
744 #endif
745
746 /* Wait for pending write to complete */
747 gicd_wait_for_pending_write(gicd_base);
748
749 /* Save the GICD_CTLR */
750 dist_ctx->gicd_ctlr = gicd_read_ctlr(gicd_base);
751
752 /* Save GICD_IGROUPR for INTIDs 32 - 1019 */
753 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
754
755 /* Save GICD_IGROUPRE for INTIDs 4096 - 5119 */
756 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
757
758 /* Save GICD_ISENABLER for INT_IDs 32 - 1019 */
759 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
760
761 /* Save GICD_ISENABLERE for INT_IDs 4096 - 5119 */
762 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
763
764 /* Save GICD_ISPENDR for INTIDs 32 - 1019 */
765 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
766
767 /* Save GICD_ISPENDRE for INTIDs 4096 - 5119 */
768 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
769
770 /* Save GICD_ISACTIVER for INTIDs 32 - 1019 */
771 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
772
773 /* Save GICD_ISACTIVERE for INTIDs 4096 - 5119 */
774 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
775
776 /* Save GICD_IPRIORITYR for INTIDs 32 - 1019 */
777 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
778
779 /* Save GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
780 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
781
782 /* Save GICD_ICFGR for INTIDs 32 - 1019 */
783 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
784
785 /* Save GICD_ICFGRE for INTIDs 4096 - 5119 */
786 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
787
788 /* Save GICD_IGRPMODR for INTIDs 32 - 1019 */
789 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
790
791 /* Save GICD_IGRPMODRE for INTIDs 4096 - 5119 */
792 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
793
794 /* Save GICD_NSACR for INTIDs 32 - 1019 */
795 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
796
797 /* Save GICD_NSACRE for INTIDs 4096 - 5119 */
798 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
799
800 /* Save GICD_IROUTER for INTIDs 32 - 1019 */
801 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
802
803 /* Save GICD_IROUTERE for INTIDs 4096 - 5119 */
804 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
805
806 /*
807 * GICD_ITARGETSR<n> and GICD_SPENDSGIR<n> are RAZ/WI when
808 * GICD_CTLR.ARE_(S|NS) bits are set which is the case for our GICv3
809 * driver.
810 */
811 }
812
813 /*****************************************************************************
814 * Function to restore the GIC Distributor register context. We disable G0, G1S
815 * and G1NS interrupt groups before we start restore of the Distributor. This
816 * function must be invoked prior to Redistributor restore and CPU interface
817 * enable. The pending and active interrupts are restored after the interrupts
818 * are fully configured and enabled.
819 *****************************************************************************/
gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)820 void gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)
821 {
822 assert(gicv3_driver_data != NULL);
823 assert(gicv3_driver_data->gicd_base != 0U);
824 assert(IS_IN_EL3());
825 assert(dist_ctx != NULL);
826
827 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
828
829 /*
830 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
831 * the ARE_S bit. The Distributor might generate a system error
832 * otherwise.
833 */
834 gicd_clr_ctlr(gicd_base,
835 CTLR_ENABLE_G0_BIT |
836 CTLR_ENABLE_G1S_BIT |
837 CTLR_ENABLE_G1NS_BIT,
838 RWP_TRUE);
839
840 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
841 gicd_set_ctlr(gicd_base, CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
842
843 unsigned int num_ints = gicv3_get_spi_limit(gicd_base);
844 #if GIC_EXT_INTID
845 unsigned int num_eints = gicv3_get_espi_limit(gicd_base);
846 #endif
847 /* Restore GICD_IGROUPR for INTIDs 32 - 1019 */
848 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
849
850 /* Restore GICD_IGROUPRE for INTIDs 4096 - 5119 */
851 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
852
853 /* Restore GICD_IPRIORITYR for INTIDs 32 - 1019 */
854 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
855
856 /* Restore GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
857 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
858
859 /* Restore GICD_ICFGR for INTIDs 32 - 1019 */
860 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
861
862 /* Restore GICD_ICFGRE for INTIDs 4096 - 5119 */
863 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
864
865 /* Restore GICD_IGRPMODR for INTIDs 32 - 1019 */
866 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
867
868 /* Restore GICD_IGRPMODRE for INTIDs 4096 - 5119 */
869 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
870
871 /* Restore GICD_NSACR for INTIDs 32 - 1019 */
872 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
873
874 /* Restore GICD_NSACRE for INTIDs 4096 - 5119 */
875 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
876
877 /* Restore GICD_IROUTER for INTIDs 32 - 1019 */
878 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
879
880 /* Restore GICD_IROUTERE for INTIDs 4096 - 5119 */
881 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
882
883 /*
884 * Restore ISENABLER(E), ISPENDR(E) and ISACTIVER(E) after
885 * the interrupts are configured.
886 */
887
888 /* Restore GICD_ISENABLER for INT_IDs 32 - 1019 */
889 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
890
891 /* Restore GICD_ISENABLERE for INT_IDs 4096 - 5119 */
892 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
893
894 /* Restore GICD_ISPENDR for INTIDs 32 - 1019 */
895 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
896
897 /* Restore GICD_ISPENDRE for INTIDs 4096 - 5119 */
898 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
899
900 /* Restore GICD_ISACTIVER for INTIDs 32 - 1019 */
901 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
902
903 /* Restore GICD_ISACTIVERE for INTIDs 4096 - 5119 */
904 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
905
906 /* Restore the GICD_CTLR */
907 gicd_write_ctlr(gicd_base, dist_ctx->gicd_ctlr);
908 gicd_wait_for_pending_write(gicd_base);
909 }
910
911 /*******************************************************************************
912 * This function gets the priority of the interrupt the processor is currently
913 * servicing.
914 ******************************************************************************/
gicv3_get_running_priority(void)915 unsigned int gicv3_get_running_priority(void)
916 {
917 return (unsigned int)read_icc_rpr_el1();
918 }
919
920 /*******************************************************************************
921 * This function checks if the interrupt identified by id is active (whether the
922 * state is either active, or active and pending). The proc_num is used if the
923 * interrupt is SGI or (E)PPI and programs the corresponding Redistributor
924 * interface.
925 ******************************************************************************/
gicv3_get_interrupt_active(unsigned int id,unsigned int proc_num)926 unsigned int gicv3_get_interrupt_active(unsigned int id, unsigned int proc_num)
927 {
928 assert(gicv3_driver_data != NULL);
929 assert(gicv3_driver_data->gicd_base != 0U);
930 assert(proc_num < gicv3_driver_data->rdistif_num);
931 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
932
933 /* Check interrupt ID */
934 if (is_sgi_ppi(id)) {
935 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
936 return gicr_get_isactiver(
937 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
938 }
939
940 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
941 return gicd_get_isactiver(gicv3_driver_data->gicd_base, id);
942 }
943
944 /*******************************************************************************
945 * This function enables the interrupt identified by id. The proc_num
946 * is used if the interrupt is SGI or PPI, and programs the corresponding
947 * Redistributor interface.
948 ******************************************************************************/
gicv3_enable_interrupt(unsigned int id,unsigned int proc_num)949 void gicv3_enable_interrupt(unsigned int id, unsigned int proc_num)
950 {
951 assert(gicv3_driver_data != NULL);
952 assert(gicv3_driver_data->gicd_base != 0U);
953 assert(proc_num < gicv3_driver_data->rdistif_num);
954 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
955
956 /*
957 * Ensure that any shared variable updates depending on out of band
958 * interrupt trigger are observed before enabling interrupt.
959 */
960 dsbishst();
961
962 /* Check interrupt ID */
963 if (is_sgi_ppi(id)) {
964 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
965 gicr_set_isenabler(
966 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
967 } else {
968 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
969 gicd_set_isenabler(gicv3_driver_data->gicd_base, id);
970 }
971 }
972
973 /*******************************************************************************
974 * This function disables the interrupt identified by id. The proc_num
975 * is used if the interrupt is SGI or PPI, and programs the corresponding
976 * Redistributor interface.
977 ******************************************************************************/
gicv3_disable_interrupt(unsigned int id,unsigned int proc_num)978 void gicv3_disable_interrupt(unsigned int id, unsigned int proc_num)
979 {
980 assert(gicv3_driver_data != NULL);
981 assert(gicv3_driver_data->gicd_base != 0U);
982 assert(proc_num < gicv3_driver_data->rdistif_num);
983 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
984
985 /*
986 * Disable interrupt, and ensure that any shared variable updates
987 * depending on out of band interrupt trigger are observed afterwards.
988 */
989
990 /* Check interrupt ID */
991 if (is_sgi_ppi(id)) {
992 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
993 gicr_set_icenabler(
994 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
995
996 /* Write to clear enable requires waiting for pending writes */
997 gicr_wait_for_pending_write(
998 gicv3_driver_data->rdistif_base_addrs[proc_num]);
999 } else {
1000 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1001 gicd_set_icenabler(gicv3_driver_data->gicd_base, id);
1002
1003 /* Write to clear enable requires waiting for pending writes */
1004 gicd_wait_for_pending_write(gicv3_driver_data->gicd_base);
1005 }
1006
1007 dsbishst();
1008 }
1009
1010 /*******************************************************************************
1011 * This function sets the interrupt priority as supplied for the given interrupt
1012 * id.
1013 ******************************************************************************/
gicv3_set_interrupt_priority(unsigned int id,unsigned int proc_num,unsigned int priority)1014 void gicv3_set_interrupt_priority(unsigned int id, unsigned int proc_num,
1015 unsigned int priority)
1016 {
1017 uintptr_t gicr_base;
1018
1019 assert(gicv3_driver_data != NULL);
1020 assert(gicv3_driver_data->gicd_base != 0U);
1021 assert(proc_num < gicv3_driver_data->rdistif_num);
1022 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1023
1024 /* Check interrupt ID */
1025 if (is_sgi_ppi(id)) {
1026 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1027 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1028 gicr_set_ipriorityr(gicr_base, id, priority);
1029 } else {
1030 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1031 gicd_set_ipriorityr(gicv3_driver_data->gicd_base, id, priority);
1032 }
1033 }
1034
1035 /*******************************************************************************
1036 * This function assigns group for the interrupt identified by id. The proc_num
1037 * is used if the interrupt is SGI or (E)PPI, and programs the corresponding
1038 * Redistributor interface. The group can be any of GICV3_INTR_GROUP*
1039 ******************************************************************************/
gicv3_set_interrupt_type(unsigned int id,unsigned int proc_num,unsigned int type)1040 void gicv3_set_interrupt_type(unsigned int id, unsigned int proc_num,
1041 unsigned int type)
1042 {
1043 bool igroup = false, grpmod = false;
1044 uintptr_t gicr_base;
1045
1046 assert(gicv3_driver_data != NULL);
1047 assert(gicv3_driver_data->gicd_base != 0U);
1048 assert(proc_num < gicv3_driver_data->rdistif_num);
1049 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1050
1051 switch (type) {
1052 case INTR_GROUP1S:
1053 igroup = false;
1054 grpmod = true;
1055 break;
1056 case INTR_GROUP0:
1057 igroup = false;
1058 grpmod = false;
1059 break;
1060 case INTR_GROUP1NS:
1061 igroup = true;
1062 grpmod = false;
1063 break;
1064 default:
1065 assert(false);
1066 break;
1067 }
1068
1069 /* Check interrupt ID */
1070 if (is_sgi_ppi(id)) {
1071 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1072 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1073
1074 igroup ? gicr_set_igroupr(gicr_base, id) :
1075 gicr_clr_igroupr(gicr_base, id);
1076 grpmod ? gicr_set_igrpmodr(gicr_base, id) :
1077 gicr_clr_igrpmodr(gicr_base, id);
1078 } else {
1079 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1080
1081 /* Serialize read-modify-write to Distributor registers */
1082 spin_lock(&gic_lock);
1083
1084 igroup ? gicd_set_igroupr(gicv3_driver_data->gicd_base, id) :
1085 gicd_clr_igroupr(gicv3_driver_data->gicd_base, id);
1086 grpmod ? gicd_set_igrpmodr(gicv3_driver_data->gicd_base, id) :
1087 gicd_clr_igrpmodr(gicv3_driver_data->gicd_base, id);
1088
1089 spin_unlock(&gic_lock);
1090 }
1091 }
1092
1093 /*******************************************************************************
1094 * This function raises the specified Secure Group 0 SGI.
1095 *
1096 * The target parameter must be a valid MPIDR in the system.
1097 ******************************************************************************/
gicv3_raise_secure_g0_sgi(unsigned int sgi_num,u_register_t target)1098 void gicv3_raise_secure_g0_sgi(unsigned int sgi_num, u_register_t target)
1099 {
1100 unsigned int tgt, aff3, aff2, aff1, aff0;
1101 uint64_t sgi_val;
1102
1103 /* Verify interrupt number is in the SGI range */
1104 assert((sgi_num >= MIN_SGI_ID) && (sgi_num < MIN_PPI_ID));
1105
1106 /* Extract affinity fields from target */
1107 aff0 = MPIDR_AFFLVL0_VAL(target);
1108 aff1 = MPIDR_AFFLVL1_VAL(target);
1109 aff2 = MPIDR_AFFLVL2_VAL(target);
1110 aff3 = MPIDR_AFFLVL3_VAL(target);
1111
1112 /*
1113 * Make target list from affinity 0, and ensure GICv3 SGI can target
1114 * this PE.
1115 */
1116 assert(aff0 < GICV3_MAX_SGI_TARGETS);
1117 tgt = BIT_32(aff0);
1118
1119 /* Raise SGI to PE specified by its affinity */
1120 sgi_val = GICV3_SGIR_VALUE(aff3, aff2, aff1, sgi_num, SGIR_IRM_TO_AFF,
1121 tgt);
1122
1123 /*
1124 * Ensure that any shared variable updates depending on out of band
1125 * interrupt trigger are observed before raising SGI.
1126 */
1127 dsbishst();
1128 write_icc_sgi0r_el1(sgi_val);
1129 isb();
1130 }
1131
1132 /*******************************************************************************
1133 * This function sets the interrupt routing for the given (E)SPI interrupt id.
1134 * The interrupt routing is specified in routing mode and mpidr.
1135 *
1136 * The routing mode can be either of:
1137 * - GICV3_IRM_ANY
1138 * - GICV3_IRM_PE
1139 *
1140 * The mpidr is the affinity of the PE to which the interrupt will be routed,
1141 * and is ignored for routing mode GICV3_IRM_ANY.
1142 ******************************************************************************/
gicv3_set_spi_routing(unsigned int id,unsigned int irm,u_register_t mpidr)1143 void gicv3_set_spi_routing(unsigned int id, unsigned int irm, u_register_t mpidr)
1144 {
1145 unsigned long long aff;
1146 uint64_t router;
1147
1148 assert(gicv3_driver_data != NULL);
1149 assert(gicv3_driver_data->gicd_base != 0U);
1150
1151 assert((irm == GICV3_IRM_ANY) || (irm == GICV3_IRM_PE));
1152
1153 assert(IS_SPI(id));
1154
1155 aff = gicd_irouter_val_from_mpidr(mpidr, irm);
1156 gicd_write_irouter(gicv3_driver_data->gicd_base, id, aff);
1157
1158 /*
1159 * In implementations that do not require 1 of N distribution of SPIs,
1160 * IRM might be RAZ/WI. Read back and verify IRM bit.
1161 */
1162 if (irm == GICV3_IRM_ANY) {
1163 router = gicd_read_irouter(gicv3_driver_data->gicd_base, id);
1164 if (((router >> IROUTER_IRM_SHIFT) & IROUTER_IRM_MASK) == 0U) {
1165 ERROR("GICv3 implementation doesn't support routing ANY\n");
1166 panic();
1167 }
1168 }
1169 }
1170
1171 /*******************************************************************************
1172 * This function clears the pending status of an interrupt identified by id.
1173 * The proc_num is used if the interrupt is SGI or (E)PPI, and programs the
1174 * corresponding Redistributor interface.
1175 ******************************************************************************/
gicv3_clear_interrupt_pending(unsigned int id,unsigned int proc_num)1176 void gicv3_clear_interrupt_pending(unsigned int id, unsigned int proc_num)
1177 {
1178 assert(gicv3_driver_data != NULL);
1179 assert(gicv3_driver_data->gicd_base != 0U);
1180 assert(proc_num < gicv3_driver_data->rdistif_num);
1181 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1182
1183 /*
1184 * Clear pending interrupt, and ensure that any shared variable updates
1185 * depending on out of band interrupt trigger are observed afterwards.
1186 */
1187
1188 /* Check interrupt ID */
1189 if (is_sgi_ppi(id)) {
1190 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1191 gicr_set_icpendr(
1192 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1193 } else {
1194 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1195 gicd_set_icpendr(gicv3_driver_data->gicd_base, id);
1196 }
1197
1198 dsbishst();
1199 }
1200
1201 /*******************************************************************************
1202 * This function sets the pending status of an interrupt identified by id.
1203 * The proc_num is used if the interrupt is SGI or PPI and programs the
1204 * corresponding Redistributor interface.
1205 ******************************************************************************/
gicv3_set_interrupt_pending(unsigned int id,unsigned int proc_num)1206 void gicv3_set_interrupt_pending(unsigned int id, unsigned int proc_num)
1207 {
1208 assert(gicv3_driver_data != NULL);
1209 assert(gicv3_driver_data->gicd_base != 0U);
1210 assert(proc_num < gicv3_driver_data->rdistif_num);
1211 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1212
1213 /*
1214 * Ensure that any shared variable updates depending on out of band
1215 * interrupt trigger are observed before setting interrupt pending.
1216 */
1217 dsbishst();
1218
1219 /* Check interrupt ID */
1220 if (is_sgi_ppi(id)) {
1221 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1222 gicr_set_ispendr(
1223 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1224 } else {
1225 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1226 gicd_set_ispendr(gicv3_driver_data->gicd_base, id);
1227 }
1228 }
1229
1230 /*******************************************************************************
1231 * This function sets the PMR register with the supplied value. Returns the
1232 * original PMR.
1233 ******************************************************************************/
gicv3_set_pmr(unsigned int mask)1234 unsigned int gicv3_set_pmr(unsigned int mask)
1235 {
1236 unsigned int old_mask;
1237
1238 old_mask = (unsigned int)read_icc_pmr_el1();
1239
1240 /*
1241 * Order memory updates w.r.t. PMR write, and ensure they're visible
1242 * before potential out of band interrupt trigger because of PMR update.
1243 * PMR system register writes are self-synchronizing, so no ISB required
1244 * thereafter.
1245 */
1246 dsbishst();
1247 write_icc_pmr_el1(mask);
1248
1249 return old_mask;
1250 }
1251
1252 /*******************************************************************************
1253 * This function delegates the responsibility of discovering the corresponding
1254 * Redistributor frames to each CPU itself. It is a modified version of
1255 * gicv3_rdistif_base_addrs_probe() and is executed by each CPU in the platform
1256 * unlike the previous way in which only the Primary CPU did the discovery of
1257 * all the Redistributor frames for every CPU. It also handles the scenario in
1258 * which the frames of various CPUs are not contiguous in physical memory.
1259 ******************************************************************************/
gicv3_rdistif_probe(const uintptr_t gicr_frame)1260 int gicv3_rdistif_probe(const uintptr_t gicr_frame)
1261 {
1262 u_register_t mpidr, mpidr_self;
1263 unsigned int proc_num;
1264 uint64_t typer_val;
1265 uintptr_t rdistif_base;
1266 bool gicr_frame_found = false;
1267
1268 assert(gicv3_driver_data->gicr_base == 0U);
1269
1270 /* Ensure this function is called with Data Cache enabled */
1271 #ifndef __aarch64__
1272 assert((read_sctlr() & SCTLR_C_BIT) != 0U);
1273 #else
1274 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1275 #endif /* !__aarch64__ */
1276
1277 mpidr_self = read_mpidr_el1() & MPIDR_AFFINITY_MASK;
1278 rdistif_base = gicr_frame;
1279 do {
1280 typer_val = gicr_read_typer(rdistif_base);
1281 mpidr = mpidr_from_gicr_typer(typer_val);
1282 if (gicv3_driver_data->mpidr_to_core_pos != NULL) {
1283 proc_num = gicv3_driver_data->mpidr_to_core_pos(mpidr);
1284 } else {
1285 proc_num = (unsigned int)(typer_val >>
1286 TYPER_PROC_NUM_SHIFT) & TYPER_PROC_NUM_MASK;
1287 }
1288 if (mpidr == mpidr_self) {
1289 /* The base address doesn't need to be initialized on
1290 * every warm boot.
1291 */
1292 if (gicv3_driver_data->rdistif_base_addrs[proc_num]
1293 != 0U) {
1294 return 0;
1295 }
1296 gicv3_driver_data->rdistif_base_addrs[proc_num] =
1297 rdistif_base;
1298 gicr_frame_found = true;
1299 break;
1300 }
1301 rdistif_base += (uintptr_t)(ULL(1) << GICR_PCPUBASE_SHIFT);
1302 } while ((typer_val & TYPER_LAST_BIT) == 0U);
1303
1304 if (!gicr_frame_found) {
1305 return -1;
1306 }
1307
1308 /*
1309 * Flush the driver data to ensure coherency. This is
1310 * not required if platform has HW_ASSISTED_COHERENCY
1311 * enabled.
1312 */
1313 #if !HW_ASSISTED_COHERENCY
1314 /*
1315 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
1316 */
1317 flush_dcache_range((uintptr_t)&(gicv3_driver_data->rdistif_base_addrs[proc_num]),
1318 sizeof(*(gicv3_driver_data->rdistif_base_addrs)));
1319 #endif
1320 return 0; /* Found matching GICR frame */
1321 }
1322
1323 /******************************************************************************
1324 * This function checks the interrupt ID and returns true for SGIs and (E)PPIs
1325 * and false for (E)SPIs IDs.
1326 *****************************************************************************/
is_sgi_ppi(unsigned int id)1327 static bool is_sgi_ppi(unsigned int id)
1328 {
1329 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
1330 if (IS_SGI_PPI(id)) {
1331 return true;
1332 }
1333
1334 /* SPIs: 32-1019, ESPIs: 4096-5119 */
1335 if (IS_SPI(id)) {
1336 return false;
1337 }
1338
1339 assert(false);
1340 panic();
1341 }
1342