1 /*
2 * Copyright (c) 2015-2021, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8
9 #include <arch.h>
10 #include <arch_helpers.h>
11 #include <common/debug.h>
12 #include <common/interrupt_props.h>
13 #include <drivers/arm/gicv3.h>
14 #include <lib/spinlock.h>
15
16 #include "gicv3_private.h"
17
18 const gicv3_driver_data_t *gicv3_driver_data;
19
20 /*
21 * Spinlock to guard registers needing read-modify-write. APIs protected by this
22 * spinlock are used either at boot time (when only a single CPU is active), or
23 * when the system is fully coherent.
24 */
25 static spinlock_t gic_lock;
26
27 /*
28 * Redistributor power operations are weakly bound so that they can be
29 * overridden
30 */
31 #pragma weak gicv3_rdistif_off
32 #pragma weak gicv3_rdistif_on
33
34 /* Check interrupt ID for SGI/(E)PPI and (E)SPIs */
35 static bool is_sgi_ppi(unsigned int id);
36
37 /*
38 * Helper macros to save and restore GICR and GICD registers
39 * corresponding to their numbers to and from the context
40 */
41 #define RESTORE_GICR_REG(base, ctx, name, i) \
42 gicr_write_##name((base), (i), (ctx)->gicr_##name[(i)])
43
44 #define SAVE_GICR_REG(base, ctx, name, i) \
45 (ctx)->gicr_##name[(i)] = gicr_read_##name((base), (i))
46
47 /* Helper macros to save and restore GICD registers to and from the context */
48 #define RESTORE_GICD_REGS(base, ctx, intr_num, reg, REG) \
49 do { \
50 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
51 int_id += (1U << REG##R_SHIFT)) { \
52 gicd_write_##reg((base), int_id, \
53 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
54 REG##R_SHIFT]); \
55 } \
56 } while (false)
57
58 #define SAVE_GICD_REGS(base, ctx, intr_num, reg, REG) \
59 do { \
60 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
61 int_id += (1U << REG##R_SHIFT)) { \
62 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
63 REG##R_SHIFT] = gicd_read_##reg((base), int_id); \
64 } \
65 } while (false)
66
67 #if GIC_EXT_INTID
68 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
69 do { \
70 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
71 int_id += (1U << REG##R_SHIFT)) { \
72 gicd_write_##reg((base), int_id, \
73 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - \
74 round_up(TOTAL_SPI_INTR_NUM, 1U << REG##R_SHIFT)))\
75 >> REG##R_SHIFT]); \
76 } \
77 } while (false)
78
79 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
80 do { \
81 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
82 int_id += (1U << REG##R_SHIFT)) { \
83 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - \
84 round_up(TOTAL_SPI_INTR_NUM, 1U << REG##R_SHIFT)))\
85 >> REG##R_SHIFT] = gicd_read_##reg((base), int_id);\
86 } \
87 } while (false)
88 #else
89 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG)
90 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG)
91 #endif /* GIC_EXT_INTID */
92
93 /*******************************************************************************
94 * This function initialises the ARM GICv3 driver in EL3 with provided platform
95 * inputs.
96 ******************************************************************************/
gicv3_driver_init(const gicv3_driver_data_t * plat_driver_data)97 void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
98 {
99 unsigned int gic_version;
100 unsigned int gicv2_compat;
101
102 assert(plat_driver_data != NULL);
103 assert(plat_driver_data->gicd_base != 0U);
104 assert(plat_driver_data->rdistif_num != 0U);
105 assert(plat_driver_data->rdistif_base_addrs != NULL);
106
107 assert(IS_IN_EL3());
108
109 assert((plat_driver_data->interrupt_props_num != 0U) ?
110 (plat_driver_data->interrupt_props != NULL) : 1);
111
112 /* Check for system register support */
113 #ifndef __aarch64__
114 assert((read_id_pfr1() &
115 (ID_PFR1_GIC_MASK << ID_PFR1_GIC_SHIFT)) != 0U);
116 #else
117 assert((read_id_aa64pfr0_el1() &
118 (ID_AA64PFR0_GIC_MASK << ID_AA64PFR0_GIC_SHIFT)) != 0U);
119 #endif /* !__aarch64__ */
120
121 gic_version = gicd_read_pidr2(plat_driver_data->gicd_base);
122 gic_version >>= PIDR2_ARCH_REV_SHIFT;
123 gic_version &= PIDR2_ARCH_REV_MASK;
124
125 /* Check GIC version */
126 #if !GIC_ENABLE_V4_EXTN
127 assert(gic_version == ARCH_REV_GICV3);
128 #endif
129 /*
130 * Find out whether the GIC supports the GICv2 compatibility mode.
131 * The ARE_S bit resets to 0 if supported
132 */
133 gicv2_compat = gicd_read_ctlr(plat_driver_data->gicd_base);
134 gicv2_compat >>= CTLR_ARE_S_SHIFT;
135 gicv2_compat = gicv2_compat & CTLR_ARE_S_MASK;
136
137 if (plat_driver_data->gicr_base != 0U) {
138 /*
139 * Find the base address of each implemented Redistributor interface.
140 * The number of interfaces should be equal to the number of CPUs in the
141 * system. The memory for saving these addresses has to be allocated by
142 * the platform port
143 */
144 gicv3_rdistif_base_addrs_probe(plat_driver_data->rdistif_base_addrs,
145 plat_driver_data->rdistif_num,
146 plat_driver_data->gicr_base,
147 plat_driver_data->mpidr_to_core_pos);
148 #if !HW_ASSISTED_COHERENCY
149 /*
150 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
151 */
152 flush_dcache_range((uintptr_t)(plat_driver_data->rdistif_base_addrs),
153 plat_driver_data->rdistif_num *
154 sizeof(*(plat_driver_data->rdistif_base_addrs)));
155 #endif
156 }
157 gicv3_driver_data = plat_driver_data;
158
159 /*
160 * The GIC driver data is initialized by the primary CPU with caches
161 * enabled. When the secondary CPU boots up, it initializes the
162 * GICC/GICR interface with the caches disabled. Hence flush the
163 * driver data to ensure coherency. This is not required if the
164 * platform has HW_ASSISTED_COHERENCY enabled.
165 */
166 #if !HW_ASSISTED_COHERENCY
167 flush_dcache_range((uintptr_t)&gicv3_driver_data,
168 sizeof(gicv3_driver_data));
169 flush_dcache_range((uintptr_t)gicv3_driver_data,
170 sizeof(*gicv3_driver_data));
171 #endif
172 INFO("GICv%u with%s legacy support detected.\n", gic_version,
173 (gicv2_compat == 0U) ? "" : "out");
174 INFO("ARM GICv%u driver initialized in EL3\n", gic_version);
175 }
176
177 /*******************************************************************************
178 * This function initialises the GIC distributor interface based upon the data
179 * provided by the platform while initialising the driver.
180 ******************************************************************************/
gicv3_distif_init(void)181 void __init gicv3_distif_init(void)
182 {
183 unsigned int bitmap;
184
185 assert(gicv3_driver_data != NULL);
186 assert(gicv3_driver_data->gicd_base != 0U);
187
188 assert(IS_IN_EL3());
189
190 /*
191 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
192 * the ARE_S bit. The Distributor might generate a system error
193 * otherwise.
194 */
195 gicd_clr_ctlr(gicv3_driver_data->gicd_base,
196 CTLR_ENABLE_G0_BIT |
197 CTLR_ENABLE_G1S_BIT |
198 CTLR_ENABLE_G1NS_BIT,
199 RWP_TRUE);
200
201 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
202 gicd_set_ctlr(gicv3_driver_data->gicd_base,
203 CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
204
205 /* Set the default attribute of all (E)SPIs */
206 gicv3_spis_config_defaults(gicv3_driver_data->gicd_base);
207
208 bitmap = gicv3_secure_spis_config_props(
209 gicv3_driver_data->gicd_base,
210 gicv3_driver_data->interrupt_props,
211 gicv3_driver_data->interrupt_props_num);
212
213 /* Enable the secure (E)SPIs now that they have been configured */
214 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
215 }
216
217 /*******************************************************************************
218 * This function initialises the GIC Redistributor interface of the calling CPU
219 * (identified by the 'proc_num' parameter) based upon the data provided by the
220 * platform while initialising the driver.
221 ******************************************************************************/
gicv3_rdistif_init(unsigned int proc_num)222 void gicv3_rdistif_init(unsigned int proc_num)
223 {
224 uintptr_t gicr_base;
225 unsigned int bitmap;
226 uint32_t ctlr;
227
228 assert(gicv3_driver_data != NULL);
229 assert(proc_num < gicv3_driver_data->rdistif_num);
230 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
231 assert(gicv3_driver_data->gicd_base != 0U);
232
233 ctlr = gicd_read_ctlr(gicv3_driver_data->gicd_base);
234 assert((ctlr & CTLR_ARE_S_BIT) != 0U);
235
236 assert(IS_IN_EL3());
237
238 /* Power on redistributor */
239 gicv3_rdistif_on(proc_num);
240
241 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
242 assert(gicr_base != 0U);
243
244 /* Set the default attribute of all SGIs and (E)PPIs */
245 gicv3_ppi_sgi_config_defaults(gicr_base);
246
247 bitmap = gicv3_secure_ppi_sgi_config_props(gicr_base,
248 gicv3_driver_data->interrupt_props,
249 gicv3_driver_data->interrupt_props_num);
250
251 /* Enable interrupt groups as required, if not already */
252 if ((ctlr & bitmap) != bitmap) {
253 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
254 }
255 }
256
257 /*******************************************************************************
258 * Functions to perform power operations on GIC Redistributor
259 ******************************************************************************/
gicv3_rdistif_off(unsigned int proc_num)260 void gicv3_rdistif_off(unsigned int proc_num)
261 {
262 }
263
gicv3_rdistif_on(unsigned int proc_num)264 void gicv3_rdistif_on(unsigned int proc_num)
265 {
266 }
267
268 /*******************************************************************************
269 * This function enables the GIC CPU interface of the calling CPU using only
270 * system register accesses.
271 ******************************************************************************/
gicv3_cpuif_enable(unsigned int proc_num)272 void gicv3_cpuif_enable(unsigned int proc_num)
273 {
274 uintptr_t gicr_base;
275 u_register_t scr_el3;
276 unsigned int icc_sre_el3;
277
278 assert(gicv3_driver_data != NULL);
279 assert(proc_num < gicv3_driver_data->rdistif_num);
280 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
281 assert(IS_IN_EL3());
282
283 /* Mark the connected core as awake */
284 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
285 gicv3_rdistif_mark_core_awake(gicr_base);
286
287 /* Disable the legacy interrupt bypass */
288 icc_sre_el3 = ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT;
289
290 /*
291 * Enable system register access for EL3 and allow lower exception
292 * levels to configure the same for themselves. If the legacy mode is
293 * not supported, the SRE bit is RAO/WI
294 */
295 icc_sre_el3 |= (ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT);
296 write_icc_sre_el3(read_icc_sre_el3() | icc_sre_el3);
297
298 scr_el3 = read_scr_el3();
299
300 /*
301 * Switch to NS state to write Non secure ICC_SRE_EL1 and
302 * ICC_SRE_EL2 registers.
303 */
304 write_scr_el3(scr_el3 | SCR_NS_BIT);
305 isb();
306
307 write_icc_sre_el2(read_icc_sre_el2() | icc_sre_el3);
308 write_icc_sre_el1(ICC_SRE_SRE_BIT);
309 isb();
310
311 /* Switch to secure state. */
312 write_scr_el3(scr_el3 & (~SCR_NS_BIT));
313 isb();
314
315 /* Write the secure ICC_SRE_EL1 register */
316 write_icc_sre_el1(ICC_SRE_SRE_BIT);
317 isb();
318
319 /* Program the idle priority in the PMR */
320 write_icc_pmr_el1(GIC_PRI_MASK);
321
322 /* Enable Group0 interrupts */
323 write_icc_igrpen0_el1(IGRPEN1_EL1_ENABLE_G0_BIT);
324
325 /* Enable Group1 Secure interrupts */
326 write_icc_igrpen1_el3(read_icc_igrpen1_el3() |
327 IGRPEN1_EL3_ENABLE_G1S_BIT);
328 isb();
329 /* Add DSB to ensure visibility of System register writes */
330 dsb();
331 }
332
333 /*******************************************************************************
334 * This function disables the GIC CPU interface of the calling CPU using
335 * only system register accesses.
336 ******************************************************************************/
gicv3_cpuif_disable(unsigned int proc_num)337 void gicv3_cpuif_disable(unsigned int proc_num)
338 {
339 uintptr_t gicr_base;
340
341 assert(gicv3_driver_data != NULL);
342 assert(proc_num < gicv3_driver_data->rdistif_num);
343 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
344
345 assert(IS_IN_EL3());
346
347 /* Disable legacy interrupt bypass */
348 write_icc_sre_el3(read_icc_sre_el3() |
349 (ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT));
350
351 /* Disable Group0 interrupts */
352 write_icc_igrpen0_el1(read_icc_igrpen0_el1() &
353 ~IGRPEN1_EL1_ENABLE_G0_BIT);
354
355 /* Disable Group1 Secure and Non-Secure interrupts */
356 write_icc_igrpen1_el3(read_icc_igrpen1_el3() &
357 ~(IGRPEN1_EL3_ENABLE_G1NS_BIT |
358 IGRPEN1_EL3_ENABLE_G1S_BIT));
359
360 /* Synchronise accesses to group enable registers */
361 isb();
362 /* Add DSB to ensure visibility of System register writes */
363 dsb();
364
365 /* Mark the connected core as asleep */
366 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
367 assert(gicr_base != 0U);
368 gicv3_rdistif_mark_core_asleep(gicr_base);
369 }
370
371 /*******************************************************************************
372 * This function returns the id of the highest priority pending interrupt at
373 * the GIC cpu interface.
374 ******************************************************************************/
gicv3_get_pending_interrupt_id(void)375 unsigned int gicv3_get_pending_interrupt_id(void)
376 {
377 unsigned int id;
378
379 assert(IS_IN_EL3());
380 id = (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
381
382 /*
383 * If the ID is special identifier corresponding to G1S or G1NS
384 * interrupt, then read the highest pending group 1 interrupt.
385 */
386 if ((id == PENDING_G1S_INTID) || (id == PENDING_G1NS_INTID)) {
387 return (uint32_t)read_icc_hppir1_el1() & HPPIR1_EL1_INTID_MASK;
388 }
389
390 return id;
391 }
392
393 /*******************************************************************************
394 * This function returns the type of the highest priority pending interrupt at
395 * the GIC cpu interface. The return values can be one of the following :
396 * PENDING_G1S_INTID : The interrupt type is secure Group 1.
397 * PENDING_G1NS_INTID : The interrupt type is non secure Group 1.
398 * 0 - 1019 : The interrupt type is secure Group 0.
399 * GIC_SPURIOUS_INTERRUPT : there is no pending interrupt with
400 * sufficient priority to be signaled
401 ******************************************************************************/
gicv3_get_pending_interrupt_type(void)402 unsigned int gicv3_get_pending_interrupt_type(void)
403 {
404 assert(IS_IN_EL3());
405 return (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
406 }
407
408 /*******************************************************************************
409 * This function returns the type of the interrupt id depending upon the group
410 * this interrupt has been configured under by the interrupt controller i.e.
411 * group0 or group1 Secure / Non Secure. The return value can be one of the
412 * following :
413 * INTR_GROUP0 : The interrupt type is a Secure Group 0 interrupt
414 * INTR_GROUP1S : The interrupt type is a Secure Group 1 secure interrupt
415 * INTR_GROUP1NS: The interrupt type is a Secure Group 1 non secure
416 * interrupt.
417 ******************************************************************************/
gicv3_get_interrupt_type(unsigned int id,unsigned int proc_num)418 unsigned int gicv3_get_interrupt_type(unsigned int id, unsigned int proc_num)
419 {
420 unsigned int igroup, grpmodr;
421 uintptr_t gicr_base;
422
423 assert(IS_IN_EL3());
424 assert(gicv3_driver_data != NULL);
425
426 /* Ensure the parameters are valid */
427 assert((id < PENDING_G1S_INTID) || (id >= MIN_LPI_ID));
428 assert(proc_num < gicv3_driver_data->rdistif_num);
429
430 /* All LPI interrupts are Group 1 non secure */
431 if (id >= MIN_LPI_ID) {
432 return INTR_GROUP1NS;
433 }
434
435 /* Check interrupt ID */
436 if (is_sgi_ppi(id)) {
437 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
438 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
439 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
440 igroup = gicr_get_igroupr(gicr_base, id);
441 grpmodr = gicr_get_igrpmodr(gicr_base, id);
442 } else {
443 /* SPIs: 32-1019, ESPIs: 4096-5119 */
444 assert(gicv3_driver_data->gicd_base != 0U);
445 igroup = gicd_get_igroupr(gicv3_driver_data->gicd_base, id);
446 grpmodr = gicd_get_igrpmodr(gicv3_driver_data->gicd_base, id);
447 }
448
449 /*
450 * If the IGROUP bit is set, then it is a Group 1 Non secure
451 * interrupt
452 */
453 if (igroup != 0U) {
454 return INTR_GROUP1NS;
455 }
456
457 /* If the GRPMOD bit is set, then it is a Group 1 Secure interrupt */
458 if (grpmodr != 0U) {
459 return INTR_GROUP1S;
460 }
461
462 /* Else it is a Group 0 Secure interrupt */
463 return INTR_GROUP0;
464 }
465
466 /*****************************************************************************
467 * Function to save and disable the GIC ITS register context. The power
468 * management of GIC ITS is implementation-defined and this function doesn't
469 * save any memory structures required to support ITS. As the sequence to save
470 * this state is implementation defined, it should be executed in platform
471 * specific code. Calling this function alone and then powering down the GIC and
472 * ITS without implementing the aforementioned platform specific code will
473 * corrupt the ITS state.
474 *
475 * This function must be invoked after the GIC CPU interface is disabled.
476 *****************************************************************************/
gicv3_its_save_disable(uintptr_t gits_base,gicv3_its_ctx_t * const its_ctx)477 void gicv3_its_save_disable(uintptr_t gits_base,
478 gicv3_its_ctx_t * const its_ctx)
479 {
480 unsigned int i;
481
482 assert(gicv3_driver_data != NULL);
483 assert(IS_IN_EL3());
484 assert(its_ctx != NULL);
485 assert(gits_base != 0U);
486
487 its_ctx->gits_ctlr = gits_read_ctlr(gits_base);
488
489 /* Disable the ITS */
490 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
491
492 /* Wait for quiescent state */
493 gits_wait_for_quiescent_bit(gits_base);
494
495 its_ctx->gits_cbaser = gits_read_cbaser(gits_base);
496 its_ctx->gits_cwriter = gits_read_cwriter(gits_base);
497
498 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
499 its_ctx->gits_baser[i] = gits_read_baser(gits_base, i);
500 }
501 }
502
503 /*****************************************************************************
504 * Function to restore the GIC ITS register context. The power
505 * management of GIC ITS is implementation defined and this function doesn't
506 * restore any memory structures required to support ITS. The assumption is
507 * that these structures are in memory and are retained during system suspend.
508 *
509 * This must be invoked before the GIC CPU interface is enabled.
510 *****************************************************************************/
gicv3_its_restore(uintptr_t gits_base,const gicv3_its_ctx_t * const its_ctx)511 void gicv3_its_restore(uintptr_t gits_base,
512 const gicv3_its_ctx_t * const its_ctx)
513 {
514 unsigned int i;
515
516 assert(gicv3_driver_data != NULL);
517 assert(IS_IN_EL3());
518 assert(its_ctx != NULL);
519 assert(gits_base != 0U);
520
521 /* Assert that the GITS is disabled and quiescent */
522 assert((gits_read_ctlr(gits_base) & GITS_CTLR_ENABLED_BIT) == 0U);
523 assert((gits_read_ctlr(gits_base) & GITS_CTLR_QUIESCENT_BIT) != 0U);
524
525 gits_write_cbaser(gits_base, its_ctx->gits_cbaser);
526 gits_write_cwriter(gits_base, its_ctx->gits_cwriter);
527
528 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
529 gits_write_baser(gits_base, i, its_ctx->gits_baser[i]);
530 }
531
532 /* Restore the ITS CTLR but leave the ITS disabled */
533 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
534 }
535
536 /*****************************************************************************
537 * Function to save the GIC Redistributor register context. This function
538 * must be invoked after CPU interface disable and prior to Distributor save.
539 *****************************************************************************/
gicv3_rdistif_save(unsigned int proc_num,gicv3_redist_ctx_t * const rdist_ctx)540 void gicv3_rdistif_save(unsigned int proc_num,
541 gicv3_redist_ctx_t * const rdist_ctx)
542 {
543 uintptr_t gicr_base;
544 unsigned int i, ppi_regs_num, regs_num;
545
546 assert(gicv3_driver_data != NULL);
547 assert(proc_num < gicv3_driver_data->rdistif_num);
548 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
549 assert(IS_IN_EL3());
550 assert(rdist_ctx != NULL);
551
552 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
553
554 #if GIC_EXT_INTID
555 /* Calculate number of PPI registers */
556 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
557 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
558 /* All other values except PPInum [0-2] are reserved */
559 if (ppi_regs_num > 3U) {
560 ppi_regs_num = 1U;
561 }
562 #else
563 ppi_regs_num = 1U;
564 #endif
565 /*
566 * Wait for any write to GICR_CTLR to complete before trying to save any
567 * state.
568 */
569 gicr_wait_for_pending_write(gicr_base);
570
571 rdist_ctx->gicr_ctlr = gicr_read_ctlr(gicr_base);
572
573 rdist_ctx->gicr_propbaser = gicr_read_propbaser(gicr_base);
574 rdist_ctx->gicr_pendbaser = gicr_read_pendbaser(gicr_base);
575
576 /* 32 interrupt IDs per register */
577 for (i = 0U; i < ppi_regs_num; ++i) {
578 SAVE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
579 SAVE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
580 SAVE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
581 SAVE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
582 SAVE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
583 }
584
585 /* 16 interrupt IDs per GICR_ICFGR register */
586 regs_num = ppi_regs_num << 1;
587 for (i = 0U; i < regs_num; ++i) {
588 SAVE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
589 }
590
591 rdist_ctx->gicr_nsacr = gicr_read_nsacr(gicr_base);
592
593 /* 4 interrupt IDs per GICR_IPRIORITYR register */
594 regs_num = ppi_regs_num << 3;
595 for (i = 0U; i < regs_num; ++i) {
596 rdist_ctx->gicr_ipriorityr[i] =
597 gicr_ipriorityr_read(gicr_base, i);
598 }
599
600 /*
601 * Call the pre-save hook that implements the IMP DEF sequence that may
602 * be required on some GIC implementations. As this may need to access
603 * the Redistributor registers, we pass it proc_num.
604 */
605 gicv3_distif_pre_save(proc_num);
606 }
607
608 /*****************************************************************************
609 * Function to restore the GIC Redistributor register context. We disable
610 * LPI and per-cpu interrupts before we start restore of the Redistributor.
611 * This function must be invoked after Distributor restore but prior to
612 * CPU interface enable. The pending and active interrupts are restored
613 * after the interrupts are fully configured and enabled.
614 *****************************************************************************/
gicv3_rdistif_init_restore(unsigned int proc_num,const gicv3_redist_ctx_t * const rdist_ctx)615 void gicv3_rdistif_init_restore(unsigned int proc_num,
616 const gicv3_redist_ctx_t * const rdist_ctx)
617 {
618 uintptr_t gicr_base;
619 unsigned int i, ppi_regs_num, regs_num;
620
621 assert(gicv3_driver_data != NULL);
622 assert(proc_num < gicv3_driver_data->rdistif_num);
623 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
624 assert(IS_IN_EL3());
625 assert(rdist_ctx != NULL);
626
627 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
628
629 #if GIC_EXT_INTID
630 /* Calculate number of PPI registers */
631 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
632 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
633 /* All other values except PPInum [0-2] are reserved */
634 if (ppi_regs_num > 3U) {
635 ppi_regs_num = 1U;
636 }
637 #else
638 ppi_regs_num = 1U;
639 #endif
640 /* Power on redistributor */
641 gicv3_rdistif_on(proc_num);
642
643 /*
644 * Call the post-restore hook that implements the IMP DEF sequence that
645 * may be required on some GIC implementations. As this may need to
646 * access the Redistributor registers, we pass it proc_num.
647 */
648 gicv3_distif_post_restore(proc_num);
649
650 /*
651 * Disable all SGIs (imp. def.)/(E)PPIs before configuring them.
652 * This is a more scalable approach as it avoids clearing the enable
653 * bits in the GICD_CTLR.
654 */
655 for (i = 0U; i < ppi_regs_num; ++i) {
656 gicr_write_icenabler(gicr_base, i, ~0U);
657 }
658
659 /* Wait for pending writes to GICR_ICENABLER */
660 gicr_wait_for_pending_write(gicr_base);
661
662 /*
663 * Disable the LPIs to avoid unpredictable behavior when writing to
664 * GICR_PROPBASER and GICR_PENDBASER.
665 */
666 gicr_write_ctlr(gicr_base,
667 rdist_ctx->gicr_ctlr & ~(GICR_CTLR_EN_LPIS_BIT));
668
669 /* Restore registers' content */
670 gicr_write_propbaser(gicr_base, rdist_ctx->gicr_propbaser);
671 gicr_write_pendbaser(gicr_base, rdist_ctx->gicr_pendbaser);
672
673 /* 32 interrupt IDs per register */
674 for (i = 0U; i < ppi_regs_num; ++i) {
675 RESTORE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
676 RESTORE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
677 }
678
679 /* 4 interrupt IDs per GICR_IPRIORITYR register */
680 regs_num = ppi_regs_num << 3;
681 for (i = 0U; i < regs_num; ++i) {
682 gicr_ipriorityr_write(gicr_base, i,
683 rdist_ctx->gicr_ipriorityr[i]);
684 }
685
686 /* 16 interrupt IDs per GICR_ICFGR register */
687 regs_num = ppi_regs_num << 1;
688 for (i = 0U; i < regs_num; ++i) {
689 RESTORE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
690 }
691
692 gicr_write_nsacr(gicr_base, rdist_ctx->gicr_nsacr);
693
694 /* Restore after group and priorities are set.
695 * 32 interrupt IDs per register
696 */
697 for (i = 0U; i < ppi_regs_num; ++i) {
698 RESTORE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
699 RESTORE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
700 }
701
702 /*
703 * Wait for all writes to the Distributor to complete before enabling
704 * the SGI and (E)PPIs.
705 */
706 gicr_wait_for_upstream_pending_write(gicr_base);
707
708 /* 32 interrupt IDs per GICR_ISENABLER register */
709 for (i = 0U; i < ppi_regs_num; ++i) {
710 RESTORE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
711 }
712
713 /*
714 * Restore GICR_CTLR.Enable_LPIs bit and wait for pending writes in case
715 * the first write to GICR_CTLR was still in flight (this write only
716 * restores GICR_CTLR.Enable_LPIs and no waiting is required for this
717 * bit).
718 */
719 gicr_write_ctlr(gicr_base, rdist_ctx->gicr_ctlr);
720 gicr_wait_for_pending_write(gicr_base);
721 }
722
723 /*****************************************************************************
724 * Function to save the GIC Distributor register context. This function
725 * must be invoked after CPU interface disable and Redistributor save.
726 *****************************************************************************/
gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)727 void gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)
728 {
729 assert(gicv3_driver_data != NULL);
730 assert(gicv3_driver_data->gicd_base != 0U);
731 assert(IS_IN_EL3());
732 assert(dist_ctx != NULL);
733
734 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
735 unsigned int num_ints = gicv3_get_spi_limit(gicd_base);
736 #if GIC_EXT_INTID
737 unsigned int num_eints = gicv3_get_espi_limit(gicd_base);
738 #endif
739
740 /* Wait for pending write to complete */
741 gicd_wait_for_pending_write(gicd_base);
742
743 /* Save the GICD_CTLR */
744 dist_ctx->gicd_ctlr = gicd_read_ctlr(gicd_base);
745
746 /* Save GICD_IGROUPR for INTIDs 32 - 1019 */
747 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
748
749 /* Save GICD_IGROUPRE for INTIDs 4096 - 5119 */
750 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
751
752 /* Save GICD_ISENABLER for INT_IDs 32 - 1019 */
753 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
754
755 /* Save GICD_ISENABLERE for INT_IDs 4096 - 5119 */
756 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
757
758 /* Save GICD_ISPENDR for INTIDs 32 - 1019 */
759 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
760
761 /* Save GICD_ISPENDRE for INTIDs 4096 - 5119 */
762 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
763
764 /* Save GICD_ISACTIVER for INTIDs 32 - 1019 */
765 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
766
767 /* Save GICD_ISACTIVERE for INTIDs 4096 - 5119 */
768 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
769
770 /* Save GICD_IPRIORITYR for INTIDs 32 - 1019 */
771 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
772
773 /* Save GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
774 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
775
776 /* Save GICD_ICFGR for INTIDs 32 - 1019 */
777 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
778
779 /* Save GICD_ICFGRE for INTIDs 4096 - 5119 */
780 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
781
782 /* Save GICD_IGRPMODR for INTIDs 32 - 1019 */
783 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
784
785 /* Save GICD_IGRPMODRE for INTIDs 4096 - 5119 */
786 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
787
788 /* Save GICD_NSACR for INTIDs 32 - 1019 */
789 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
790
791 /* Save GICD_NSACRE for INTIDs 4096 - 5119 */
792 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
793
794 /* Save GICD_IROUTER for INTIDs 32 - 1019 */
795 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
796
797 /* Save GICD_IROUTERE for INTIDs 4096 - 5119 */
798 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
799
800 /*
801 * GICD_ITARGETSR<n> and GICD_SPENDSGIR<n> are RAZ/WI when
802 * GICD_CTLR.ARE_(S|NS) bits are set which is the case for our GICv3
803 * driver.
804 */
805 }
806
807 /*****************************************************************************
808 * Function to restore the GIC Distributor register context. We disable G0, G1S
809 * and G1NS interrupt groups before we start restore of the Distributor. This
810 * function must be invoked prior to Redistributor restore and CPU interface
811 * enable. The pending and active interrupts are restored after the interrupts
812 * are fully configured and enabled.
813 *****************************************************************************/
gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)814 void gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)
815 {
816 assert(gicv3_driver_data != NULL);
817 assert(gicv3_driver_data->gicd_base != 0U);
818 assert(IS_IN_EL3());
819 assert(dist_ctx != NULL);
820
821 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
822
823 /*
824 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
825 * the ARE_S bit. The Distributor might generate a system error
826 * otherwise.
827 */
828 gicd_clr_ctlr(gicd_base,
829 CTLR_ENABLE_G0_BIT |
830 CTLR_ENABLE_G1S_BIT |
831 CTLR_ENABLE_G1NS_BIT,
832 RWP_TRUE);
833
834 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
835 gicd_set_ctlr(gicd_base, CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
836
837 unsigned int num_ints = gicv3_get_spi_limit(gicd_base);
838 #if GIC_EXT_INTID
839 unsigned int num_eints = gicv3_get_espi_limit(gicd_base);
840 #endif
841 /* Restore GICD_IGROUPR for INTIDs 32 - 1019 */
842 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
843
844 /* Restore GICD_IGROUPRE for INTIDs 4096 - 5119 */
845 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
846
847 /* Restore GICD_IPRIORITYR for INTIDs 32 - 1019 */
848 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
849
850 /* Restore GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
851 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
852
853 /* Restore GICD_ICFGR for INTIDs 32 - 1019 */
854 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
855
856 /* Restore GICD_ICFGRE for INTIDs 4096 - 5119 */
857 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
858
859 /* Restore GICD_IGRPMODR for INTIDs 32 - 1019 */
860 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
861
862 /* Restore GICD_IGRPMODRE for INTIDs 4096 - 5119 */
863 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
864
865 /* Restore GICD_NSACR for INTIDs 32 - 1019 */
866 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
867
868 /* Restore GICD_NSACRE for INTIDs 4096 - 5119 */
869 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
870
871 /* Restore GICD_IROUTER for INTIDs 32 - 1019 */
872 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
873
874 /* Restore GICD_IROUTERE for INTIDs 4096 - 5119 */
875 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
876
877 /*
878 * Restore ISENABLER(E), ISPENDR(E) and ISACTIVER(E) after
879 * the interrupts are configured.
880 */
881
882 /* Restore GICD_ISENABLER for INT_IDs 32 - 1019 */
883 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
884
885 /* Restore GICD_ISENABLERE for INT_IDs 4096 - 5119 */
886 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
887
888 /* Restore GICD_ISPENDR for INTIDs 32 - 1019 */
889 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
890
891 /* Restore GICD_ISPENDRE for INTIDs 4096 - 5119 */
892 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
893
894 /* Restore GICD_ISACTIVER for INTIDs 32 - 1019 */
895 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
896
897 /* Restore GICD_ISACTIVERE for INTIDs 4096 - 5119 */
898 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
899
900 /* Restore the GICD_CTLR */
901 gicd_write_ctlr(gicd_base, dist_ctx->gicd_ctlr);
902 gicd_wait_for_pending_write(gicd_base);
903 }
904
905 /*******************************************************************************
906 * This function gets the priority of the interrupt the processor is currently
907 * servicing.
908 ******************************************************************************/
gicv3_get_running_priority(void)909 unsigned int gicv3_get_running_priority(void)
910 {
911 return (unsigned int)read_icc_rpr_el1();
912 }
913
914 /*******************************************************************************
915 * This function checks if the interrupt identified by id is active (whether the
916 * state is either active, or active and pending). The proc_num is used if the
917 * interrupt is SGI or (E)PPI and programs the corresponding Redistributor
918 * interface.
919 ******************************************************************************/
gicv3_get_interrupt_active(unsigned int id,unsigned int proc_num)920 unsigned int gicv3_get_interrupt_active(unsigned int id, unsigned int proc_num)
921 {
922 assert(gicv3_driver_data != NULL);
923 assert(gicv3_driver_data->gicd_base != 0U);
924 assert(proc_num < gicv3_driver_data->rdistif_num);
925 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
926
927 /* Check interrupt ID */
928 if (is_sgi_ppi(id)) {
929 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
930 return gicr_get_isactiver(
931 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
932 }
933
934 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
935 return gicd_get_isactiver(gicv3_driver_data->gicd_base, id);
936 }
937
938 /*******************************************************************************
939 * This function enables the interrupt identified by id. The proc_num
940 * is used if the interrupt is SGI or PPI, and programs the corresponding
941 * Redistributor interface.
942 ******************************************************************************/
gicv3_enable_interrupt(unsigned int id,unsigned int proc_num)943 void gicv3_enable_interrupt(unsigned int id, unsigned int proc_num)
944 {
945 assert(gicv3_driver_data != NULL);
946 assert(gicv3_driver_data->gicd_base != 0U);
947 assert(proc_num < gicv3_driver_data->rdistif_num);
948 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
949
950 /*
951 * Ensure that any shared variable updates depending on out of band
952 * interrupt trigger are observed before enabling interrupt.
953 */
954 dsbishst();
955
956 /* Check interrupt ID */
957 if (is_sgi_ppi(id)) {
958 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
959 gicr_set_isenabler(
960 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
961 } else {
962 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
963 gicd_set_isenabler(gicv3_driver_data->gicd_base, id);
964 }
965 }
966
967 /*******************************************************************************
968 * This function disables the interrupt identified by id. The proc_num
969 * is used if the interrupt is SGI or PPI, and programs the corresponding
970 * Redistributor interface.
971 ******************************************************************************/
gicv3_disable_interrupt(unsigned int id,unsigned int proc_num)972 void gicv3_disable_interrupt(unsigned int id, unsigned int proc_num)
973 {
974 assert(gicv3_driver_data != NULL);
975 assert(gicv3_driver_data->gicd_base != 0U);
976 assert(proc_num < gicv3_driver_data->rdistif_num);
977 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
978
979 /*
980 * Disable interrupt, and ensure that any shared variable updates
981 * depending on out of band interrupt trigger are observed afterwards.
982 */
983
984 /* Check interrupt ID */
985 if (is_sgi_ppi(id)) {
986 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
987 gicr_set_icenabler(
988 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
989
990 /* Write to clear enable requires waiting for pending writes */
991 gicr_wait_for_pending_write(
992 gicv3_driver_data->rdistif_base_addrs[proc_num]);
993 } else {
994 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
995 gicd_set_icenabler(gicv3_driver_data->gicd_base, id);
996
997 /* Write to clear enable requires waiting for pending writes */
998 gicd_wait_for_pending_write(gicv3_driver_data->gicd_base);
999 }
1000
1001 dsbishst();
1002 }
1003
1004 /*******************************************************************************
1005 * This function sets the interrupt priority as supplied for the given interrupt
1006 * id.
1007 ******************************************************************************/
gicv3_set_interrupt_priority(unsigned int id,unsigned int proc_num,unsigned int priority)1008 void gicv3_set_interrupt_priority(unsigned int id, unsigned int proc_num,
1009 unsigned int priority)
1010 {
1011 uintptr_t gicr_base;
1012
1013 assert(gicv3_driver_data != NULL);
1014 assert(gicv3_driver_data->gicd_base != 0U);
1015 assert(proc_num < gicv3_driver_data->rdistif_num);
1016 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1017
1018 /* Check interrupt ID */
1019 if (is_sgi_ppi(id)) {
1020 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1021 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1022 gicr_set_ipriorityr(gicr_base, id, priority);
1023 } else {
1024 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1025 gicd_set_ipriorityr(gicv3_driver_data->gicd_base, id, priority);
1026 }
1027 }
1028
1029 /*******************************************************************************
1030 * This function assigns group for the interrupt identified by id. The proc_num
1031 * is used if the interrupt is SGI or (E)PPI, and programs the corresponding
1032 * Redistributor interface. The group can be any of GICV3_INTR_GROUP*
1033 ******************************************************************************/
gicv3_set_interrupt_type(unsigned int id,unsigned int proc_num,unsigned int type)1034 void gicv3_set_interrupt_type(unsigned int id, unsigned int proc_num,
1035 unsigned int type)
1036 {
1037 bool igroup = false, grpmod = false;
1038 uintptr_t gicr_base;
1039
1040 assert(gicv3_driver_data != NULL);
1041 assert(gicv3_driver_data->gicd_base != 0U);
1042 assert(proc_num < gicv3_driver_data->rdistif_num);
1043 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1044
1045 switch (type) {
1046 case INTR_GROUP1S:
1047 igroup = false;
1048 grpmod = true;
1049 break;
1050 case INTR_GROUP0:
1051 igroup = false;
1052 grpmod = false;
1053 break;
1054 case INTR_GROUP1NS:
1055 igroup = true;
1056 grpmod = false;
1057 break;
1058 default:
1059 assert(false);
1060 break;
1061 }
1062
1063 /* Check interrupt ID */
1064 if (is_sgi_ppi(id)) {
1065 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1066 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1067
1068 igroup ? gicr_set_igroupr(gicr_base, id) :
1069 gicr_clr_igroupr(gicr_base, id);
1070 grpmod ? gicr_set_igrpmodr(gicr_base, id) :
1071 gicr_clr_igrpmodr(gicr_base, id);
1072 } else {
1073 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1074
1075 /* Serialize read-modify-write to Distributor registers */
1076 spin_lock(&gic_lock);
1077
1078 igroup ? gicd_set_igroupr(gicv3_driver_data->gicd_base, id) :
1079 gicd_clr_igroupr(gicv3_driver_data->gicd_base, id);
1080 grpmod ? gicd_set_igrpmodr(gicv3_driver_data->gicd_base, id) :
1081 gicd_clr_igrpmodr(gicv3_driver_data->gicd_base, id);
1082
1083 spin_unlock(&gic_lock);
1084 }
1085 }
1086
1087 /*******************************************************************************
1088 * This function raises the specified Secure Group 0 SGI.
1089 *
1090 * The target parameter must be a valid MPIDR in the system.
1091 ******************************************************************************/
gicv3_raise_secure_g0_sgi(unsigned int sgi_num,u_register_t target)1092 void gicv3_raise_secure_g0_sgi(unsigned int sgi_num, u_register_t target)
1093 {
1094 unsigned int tgt, aff3, aff2, aff1, aff0;
1095 uint64_t sgi_val;
1096
1097 /* Verify interrupt number is in the SGI range */
1098 assert((sgi_num >= MIN_SGI_ID) && (sgi_num < MIN_PPI_ID));
1099
1100 /* Extract affinity fields from target */
1101 aff0 = MPIDR_AFFLVL0_VAL(target);
1102 aff1 = MPIDR_AFFLVL1_VAL(target);
1103 aff2 = MPIDR_AFFLVL2_VAL(target);
1104 aff3 = MPIDR_AFFLVL3_VAL(target);
1105
1106 /*
1107 * Make target list from affinity 0, and ensure GICv3 SGI can target
1108 * this PE.
1109 */
1110 assert(aff0 < GICV3_MAX_SGI_TARGETS);
1111 tgt = BIT_32(aff0);
1112
1113 /* Raise SGI to PE specified by its affinity */
1114 sgi_val = GICV3_SGIR_VALUE(aff3, aff2, aff1, sgi_num, SGIR_IRM_TO_AFF,
1115 tgt);
1116
1117 /*
1118 * Ensure that any shared variable updates depending on out of band
1119 * interrupt trigger are observed before raising SGI.
1120 */
1121 dsbishst();
1122 write_icc_sgi0r_el1(sgi_val);
1123 isb();
1124 }
1125
1126 /*******************************************************************************
1127 * This function sets the interrupt routing for the given (E)SPI interrupt id.
1128 * The interrupt routing is specified in routing mode and mpidr.
1129 *
1130 * The routing mode can be either of:
1131 * - GICV3_IRM_ANY
1132 * - GICV3_IRM_PE
1133 *
1134 * The mpidr is the affinity of the PE to which the interrupt will be routed,
1135 * and is ignored for routing mode GICV3_IRM_ANY.
1136 ******************************************************************************/
gicv3_set_spi_routing(unsigned int id,unsigned int irm,u_register_t mpidr)1137 void gicv3_set_spi_routing(unsigned int id, unsigned int irm, u_register_t mpidr)
1138 {
1139 unsigned long long aff;
1140 uint64_t router;
1141
1142 assert(gicv3_driver_data != NULL);
1143 assert(gicv3_driver_data->gicd_base != 0U);
1144
1145 assert((irm == GICV3_IRM_ANY) || (irm == GICV3_IRM_PE));
1146
1147 assert(IS_SPI(id));
1148
1149 aff = gicd_irouter_val_from_mpidr(mpidr, irm);
1150 gicd_write_irouter(gicv3_driver_data->gicd_base, id, aff);
1151
1152 /*
1153 * In implementations that do not require 1 of N distribution of SPIs,
1154 * IRM might be RAZ/WI. Read back and verify IRM bit.
1155 */
1156 if (irm == GICV3_IRM_ANY) {
1157 router = gicd_read_irouter(gicv3_driver_data->gicd_base, id);
1158 if (((router >> IROUTER_IRM_SHIFT) & IROUTER_IRM_MASK) == 0U) {
1159 ERROR("GICv3 implementation doesn't support routing ANY\n");
1160 panic();
1161 }
1162 }
1163 }
1164
1165 /*******************************************************************************
1166 * This function clears the pending status of an interrupt identified by id.
1167 * The proc_num is used if the interrupt is SGI or (E)PPI, and programs the
1168 * corresponding Redistributor interface.
1169 ******************************************************************************/
gicv3_clear_interrupt_pending(unsigned int id,unsigned int proc_num)1170 void gicv3_clear_interrupt_pending(unsigned int id, unsigned int proc_num)
1171 {
1172 assert(gicv3_driver_data != NULL);
1173 assert(gicv3_driver_data->gicd_base != 0U);
1174 assert(proc_num < gicv3_driver_data->rdistif_num);
1175 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1176
1177 /*
1178 * Clear pending interrupt, and ensure that any shared variable updates
1179 * depending on out of band interrupt trigger are observed afterwards.
1180 */
1181
1182 /* Check interrupt ID */
1183 if (is_sgi_ppi(id)) {
1184 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1185 gicr_set_icpendr(
1186 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1187 } else {
1188 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1189 gicd_set_icpendr(gicv3_driver_data->gicd_base, id);
1190 }
1191
1192 dsbishst();
1193 }
1194
1195 /*******************************************************************************
1196 * This function sets the pending status of an interrupt identified by id.
1197 * The proc_num is used if the interrupt is SGI or PPI and programs the
1198 * corresponding Redistributor interface.
1199 ******************************************************************************/
gicv3_set_interrupt_pending(unsigned int id,unsigned int proc_num)1200 void gicv3_set_interrupt_pending(unsigned int id, unsigned int proc_num)
1201 {
1202 assert(gicv3_driver_data != NULL);
1203 assert(gicv3_driver_data->gicd_base != 0U);
1204 assert(proc_num < gicv3_driver_data->rdistif_num);
1205 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1206
1207 /*
1208 * Ensure that any shared variable updates depending on out of band
1209 * interrupt trigger are observed before setting interrupt pending.
1210 */
1211 dsbishst();
1212
1213 /* Check interrupt ID */
1214 if (is_sgi_ppi(id)) {
1215 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1216 gicr_set_ispendr(
1217 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1218 } else {
1219 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1220 gicd_set_ispendr(gicv3_driver_data->gicd_base, id);
1221 }
1222 }
1223
1224 /*******************************************************************************
1225 * This function sets the PMR register with the supplied value. Returns the
1226 * original PMR.
1227 ******************************************************************************/
gicv3_set_pmr(unsigned int mask)1228 unsigned int gicv3_set_pmr(unsigned int mask)
1229 {
1230 unsigned int old_mask;
1231
1232 old_mask = (unsigned int)read_icc_pmr_el1();
1233
1234 /*
1235 * Order memory updates w.r.t. PMR write, and ensure they're visible
1236 * before potential out of band interrupt trigger because of PMR update.
1237 * PMR system register writes are self-synchronizing, so no ISB required
1238 * thereafter.
1239 */
1240 dsbishst();
1241 write_icc_pmr_el1(mask);
1242
1243 return old_mask;
1244 }
1245
1246 /*******************************************************************************
1247 * This function delegates the responsibility of discovering the corresponding
1248 * Redistributor frames to each CPU itself. It is a modified version of
1249 * gicv3_rdistif_base_addrs_probe() and is executed by each CPU in the platform
1250 * unlike the previous way in which only the Primary CPU did the discovery of
1251 * all the Redistributor frames for every CPU. It also handles the scenario in
1252 * which the frames of various CPUs are not contiguous in physical memory.
1253 ******************************************************************************/
gicv3_rdistif_probe(const uintptr_t gicr_frame)1254 int gicv3_rdistif_probe(const uintptr_t gicr_frame)
1255 {
1256 u_register_t mpidr, mpidr_self;
1257 unsigned int proc_num;
1258 uint64_t typer_val;
1259 uintptr_t rdistif_base;
1260 bool gicr_frame_found = false;
1261
1262 assert(gicv3_driver_data->gicr_base == 0U);
1263
1264 /* Ensure this function is called with Data Cache enabled */
1265 #ifndef __aarch64__
1266 assert((read_sctlr() & SCTLR_C_BIT) != 0U);
1267 #else
1268 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1269 #endif /* !__aarch64__ */
1270
1271 mpidr_self = read_mpidr_el1() & MPIDR_AFFINITY_MASK;
1272 rdistif_base = gicr_frame;
1273 do {
1274 typer_val = gicr_read_typer(rdistif_base);
1275 mpidr = mpidr_from_gicr_typer(typer_val);
1276 if (gicv3_driver_data->mpidr_to_core_pos != NULL) {
1277 proc_num = gicv3_driver_data->mpidr_to_core_pos(mpidr);
1278 } else {
1279 proc_num = (unsigned int)(typer_val >>
1280 TYPER_PROC_NUM_SHIFT) & TYPER_PROC_NUM_MASK;
1281 }
1282 if (mpidr == mpidr_self) {
1283 /* The base address doesn't need to be initialized on
1284 * every warm boot.
1285 */
1286 if (gicv3_driver_data->rdistif_base_addrs[proc_num]
1287 != 0U) {
1288 return 0;
1289 }
1290 gicv3_driver_data->rdistif_base_addrs[proc_num] =
1291 rdistif_base;
1292 gicr_frame_found = true;
1293 break;
1294 }
1295 rdistif_base += gicv3_redist_size(typer_val);
1296 } while ((typer_val & TYPER_LAST_BIT) == 0U);
1297
1298 if (!gicr_frame_found) {
1299 return -1;
1300 }
1301
1302 /*
1303 * Flush the driver data to ensure coherency. This is
1304 * not required if platform has HW_ASSISTED_COHERENCY
1305 * enabled.
1306 */
1307 #if !HW_ASSISTED_COHERENCY
1308 /*
1309 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
1310 */
1311 flush_dcache_range((uintptr_t)&(gicv3_driver_data->rdistif_base_addrs[proc_num]),
1312 sizeof(*(gicv3_driver_data->rdistif_base_addrs)));
1313 #endif
1314 return 0; /* Found matching GICR frame */
1315 }
1316
1317 /******************************************************************************
1318 * This function checks the interrupt ID and returns true for SGIs and (E)PPIs
1319 * and false for (E)SPIs IDs.
1320 *****************************************************************************/
is_sgi_ppi(unsigned int id)1321 static bool is_sgi_ppi(unsigned int id)
1322 {
1323 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
1324 if (IS_SGI_PPI(id)) {
1325 return true;
1326 }
1327
1328 /* SPIs: 32-1019, ESPIs: 4096-5119 */
1329 if (IS_SPI(id)) {
1330 return false;
1331 }
1332
1333 assert(false);
1334 panic();
1335 }
1336