1 /*
2 * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <string.h>
9
10 #include <arch.h>
11 #include <arch_helpers.h>
12 #include <common/bl_common.h>
13 #include <common/debug.h>
14 #include <context.h>
15 #include <drivers/delay_timer.h>
16 #include <lib/el3_runtime/context_mgmt.h>
17 #include <lib/utils.h>
18 #include <plat/common/platform.h>
19
20 #include "psci_private.h"
21
22 /*
23 * SPD power management operations, expected to be supplied by the registered
24 * SPD on successful SP initialization
25 */
26 const spd_pm_ops_t *psci_spd_pm;
27
28 /*
29 * PSCI requested local power state map. This array is used to store the local
30 * power states requested by a CPU for power levels from level 1 to
31 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power
32 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a
33 * CPU are the same.
34 *
35 * During state coordination, the platform is passed an array containing the
36 * local states requested for a particular non cpu power domain by each cpu
37 * within the domain.
38 *
39 * TODO: Dense packing of the requested states will cause cache thrashing
40 * when multiple power domains write to it. If we allocate the requested
41 * states at each power level in a cache-line aligned per-domain memory,
42 * the cache thrashing can be avoided.
43 */
44 static plat_local_state_t
45 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT];
46
47 unsigned int psci_plat_core_count;
48
49 /*******************************************************************************
50 * Arrays that hold the platform's power domain tree information for state
51 * management of power domains.
52 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain
53 * which is an ancestor of a CPU power domain.
54 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain
55 ******************************************************************************/
56 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS]
57 #if USE_COHERENT_MEM
58 __section("tzfw_coherent_mem")
59 #endif
60 ;
61
62 /* Lock for PSCI state coordination */
63 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]);
64
65 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT];
66
67 /*******************************************************************************
68 * Pointer to functions exported by the platform to complete power mgmt. ops
69 ******************************************************************************/
70 const plat_psci_ops_t *psci_plat_pm_ops;
71
72 /******************************************************************************
73 * Check that the maximum power level supported by the platform makes sense
74 *****************************************************************************/
75 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) &&
76 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL),
77 assert_platform_max_pwrlvl_check);
78
79 /*
80 * The plat_local_state used by the platform is one of these types: RUN,
81 * RETENTION and OFF. The platform can define further sub-states for each type
82 * apart from RUN. This categorization is done to verify the sanity of the
83 * psci_power_state passed by the platform and to print debug information. The
84 * categorization is done on the basis of the following conditions:
85 *
86 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN.
87 *
88 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is
89 * STATE_TYPE_RETN.
90 *
91 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is
92 * STATE_TYPE_OFF.
93 */
94 typedef enum plat_local_state_type {
95 STATE_TYPE_RUN = 0,
96 STATE_TYPE_RETN,
97 STATE_TYPE_OFF
98 } plat_local_state_type_t;
99
100 /* Function used to categorize plat_local_state. */
find_local_state_type(plat_local_state_t state)101 static plat_local_state_type_t find_local_state_type(plat_local_state_t state)
102 {
103 if (state != 0U) {
104 if (state > PLAT_MAX_RET_STATE) {
105 return STATE_TYPE_OFF;
106 } else {
107 return STATE_TYPE_RETN;
108 }
109 } else {
110 return STATE_TYPE_RUN;
111 }
112 }
113
114 /******************************************************************************
115 * Check that the maximum retention level supported by the platform is less
116 * than the maximum off level.
117 *****************************************************************************/
118 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE,
119 assert_platform_max_off_and_retn_state_check);
120
121 /******************************************************************************
122 * This function ensures that the power state parameter in a CPU_SUSPEND request
123 * is valid. If so, it returns the requested states for each power level.
124 *****************************************************************************/
psci_validate_power_state(unsigned int power_state,psci_power_state_t * state_info)125 int psci_validate_power_state(unsigned int power_state,
126 psci_power_state_t *state_info)
127 {
128 /* Check SBZ bits in power state are zero */
129 if (psci_check_power_state(power_state) != 0U)
130 return PSCI_E_INVALID_PARAMS;
131
132 assert(psci_plat_pm_ops->validate_power_state != NULL);
133
134 /* Validate the power_state using platform pm_ops */
135 return psci_plat_pm_ops->validate_power_state(power_state, state_info);
136 }
137
138 /******************************************************************************
139 * This function retrieves the `psci_power_state_t` for system suspend from
140 * the platform.
141 *****************************************************************************/
psci_query_sys_suspend_pwrstate(psci_power_state_t * state_info)142 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info)
143 {
144 /*
145 * Assert that the required pm_ops hook is implemented to ensure that
146 * the capability detected during psci_setup() is valid.
147 */
148 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL);
149
150 /*
151 * Query the platform for the power_state required for system suspend
152 */
153 psci_plat_pm_ops->get_sys_suspend_power_state(state_info);
154 }
155
156 /*******************************************************************************
157 * This function verifies that the all the other cores in the system have been
158 * turned OFF and the current CPU is the last running CPU in the system.
159 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false)
160 * otherwise.
161 ******************************************************************************/
psci_is_last_on_cpu(void)162 unsigned int psci_is_last_on_cpu(void)
163 {
164 unsigned int cpu_idx, my_idx = plat_my_core_pos();
165
166 for (cpu_idx = 0; cpu_idx < psci_plat_core_count;
167 cpu_idx++) {
168 if (cpu_idx == my_idx) {
169 assert(psci_get_aff_info_state() == AFF_STATE_ON);
170 continue;
171 }
172
173 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF)
174 return 0;
175 }
176
177 return 1;
178 }
179
180 /*******************************************************************************
181 * Routine to return the maximum power level to traverse to after a cpu has
182 * been physically powered up. It is expected to be called immediately after
183 * reset from assembler code.
184 ******************************************************************************/
get_power_on_target_pwrlvl(void)185 static unsigned int get_power_on_target_pwrlvl(void)
186 {
187 unsigned int pwrlvl;
188
189 /*
190 * Assume that this cpu was suspended and retrieve its target power
191 * level. If it is invalid then it could only have been turned off
192 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a
193 * cpu can be turned off to.
194 */
195 pwrlvl = psci_get_suspend_pwrlvl();
196 if (pwrlvl == PSCI_INVALID_PWR_LVL)
197 pwrlvl = PLAT_MAX_PWR_LVL;
198 assert(pwrlvl < PSCI_INVALID_PWR_LVL);
199 return pwrlvl;
200 }
201
202 /******************************************************************************
203 * Helper function to update the requested local power state array. This array
204 * does not store the requested state for the CPU power level. Hence an
205 * assertion is added to prevent us from accessing the CPU power level.
206 *****************************************************************************/
psci_set_req_local_pwr_state(unsigned int pwrlvl,unsigned int cpu_idx,plat_local_state_t req_pwr_state)207 static void psci_set_req_local_pwr_state(unsigned int pwrlvl,
208 unsigned int cpu_idx,
209 plat_local_state_t req_pwr_state)
210 {
211 assert(pwrlvl > PSCI_CPU_PWR_LVL);
212 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
213 (cpu_idx < psci_plat_core_count)) {
214 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state;
215 }
216 }
217
218 /******************************************************************************
219 * This function initializes the psci_req_local_pwr_states.
220 *****************************************************************************/
psci_init_req_local_pwr_states(void)221 void __init psci_init_req_local_pwr_states(void)
222 {
223 /* Initialize the requested state of all non CPU power domains as OFF */
224 unsigned int pwrlvl;
225 unsigned int core;
226
227 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) {
228 for (core = 0; core < psci_plat_core_count; core++) {
229 psci_req_local_pwr_states[pwrlvl][core] =
230 PLAT_MAX_OFF_STATE;
231 }
232 }
233 }
234
235 /******************************************************************************
236 * Helper function to return a reference to an array containing the local power
237 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the
238 * array will be the number of cpu power domains of which this power domain is
239 * an ancestor. These requested states will be used to determine a suitable
240 * target state for this power domain during psci state coordination. An
241 * assertion is added to prevent us from accessing the CPU power level.
242 *****************************************************************************/
psci_get_req_local_pwr_states(unsigned int pwrlvl,unsigned int cpu_idx)243 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl,
244 unsigned int cpu_idx)
245 {
246 assert(pwrlvl > PSCI_CPU_PWR_LVL);
247
248 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
249 (cpu_idx < psci_plat_core_count)) {
250 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx];
251 } else
252 return NULL;
253 }
254
255 /*
256 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent
257 * memory.
258 *
259 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory,
260 * it's accessed by both cached and non-cached participants. To serve the common
261 * minimum, perform a cache flush before read and after write so that non-cached
262 * participants operate on latest data in main memory.
263 *
264 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent
265 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent.
266 * In both cases, no cache operations are required.
267 */
268
269 /*
270 * Retrieve local state of non-CPU power domain node from a non-cached CPU,
271 * after any required cache maintenance operation.
272 */
get_non_cpu_pd_node_local_state(unsigned int parent_idx)273 static plat_local_state_t get_non_cpu_pd_node_local_state(
274 unsigned int parent_idx)
275 {
276 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
277 flush_dcache_range(
278 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
279 sizeof(psci_non_cpu_pd_nodes[parent_idx]));
280 #endif
281 return psci_non_cpu_pd_nodes[parent_idx].local_state;
282 }
283
284 /*
285 * Update local state of non-CPU power domain node from a cached CPU; perform
286 * any required cache maintenance operation afterwards.
287 */
set_non_cpu_pd_node_local_state(unsigned int parent_idx,plat_local_state_t state)288 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx,
289 plat_local_state_t state)
290 {
291 psci_non_cpu_pd_nodes[parent_idx].local_state = state;
292 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
293 flush_dcache_range(
294 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
295 sizeof(psci_non_cpu_pd_nodes[parent_idx]));
296 #endif
297 }
298
299 /******************************************************************************
300 * Helper function to return the current local power state of each power domain
301 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This
302 * function will be called after a cpu is powered on to find the local state
303 * each power domain has emerged from.
304 *****************************************************************************/
psci_get_target_local_pwr_states(unsigned int end_pwrlvl,psci_power_state_t * target_state)305 void psci_get_target_local_pwr_states(unsigned int end_pwrlvl,
306 psci_power_state_t *target_state)
307 {
308 unsigned int parent_idx, lvl;
309 plat_local_state_t *pd_state = target_state->pwr_domain_state;
310
311 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state();
312 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
313
314 /* Copy the local power state from node to state_info */
315 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
316 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx);
317 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
318 }
319
320 /* Set the the higher levels to RUN */
321 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++)
322 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
323 }
324
325 /******************************************************************************
326 * Helper function to set the target local power state that each power domain
327 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will
328 * enter. This function will be called after coordination of requested power
329 * states has been done for each power level.
330 *****************************************************************************/
psci_set_target_local_pwr_states(unsigned int end_pwrlvl,const psci_power_state_t * target_state)331 static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl,
332 const psci_power_state_t *target_state)
333 {
334 unsigned int parent_idx, lvl;
335 const plat_local_state_t *pd_state = target_state->pwr_domain_state;
336
337 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]);
338
339 /*
340 * Need to flush as local_state might be accessed with Data Cache
341 * disabled during power on
342 */
343 psci_flush_cpu_data(psci_svc_cpu_data.local_state);
344
345 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
346
347 /* Copy the local_state from state_info */
348 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) {
349 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]);
350 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
351 }
352 }
353
354
355 /*******************************************************************************
356 * PSCI helper function to get the parent nodes corresponding to a cpu_index.
357 ******************************************************************************/
psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx,unsigned int end_lvl,unsigned int * node_index)358 void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx,
359 unsigned int end_lvl,
360 unsigned int *node_index)
361 {
362 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node;
363 unsigned int i;
364 unsigned int *node = node_index;
365
366 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) {
367 *node = parent_node;
368 node++;
369 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node;
370 }
371 }
372
373 /******************************************************************************
374 * This function is invoked post CPU power up and initialization. It sets the
375 * affinity info state, target power state and requested power state for the
376 * current CPU and all its ancestor power domains to RUN.
377 *****************************************************************************/
psci_set_pwr_domains_to_run(unsigned int end_pwrlvl)378 void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl)
379 {
380 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl;
381 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
382
383 /* Reset the local_state to RUN for the non cpu power domains. */
384 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
385 set_non_cpu_pd_node_local_state(parent_idx,
386 PSCI_LOCAL_STATE_RUN);
387 psci_set_req_local_pwr_state(lvl,
388 cpu_idx,
389 PSCI_LOCAL_STATE_RUN);
390 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
391 }
392
393 /* Set the affinity info state to ON */
394 psci_set_aff_info_state(AFF_STATE_ON);
395
396 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN);
397 psci_flush_cpu_data(psci_svc_cpu_data);
398 }
399
400 /******************************************************************************
401 * This function is passed the local power states requested for each power
402 * domain (state_info) between the current CPU domain and its ancestors until
403 * the target power level (end_pwrlvl). It updates the array of requested power
404 * states with this information.
405 *
406 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it
407 * retrieves the states requested by all the cpus of which the power domain at
408 * that level is an ancestor. It passes this information to the platform to
409 * coordinate and return the target power state. If the target state for a level
410 * is RUN then subsequent levels are not considered. At the CPU level, state
411 * coordination is not required. Hence, the requested and the target states are
412 * the same.
413 *
414 * The 'state_info' is updated with the target state for each level between the
415 * CPU and the 'end_pwrlvl' and returned to the caller.
416 *
417 * This function will only be invoked with data cache enabled and while
418 * powering down a core.
419 *****************************************************************************/
psci_do_state_coordination(unsigned int end_pwrlvl,psci_power_state_t * state_info)420 void psci_do_state_coordination(unsigned int end_pwrlvl,
421 psci_power_state_t *state_info)
422 {
423 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
424 unsigned int start_idx;
425 unsigned int ncpus;
426 plat_local_state_t target_state, *req_states;
427
428 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL);
429 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
430
431 /* For level 0, the requested state will be equivalent
432 to target state */
433 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
434
435 /* First update the requested power state */
436 psci_set_req_local_pwr_state(lvl, cpu_idx,
437 state_info->pwr_domain_state[lvl]);
438
439 /* Get the requested power states for this power level */
440 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx;
441 req_states = psci_get_req_local_pwr_states(lvl, start_idx);
442
443 /*
444 * Let the platform coordinate amongst the requested states at
445 * this power level and return the target local power state.
446 */
447 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus;
448 target_state = plat_get_target_pwr_state(lvl,
449 req_states,
450 ncpus);
451
452 state_info->pwr_domain_state[lvl] = target_state;
453
454 /* Break early if the negotiated target power state is RUN */
455 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0)
456 break;
457
458 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
459 }
460
461 /*
462 * This is for cases when we break out of the above loop early because
463 * the target power state is RUN at a power level < end_pwlvl.
464 * We update the requested power state from state_info and then
465 * set the target state as RUN.
466 */
467 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) {
468 psci_set_req_local_pwr_state(lvl, cpu_idx,
469 state_info->pwr_domain_state[lvl]);
470 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
471
472 }
473
474 /* Update the target state in the power domain nodes */
475 psci_set_target_local_pwr_states(end_pwrlvl, state_info);
476 }
477
478 /******************************************************************************
479 * This function validates a suspend request by making sure that if a standby
480 * state is requested then no power level is turned off and the highest power
481 * level is placed in a standby/retention state.
482 *
483 * It also ensures that the state level X will enter is not shallower than the
484 * state level X + 1 will enter.
485 *
486 * This validation will be enabled only for DEBUG builds as the platform is
487 * expected to perform these validations as well.
488 *****************************************************************************/
psci_validate_suspend_req(const psci_power_state_t * state_info,unsigned int is_power_down_state)489 int psci_validate_suspend_req(const psci_power_state_t *state_info,
490 unsigned int is_power_down_state)
491 {
492 unsigned int max_off_lvl, target_lvl, max_retn_lvl;
493 plat_local_state_t state;
494 plat_local_state_type_t req_state_type, deepest_state_type;
495 int i;
496
497 /* Find the target suspend power level */
498 target_lvl = psci_find_target_suspend_lvl(state_info);
499 if (target_lvl == PSCI_INVALID_PWR_LVL)
500 return PSCI_E_INVALID_PARAMS;
501
502 /* All power domain levels are in a RUN state to begin with */
503 deepest_state_type = STATE_TYPE_RUN;
504
505 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) {
506 state = state_info->pwr_domain_state[i];
507 req_state_type = find_local_state_type(state);
508
509 /*
510 * While traversing from the highest power level to the lowest,
511 * the state requested for lower levels has to be the same or
512 * deeper i.e. equal to or greater than the state at the higher
513 * levels. If this condition is true, then the requested state
514 * becomes the deepest state encountered so far.
515 */
516 if (req_state_type < deepest_state_type)
517 return PSCI_E_INVALID_PARAMS;
518 deepest_state_type = req_state_type;
519 }
520
521 /* Find the highest off power level */
522 max_off_lvl = psci_find_max_off_lvl(state_info);
523
524 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */
525 max_retn_lvl = PSCI_INVALID_PWR_LVL;
526 if (target_lvl != max_off_lvl)
527 max_retn_lvl = target_lvl;
528
529 /*
530 * If this is not a request for a power down state then max off level
531 * has to be invalid and max retention level has to be a valid power
532 * level.
533 */
534 if ((is_power_down_state == 0U) &&
535 ((max_off_lvl != PSCI_INVALID_PWR_LVL) ||
536 (max_retn_lvl == PSCI_INVALID_PWR_LVL)))
537 return PSCI_E_INVALID_PARAMS;
538
539 return PSCI_E_SUCCESS;
540 }
541
542 /******************************************************************************
543 * This function finds the highest power level which will be powered down
544 * amongst all the power levels specified in the 'state_info' structure
545 *****************************************************************************/
psci_find_max_off_lvl(const psci_power_state_t * state_info)546 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info)
547 {
548 int i;
549
550 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
551 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0)
552 return (unsigned int) i;
553 }
554
555 return PSCI_INVALID_PWR_LVL;
556 }
557
558 /******************************************************************************
559 * This functions finds the level of the highest power domain which will be
560 * placed in a low power state during a suspend operation.
561 *****************************************************************************/
psci_find_target_suspend_lvl(const psci_power_state_t * state_info)562 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info)
563 {
564 int i;
565
566 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
567 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0)
568 return (unsigned int) i;
569 }
570
571 return PSCI_INVALID_PWR_LVL;
572 }
573
574 /*******************************************************************************
575 * This function is passed the highest level in the topology tree that the
576 * operation should be applied to and a list of node indexes. It picks up locks
577 * from the node index list in order of increasing power domain level in the
578 * range specified.
579 ******************************************************************************/
psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl,const unsigned int * parent_nodes)580 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl,
581 const unsigned int *parent_nodes)
582 {
583 unsigned int parent_idx;
584 unsigned int level;
585
586 /* No locking required for level 0. Hence start locking from level 1 */
587 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) {
588 parent_idx = parent_nodes[level - 1U];
589 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]);
590 }
591 }
592
593 /*******************************************************************************
594 * This function is passed the highest level in the topology tree that the
595 * operation should be applied to and a list of node indexes. It releases the
596 * locks in order of decreasing power domain level in the range specified.
597 ******************************************************************************/
psci_release_pwr_domain_locks(unsigned int end_pwrlvl,const unsigned int * parent_nodes)598 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl,
599 const unsigned int *parent_nodes)
600 {
601 unsigned int parent_idx;
602 unsigned int level;
603
604 /* Unlock top down. No unlocking required for level 0. */
605 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) {
606 parent_idx = parent_nodes[level - 1U];
607 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]);
608 }
609 }
610
611 /*******************************************************************************
612 * Simple routine to determine whether a mpidr is valid or not.
613 ******************************************************************************/
psci_validate_mpidr(u_register_t mpidr)614 int psci_validate_mpidr(u_register_t mpidr)
615 {
616 if (plat_core_pos_by_mpidr(mpidr) < 0)
617 return PSCI_E_INVALID_PARAMS;
618
619 return PSCI_E_SUCCESS;
620 }
621
622 /*******************************************************************************
623 * This function determines the full entrypoint information for the requested
624 * PSCI entrypoint on power on/resume and returns it.
625 ******************************************************************************/
626 #ifdef __aarch64__
psci_get_ns_ep_info(entry_point_info_t * ep,uintptr_t entrypoint,u_register_t context_id)627 static int psci_get_ns_ep_info(entry_point_info_t *ep,
628 uintptr_t entrypoint,
629 u_register_t context_id)
630 {
631 u_register_t ep_attr, sctlr;
632 unsigned int daif, ee, mode;
633 u_register_t ns_scr_el3 = read_scr_el3();
634 u_register_t ns_sctlr_el1 = read_sctlr_el1();
635
636 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
637 read_sctlr_el2() : ns_sctlr_el1;
638 ee = 0;
639
640 ep_attr = NON_SECURE | EP_ST_DISABLE;
641 if ((sctlr & SCTLR_EE_BIT) != 0U) {
642 ep_attr |= EP_EE_BIG;
643 ee = 1;
644 }
645 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
646
647 ep->pc = entrypoint;
648 zeromem(&ep->args, sizeof(ep->args));
649 ep->args.arg0 = context_id;
650
651 /*
652 * Figure out whether the cpu enters the non-secure address space
653 * in aarch32 or aarch64
654 */
655 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) {
656
657 /*
658 * Check whether a Thumb entry point has been provided for an
659 * aarch64 EL
660 */
661 if ((entrypoint & 0x1UL) != 0UL)
662 return PSCI_E_INVALID_ADDRESS;
663
664 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1;
665
666 ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX,
667 DISABLE_ALL_EXCEPTIONS);
668 } else {
669
670 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
671 MODE32_hyp : MODE32_svc;
672
673 /*
674 * TODO: Choose async. exception bits if HYP mode is not
675 * implemented according to the values of SCR.{AW, FW} bits
676 */
677 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT;
678
679 ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee,
680 daif);
681 }
682
683 return PSCI_E_SUCCESS;
684 }
685 #else /* !__aarch64__ */
psci_get_ns_ep_info(entry_point_info_t * ep,uintptr_t entrypoint,u_register_t context_id)686 static int psci_get_ns_ep_info(entry_point_info_t *ep,
687 uintptr_t entrypoint,
688 u_register_t context_id)
689 {
690 u_register_t ep_attr;
691 unsigned int aif, ee, mode;
692 u_register_t scr = read_scr();
693 u_register_t ns_sctlr, sctlr;
694
695 /* Switch to non secure state */
696 write_scr(scr | SCR_NS_BIT);
697 isb();
698 ns_sctlr = read_sctlr();
699
700 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr;
701
702 /* Return to original state */
703 write_scr(scr);
704 isb();
705 ee = 0;
706
707 ep_attr = NON_SECURE | EP_ST_DISABLE;
708 if (sctlr & SCTLR_EE_BIT) {
709 ep_attr |= EP_EE_BIG;
710 ee = 1;
711 }
712 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
713
714 ep->pc = entrypoint;
715 zeromem(&ep->args, sizeof(ep->args));
716 ep->args.arg0 = context_id;
717
718 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc;
719
720 /*
721 * TODO: Choose async. exception bits if HYP mode is not
722 * implemented according to the values of SCR.{AW, FW} bits
723 */
724 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT;
725
726 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif);
727
728 return PSCI_E_SUCCESS;
729 }
730
731 #endif /* __aarch64__ */
732
733 /*******************************************************************************
734 * This function validates the entrypoint with the platform layer if the
735 * appropriate pm_ops hook is exported by the platform and returns the
736 * 'entry_point_info'.
737 ******************************************************************************/
psci_validate_entry_point(entry_point_info_t * ep,uintptr_t entrypoint,u_register_t context_id)738 int psci_validate_entry_point(entry_point_info_t *ep,
739 uintptr_t entrypoint,
740 u_register_t context_id)
741 {
742 int rc;
743
744 /* Validate the entrypoint using platform psci_ops */
745 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) {
746 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint);
747 if (rc != PSCI_E_SUCCESS)
748 return PSCI_E_INVALID_ADDRESS;
749 }
750
751 /*
752 * Verify and derive the re-entry information for
753 * the non-secure world from the non-secure state from
754 * where this call originated.
755 */
756 rc = psci_get_ns_ep_info(ep, entrypoint, context_id);
757 return rc;
758 }
759
760 /*******************************************************************************
761 * Generic handler which is called when a cpu is physically powered on. It
762 * traverses the node information and finds the highest power level powered
763 * off and performs generic, architectural, platform setup and state management
764 * to power on that power level and power levels below it.
765 * e.g. For a cpu that's been powered on, it will call the platform specific
766 * code to enable the gic cpu interface and for a cluster it will enable
767 * coherency at the interconnect level in addition to gic cpu interface.
768 ******************************************************************************/
psci_warmboot_entrypoint(void)769 void psci_warmboot_entrypoint(void)
770 {
771 unsigned int end_pwrlvl;
772 unsigned int cpu_idx = plat_my_core_pos();
773 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
774 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
775
776 /*
777 * Verify that we have been explicitly turned ON or resumed from
778 * suspend.
779 */
780 if (psci_get_aff_info_state() == AFF_STATE_OFF) {
781 ERROR("Unexpected affinity info state.\n");
782 panic();
783 }
784
785 /*
786 * Get the maximum power domain level to traverse to after this cpu
787 * has been physically powered up.
788 */
789 end_pwrlvl = get_power_on_target_pwrlvl();
790
791 /* Get the parent nodes */
792 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes);
793
794 /*
795 * This function acquires the lock corresponding to each power level so
796 * that by the time all locks are taken, the system topology is snapshot
797 * and state management can be done safely.
798 */
799 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
800
801 psci_get_target_local_pwr_states(end_pwrlvl, &state_info);
802
803 #if ENABLE_PSCI_STAT
804 plat_psci_stat_accounting_stop(&state_info);
805 #endif
806
807 /*
808 * This CPU could be resuming from suspend or it could have just been
809 * turned on. To distinguish between these 2 cases, we examine the
810 * affinity state of the CPU:
811 * - If the affinity state is ON_PENDING then it has just been
812 * turned on.
813 * - Else it is resuming from suspend.
814 *
815 * Depending on the type of warm reset identified, choose the right set
816 * of power management handler and perform the generic, architecture
817 * and platform specific handling.
818 */
819 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING)
820 psci_cpu_on_finish(cpu_idx, &state_info);
821 else
822 psci_cpu_suspend_finish(cpu_idx, &state_info);
823
824 /*
825 * Set the requested and target state of this CPU and all the higher
826 * power domains which are ancestors of this CPU to run.
827 */
828 psci_set_pwr_domains_to_run(end_pwrlvl);
829
830 #if ENABLE_PSCI_STAT
831 /*
832 * Update PSCI stats.
833 * Caches are off when writing stats data on the power down path.
834 * Since caches are now enabled, it's necessary to do cache
835 * maintenance before reading that same data.
836 */
837 psci_stats_update_pwr_up(end_pwrlvl, &state_info);
838 #endif
839
840 /*
841 * This loop releases the lock corresponding to each power level
842 * in the reverse order to which they were acquired.
843 */
844 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);
845 }
846
847 /*******************************************************************************
848 * This function initializes the set of hooks that PSCI invokes as part of power
849 * management operation. The power management hooks are expected to be provided
850 * by the SPD, after it finishes all its initialization
851 ******************************************************************************/
psci_register_spd_pm_hook(const spd_pm_ops_t * pm)852 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm)
853 {
854 assert(pm != NULL);
855 psci_spd_pm = pm;
856
857 if (pm->svc_migrate != NULL)
858 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64);
859
860 if (pm->svc_migrate_info != NULL)
861 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64)
862 | define_psci_cap(PSCI_MIG_INFO_TYPE);
863 }
864
865 /*******************************************************************************
866 * This function invokes the migrate info hook in the spd_pm_ops. It performs
867 * the necessary return value validation. If the Secure Payload is UP and
868 * migrate capable, it returns the mpidr of the CPU on which the Secure payload
869 * is resident through the mpidr parameter. Else the value of the parameter on
870 * return is undefined.
871 ******************************************************************************/
psci_spd_migrate_info(u_register_t * mpidr)872 int psci_spd_migrate_info(u_register_t *mpidr)
873 {
874 int rc;
875
876 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL))
877 return PSCI_E_NOT_SUPPORTED;
878
879 rc = psci_spd_pm->svc_migrate_info(mpidr);
880
881 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) ||
882 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED));
883
884 return rc;
885 }
886
887
888 /*******************************************************************************
889 * This function prints the state of all power domains present in the
890 * system
891 ******************************************************************************/
psci_print_power_domain_map(void)892 void psci_print_power_domain_map(void)
893 {
894 #if LOG_LEVEL >= LOG_LEVEL_INFO
895 unsigned int idx;
896 plat_local_state_t state;
897 plat_local_state_type_t state_type;
898
899 /* This array maps to the PSCI_STATE_X definitions in psci.h */
900 static const char * const psci_state_type_str[] = {
901 "ON",
902 "RETENTION",
903 "OFF",
904 };
905
906 INFO("PSCI Power Domain Map:\n");
907 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count);
908 idx++) {
909 state_type = find_local_state_type(
910 psci_non_cpu_pd_nodes[idx].local_state);
911 INFO(" Domain Node : Level %u, parent_node %d,"
912 " State %s (0x%x)\n",
913 psci_non_cpu_pd_nodes[idx].level,
914 psci_non_cpu_pd_nodes[idx].parent_node,
915 psci_state_type_str[state_type],
916 psci_non_cpu_pd_nodes[idx].local_state);
917 }
918
919 for (idx = 0; idx < psci_plat_core_count; idx++) {
920 state = psci_get_cpu_local_state_by_idx(idx);
921 state_type = find_local_state_type(state);
922 INFO(" CPU Node : MPID 0x%llx, parent_node %d,"
923 " State %s (0x%x)\n",
924 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr,
925 psci_cpu_pd_nodes[idx].parent_node,
926 psci_state_type_str[state_type],
927 psci_get_cpu_local_state_by_idx(idx));
928 }
929 #endif
930 }
931
932 /******************************************************************************
933 * Return whether any secondaries were powered up with CPU_ON call. A CPU that
934 * have ever been powered up would have set its MPDIR value to something other
935 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to
936 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is
937 * meaningful only when called on the primary CPU during early boot.
938 *****************************************************************************/
psci_secondaries_brought_up(void)939 int psci_secondaries_brought_up(void)
940 {
941 unsigned int idx, n_valid = 0U;
942
943 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) {
944 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR)
945 n_valid++;
946 }
947
948 assert(n_valid > 0U);
949
950 return (n_valid > 1U) ? 1 : 0;
951 }
952
953 /*******************************************************************************
954 * Initiate power down sequence, by calling power down operations registered for
955 * this CPU.
956 ******************************************************************************/
psci_do_pwrdown_sequence(unsigned int power_level)957 void psci_do_pwrdown_sequence(unsigned int power_level)
958 {
959 #if HW_ASSISTED_COHERENCY
960 /*
961 * With hardware-assisted coherency, the CPU drivers only initiate the
962 * power down sequence, without performing cache-maintenance operations
963 * in software. Data caches enabled both before and after this call.
964 */
965 prepare_cpu_pwr_dwn(power_level);
966 #else
967 /*
968 * Without hardware-assisted coherency, the CPU drivers disable data
969 * caches, then perform cache-maintenance operations in software.
970 *
971 * This also calls prepare_cpu_pwr_dwn() to initiate power down
972 * sequence, but that function will return with data caches disabled.
973 * We must ensure that the stack memory is flushed out to memory before
974 * we start popping from it again.
975 */
976 psci_do_pwrdown_cache_maintenance(power_level);
977 #endif
978 }
979
980 /*******************************************************************************
981 * This function invokes the callback 'stop_func()' with the 'mpidr' of each
982 * online PE. Caller can pass suitable method to stop a remote core.
983 *
984 * 'wait_ms' is the timeout value in milliseconds for the other cores to
985 * transition to power down state. Passing '0' makes it non-blocking.
986 *
987 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the
988 * given timeout.
989 ******************************************************************************/
psci_stop_other_cores(unsigned int wait_ms,void (* stop_func)(u_register_t mpidr))990 int psci_stop_other_cores(unsigned int wait_ms,
991 void (*stop_func)(u_register_t mpidr))
992 {
993 unsigned int idx, this_cpu_idx;
994
995 this_cpu_idx = plat_my_core_pos();
996
997 /* Invoke stop_func for each core */
998 for (idx = 0U; idx < psci_plat_core_count; idx++) {
999 /* skip current CPU */
1000 if (idx == this_cpu_idx) {
1001 continue;
1002 }
1003
1004 /* Check if the CPU is ON */
1005 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) {
1006 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr);
1007 }
1008 }
1009
1010 /* Need to wait for other cores to shutdown */
1011 if (wait_ms != 0U) {
1012 while ((wait_ms-- != 0U) && (psci_is_last_on_cpu() != 0U)) {
1013 mdelay(1U);
1014 }
1015
1016 if (psci_is_last_on_cpu() != 0U) {
1017 WARN("Failed to stop all cores!\n");
1018 psci_print_power_domain_map();
1019 return PSCI_E_DENIED;
1020 }
1021 }
1022
1023 return PSCI_E_SUCCESS;
1024 }
1025