Lines Matching +full:p +full:- +full:state
1 // SPDX-License-Identifier: GPL-2.0-or-later
13 #include <asm/asm-offsets.h>
17 #include <asm/mips-cps.h>
20 #include <asm/pm-cps.h>
21 #include <asm/smp-cps.h>
25 * cps_nc_entry_fn - type of a generated non-coherent state entry function
27 * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count
29 * The code entering & exiting non-coherent states is generated at runtime
32 * core-specific code particularly for cache routines. If coupled_coherence
33 * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state,
34 * returns the number of VPEs that were in the wait state at the point this
41 * The entry point of the generated non-coherent idle state entry/exit
42 * functions. Actually per-core rather than per-CPU.
51 * Indicates the number of coupled VPEs ready to operate in a non-coherent
52 * state. Actually per-core rather than per-CPU.
60 * Used to synchronize entry to deep idle states. Actually per-core rather
61 * than per-CPU.
65 /* Saved CPU state across the CPS_PM_POWER_GATED state */
79 bool cps_pm_support_state(enum cps_pm_state state) in cps_pm_support_state() argument
81 return test_bit(state, state_support); in cps_pm_support_state()
110 int cps_pm_enter_state(enum cps_pm_state state) in cps_pm_enter_state() argument
122 /* Check that there is an entry function for this state */ in cps_pm_enter_state()
123 entry = per_cpu(nc_asm_enter, core)[state]; in cps_pm_enter_state()
125 return -EINVAL; in cps_pm_enter_state()
142 if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { in cps_pm_enter_state()
145 return -EINVAL; in cps_pm_enter_state()
148 vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(¤t_cpu_data)]; in cps_pm_enter_state()
149 vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; in cps_pm_enter_state()
150 vpe_cfg->gp = (unsigned long)current_thread_info(); in cps_pm_enter_state()
151 vpe_cfg->sp = 0; in cps_pm_enter_state()
158 /* Create a non-coherent mapping of the core ready_count */ in cps_pm_enter_state()
165 /* Ensure ready_count is zero-initialised before the assembly runs */ in cps_pm_enter_state()
172 /* Remove the non-coherent mapping of ready_count */ in cps_pm_enter_state()
179 * If this VPE is the first to leave the non-coherent wait state then in cps_pm_enter_state()
184 * idle state. in cps_pm_enter_state()
186 if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) in cps_pm_enter_state()
197 unsigned cache_size = cache->ways << cache->waybit; in cps_gen_cache_routine()
202 if (cache->flags & MIPS_CACHE_NOT_PRESENT) in cps_gen_cache_routine()
221 uasm_i_addiu(pp, t0, t0, cache->linesz); in cps_gen_cache_routine()
223 uasm_i_cache(pp, op, i * cache->linesz, t0); in cps_gen_cache_routine()
229 uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz); in cps_gen_cache_routine()
244 unsigned line_size = cpu_info->dcache.linesz; in cps_gen_flush_fsb()
246 unsigned revision = cpu_info->processor_id & PRID_REV_MASK; in cps_gen_flush_fsb()
252 switch (__get_cpu_type(cpu_info->cputype)) { in cps_gen_flush_fsb()
264 return -1; in cps_gen_flush_fsb()
274 * stuck in the D3 (ClrBus) state whilst entering a low power state. in cps_gen_flush_fsb()
299 * Invalidate the new D-cache entries so that the cache will need in cps_gen_flush_fsb()
342 static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) in cps_gen_entry_code() argument
346 u32 *buf, *p; in cps_gen_entry_code() local
368 p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); in cps_gen_entry_code()
376 if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { in cps_gen_entry_code()
382 * Save CPU state. Note the non-standard calling convention in cps_gen_entry_code()
386 UASM_i_LA(&p, t0, (long)mips_cps_pm_save); in cps_gen_entry_code()
387 uasm_i_jalr(&p, v0, t0); in cps_gen_entry_code()
388 uasm_i_nop(&p); in cps_gen_entry_code()
396 UASM_i_LA(&p, r_pcohctl, (long)addr_gcr_cl_coherence()); in cps_gen_entry_code()
400 uasm_i_sync(&p, __SYNC_mb); in cps_gen_entry_code()
401 uasm_build_label(&l, p, lbl_incready); in cps_gen_entry_code()
402 uasm_i_ll(&p, t1, 0, r_nc_count); in cps_gen_entry_code()
403 uasm_i_addiu(&p, t2, t1, 1); in cps_gen_entry_code()
404 uasm_i_sc(&p, t2, 0, r_nc_count); in cps_gen_entry_code()
405 uasm_il_beqz(&p, &r, t2, lbl_incready); in cps_gen_entry_code()
406 uasm_i_addiu(&p, t1, t1, 1); in cps_gen_entry_code()
409 uasm_i_sync(&p, __SYNC_mb); in cps_gen_entry_code()
412 * If this is the last VPE to become ready for non-coherence in cps_gen_entry_code()
415 uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence); in cps_gen_entry_code()
416 uasm_i_nop(&p); in cps_gen_entry_code()
418 if (state < CPS_PM_POWER_GATED) { in cps_gen_entry_code()
421 * for non-coherence. It needs to wait until coherence in cps_gen_entry_code()
425 uasm_i_addiu(&p, t1, zero, -1); in cps_gen_entry_code()
426 uasm_build_label(&l, p, lbl_poll_cont); in cps_gen_entry_code()
427 uasm_i_lw(&p, t0, 0, r_nc_count); in cps_gen_entry_code()
428 uasm_il_bltz(&p, &r, t0, lbl_secondary_cont); in cps_gen_entry_code()
429 uasm_i_ehb(&p); in cps_gen_entry_code()
431 uasm_i_yield(&p, zero, t1); in cps_gen_entry_code()
432 uasm_il_b(&p, &r, lbl_poll_cont); in cps_gen_entry_code()
433 uasm_i_nop(&p); in cps_gen_entry_code()
441 uasm_i_addiu(&p, t0, zero, TCHALT_H); in cps_gen_entry_code()
442 uasm_i_mtc0(&p, t0, 2, 4); in cps_gen_entry_code()
448 uasm_i_addiu(&p, t0, zero, 1 << vpe_id); in cps_gen_entry_code()
449 UASM_i_LA(&p, t1, (long)addr_cpc_cl_vp_stop()); in cps_gen_entry_code()
450 uasm_i_sw(&p, t0, 0, t1); in cps_gen_entry_code()
454 uasm_build_label(&l, p, lbl_secondary_hang); in cps_gen_entry_code()
455 uasm_il_b(&p, &r, lbl_secondary_hang); in cps_gen_entry_code()
456 uasm_i_nop(&p); in cps_gen_entry_code()
461 * This is the point of no return - this VPE will now proceed to in cps_gen_entry_code()
465 uasm_build_label(&l, p, lbl_disable_coherence); in cps_gen_entry_code()
468 cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, in cps_gen_entry_code()
472 cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, in cps_gen_entry_code()
476 uasm_i_sync(&p, __SYNC_full); in cps_gen_entry_code()
477 uasm_i_ehb(&p); in cps_gen_entry_code()
485 uasm_i_addiu(&p, t0, zero, 1 << cpu_core(&cpu_data[cpu])); in cps_gen_entry_code()
486 uasm_i_sw(&p, t0, 0, r_pcohctl); in cps_gen_entry_code()
487 uasm_i_lw(&p, t0, 0, r_pcohctl); in cps_gen_entry_code()
490 uasm_i_sync(&p, __SYNC_full); in cps_gen_entry_code()
491 uasm_i_ehb(&p); in cps_gen_entry_code()
495 uasm_i_sw(&p, zero, 0, r_pcohctl); in cps_gen_entry_code()
496 uasm_i_lw(&p, t0, 0, r_pcohctl); in cps_gen_entry_code()
498 if (state >= CPS_PM_CLOCK_GATED) { in cps_gen_entry_code()
499 err = cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu], in cps_gen_entry_code()
505 switch (state) { in cps_gen_entry_code()
518 UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd()); in cps_gen_entry_code()
519 uasm_i_addiu(&p, t1, zero, cpc_cmd); in cps_gen_entry_code()
520 uasm_i_sw(&p, t1, 0, t0); in cps_gen_entry_code()
522 if (state == CPS_PM_POWER_GATED) { in cps_gen_entry_code()
524 uasm_build_label(&l, p, lbl_hang); in cps_gen_entry_code()
525 uasm_il_b(&p, &r, lbl_hang); in cps_gen_entry_code()
526 uasm_i_nop(&p); in cps_gen_entry_code()
537 uasm_i_sync(&p, __SYNC_full); in cps_gen_entry_code()
538 uasm_i_ehb(&p); in cps_gen_entry_code()
541 if (state == CPS_PM_NC_WAIT) { in cps_gen_entry_code()
548 cps_gen_set_top_bit(&p, &l, &r, r_nc_count, in cps_gen_entry_code()
556 uasm_build_label(&l, p, lbl_secondary_cont); in cps_gen_entry_code()
559 uasm_i_wait(&p, 0); in cps_gen_entry_code()
563 * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs in cps_gen_entry_code()
564 * will run this. The first will actually re-enable coherence & the in cps_gen_entry_code()
567 uasm_i_addiu(&p, t0, zero, mips_cm_revision() < CM_REV_CM3 in cps_gen_entry_code()
571 uasm_i_sw(&p, t0, 0, r_pcohctl); in cps_gen_entry_code()
572 uasm_i_lw(&p, t0, 0, r_pcohctl); in cps_gen_entry_code()
575 uasm_i_sync(&p, __SYNC_full); in cps_gen_entry_code()
576 uasm_i_ehb(&p); in cps_gen_entry_code()
578 if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { in cps_gen_entry_code()
580 uasm_build_label(&l, p, lbl_decready); in cps_gen_entry_code()
581 uasm_i_sync(&p, __SYNC_mb); in cps_gen_entry_code()
582 uasm_i_ll(&p, t1, 0, r_nc_count); in cps_gen_entry_code()
583 uasm_i_addiu(&p, t2, t1, -1); in cps_gen_entry_code()
584 uasm_i_sc(&p, t2, 0, r_nc_count); in cps_gen_entry_code()
585 uasm_il_beqz(&p, &r, t2, lbl_decready); in cps_gen_entry_code()
586 uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1); in cps_gen_entry_code()
589 uasm_i_sync(&p, __SYNC_mb); in cps_gen_entry_code()
592 if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { in cps_gen_entry_code()
598 cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); in cps_gen_entry_code()
602 * power-up command to the CPC in order to resume operation. in cps_gen_entry_code()
604 * idle state and the one that disables coherence might as well in cps_gen_entry_code()
605 * be the one to re-enable it. The rest will continue from here in cps_gen_entry_code()
608 uasm_build_label(&l, p, lbl_secondary_cont); in cps_gen_entry_code()
611 uasm_i_sync(&p, __SYNC_mb); in cps_gen_entry_code()
615 uasm_i_jr(&p, ra); in cps_gen_entry_code()
616 uasm_i_nop(&p); in cps_gen_entry_code()
620 BUG_ON((p - buf) > max_instrs); in cps_gen_entry_code()
621 BUG_ON((l - labels) > ARRAY_SIZE(labels)); in cps_gen_entry_code()
622 BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); in cps_gen_entry_code()
628 local_flush_icache_range((unsigned long)buf, (unsigned long)p); in cps_gen_entry_code()
638 enum cps_pm_state state; in cps_pm_online_cpu() local
642 for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) { in cps_pm_online_cpu()
643 if (per_cpu(nc_asm_enter, core)[state]) in cps_pm_online_cpu()
645 if (!test_bit(state, state_support)) in cps_pm_online_cpu()
648 entry_fn = cps_gen_entry_code(cpu, state); in cps_pm_online_cpu()
650 pr_err("Failed to generate core %u state %u entry\n", in cps_pm_online_cpu()
651 core, state); in cps_pm_online_cpu()
652 clear_bit(state, state_support); in cps_pm_online_cpu()
655 per_cpu(nc_asm_enter, core)[state] = entry_fn; in cps_pm_online_cpu()
662 return -ENOMEM; in cps_pm_online_cpu()
681 * instead put the cores into clock-off state. In this state in cps_pm_power_notifier()
689 pr_warn("JTAG probe is connected - abort suspend\n"); in cps_pm_power_notifier()
700 /* A CM is required for all non-coherent states */ in cps_pm_init()
702 pr_warn("pm-cps: no CM, non-coherent states unavailable\n"); in cps_pm_init()
708 * non-coherent core then the VPE may end up processing interrupts in cps_pm_init()
709 * whilst non-coherent. That would be bad. in cps_pm_init()
714 pr_warn("pm-cps: non-coherent wait unavailable\n"); in cps_pm_init()
722 pr_warn("pm-cps: CPC does not support clock gating\n"); in cps_pm_init()
728 pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n"); in cps_pm_init()
730 pr_warn("pm-cps: no CPC, clock & power gating unavailable\n"); in cps_pm_init()