Lines Matching +full:idle +full:- +full:state +full:- +full:name
1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_idle.c - native hardware idle loop for modern Intel processors
5 * Copyright (c) 2013 - 2020, Intel Corporation.
20 * All CPUs have same idle states as boot CPU
23 * for preventing entry into deep C-states
25 * CPU will flush caches as needed when entering a C-state via MWAIT
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
39 /* un-comment DEBUG to enable pr_debug() statements */
55 #include <asm/intel-family.h>
56 #include <asm/nospec-branch.h>
64 .name = "intel_idle",
68 static int max_cstate = CPUIDLE_STATE_MAX - 1;
87 * Hardware C-state auto-demotion may not always be optimal.
102 * Enable interrupts before entering the C-state. On some platforms and for
103 * some C-states, this may measurably decrease interrupt latency.
108 * Enable this state by default even if the ACPI _CST does not list it.
113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
119 * Initialize large xstate for the C6-state entrance.
124 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
125 * the C-state (top nibble) and sub-state (bottom nibble)
128 * We store the hint at the top of our "flags" for each state.
137 struct cpuidle_state *state = &drv->states[index]; in __intel_idle() local
138 unsigned long eax = flg2MWAIT(state->flags); in __intel_idle()
147 * intel_idle - Ask the processor to enter the given idle state.
150 * @index: Target idle state index.
153 * @dev is idle and it can try to enter the idle state corresponding to @index.
155 * If the local APIC timer is not known to be reliable in the target idle state,
156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
198 * intel_idle_s2idle - Ask the processor to enter the given idle state.
201 * @index: Target idle state index.
204 * @dev is idle and it can try to enter the idle state corresponding to @index.
206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
213 struct cpuidle_state *state = &drv->states[index]; in intel_idle_s2idle() local
214 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_s2idle()
216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) in intel_idle_s2idle()
231 .name = "C1",
239 .name = "C1E",
247 .name = "C3",
255 .name = "C6",
268 .name = "C1",
276 .name = "C1E",
284 .name = "C3",
292 .name = "C6",
300 .name = "C7",
313 .name = "C1",
321 .name = "C6N",
329 .name = "C6S",
337 .name = "C7",
345 .name = "C7S",
358 .name = "C1",
366 .name = "C6N",
374 .name = "C6S",
382 .name = "C7",
390 .name = "C7S",
403 .name = "C1",
411 .name = "C1E",
419 .name = "C3",
427 .name = "C6",
435 .name = "C7",
448 .name = "C1",
456 .name = "C1E",
464 .name = "C3",
472 .name = "C6",
485 .name = "C1",
493 .name = "C1E",
501 .name = "C3",
509 .name = "C6",
522 .name = "C1",
530 .name = "C1E",
538 .name = "C3",
546 .name = "C6",
559 .name = "C1",
567 .name = "C1E",
575 .name = "C3",
583 .name = "C6",
591 .name = "C7s",
599 .name = "C8",
607 .name = "C9",
615 .name = "C10",
627 .name = "C1",
635 .name = "C1E",
643 .name = "C3",
651 .name = "C6",
659 .name = "C7s",
667 .name = "C8",
675 .name = "C9",
683 .name = "C10",
696 .name = "C1",
704 .name = "C1E",
712 .name = "C3",
720 .name = "C6",
728 .name = "C7s",
736 .name = "C8",
744 .name = "C9",
752 .name = "C10",
765 .name = "C1",
773 .name = "C1E",
781 .name = "C6",
794 .name = "C1",
802 .name = "C1E",
810 .name = "C6",
833 .name = "C1",
841 .name = "C1E",
849 .name = "C6",
857 .name = "C8",
865 .name = "C10",
878 .name = "C1",
886 .name = "C1E",
894 .name = "C6",
902 .name = "C8",
910 .name = "C10",
923 .name = "C1",
931 .name = "C1E",
939 .name = "C6",
947 .name = "C8",
955 .name = "C10",
968 .name = "C1",
976 .name = "C1E",
984 .name = "C6",
998 .name = "C1E",
1006 .name = "C2",
1014 .name = "C4",
1022 .name = "C6",
1034 .name = "C1",
1042 .name = "C4",
1050 .name = "C6",
1058 .name = "C7",
1066 .name = "C9",
1078 .name = "C1",
1086 .name = "C6",
1098 .name = "C1",
1106 .name = "C6",
1119 .name = "C1",
1127 .name = "C1E",
1135 .name = "C6",
1143 .name = "C7s",
1151 .name = "C8",
1159 .name = "C9",
1167 .name = "C10",
1180 .name = "C1",
1188 .name = "C1E",
1196 .name = "C6",
1213 .name = "C1",
1221 .name = "C1E",
1229 .name = "C6",
1447 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) in intel_idle_state_needs_timer_stop() argument
1449 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_state_needs_timer_stop()
1455 * Switch over to one-shot tick broadcast if the target C-state in intel_idle_state_needs_timer_stop()
1466 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1470 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1475 * intel_idle_cst_usable - Check if the _CST information can be used.
1477 * Check if all of the C-states listed by _CST in the max_cstate range are
1490 if (cx->entry_method != ACPI_CSTATE_FFH) in intel_idle_cst_usable()
1512 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) in intel_idle_acpi_cst_extract()
1541 struct cpuidle_state *state; in intel_idle_init_cstates_acpi() local
1543 if (intel_idle_max_cstate_reached(cstate - 1)) in intel_idle_init_cstates_acpi()
1548 state = &drv->states[drv->state_count++]; in intel_idle_init_cstates_acpi()
1550 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); in intel_idle_init_cstates_acpi()
1551 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); in intel_idle_init_cstates_acpi()
1552 state->exit_latency = cx->latency; in intel_idle_init_cstates_acpi()
1554 * For C1-type C-states use the same number for both the exit in intel_idle_init_cstates_acpi()
1556 * C1 in the majority of the static C-states tables above. in intel_idle_init_cstates_acpi()
1557 * For the other types of C-states, however, set the target in intel_idle_init_cstates_acpi()
1559 * a reasonable balance between energy-efficiency and in intel_idle_init_cstates_acpi()
1562 state->target_residency = cx->latency; in intel_idle_init_cstates_acpi()
1563 if (cx->type > ACPI_STATE_C1) in intel_idle_init_cstates_acpi()
1564 state->target_residency *= 3; in intel_idle_init_cstates_acpi()
1566 state->flags = MWAIT2flg(cx->address); in intel_idle_init_cstates_acpi()
1567 if (cx->type > ACPI_STATE_C2) in intel_idle_init_cstates_acpi()
1568 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; in intel_idle_init_cstates_acpi()
1571 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_acpi()
1573 if (intel_idle_state_needs_timer_stop(state)) in intel_idle_init_cstates_acpi()
1574 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_acpi()
1576 state->enter = intel_idle; in intel_idle_init_cstates_acpi()
1577 state->enter_s2idle = intel_idle_s2idle; in intel_idle_init_cstates_acpi()
1586 * If there are no _CST C-states, do not disable any C-states by in intel_idle_off_by_default()
1612 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1614 * Tune IVT multi-socket targets.
1619 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ in ivt_idle_state_table_update()
1641 * irtl_2_usec - IRTL to microseconds conversion.
1662 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1710 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1712 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1730 /* PC10 is not enabled in PKG C-state limit */ in sklh_idle_state_table_update()
1747 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ in sklh_idle_state_table_update()
1748 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ in sklh_idle_state_table_update()
1752 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1753 * idle states table.
1762 * 000b: C0/C1 (no package C-state support) in skx_idle_state_table_update()
1764 * 010b: C6 (non-retention) in skx_idle_state_table_update()
1766 * 111b: No Package C state limits. in skx_idle_state_table_update()
1782 * adl_idle_state_table_update - Adjust AlderLake idle states table.
1801 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1808 * By default, the C6 state assumes the worst-case scenario of package in spr_idle_state_table_update()
1827 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ in intel_idle_verify_cstate()
1832 mark_tsc_unstable("TSC halts in idle states deeper than C2"); in intel_idle_verify_cstate()
1837 static void state_update_enter_method(struct cpuidle_state *state, int cstate) in state_update_enter_method() argument
1839 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { in state_update_enter_method()
1844 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); in state_update_enter_method()
1845 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
1846 state->enter = intel_idle_xstate; in state_update_enter_method()
1851 state->flags & CPUIDLE_FLAG_IBRS) { in state_update_enter_method()
1853 * IBRS mitigation requires that C-states are entered in state_update_enter_method()
1856 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
1857 state->enter = intel_idle_ibrs; in state_update_enter_method()
1861 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { in state_update_enter_method()
1862 state->enter = intel_idle_irq; in state_update_enter_method()
1867 pr_info("forced intel_idle_irq for state %d\n", cstate); in state_update_enter_method()
1868 state->enter = intel_idle_irq; in state_update_enter_method()
1902 struct cpuidle_state *state; in intel_idle_init_cstates_icpu() local
1912 /* If marked as unusable, skip this state. */ in intel_idle_init_cstates_icpu()
1914 pr_debug("state %s is disabled\n", in intel_idle_init_cstates_icpu()
1915 cpuidle_state_table[cstate].name); in intel_idle_init_cstates_icpu()
1924 drv->states[drv->state_count] = cpuidle_state_table[cstate]; in intel_idle_init_cstates_icpu()
1925 state = &drv->states[drv->state_count]; in intel_idle_init_cstates_icpu()
1927 state_update_enter_method(state, cstate); in intel_idle_init_cstates_icpu()
1930 if ((disabled_states_mask & BIT(drv->state_count)) || in intel_idle_init_cstates_icpu()
1931 ((icpu->use_acpi || force_use_acpi) && in intel_idle_init_cstates_icpu()
1933 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) in intel_idle_init_cstates_icpu()
1934 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_icpu()
1936 if (intel_idle_state_needs_timer_stop(state)) in intel_idle_init_cstates_icpu()
1937 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_icpu()
1939 drv->state_count++; in intel_idle_init_cstates_icpu()
1942 if (icpu->byt_auto_demotion_disable_flag) { in intel_idle_init_cstates_icpu()
1949 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
1957 drv->states[0].flags |= CPUIDLE_FLAG_OFF; in intel_idle_cpuidle_driver_init()
1959 drv->state_count = 1; in intel_idle_cpuidle_driver_init()
1995 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2006 dev->cpu = cpu; in intel_idle_cpu_init()
2010 return -EIO; in intel_idle_cpu_init()
2037 if (!dev->registered) in intel_idle_cpu_online()
2044 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2060 /* Do not load intel_idle at all for now if idle= is passed */ in intel_idle_init()
2062 return -ENODEV; in intel_idle_init()
2066 return -EPERM; in intel_idle_init()
2073 return -ENODEV; in intel_idle_init()
2078 return -ENODEV; in intel_idle_init()
2082 return -ENODEV; in intel_idle_init()
2089 return -ENODEV; in intel_idle_init()
2093 icpu = (const struct idle_cpu *)id->driver_data; in intel_idle_init()
2095 cpuidle_state_table = icpu->state_table; in intel_idle_init()
2096 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; in intel_idle_init()
2097 if (icpu->disable_promotion_to_c1e) in intel_idle_init()
2099 if (icpu->use_acpi || force_use_acpi) in intel_idle_init()
2102 return -ENODEV; in intel_idle_init()
2110 return -ENOMEM; in intel_idle_init()
2118 drv ? drv->name : "none"); in intel_idle_init()
2122 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", in intel_idle_init()
2128 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); in intel_idle_init()
2144 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2145 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2151 * idle states to be disabled by default (as reflected by the names of the
2152 * corresponding idle state directories in sysfs, "state0", "state1" ...
2153 * "state<i>" ..., where <i> is the index of the given state).
2156 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2158 * Some platforms come with mutually exclusive C-states, so that if one is
2159 * enabled, the other C-states must not be used. Example: C1 and C1E on
2161 * preferred C-states among the groups of mutually exclusive C-states - the
2162 * selected C-states will be registered, the other C-states from the mutually
2164 * exclusive C-states, this parameter has no effect.
2167 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2169 * Debugging option that forces the driver to enter all C-states with
2170 * interrupts enabled. Does not apply to C-states with