1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * kernel/power/suspend.c - Suspend to RAM and standby functionality.
4 *
5 * Copyright (c) 2003 Patrick Mochel
6 * Copyright (c) 2003 Open Source Development Lab
7 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
8 */
9
10 #define pr_fmt(fmt) "PM: " fmt
11
12 #include <linux/string.h>
13 #include <linux/delay.h>
14 #include <linux/errno.h>
15 #include <linux/init.h>
16 #include <linux/console.h>
17 #include <linux/cpu.h>
18 #include <linux/cpuidle.h>
19 #include <linux/gfp.h>
20 #include <linux/io.h>
21 #include <linux/kernel.h>
22 #include <linux/list.h>
23 #include <linux/mm.h>
24 #include <linux/slab.h>
25 #include <linux/export.h>
26 #include <linux/suspend.h>
27 #include <linux/syscore_ops.h>
28 #include <linux/swait.h>
29 #include <linux/ftrace.h>
30 #include <trace/events/power.h>
31 #include <linux/compiler.h>
32 #include <linux/moduleparam.h>
33 #include <linux/wakeup_reason.h>
34 #include <trace/hooks/suspend.h>
35
36 #include "power.h"
37
38 const char * const pm_labels[] = {
39 [PM_SUSPEND_TO_IDLE] = "freeze",
40 [PM_SUSPEND_STANDBY] = "standby",
41 [PM_SUSPEND_MEM] = "mem",
42 };
43 const char *pm_states[PM_SUSPEND_MAX];
44 static const char * const mem_sleep_labels[] = {
45 [PM_SUSPEND_TO_IDLE] = "s2idle",
46 [PM_SUSPEND_STANDBY] = "shallow",
47 [PM_SUSPEND_MEM] = "deep",
48 };
49 const char *mem_sleep_states[PM_SUSPEND_MAX];
50
51 suspend_state_t mem_sleep_current = PM_SUSPEND_TO_IDLE;
52 suspend_state_t mem_sleep_default = PM_SUSPEND_MAX;
53 suspend_state_t pm_suspend_target_state;
54 EXPORT_SYMBOL_GPL(pm_suspend_target_state);
55
56 unsigned int pm_suspend_global_flags;
57 EXPORT_SYMBOL_GPL(pm_suspend_global_flags);
58
59 static const struct platform_suspend_ops *suspend_ops;
60 static const struct platform_s2idle_ops *s2idle_ops;
61 static DECLARE_SWAIT_QUEUE_HEAD(s2idle_wait_head);
62
63 enum s2idle_states __read_mostly s2idle_state;
64 static DEFINE_RAW_SPINLOCK(s2idle_lock);
65
66 /**
67 * pm_suspend_default_s2idle - Check if suspend-to-idle is the default suspend.
68 *
69 * Return 'true' if suspend-to-idle has been selected as the default system
70 * suspend method.
71 */
pm_suspend_default_s2idle(void)72 bool pm_suspend_default_s2idle(void)
73 {
74 return mem_sleep_current == PM_SUSPEND_TO_IDLE;
75 }
76 EXPORT_SYMBOL_GPL(pm_suspend_default_s2idle);
77
s2idle_set_ops(const struct platform_s2idle_ops * ops)78 void s2idle_set_ops(const struct platform_s2idle_ops *ops)
79 {
80 unsigned int sleep_flags;
81
82 sleep_flags = lock_system_sleep();
83 s2idle_ops = ops;
84 unlock_system_sleep(sleep_flags);
85 }
86
s2idle_begin(void)87 static void s2idle_begin(void)
88 {
89 s2idle_state = S2IDLE_STATE_NONE;
90 }
91
s2idle_enter(void)92 static void s2idle_enter(void)
93 {
94 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true);
95
96 raw_spin_lock_irq(&s2idle_lock);
97 if (pm_wakeup_pending())
98 goto out;
99
100 s2idle_state = S2IDLE_STATE_ENTER;
101 raw_spin_unlock_irq(&s2idle_lock);
102
103 cpus_read_lock();
104
105 /* Push all the CPUs into the idle loop. */
106 wake_up_all_idle_cpus();
107 /* Make the current CPU wait so it can enter the idle loop too. */
108 swait_event_exclusive(s2idle_wait_head,
109 s2idle_state == S2IDLE_STATE_WAKE);
110
111 /*
112 * Kick all CPUs to ensure that they resume their timers and restore
113 * consistent system state.
114 */
115 wake_up_all_idle_cpus();
116
117 cpus_read_unlock();
118
119 raw_spin_lock_irq(&s2idle_lock);
120
121 out:
122 s2idle_state = S2IDLE_STATE_NONE;
123 raw_spin_unlock_irq(&s2idle_lock);
124
125 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, false);
126 }
127
s2idle_loop(void)128 static void s2idle_loop(void)
129 {
130 pm_pr_dbg("suspend-to-idle\n");
131
132 /*
133 * Suspend-to-idle equals:
134 * frozen processes + suspended devices + idle processors.
135 * Thus s2idle_enter() should be called right after all devices have
136 * been suspended.
137 *
138 * Wakeups during the noirq suspend of devices may be spurious, so try
139 * to avoid them upfront.
140 */
141 for (;;) {
142 if (s2idle_ops && s2idle_ops->wake) {
143 if (s2idle_ops->wake())
144 break;
145 } else if (pm_wakeup_pending()) {
146 break;
147 }
148
149 clear_wakeup_reasons();
150
151 if (s2idle_ops && s2idle_ops->check)
152 s2idle_ops->check();
153
154 s2idle_enter();
155 }
156
157 pm_pr_dbg("resume from suspend-to-idle\n");
158 }
159
s2idle_wake(void)160 void s2idle_wake(void)
161 {
162 unsigned long flags;
163
164 raw_spin_lock_irqsave(&s2idle_lock, flags);
165 if (s2idle_state > S2IDLE_STATE_NONE) {
166 s2idle_state = S2IDLE_STATE_WAKE;
167 swake_up_one(&s2idle_wait_head);
168 }
169 raw_spin_unlock_irqrestore(&s2idle_lock, flags);
170 }
171 EXPORT_SYMBOL_GPL(s2idle_wake);
172
valid_state(suspend_state_t state)173 static bool valid_state(suspend_state_t state)
174 {
175 /*
176 * The PM_SUSPEND_STANDBY and PM_SUSPEND_MEM states require low-level
177 * support and need to be valid to the low-level implementation.
178 *
179 * No ->valid() or ->enter() callback implies that none are valid.
180 */
181 return suspend_ops && suspend_ops->valid && suspend_ops->valid(state) &&
182 suspend_ops->enter;
183 }
184
pm_states_init(void)185 void __init pm_states_init(void)
186 {
187 /* "mem" and "freeze" are always present in /sys/power/state. */
188 pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM];
189 pm_states[PM_SUSPEND_TO_IDLE] = pm_labels[PM_SUSPEND_TO_IDLE];
190 /*
191 * Suspend-to-idle should be supported even without any suspend_ops,
192 * initialize mem_sleep_states[] accordingly here.
193 */
194 mem_sleep_states[PM_SUSPEND_TO_IDLE] = mem_sleep_labels[PM_SUSPEND_TO_IDLE];
195 }
196
mem_sleep_default_setup(char * str)197 static int __init mem_sleep_default_setup(char *str)
198 {
199 suspend_state_t state;
200
201 for (state = PM_SUSPEND_TO_IDLE; state <= PM_SUSPEND_MEM; state++)
202 if (mem_sleep_labels[state] &&
203 !strcmp(str, mem_sleep_labels[state])) {
204 mem_sleep_default = state;
205 mem_sleep_current = state;
206 break;
207 }
208
209 return 1;
210 }
211 __setup("mem_sleep_default=", mem_sleep_default_setup);
212
213 /**
214 * suspend_set_ops - Set the global suspend method table.
215 * @ops: Suspend operations to use.
216 */
suspend_set_ops(const struct platform_suspend_ops * ops)217 void suspend_set_ops(const struct platform_suspend_ops *ops)
218 {
219 unsigned int sleep_flags;
220
221 sleep_flags = lock_system_sleep();
222
223 suspend_ops = ops;
224
225 if (valid_state(PM_SUSPEND_STANDBY)) {
226 mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY];
227 pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY];
228 if (mem_sleep_default == PM_SUSPEND_STANDBY)
229 mem_sleep_current = PM_SUSPEND_STANDBY;
230 }
231 if (valid_state(PM_SUSPEND_MEM)) {
232 mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM];
233 if (mem_sleep_default >= PM_SUSPEND_MEM)
234 mem_sleep_current = PM_SUSPEND_MEM;
235 }
236
237 unlock_system_sleep(sleep_flags);
238 }
239 EXPORT_SYMBOL_GPL(suspend_set_ops);
240
241 /**
242 * suspend_valid_only_mem - Generic memory-only valid callback.
243 * @state: Target system sleep state.
244 *
245 * Platform drivers that implement mem suspend only and only need to check for
246 * that in their .valid() callback can use this instead of rolling their own
247 * .valid() callback.
248 */
suspend_valid_only_mem(suspend_state_t state)249 int suspend_valid_only_mem(suspend_state_t state)
250 {
251 return state == PM_SUSPEND_MEM;
252 }
253 EXPORT_SYMBOL_GPL(suspend_valid_only_mem);
254
sleep_state_supported(suspend_state_t state)255 static bool sleep_state_supported(suspend_state_t state)
256 {
257 return state == PM_SUSPEND_TO_IDLE ||
258 (valid_state(state) && !cxl_mem_active());
259 }
260
platform_suspend_prepare(suspend_state_t state)261 static int platform_suspend_prepare(suspend_state_t state)
262 {
263 return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare ?
264 suspend_ops->prepare() : 0;
265 }
266
platform_suspend_prepare_late(suspend_state_t state)267 static int platform_suspend_prepare_late(suspend_state_t state)
268 {
269 return state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->prepare ?
270 s2idle_ops->prepare() : 0;
271 }
272
platform_suspend_prepare_noirq(suspend_state_t state)273 static int platform_suspend_prepare_noirq(suspend_state_t state)
274 {
275 if (state == PM_SUSPEND_TO_IDLE)
276 return s2idle_ops && s2idle_ops->prepare_late ?
277 s2idle_ops->prepare_late() : 0;
278
279 return suspend_ops->prepare_late ? suspend_ops->prepare_late() : 0;
280 }
281
platform_resume_noirq(suspend_state_t state)282 static void platform_resume_noirq(suspend_state_t state)
283 {
284 if (state == PM_SUSPEND_TO_IDLE) {
285 if (s2idle_ops && s2idle_ops->restore_early)
286 s2idle_ops->restore_early();
287 } else if (suspend_ops->wake) {
288 suspend_ops->wake();
289 }
290 }
291
platform_resume_early(suspend_state_t state)292 static void platform_resume_early(suspend_state_t state)
293 {
294 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->restore)
295 s2idle_ops->restore();
296 }
297
platform_resume_finish(suspend_state_t state)298 static void platform_resume_finish(suspend_state_t state)
299 {
300 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->finish)
301 suspend_ops->finish();
302 }
303
platform_suspend_begin(suspend_state_t state)304 static int platform_suspend_begin(suspend_state_t state)
305 {
306 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->begin)
307 return s2idle_ops->begin();
308 else if (suspend_ops && suspend_ops->begin)
309 return suspend_ops->begin(state);
310 else
311 return 0;
312 }
313
platform_resume_end(suspend_state_t state)314 static void platform_resume_end(suspend_state_t state)
315 {
316 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->end)
317 s2idle_ops->end();
318 else if (suspend_ops && suspend_ops->end)
319 suspend_ops->end();
320 }
321
platform_recover(suspend_state_t state)322 static void platform_recover(suspend_state_t state)
323 {
324 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->recover)
325 suspend_ops->recover();
326 }
327
platform_suspend_again(suspend_state_t state)328 static bool platform_suspend_again(suspend_state_t state)
329 {
330 return state != PM_SUSPEND_TO_IDLE && suspend_ops->suspend_again ?
331 suspend_ops->suspend_again() : false;
332 }
333
334 #ifdef CONFIG_PM_DEBUG
335 static unsigned int pm_test_delay = 5;
336 module_param(pm_test_delay, uint, 0644);
337 MODULE_PARM_DESC(pm_test_delay,
338 "Number of seconds to wait before resuming from suspend test");
339 #endif
340
suspend_test(int level)341 static int suspend_test(int level)
342 {
343 #ifdef CONFIG_PM_DEBUG
344 if (pm_test_level == level) {
345 pr_info("suspend debug: Waiting for %d second(s).\n",
346 pm_test_delay);
347 mdelay(pm_test_delay * 1000);
348 return 1;
349 }
350 #endif /* !CONFIG_PM_DEBUG */
351 return 0;
352 }
353
354 /**
355 * suspend_prepare - Prepare for entering system sleep state.
356 * @state: Target system sleep state.
357 *
358 * Common code run for every system sleep state that can be entered (except for
359 * hibernation). Run suspend notifiers, allocate the "suspend" console and
360 * freeze processes.
361 */
suspend_prepare(suspend_state_t state)362 static int suspend_prepare(suspend_state_t state)
363 {
364 int error;
365
366 if (!sleep_state_supported(state))
367 return -EPERM;
368
369 pm_prepare_console();
370
371 error = pm_notifier_call_chain_robust(PM_SUSPEND_PREPARE, PM_POST_SUSPEND);
372 if (error)
373 goto Restore;
374
375 trace_suspend_resume(TPS("freeze_processes"), 0, true);
376 error = suspend_freeze_processes();
377 trace_suspend_resume(TPS("freeze_processes"), 0, false);
378 if (!error)
379 return 0;
380
381 log_suspend_abort_reason("One or more tasks refusing to freeze");
382 dpm_save_failed_step(SUSPEND_FREEZE);
383 pm_notifier_call_chain(PM_POST_SUSPEND);
384 Restore:
385 pm_restore_console();
386 return error;
387 }
388
389 /* default implementation */
arch_suspend_disable_irqs(void)390 void __weak arch_suspend_disable_irqs(void)
391 {
392 local_irq_disable();
393 }
394
395 /* default implementation */
arch_suspend_enable_irqs(void)396 void __weak arch_suspend_enable_irqs(void)
397 {
398 local_irq_enable();
399 }
400
401 /**
402 * suspend_enter - Make the system enter the given sleep state.
403 * @state: System sleep state to enter.
404 * @wakeup: Returns information that the sleep state should not be re-entered.
405 *
406 * This function should be called after devices have been suspended.
407 */
suspend_enter(suspend_state_t state,bool * wakeup)408 static int suspend_enter(suspend_state_t state, bool *wakeup)
409 {
410 int error, last_dev;
411
412 error = platform_suspend_prepare(state);
413 if (error)
414 goto Platform_finish;
415
416 error = dpm_suspend_late(PMSG_SUSPEND);
417 if (error) {
418 last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1;
419 last_dev %= REC_FAILED_NUM;
420 pr_err("late suspend of devices failed\n");
421 log_suspend_abort_reason("late suspend of %s device failed",
422 suspend_stats.failed_devs[last_dev]);
423 goto Platform_finish;
424 }
425 error = platform_suspend_prepare_late(state);
426 if (error)
427 goto Devices_early_resume;
428
429 error = dpm_suspend_noirq(PMSG_SUSPEND);
430 if (error) {
431 last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1;
432 last_dev %= REC_FAILED_NUM;
433 pr_err("noirq suspend of devices failed\n");
434 log_suspend_abort_reason("noirq suspend of %s device failed",
435 suspend_stats.failed_devs[last_dev]);
436 goto Platform_early_resume;
437 }
438 error = platform_suspend_prepare_noirq(state);
439 if (error)
440 goto Platform_wake;
441
442 if (suspend_test(TEST_PLATFORM))
443 goto Platform_wake;
444
445 if (state == PM_SUSPEND_TO_IDLE) {
446 s2idle_loop();
447 goto Platform_wake;
448 }
449
450 error = pm_sleep_disable_secondary_cpus();
451 if (error || suspend_test(TEST_CPUS)) {
452 log_suspend_abort_reason("Disabling non-boot cpus failed");
453 goto Enable_cpus;
454 }
455
456 arch_suspend_disable_irqs();
457 BUG_ON(!irqs_disabled());
458
459 system_state = SYSTEM_SUSPEND;
460
461 error = syscore_suspend();
462 if (!error) {
463 *wakeup = pm_wakeup_pending();
464 if (!(suspend_test(TEST_CORE) || *wakeup)) {
465 trace_suspend_resume(TPS("machine_suspend"),
466 state, true);
467 error = suspend_ops->enter(state);
468 trace_suspend_resume(TPS("machine_suspend"),
469 state, false);
470 trace_android_vh_early_resume_begin(NULL);
471 } else if (*wakeup) {
472 error = -EBUSY;
473 }
474 syscore_resume();
475 }
476
477 system_state = SYSTEM_RUNNING;
478
479 arch_suspend_enable_irqs();
480 BUG_ON(irqs_disabled());
481
482 Enable_cpus:
483 pm_sleep_enable_secondary_cpus();
484
485 Platform_wake:
486 platform_resume_noirq(state);
487 dpm_resume_noirq(PMSG_RESUME);
488
489 Platform_early_resume:
490 platform_resume_early(state);
491
492 Devices_early_resume:
493 dpm_resume_early(PMSG_RESUME);
494
495 Platform_finish:
496 platform_resume_finish(state);
497 return error;
498 }
499
500 /**
501 * suspend_devices_and_enter - Suspend devices and enter system sleep state.
502 * @state: System sleep state to enter.
503 */
suspend_devices_and_enter(suspend_state_t state)504 int suspend_devices_and_enter(suspend_state_t state)
505 {
506 int error;
507 bool wakeup = false;
508
509 if (!sleep_state_supported(state))
510 return -ENOSYS;
511
512 pm_suspend_target_state = state;
513
514 if (state == PM_SUSPEND_TO_IDLE)
515 pm_set_suspend_no_platform();
516
517 error = platform_suspend_begin(state);
518 if (error)
519 goto Close;
520
521 suspend_console();
522 suspend_test_start();
523 error = dpm_suspend_start(PMSG_SUSPEND);
524 if (error) {
525 pr_err("Some devices failed to suspend, or early wake event detected\n");
526 log_suspend_abort_reason(
527 "Some devices failed to suspend, or early wake event detected");
528 goto Recover_platform;
529 }
530 suspend_test_finish("suspend devices");
531 if (suspend_test(TEST_DEVICES))
532 goto Recover_platform;
533
534 do {
535 error = suspend_enter(state, &wakeup);
536 } while (!error && !wakeup && platform_suspend_again(state));
537
538 Resume_devices:
539 trace_android_vh_resume_begin(NULL);
540 suspend_test_start();
541 dpm_resume_end(PMSG_RESUME);
542 suspend_test_finish("resume devices");
543 trace_suspend_resume(TPS("resume_console"), state, true);
544 resume_console();
545 trace_suspend_resume(TPS("resume_console"), state, false);
546
547 Close:
548 platform_resume_end(state);
549 pm_suspend_target_state = PM_SUSPEND_ON;
550 trace_android_vh_resume_end(NULL);
551 return error;
552
553 Recover_platform:
554 platform_recover(state);
555 goto Resume_devices;
556 }
557
558 /**
559 * suspend_finish - Clean up before finishing the suspend sequence.
560 *
561 * Call platform code to clean up, restart processes, and free the console that
562 * we've allocated. This routine is not called for hibernation.
563 */
suspend_finish(void)564 static void suspend_finish(void)
565 {
566 suspend_thaw_processes();
567 pm_notifier_call_chain(PM_POST_SUSPEND);
568 pm_restore_console();
569 }
570
571 /**
572 * enter_state - Do common work needed to enter system sleep state.
573 * @state: System sleep state to enter.
574 *
575 * Make sure that no one else is trying to put the system into a sleep state.
576 * Fail if that's not the case. Otherwise, prepare for system suspend, make the
577 * system enter the given sleep state and clean up after wakeup.
578 */
enter_state(suspend_state_t state)579 static int enter_state(suspend_state_t state)
580 {
581 int error;
582
583 trace_suspend_resume(TPS("suspend_enter"), state, true);
584 if (state == PM_SUSPEND_TO_IDLE) {
585 #ifdef CONFIG_PM_DEBUG
586 if (pm_test_level != TEST_NONE && pm_test_level <= TEST_CPUS) {
587 pr_warn("Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n");
588 return -EAGAIN;
589 }
590 #endif
591 } else if (!valid_state(state)) {
592 return -EINVAL;
593 }
594 if (!mutex_trylock(&system_transition_mutex))
595 return -EBUSY;
596
597 if (state == PM_SUSPEND_TO_IDLE)
598 s2idle_begin();
599
600 if (sync_on_suspend_enabled) {
601 trace_suspend_resume(TPS("sync_filesystems"), 0, true);
602 ksys_sync_helper();
603 trace_suspend_resume(TPS("sync_filesystems"), 0, false);
604 }
605
606 pm_pr_dbg("Preparing system for sleep (%s)\n", mem_sleep_labels[state]);
607 pm_suspend_clear_flags();
608 error = suspend_prepare(state);
609 if (error)
610 goto Unlock;
611
612 if (suspend_test(TEST_FREEZER))
613 goto Finish;
614
615 trace_suspend_resume(TPS("suspend_enter"), state, false);
616 pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]);
617 pm_restrict_gfp_mask();
618 error = suspend_devices_and_enter(state);
619 pm_restore_gfp_mask();
620
621 Finish:
622 events_check_enabled = false;
623 pm_pr_dbg("Finishing wakeup.\n");
624 suspend_finish();
625 Unlock:
626 mutex_unlock(&system_transition_mutex);
627 return error;
628 }
629
630 /**
631 * pm_suspend - Externally visible function for suspending the system.
632 * @state: System sleep state to enter.
633 *
634 * Check if the value of @state represents one of the supported states,
635 * execute enter_state() and update system suspend statistics.
636 */
pm_suspend(suspend_state_t state)637 int pm_suspend(suspend_state_t state)
638 {
639 int error;
640
641 if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX)
642 return -EINVAL;
643
644 pr_info("suspend entry (%s)\n", mem_sleep_labels[state]);
645 error = enter_state(state);
646 if (error) {
647 suspend_stats.fail++;
648 dpm_save_failed_errno(error);
649 } else {
650 suspend_stats.success++;
651 }
652 pr_info("suspend exit\n");
653 return error;
654 }
655 EXPORT_SYMBOL(pm_suspend);
656