1 #include <linux/ftrace.h>
2 #include <linux/percpu.h>
3 #include <linux/slab.h>
4 #include <asm/alternative.h>
5 #include <asm/cacheflush.h>
6 #include <asm/cpufeature.h>
7 #include <asm/debug-monitors.h>
8 #include <asm/exec.h>
9 #include <asm/pgtable.h>
10 #include <asm/memory.h>
11 #include <asm/mmu_context.h>
12 #include <asm/smp_plat.h>
13 #include <asm/suspend.h>
14 #include <asm/tlbflush.h>
15
16 extern int __cpu_suspend_enter(unsigned long arg, int (*fn)(unsigned long));
17 /*
18 * This is called by __cpu_suspend_enter() to save the state, and do whatever
19 * flushing is required to ensure that when the CPU goes to sleep we have
20 * the necessary data available when the caches are not searched.
21 *
22 * ptr: CPU context virtual address
23 * save_ptr: address of the location where the context physical address
24 * must be saved
25 */
__cpu_suspend_save(struct cpu_suspend_ctx * ptr,phys_addr_t * save_ptr)26 void notrace __cpu_suspend_save(struct cpu_suspend_ctx *ptr,
27 phys_addr_t *save_ptr)
28 {
29 *save_ptr = virt_to_phys(ptr);
30
31 cpu_do_suspend(ptr);
32 /*
33 * Only flush the context that must be retrieved with the MMU
34 * off. VA primitives ensure the flush is applied to all
35 * cache levels so context is pushed to DRAM.
36 */
37 __flush_dcache_area(ptr, sizeof(*ptr));
38 __flush_dcache_area(save_ptr, sizeof(*save_ptr));
39 }
40
41 /*
42 * This hook is provided so that cpu_suspend code can restore HW
43 * breakpoints as early as possible in the resume path, before reenabling
44 * debug exceptions. Code cannot be run from a CPU PM notifier since by the
45 * time the notifier runs debug exceptions might have been enabled already,
46 * with HW breakpoints registers content still in an unknown state.
47 */
48 static void (*hw_breakpoint_restore)(void *);
cpu_suspend_set_dbg_restorer(void (* hw_bp_restore)(void *))49 void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
50 {
51 /* Prevent multiple restore hook initializations */
52 if (WARN_ON(hw_breakpoint_restore))
53 return;
54 hw_breakpoint_restore = hw_bp_restore;
55 }
56
57 /*
58 * cpu_suspend
59 *
60 * arg: argument to pass to the finisher function
61 * fn: finisher function pointer
62 *
63 */
cpu_suspend(unsigned long arg,int (* fn)(unsigned long))64 int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
65 {
66 int ret;
67 unsigned long flags;
68
69 /*
70 * From this point debug exceptions are disabled to prevent
71 * updates to mdscr register (saved and restored along with
72 * general purpose registers) from kernel debuggers.
73 */
74 local_dbg_save(flags);
75
76 /*
77 * Function graph tracer state gets incosistent when the kernel
78 * calls functions that never return (aka suspend finishers) hence
79 * disable graph tracing during their execution.
80 */
81 pause_graph_tracing();
82
83 /*
84 * mm context saved on the stack, it will be restored when
85 * the cpu comes out of reset through the identity mapped
86 * page tables, so that the thread address space is properly
87 * set-up on function return.
88 */
89 ret = __cpu_suspend_enter(arg, fn);
90 if (ret == 0) {
91 /*
92 * We are resuming from reset with the idmap active in TTBR0_EL1.
93 * We must uninstall the idmap and restore the expected MMU
94 * state before we can possibly return to userspace.
95 */
96 cpu_uninstall_idmap();
97
98 /*
99 * PSTATE was not saved over suspend/resume, re-enable any
100 * detected features that might not have been set correctly.
101 */
102 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
103 CONFIG_ARM64_PAN));
104 uao_thread_switch(current);
105
106 /*
107 * Restore HW breakpoint registers to sane values
108 * before debug exceptions are possibly reenabled
109 * through local_dbg_restore.
110 */
111 if (hw_breakpoint_restore)
112 hw_breakpoint_restore(NULL);
113 }
114
115 unpause_graph_tracing();
116
117 /*
118 * Restore pstate flags. OS lock and mdscr have been already
119 * restored, so from this point onwards, debugging is fully
120 * renabled if it was enabled when core started shutdown.
121 */
122 local_dbg_restore(flags);
123
124 return ret;
125 }
126
127 struct sleep_save_sp sleep_save_sp;
128
cpu_suspend_init(void)129 static int __init cpu_suspend_init(void)
130 {
131 void *ctx_ptr;
132
133 /* ctx_ptr is an array of physical addresses */
134 ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(phys_addr_t), GFP_KERNEL);
135
136 if (WARN_ON(!ctx_ptr))
137 return -ENOMEM;
138
139 sleep_save_sp.save_ptr_stash = ctx_ptr;
140 sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr);
141 __flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp));
142
143 return 0;
144 }
145 early_initcall(cpu_suspend_init);
146