1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2002 Richard Henderson
4 * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6 * Copyright (C) 2024 Mike Rapoport IBM.
7 */
8
9 #include <linux/mm.h>
10 #include <linux/vmalloc.h>
11 #include <linux/execmem.h>
12 #include <linux/moduleloader.h>
13
14 #include "internal.h"
15
16 static struct execmem_info *execmem_info __ro_after_init;
17 static struct execmem_info default_execmem_info __ro_after_init;
18
19 #ifdef CONFIG_MMU
execmem_vmap(size_t size)20 struct vm_struct *execmem_vmap(size_t size)
21 {
22 struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
23 struct vm_struct *area;
24
25 area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
26 range->start, range->end, NUMA_NO_NODE,
27 GFP_KERNEL, __builtin_return_address(0));
28 if (!area && range->fallback_start)
29 area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
30 range->fallback_start, range->fallback_end,
31 NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));
32
33 return area;
34 }
35 #endif /* CONFIG_MMU */
36
__execmem_alloc(struct execmem_range * range,size_t size)37 static void *__execmem_alloc(struct execmem_range *range, size_t size)
38 {
39 bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
40 unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
41 gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
42 unsigned long start = range->start;
43 unsigned long end = range->end;
44 unsigned int align = range->alignment;
45 pgprot_t pgprot = range->pgprot;
46 void *p;
47
48 if (kasan)
49 vm_flags |= VM_DEFER_KMEMLEAK;
50
51 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
52 pgprot, vm_flags, NUMA_NO_NODE,
53 __builtin_return_address(0));
54 if (!p && range->fallback_start) {
55 start = range->fallback_start;
56 end = range->fallback_end;
57 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
58 pgprot, vm_flags, NUMA_NO_NODE,
59 __builtin_return_address(0));
60 }
61
62 if (!p) {
63 pr_warn_ratelimited("execmem: unable to allocate memory\n");
64 return NULL;
65 }
66
67 if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
68 vfree(p);
69 return NULL;
70 }
71
72 return kasan_reset_tag(p);
73 }
74
execmem_alloc(enum execmem_type type,size_t size)75 void *execmem_alloc(enum execmem_type type, size_t size)
76 {
77 struct execmem_range *range = &execmem_info->ranges[type];
78
79 return __execmem_alloc(range, size);
80 }
81
execmem_free(void * ptr)82 void execmem_free(void *ptr)
83 {
84 /*
85 * This memory may be RO, and freeing RO memory in an interrupt is not
86 * supported by vmalloc.
87 */
88 WARN_ON(in_interrupt());
89 vfree(ptr);
90 }
91
execmem_validate(struct execmem_info * info)92 static bool execmem_validate(struct execmem_info *info)
93 {
94 struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
95
96 if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
97 pr_crit("Invalid parameters for execmem allocator, module loading will fail");
98 return false;
99 }
100
101 return true;
102 }
103
execmem_init_missing(struct execmem_info * info)104 static void execmem_init_missing(struct execmem_info *info)
105 {
106 struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
107
108 for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
109 struct execmem_range *r = &info->ranges[i];
110
111 if (!r->start) {
112 if (i == EXECMEM_MODULE_DATA)
113 r->pgprot = PAGE_KERNEL;
114 else
115 r->pgprot = default_range->pgprot;
116 r->alignment = default_range->alignment;
117 r->start = default_range->start;
118 r->end = default_range->end;
119 r->flags = default_range->flags;
120 r->fallback_start = default_range->fallback_start;
121 r->fallback_end = default_range->fallback_end;
122 }
123 }
124 }
125
execmem_arch_setup(void)126 struct execmem_info * __weak execmem_arch_setup(void)
127 {
128 return NULL;
129 }
130
__execmem_init(void)131 static void __init __execmem_init(void)
132 {
133 struct execmem_info *info = execmem_arch_setup();
134
135 if (!info) {
136 info = execmem_info = &default_execmem_info;
137 info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
138 info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
139 info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
140 info->ranges[EXECMEM_DEFAULT].alignment = 1;
141 }
142
143 if (!execmem_validate(info))
144 return;
145
146 execmem_init_missing(info);
147
148 execmem_info = info;
149 }
150
151 #ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
execmem_late_init(void)152 static int __init execmem_late_init(void)
153 {
154 __execmem_init();
155 return 0;
156 }
157 core_initcall(execmem_late_init);
158 #else
execmem_init(void)159 void __init execmem_init(void)
160 {
161 __execmem_init();
162 }
163 #endif
164