Lines Matching refs:mod
115 static void mod_update_bounds(struct module *mod) in mod_update_bounds() argument
117 __mod_update_bounds(mod->core_layout.base, mod->core_layout.size, &mod_tree); in mod_update_bounds()
118 if (mod->init_layout.size) in mod_update_bounds()
119 __mod_update_bounds(mod->init_layout.base, mod->init_layout.size, &mod_tree); in mod_update_bounds()
121 __mod_update_bounds(mod->data_layout.base, mod->data_layout.size, &mod_data_tree); in mod_update_bounds()
151 static inline int strong_try_module_get(struct module *mod) in strong_try_module_get() argument
153 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); in strong_try_module_get()
154 if (mod && mod->state == MODULE_STATE_COMING) in strong_try_module_get()
156 if (try_module_get(mod)) in strong_try_module_get()
162 static inline void add_taint_module(struct module *mod, unsigned flag, in add_taint_module() argument
166 set_bit(flag, &mod->taints); in add_taint_module()
173 void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) in __module_put_and_kthread_exit() argument
175 module_put(mod); in __module_put_and_kthread_exit()
310 struct module *mod; in find_symbol() local
319 list_for_each_entry_rcu(mod, &modules, list, in find_symbol()
322 { mod->syms, mod->syms + mod->num_syms, mod->crcs, in find_symbol()
324 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, in find_symbol()
325 mod->gpl_crcs, in find_symbol()
329 if (mod->state == MODULE_STATE_UNFORMED) in find_symbol()
333 if (find_exported_symbol_in_section(&arr[i], mod, fsa)) in find_symbol()
348 struct module *mod; in find_module_all() local
352 list_for_each_entry_rcu(mod, &modules, list, in find_module_all()
354 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) in find_module_all()
356 if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) in find_module_all()
357 return mod; in find_module_all()
369 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
371 return mod->percpu; in mod_percpu()
374 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
384 mod->name, align, PAGE_SIZE); in percpu_modalloc()
388 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); in percpu_modalloc()
389 if (!mod->percpu) { in percpu_modalloc()
391 mod->name, (unsigned long)pcpusec->sh_size); in percpu_modalloc()
394 mod->percpu_size = pcpusec->sh_size; in percpu_modalloc()
398 static void percpu_modfree(struct module *mod) in percpu_modfree() argument
400 free_percpu(mod->percpu); in percpu_modfree()
408 static void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
414 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); in percpu_modcopy()
419 struct module *mod; in __is_module_percpu_address() local
424 list_for_each_entry_rcu(mod, &modules, list) { in __is_module_percpu_address()
425 if (mod->state == MODULE_STATE_UNFORMED) in __is_module_percpu_address()
427 if (!mod->percpu_size) in __is_module_percpu_address()
430 void *start = per_cpu_ptr(mod->percpu, cpu); in __is_module_percpu_address()
433 if (va >= start && va < start + mod->percpu_size) { in __is_module_percpu_address()
437 per_cpu_ptr(mod->percpu, in __is_module_percpu_address()
465 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
469 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
476 static inline void percpu_modfree(struct module *mod) in percpu_modfree() argument
483 static inline void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
502 static void setup_modinfo_##field(struct module *mod, const char *s) \
504 mod->field = kstrdup(s, GFP_KERNEL); \
509 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
511 static int modinfo_##field##_exists(struct module *mod) \
513 return mod->field != NULL; \
515 static void free_modinfo_##field(struct module *mod) \
517 kfree(mod->field); \
518 mod->field = NULL; \
545 static int module_unload_init(struct module *mod) in module_unload_init() argument
551 atomic_set(&mod->refcnt, MODULE_REF_BASE); in module_unload_init()
553 INIT_LIST_HEAD(&mod->source_list); in module_unload_init()
554 INIT_LIST_HEAD(&mod->target_list); in module_unload_init()
557 atomic_inc(&mod->refcnt); in module_unload_init()
622 static void module_unload_free(struct module *mod) in module_unload_free() argument
627 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { in module_unload_free()
629 pr_debug("%s unusing %s\n", mod->name, i->name); in module_unload_free()
654 static int try_release_module_ref(struct module *mod) in try_release_module_ref() argument
659 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); in try_release_module_ref()
663 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); in try_release_module_ref()
668 static int try_stop_module(struct module *mod, int flags, int *forced) in try_stop_module() argument
671 if (try_release_module_ref(mod) != 0) { in try_stop_module()
678 mod->state = MODULE_STATE_GOING; in try_stop_module()
691 int module_refcount(struct module *mod) in module_refcount() argument
693 return atomic_read(&mod->refcnt) - MODULE_REF_BASE; in module_refcount()
698 static void free_module(struct module *mod);
703 struct module *mod; in SYSCALL_DEFINE2() local
720 mod = find_module(name); in SYSCALL_DEFINE2()
721 if (!mod) { in SYSCALL_DEFINE2()
726 if (!list_empty(&mod->source_list)) { in SYSCALL_DEFINE2()
733 if (mod->state != MODULE_STATE_LIVE) { in SYSCALL_DEFINE2()
735 pr_debug("%s already dying\n", mod->name); in SYSCALL_DEFINE2()
741 if (mod->init && !mod->exit) { in SYSCALL_DEFINE2()
750 ret = try_stop_module(mod, flags, &forced); in SYSCALL_DEFINE2()
756 if (mod->exit != NULL) in SYSCALL_DEFINE2()
757 mod->exit(); in SYSCALL_DEFINE2()
759 MODULE_STATE_GOING, mod); in SYSCALL_DEFINE2()
760 klp_module_going(mod); in SYSCALL_DEFINE2()
761 ftrace_release_mod(mod); in SYSCALL_DEFINE2()
766 strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); in SYSCALL_DEFINE2()
767 …strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.ta… in SYSCALL_DEFINE2()
769 free_module(mod); in SYSCALL_DEFINE2()
816 return sprintf(buffer, "%i\n", module_refcount(mk->mod)); in show_refcnt()
867 static inline void module_unload_free(struct module *mod) in module_unload_free() argument
876 static inline int module_unload_init(struct module *mod) in module_unload_init() argument
900 switch (mk->mod->state) { in show_initstate()
935 return sprintf(buffer, "%u\n", mk->mod->core_layout.size); in show_coresize()
945 return sprintf(buffer, "%u\n", mk->mod->data_layout.size); in show_datasize()
955 return sprintf(buffer, "%u\n", mk->mod->init_layout.size); in show_initsize()
966 l = module_flags_taint(mk->mod->taints, buffer); in show_taint()
996 int try_to_force_load(struct module *mod, const char *reason) in try_to_force_load() argument
1000 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); in try_to_force_load()
1001 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); in try_to_force_load()
1014 struct module *mod) in verify_namespace_is_imported() argument
1034 mod->name, kernel_symbol_name(sym), namespace); in verify_namespace_is_imported()
1042 static bool inherit_taint(struct module *mod, struct module *owner, const char *name) in inherit_taint() argument
1047 if (mod->using_gplonly_symbols) { in inherit_taint()
1049 mod->name, name, owner->name); in inherit_taint()
1053 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { in inherit_taint()
1055 mod->name, name, owner->name); in inherit_taint()
1056 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); in inherit_taint()
1062 static const struct kernel_symbol *resolve_symbol(struct module *mod, in resolve_symbol() argument
1069 .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), in resolve_symbol()
1085 mod->using_gplonly_symbols = true; in resolve_symbol()
1087 if (!inherit_taint(mod, fsa.owner, name)) { in resolve_symbol()
1092 if (!check_version(info, name, mod, fsa.crc)) { in resolve_symbol()
1097 err = verify_namespace_is_imported(info, fsa.sym, mod); in resolve_symbol()
1111 if (!mod->sig_ok && in resolve_symbol()
1118 err = ref_module(mod, fsa.owner); in resolve_symbol()
1133 resolve_symbol_wait(struct module *mod, in resolve_symbol_wait() argument
1141 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) in resolve_symbol_wait()
1145 mod->name, owner); in resolve_symbol_wait()
1160 void __weak module_arch_cleanup(struct module *mod) in module_arch_cleanup() argument
1164 void __weak module_arch_freeing_init(struct module *mod) in module_arch_freeing_init() argument
1169 static void free_module(struct module *mod) in free_module() argument
1171 trace_module_free(mod); in free_module()
1173 mod_sysfs_teardown(mod); in free_module()
1180 mod->state = MODULE_STATE_UNFORMED; in free_module()
1184 ddebug_remove_module(mod->name); in free_module()
1187 module_arch_cleanup(mod); in free_module()
1190 module_unload_free(mod); in free_module()
1193 destroy_params(mod->kp, mod->num_kp); in free_module()
1195 if (is_livepatch_module(mod)) in free_module()
1196 free_module_elf(mod); in free_module()
1201 list_del_rcu(&mod->list); in free_module()
1202 mod_tree_remove(mod); in free_module()
1204 module_bug_cleanup(mod); in free_module()
1207 if (try_add_tainted_module(mod)) in free_module()
1209 mod->name); in free_module()
1213 module_arch_freeing_init(mod); in free_module()
1214 trace_android_rvh_set_module_init_rw_nx(mod); in free_module()
1215 module_memfree(mod->init_layout.base); in free_module()
1216 kfree(mod->args); in free_module()
1217 percpu_modfree(mod); in free_module()
1220 lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); in free_module()
1223 trace_android_rvh_set_module_core_rw_nx(mod); in free_module()
1224 module_memfree(mod->core_layout.base); in free_module()
1226 vfree(mod->data_layout.base); in free_module()
1262 static int verify_exported_symbols(struct module *mod) in verify_exported_symbols() argument
1270 { mod->syms, mod->num_syms }, in verify_exported_symbols()
1271 { mod->gpl_syms, mod->num_gpl_syms }, in verify_exported_symbols()
1281 if (!mod->sig_ok && gki_is_module_protected_export( in verify_exported_symbols()
1284 mod->name, kernel_symbol_name(s)); in verify_exported_symbols()
1291 mod->name, kernel_symbol_name(s), in verify_exported_symbols()
1316 static int simplify_symbols(struct module *mod, const struct load_info *info) in simplify_symbols() argument
1340 mod->name); in simplify_symbols()
1355 ksym = resolve_symbol_wait(mod, info, name); in simplify_symbols()
1371 mod->name, name, ret); in simplify_symbols()
1375 mod->name, name, ret); in simplify_symbols()
1382 secbase = (unsigned long)mod_percpu(mod); in simplify_symbols()
1393 static int apply_relocations(struct module *mod, const struct load_info *info) in apply_relocations() argument
1411 err = klp_apply_section_relocs(mod, info->sechdrs, in apply_relocations()
1418 info->index.sym, i, mod); in apply_relocations()
1421 info->index.sym, i, mod); in apply_relocations()
1429 unsigned int __weak arch_mod_section_prepend(struct module *mod, in arch_mod_section_prepend() argument
1437 long module_get_offset(struct module *mod, unsigned int *size, in module_get_offset() argument
1442 *size += arch_mod_section_prepend(mod, section); in module_get_offset()
1463 static void layout_sections(struct module *mod, struct load_info *info) in layout_sections() argument
1494 sizep = m ? &mod->data_layout.size : &mod->core_layout.size; in layout_sections()
1495 s->sh_entsize = module_get_offset(mod, sizep, s, i); in layout_sections()
1500 mod->core_layout.size = strict_align(mod->core_layout.size); in layout_sections()
1501 mod->core_layout.text_size = mod->core_layout.size; in layout_sections()
1504 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1505 mod->data_layout.ro_size = mod->data_layout.size; in layout_sections()
1508 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1509 mod->data_layout.ro_after_init_size = mod->data_layout.size; in layout_sections()
1512 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1528 s->sh_entsize = (module_get_offset(mod, &mod->init_layout.size, s, i) in layout_sections()
1534 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1535 mod->init_layout.text_size = mod->init_layout.size; in layout_sections()
1538 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1539 mod->init_layout.ro_size = mod->init_layout.size; in layout_sections()
1546 mod->init_layout.ro_after_init_size = mod->init_layout.ro_size; in layout_sections()
1549 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1555 static void set_license(struct module *mod, const char *license) in set_license() argument
1563 mod->name, license); in set_license()
1564 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in set_license()
1619 static void setup_modinfo(struct module *mod, struct load_info *info) in setup_modinfo() argument
1626 attr->setup(mod, get_modinfo(info, attr->attr.name)); in setup_modinfo()
1630 static void free_modinfo(struct module *mod) in free_modinfo() argument
1637 attr->free(mod); in free_modinfo()
1641 static void dynamic_debug_setup(struct module *mod, struct _ddebug_info *dyndbg) in dynamic_debug_setup() argument
1645 ddebug_add_module(dyndbg, mod->name); in dynamic_debug_setup()
1648 static void dynamic_debug_remove(struct module *mod, struct _ddebug_info *dyndbg) in dynamic_debug_remove() argument
1651 ddebug_remove_module(mod->name); in dynamic_debug_remove()
1842 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) in check_modinfo_livepatch() argument
1848 if (set_livepatch_module(mod)) { in check_modinfo_livepatch()
1849 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); in check_modinfo_livepatch()
1851 mod->name); in check_modinfo_livepatch()
1856 mod->name); in check_modinfo_livepatch()
1860 static void check_modinfo_retpoline(struct module *mod, struct load_info *info) in check_modinfo_retpoline() argument
1866 mod->name); in check_modinfo_retpoline()
1969 info->index.mod = find_sec(info, ".gnu.linkonce.this_module"); in setup_load_info()
1970 if (!info->index.mod) { in setup_load_info()
1976 info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; in setup_load_info()
1983 info->name = info->mod->name; in setup_load_info()
1995 static int check_modinfo(struct module *mod, struct load_info *info, int flags) in check_modinfo() argument
2005 err = try_to_force_load(mod, "bad vermagic"); in check_modinfo()
2017 mod->name); in check_modinfo()
2018 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); in check_modinfo()
2021 check_modinfo_retpoline(mod, info); in check_modinfo()
2024 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); in check_modinfo()
2026 "is unknown, you have been warned.\n", mod->name); in check_modinfo()
2029 err = check_modinfo_livepatch(mod, info); in check_modinfo()
2034 set_license(mod, get_modinfo(info, "license")); in check_modinfo()
2039 mod->name); in check_modinfo()
2040 add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK); in check_modinfo()
2046 static int find_module_sections(struct module *mod, struct load_info *info) in find_module_sections() argument
2048 mod->kp = section_objs(info, "__param", in find_module_sections()
2049 sizeof(*mod->kp), &mod->num_kp); in find_module_sections()
2050 mod->syms = section_objs(info, "__ksymtab", in find_module_sections()
2051 sizeof(*mod->syms), &mod->num_syms); in find_module_sections()
2052 mod->crcs = section_addr(info, "__kcrctab"); in find_module_sections()
2053 mod->gpl_syms = section_objs(info, "__ksymtab_gpl", in find_module_sections()
2054 sizeof(*mod->gpl_syms), in find_module_sections()
2055 &mod->num_gpl_syms); in find_module_sections()
2056 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); in find_module_sections()
2059 mod->ctors = section_objs(info, ".ctors", in find_module_sections()
2060 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2061 if (!mod->ctors) in find_module_sections()
2062 mod->ctors = section_objs(info, ".init_array", in find_module_sections()
2063 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2070 mod->name); in find_module_sections()
2075 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, in find_module_sections()
2076 &mod->noinstr_text_size); in find_module_sections()
2079 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", in find_module_sections()
2080 sizeof(*mod->tracepoints_ptrs), in find_module_sections()
2081 &mod->num_tracepoints); in find_module_sections()
2084 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", in find_module_sections()
2085 sizeof(*mod->srcu_struct_ptrs), in find_module_sections()
2086 &mod->num_srcu_structs); in find_module_sections()
2089 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", in find_module_sections()
2090 sizeof(*mod->bpf_raw_events), in find_module_sections()
2091 &mod->num_bpf_raw_events); in find_module_sections()
2094 mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); in find_module_sections()
2097 mod->jump_entries = section_objs(info, "__jump_table", in find_module_sections()
2098 sizeof(*mod->jump_entries), in find_module_sections()
2099 &mod->num_jump_entries); in find_module_sections()
2102 mod->trace_events = section_objs(info, "_ftrace_events", in find_module_sections()
2103 sizeof(*mod->trace_events), in find_module_sections()
2104 &mod->num_trace_events); in find_module_sections()
2105 mod->trace_evals = section_objs(info, "_ftrace_eval_map", in find_module_sections()
2106 sizeof(*mod->trace_evals), in find_module_sections()
2107 &mod->num_trace_evals); in find_module_sections()
2110 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", in find_module_sections()
2111 sizeof(*mod->trace_bprintk_fmt_start), in find_module_sections()
2112 &mod->num_trace_bprintk_fmt); in find_module_sections()
2116 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, in find_module_sections()
2117 sizeof(*mod->ftrace_callsites), in find_module_sections()
2118 &mod->num_ftrace_callsites); in find_module_sections()
2121 mod->ei_funcs = section_objs(info, "_error_injection_whitelist", in find_module_sections()
2122 sizeof(*mod->ei_funcs), in find_module_sections()
2123 &mod->num_ei_funcs); in find_module_sections()
2126 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, in find_module_sections()
2127 &mod->kprobes_text_size); in find_module_sections()
2128 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", in find_module_sections()
2130 &mod->num_kprobe_blacklist); in find_module_sections()
2133 mod->printk_index_start = section_objs(info, ".printk_index", in find_module_sections()
2134 sizeof(*mod->printk_index_start), in find_module_sections()
2135 &mod->printk_index_size); in find_module_sections()
2138 mod->static_call_sites = section_objs(info, ".static_call_sites", in find_module_sections()
2139 sizeof(*mod->static_call_sites), in find_module_sections()
2140 &mod->num_static_call_sites); in find_module_sections()
2143 mod->kunit_suites = section_objs(info, ".kunit_test_suites", in find_module_sections()
2144 sizeof(*mod->kunit_suites), in find_module_sections()
2145 &mod->num_kunit_suites); in find_module_sections()
2148 mod->extable = section_objs(info, "__ex_table", in find_module_sections()
2149 sizeof(*mod->extable), &mod->num_exentries); in find_module_sections()
2152 pr_warn("%s: Ignoring obsolete parameters\n", mod->name); in find_module_sections()
2162 static int move_module(struct module *mod, struct load_info *info) in move_module() argument
2168 ptr = module_alloc(mod->core_layout.size); in move_module()
2178 memset(ptr, 0, mod->core_layout.size); in move_module()
2179 mod->core_layout.base = ptr; in move_module()
2181 if (mod->init_layout.size) { in move_module()
2182 ptr = module_alloc(mod->init_layout.size); in move_module()
2191 module_memfree(mod->core_layout.base); in move_module()
2194 memset(ptr, 0, mod->init_layout.size); in move_module()
2195 mod->init_layout.base = ptr; in move_module()
2197 mod->init_layout.base = NULL; in move_module()
2201 ptr = vzalloc(mod->data_layout.size); in move_module()
2209 module_memfree(mod->core_layout.base); in move_module()
2210 module_memfree(mod->init_layout.base); in move_module()
2214 mod->data_layout.base = ptr; in move_module()
2226 dest = mod->init_layout.base in move_module()
2229 dest = mod->data_layout.base + shdr->sh_entsize; in move_module()
2231 dest = mod->core_layout.base + shdr->sh_entsize; in move_module()
2244 static int check_module_license_and_versions(struct module *mod) in check_module_license_and_versions() argument
2253 if (strcmp(mod->name, "ndiswrapper") == 0) in check_module_license_and_versions()
2257 if (strcmp(mod->name, "driverloader") == 0) in check_module_license_and_versions()
2258 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
2262 if (strcmp(mod->name, "lve") == 0) in check_module_license_and_versions()
2263 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
2267 pr_warn("%s: module license taints kernel.\n", mod->name); in check_module_license_and_versions()
2270 if ((mod->num_syms && !mod->crcs) || in check_module_license_and_versions()
2271 (mod->num_gpl_syms && !mod->gpl_crcs)) { in check_module_license_and_versions()
2272 return try_to_force_load(mod, in check_module_license_and_versions()
2279 static void flush_module_icache(const struct module *mod) in flush_module_icache() argument
2286 if (mod->init_layout.base) in flush_module_icache()
2287 flush_icache_range((unsigned long)mod->init_layout.base, in flush_module_icache()
2288 (unsigned long)mod->init_layout.base in flush_module_icache()
2289 + mod->init_layout.size); in flush_module_icache()
2290 flush_icache_range((unsigned long)mod->core_layout.base, in flush_module_icache()
2291 (unsigned long)mod->core_layout.base + mod->core_layout.size); in flush_module_icache()
2297 struct module *mod) in module_frob_arch_sections() argument
2325 struct module *mod; in layout_and_allocate() local
2329 err = check_modinfo(info->mod, info, flags); in layout_and_allocate()
2335 info->secstrings, info->mod); in layout_and_allocate()
2340 info->secstrings, info->mod); in layout_and_allocate()
2370 layout_sections(info->mod, info); in layout_and_allocate()
2371 layout_symtab(info->mod, info); in layout_and_allocate()
2374 err = move_module(info->mod, info); in layout_and_allocate()
2379 mod = (void *)info->sechdrs[info->index.mod].sh_addr; in layout_and_allocate()
2380 kmemleak_load_module(mod, info); in layout_and_allocate()
2381 return mod; in layout_and_allocate()
2385 static void module_deallocate(struct module *mod, struct load_info *info) in module_deallocate() argument
2387 percpu_modfree(mod); in module_deallocate()
2388 module_arch_freeing_init(mod); in module_deallocate()
2389 trace_android_rvh_set_module_init_rw_nx(mod); in module_deallocate()
2390 module_memfree(mod->init_layout.base); in module_deallocate()
2391 trace_android_rvh_set_module_core_rw_nx(mod); in module_deallocate()
2392 module_memfree(mod->core_layout.base); in module_deallocate()
2394 vfree(mod->data_layout.base); in module_deallocate()
2405 static int post_relocation(struct module *mod, const struct load_info *info) in post_relocation() argument
2408 sort_extable(mod->extable, mod->extable + mod->num_exentries); in post_relocation()
2411 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, in post_relocation()
2415 add_kallsyms(mod, info); in post_relocation()
2418 return module_finalize(info->hdr, info->sechdrs, mod); in post_relocation()
2424 struct module *mod; in finished_loading() local
2434 mod = find_module_all(name, strlen(name), true); in finished_loading()
2435 ret = !mod || mod->state == MODULE_STATE_LIVE in finished_loading()
2436 || mod->state == MODULE_STATE_GOING; in finished_loading()
2443 static void do_mod_ctors(struct module *mod) in do_mod_ctors() argument
2448 for (i = 0; i < mod->num_ctors; i++) in do_mod_ctors()
2449 mod->ctors[i](); in do_mod_ctors()
2487 static noinline int do_init_module(struct module *mod) in do_init_module() argument
2497 freeinit->module_init = mod->init_layout.base; in do_init_module()
2499 do_mod_ctors(mod); in do_init_module()
2501 if (mod->init != NULL) in do_init_module()
2502 ret = do_one_initcall(mod->init); in do_init_module()
2510 __func__, mod->name, ret, __func__); in do_init_module()
2515 mod->state = MODULE_STATE_LIVE; in do_init_module()
2517 MODULE_STATE_LIVE, mod); in do_init_module()
2520 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); in do_init_module()
2530 if (!mod->async_probe_requested) in do_init_module()
2533 ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base + in do_init_module()
2534 mod->init_layout.size); in do_init_module()
2537 module_put(mod); in do_init_module()
2538 trim_init_extable(mod); in do_init_module()
2541 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); in do_init_module()
2543 module_enable_ro(mod, true); in do_init_module()
2544 trace_android_rvh_set_module_permit_after_init(mod); in do_init_module()
2545 mod_tree_remove_init(mod); in do_init_module()
2546 module_arch_freeing_init(mod); in do_init_module()
2547 trace_android_rvh_set_module_init_rw_nx(mod); in do_init_module()
2548 mod->init_layout.base = NULL; in do_init_module()
2549 mod->init_layout.size = 0; in do_init_module()
2550 mod->init_layout.ro_size = 0; in do_init_module()
2551 mod->init_layout.ro_after_init_size = 0; in do_init_module()
2552 mod->init_layout.text_size = 0; in do_init_module()
2555 mod->btf_data = NULL; in do_init_module()
2582 mod->state = MODULE_STATE_GOING; in do_init_module()
2584 module_put(mod); in do_init_module()
2586 MODULE_STATE_GOING, mod); in do_init_module()
2587 klp_module_going(mod); in do_init_module()
2588 ftrace_release_mod(mod); in do_init_module()
2589 free_module(mod); in do_init_module()
2607 static int add_unformed_module(struct module *mod) in add_unformed_module() argument
2612 mod->state = MODULE_STATE_UNFORMED; in add_unformed_module()
2615 old = find_module_all(mod->name, strlen(mod->name), true); in add_unformed_module()
2622 finished_loading(mod->name)); in add_unformed_module()
2628 old = find_module_all(mod->name, strlen(mod->name), in add_unformed_module()
2645 mod_update_bounds(mod); in add_unformed_module()
2646 list_add_rcu(&mod->list, &modules); in add_unformed_module()
2647 mod_tree_insert(mod); in add_unformed_module()
2656 static int complete_formation(struct module *mod, struct load_info *info) in complete_formation() argument
2663 err = verify_exported_symbols(mod); in complete_formation()
2668 module_bug_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
2669 module_cfi_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
2671 if (module_check_misalignment(mod)) in complete_formation()
2674 module_enable_ro(mod, false); in complete_formation()
2675 module_enable_nx(mod); in complete_formation()
2676 module_enable_x(mod); in complete_formation()
2677 trace_android_rvh_set_module_permit_before_init(mod); in complete_formation()
2683 mod->state = MODULE_STATE_COMING; in complete_formation()
2695 static int prepare_coming_module(struct module *mod) in prepare_coming_module() argument
2699 ftrace_module_enable(mod); in prepare_coming_module()
2700 err = klp_module_coming(mod); in prepare_coming_module()
2705 MODULE_STATE_COMING, MODULE_STATE_GOING, mod); in prepare_coming_module()
2708 klp_module_going(mod); in prepare_coming_module()
2716 struct module *mod = arg; in unknown_module_param_cb() local
2720 if (strtobool(val, &mod->async_probe_requested)) in unknown_module_param_cb()
2721 mod->async_probe_requested = true; in unknown_module_param_cb()
2739 struct module *mod; in load_module() local
2790 if (!check_modstruct_version(info, info->mod)) { in load_module()
2796 mod = layout_and_allocate(info, flags); in load_module()
2797 if (IS_ERR(mod)) { in load_module()
2798 err = PTR_ERR(mod); in load_module()
2802 audit_log_kern_module(mod->name); in load_module()
2805 err = add_unformed_module(mod); in load_module()
2810 mod->sig_ok = info->sig_ok; in load_module()
2811 if (!mod->sig_ok) { in load_module()
2814 "kernel\n", mod->name); in load_module()
2815 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); in load_module()
2818 mod->sig_ok = 0; in load_module()
2822 err = percpu_modalloc(mod, info); in load_module()
2827 err = module_unload_init(mod); in load_module()
2831 init_param_lock(mod); in load_module()
2837 err = find_module_sections(mod, info); in load_module()
2841 err = check_module_license_and_versions(mod); in load_module()
2846 setup_modinfo(mod, info); in load_module()
2849 err = simplify_symbols(mod, info); in load_module()
2853 err = apply_relocations(mod, info); in load_module()
2857 err = post_relocation(mod, info); in load_module()
2861 flush_module_icache(mod); in load_module()
2864 mod->args = strndup_user(uargs, ~0UL >> 1); in load_module()
2865 if (IS_ERR(mod->args)) { in load_module()
2866 err = PTR_ERR(mod->args); in load_module()
2870 init_build_id(mod, info); in load_module()
2871 dynamic_debug_setup(mod, &info->dyndbg); in load_module()
2874 ftrace_module_init(mod); in load_module()
2877 err = complete_formation(mod, info); in load_module()
2881 err = prepare_coming_module(mod); in load_module()
2885 mod->async_probe_requested = async_probe; in load_module()
2888 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, in load_module()
2889 -32768, 32767, mod, in load_module()
2896 mod->name, after_dashes); in load_module()
2900 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); in load_module()
2904 if (is_livepatch_module(mod)) { in load_module()
2905 err = copy_module_elf(mod, info); in load_module()
2914 trace_module_load(mod); in load_module()
2916 return do_init_module(mod); in load_module()
2919 mod_sysfs_teardown(mod); in load_module()
2921 mod->state = MODULE_STATE_GOING; in load_module()
2922 destroy_params(mod->kp, mod->num_kp); in load_module()
2924 MODULE_STATE_GOING, mod); in load_module()
2925 klp_module_going(mod); in load_module()
2927 mod->state = MODULE_STATE_GOING; in load_module()
2930 module_bug_cleanup(mod); in load_module()
2934 ftrace_release_mod(mod); in load_module()
2935 dynamic_debug_remove(mod, &info->dyndbg); in load_module()
2937 kfree(mod->args); in load_module()
2939 module_arch_cleanup(mod); in load_module()
2941 free_modinfo(mod); in load_module()
2943 module_unload_free(mod); in load_module()
2947 list_del_rcu(&mod->list); in load_module()
2948 mod_tree_remove(mod); in load_module()
2955 lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); in load_module()
2957 module_deallocate(mod, info); in load_module()
3025 char *module_flags(struct module *mod, char *buf, bool show_state) in module_flags() argument
3029 BUG_ON(mod->state == MODULE_STATE_UNFORMED); in module_flags()
3030 if (!mod->taints && !show_state) in module_flags()
3032 if (mod->taints || in module_flags()
3033 mod->state == MODULE_STATE_GOING || in module_flags()
3034 mod->state == MODULE_STATE_COMING) { in module_flags()
3036 bx += module_flags_taint(mod->taints, buf + bx); in module_flags()
3038 if (mod->state == MODULE_STATE_GOING && show_state) in module_flags()
3041 if (mod->state == MODULE_STATE_COMING && show_state) in module_flags()
3055 struct module *mod; in search_module_extables() local
3058 mod = __module_address(addr); in search_module_extables()
3059 if (!mod) in search_module_extables()
3062 if (!mod->num_exentries) in search_module_extables()
3065 e = search_extable(mod->extable, in search_module_extables()
3066 mod->num_exentries, in search_module_extables()
3105 struct module *mod; in __module_address() local
3119 mod = mod_find(addr, tree); in __module_address()
3120 if (mod) { in __module_address()
3121 BUG_ON(!within_module(addr, mod)); in __module_address()
3122 if (mod->state == MODULE_STATE_UNFORMED) in __module_address()
3123 mod = NULL; in __module_address()
3125 return mod; in __module_address()
3156 struct module *mod = __module_address(addr); in __module_text_address() local
3157 if (mod) { in __module_text_address()
3159 if (!within(addr, mod->init_layout.base, mod->init_layout.text_size) in __module_text_address()
3160 && !within(addr, mod->core_layout.base, mod->core_layout.text_size)) in __module_text_address()
3161 mod = NULL; in __module_text_address()
3163 return mod; in __module_text_address()
3169 struct module *mod; in print_modules() local
3175 list_for_each_entry_rcu(mod, &modules, list) { in print_modules()
3176 if (mod->state == MODULE_STATE_UNFORMED) in print_modules()
3178 pr_cont(" %s%s", mod->name, module_flags(mod, buf, true)); in print_modules()