19 static __always_inline void __enter_from_user_mode(struct pt_regs *regs)  in __enter_from_user_mode()  argument
21 	arch_enter_from_user_mode(regs);  in __enter_from_user_mode()
28 	kmsan_unpoison_entry_regs(regs);  in __enter_from_user_mode()
33 void noinstr enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
35 	__enter_from_user_mode(regs);  in enter_from_user_mode()
38 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)  in syscall_enter_audit()  argument
43 		syscall_get_arguments(current, regs, args);  in syscall_enter_audit()
48 static long syscall_trace_enter(struct pt_regs *regs, long syscall,  in syscall_trace_enter()  argument
59 		if (syscall_user_dispatch(regs))  in syscall_trace_enter()
65 		ret = ptrace_report_syscall_entry(regs);  in syscall_trace_enter()
78 	syscall = syscall_get_nr(current, regs);  in syscall_trace_enter()
81 		trace_sys_enter(regs, syscall);  in syscall_trace_enter()
86 		syscall = syscall_get_nr(current, regs);  in syscall_trace_enter()
89 	syscall_enter_audit(regs, syscall);  in syscall_trace_enter()
95 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall)  in __syscall_enter_from_user_work()  argument
100 		syscall = syscall_trace_enter(regs, syscall, work);  in __syscall_enter_from_user_work()
105 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)  in syscall_enter_from_user_mode_work()  argument
107 	return __syscall_enter_from_user_work(regs, syscall);  in syscall_enter_from_user_mode_work()
110 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)  in syscall_enter_from_user_mode()  argument
114 	__enter_from_user_mode(regs);  in syscall_enter_from_user_mode()
118 	ret = __syscall_enter_from_user_work(regs, syscall);  in syscall_enter_from_user_mode()
124 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)  in syscall_enter_from_user_mode_prepare()  argument
126 	__enter_from_user_mode(regs);  in syscall_enter_from_user_mode_prepare()
151 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }  in arch_do_signal_or_restart()  argument
153 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,  in exit_to_user_mode_loop()  argument
168 			uprobe_notify_resume(regs);  in exit_to_user_mode_loop()
174 			arch_do_signal_or_restart(regs);  in exit_to_user_mode_loop()
177 			resume_user_mode_work(regs);  in exit_to_user_mode_loop()
180 		arch_exit_to_user_mode_work(regs, ti_work);  in exit_to_user_mode_loop()
199 static void exit_to_user_mode_prepare(struct pt_regs *regs)  in exit_to_user_mode_prepare()  argument
210 		ti_work = exit_to_user_mode_loop(regs, ti_work);  in exit_to_user_mode_prepare()
212 	arch_exit_to_user_mode_prepare(regs, ti_work);  in exit_to_user_mode_prepare()
233 static void syscall_exit_work(struct pt_regs *regs, unsigned long work)  in syscall_exit_work()  argument
250 	audit_syscall_exit(regs);  in syscall_exit_work()
253 		trace_sys_exit(regs, syscall_get_return_value(current, regs));  in syscall_exit_work()
257 		ptrace_report_syscall_exit(regs, step);  in syscall_exit_work()
264 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)  in syscall_exit_to_user_mode_prepare()  argument
267 	unsigned long nr = syscall_get_nr(current, regs);  in syscall_exit_to_user_mode_prepare()
276 	rseq_syscall(regs);  in syscall_exit_to_user_mode_prepare()
284 		syscall_exit_work(regs, work);  in syscall_exit_to_user_mode_prepare()
287 static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs)  in __syscall_exit_to_user_mode_work()  argument
289 	syscall_exit_to_user_mode_prepare(regs);  in __syscall_exit_to_user_mode_work()
291 	exit_to_user_mode_prepare(regs);  in __syscall_exit_to_user_mode_work()
294 void syscall_exit_to_user_mode_work(struct pt_regs *regs)  in syscall_exit_to_user_mode_work()  argument
296 	__syscall_exit_to_user_mode_work(regs);  in syscall_exit_to_user_mode_work()
299 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)  in syscall_exit_to_user_mode()  argument
302 	__syscall_exit_to_user_mode_work(regs);  in syscall_exit_to_user_mode()
307 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)  in irqentry_enter_from_user_mode()  argument
309 	__enter_from_user_mode(regs);  in irqentry_enter_from_user_mode()
312 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)  in irqentry_exit_to_user_mode()  argument
315 	exit_to_user_mode_prepare(regs);  in irqentry_exit_to_user_mode()
320 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)  in irqentry_enter()  argument
326 	if (user_mode(regs)) {  in irqentry_enter()
327 		irqentry_enter_from_user_mode(regs);  in irqentry_enter()
363 		kmsan_unpoison_entry_regs(regs);  in irqentry_enter()
379 	kmsan_unpoison_entry_regs(regs);  in irqentry_enter()
412 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)  in irqentry_exit()  argument
417 	if (user_mode(regs)) {  in irqentry_exit()
418 		irqentry_exit_to_user_mode(regs);  in irqentry_exit()
419 	} else if (!regs_irqs_disabled(regs)) {  in irqentry_exit()
453 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)  in irqentry_nmi_enter()  argument
465 	kmsan_unpoison_entry_regs(regs);  in irqentry_nmi_enter()
473 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)  in irqentry_nmi_exit()  argument