19 static __always_inline void __enter_from_user_mode(struct pt_regs *regs)  in __enter_from_user_mode()  argument
21 	arch_enter_from_user_mode(regs);  in __enter_from_user_mode()
28 	kmsan_unpoison_entry_regs(regs);  in __enter_from_user_mode()
33 void noinstr enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
35 	__enter_from_user_mode(regs);  in enter_from_user_mode()
38 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)  in syscall_enter_audit()  argument
43 		syscall_get_arguments(current, regs, args);  in syscall_enter_audit()
48 static long syscall_trace_enter(struct pt_regs *regs, long syscall,  in syscall_trace_enter()  argument
59 		if (syscall_user_dispatch(regs))  in syscall_trace_enter()
65 		ret = ptrace_report_syscall_entry(regs);  in syscall_trace_enter()
78 	syscall = syscall_get_nr(current, regs);  in syscall_trace_enter()
81 		trace_sys_enter(regs, syscall);  in syscall_trace_enter()
83 	syscall_enter_audit(regs, syscall);  in syscall_trace_enter()
89 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall)  in __syscall_enter_from_user_work()  argument
94 		syscall = syscall_trace_enter(regs, syscall, work);  in __syscall_enter_from_user_work()
99 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)  in syscall_enter_from_user_mode_work()  argument
101 	return __syscall_enter_from_user_work(regs, syscall);  in syscall_enter_from_user_mode_work()
104 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)  in syscall_enter_from_user_mode()  argument
108 	__enter_from_user_mode(regs);  in syscall_enter_from_user_mode()
112 	ret = __syscall_enter_from_user_work(regs, syscall);  in syscall_enter_from_user_mode()
118 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)  in syscall_enter_from_user_mode_prepare()  argument
120 	__enter_from_user_mode(regs);  in syscall_enter_from_user_mode_prepare()
145 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }  in arch_do_signal_or_restart()  argument
147 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,  in exit_to_user_mode_loop()  argument
162 			uprobe_notify_resume(regs);  in exit_to_user_mode_loop()
168 			arch_do_signal_or_restart(regs);  in exit_to_user_mode_loop()
171 			resume_user_mode_work(regs);  in exit_to_user_mode_loop()
174 		arch_exit_to_user_mode_work(regs, ti_work);  in exit_to_user_mode_loop()
193 static void exit_to_user_mode_prepare(struct pt_regs *regs)  in exit_to_user_mode_prepare()  argument
204 		ti_work = exit_to_user_mode_loop(regs, ti_work);  in exit_to_user_mode_prepare()
206 	arch_exit_to_user_mode_prepare(regs, ti_work);  in exit_to_user_mode_prepare()
228 static void syscall_exit_work(struct pt_regs *regs, unsigned long work)  in syscall_exit_work()  argument
245 	audit_syscall_exit(regs);  in syscall_exit_work()
248 		trace_sys_exit(regs, syscall_get_return_value(current, regs));  in syscall_exit_work()
252 		ptrace_report_syscall_exit(regs, step);  in syscall_exit_work()
259 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)  in syscall_exit_to_user_mode_prepare()  argument
262 	unsigned long nr = syscall_get_nr(current, regs);  in syscall_exit_to_user_mode_prepare()
271 	rseq_syscall(regs);  in syscall_exit_to_user_mode_prepare()
279 		syscall_exit_work(regs, work);  in syscall_exit_to_user_mode_prepare()
282 static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs)  in __syscall_exit_to_user_mode_work()  argument
284 	syscall_exit_to_user_mode_prepare(regs);  in __syscall_exit_to_user_mode_work()
286 	exit_to_user_mode_prepare(regs);  in __syscall_exit_to_user_mode_work()
289 void syscall_exit_to_user_mode_work(struct pt_regs *regs)  in syscall_exit_to_user_mode_work()  argument
291 	__syscall_exit_to_user_mode_work(regs);  in syscall_exit_to_user_mode_work()
294 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)  in syscall_exit_to_user_mode()  argument
297 	__syscall_exit_to_user_mode_work(regs);  in syscall_exit_to_user_mode()
302 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)  in irqentry_enter_from_user_mode()  argument
304 	__enter_from_user_mode(regs);  in irqentry_enter_from_user_mode()
307 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)  in irqentry_exit_to_user_mode()  argument
310 	exit_to_user_mode_prepare(regs);  in irqentry_exit_to_user_mode()
315 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)  in irqentry_enter()  argument
321 	if (user_mode(regs)) {  in irqentry_enter()
322 		irqentry_enter_from_user_mode(regs);  in irqentry_enter()
358 		kmsan_unpoison_entry_regs(regs);  in irqentry_enter()
374 	kmsan_unpoison_entry_regs(regs);  in irqentry_enter()
407 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)  in irqentry_exit()  argument
412 	if (user_mode(regs)) {  in irqentry_exit()
413 		irqentry_exit_to_user_mode(regs);  in irqentry_exit()
414 	} else if (!regs_irqs_disabled(regs)) {  in irqentry_exit()
448 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)  in irqentry_nmi_enter()  argument
460 	kmsan_unpoison_entry_regs(regs);  in irqentry_nmi_enter()
468 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)  in irqentry_nmi_exit()  argument