14 unsigned long unwind_get_return_address(struct unwind_state *state)  in unwind_get_return_address()  argument
16 	if (unwind_done(state))  in unwind_get_return_address()
19 	return __kernel_text_address(state->ip) ? state->ip : 0;  in unwind_get_return_address()
23 unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)  in unwind_get_return_address_ptr()  argument
25 	if (unwind_done(state))  in unwind_get_return_address_ptr()
28 	return state->regs ? &state->regs->ip : state->bp + 1;  in unwind_get_return_address_ptr()
31 static void unwind_dump(struct unwind_state *state)  in unwind_dump()  argument
45 			state->stack_info.type, state->stack_info.next_sp,  in unwind_dump()
46 			state->stack_mask, state->graph_idx);  in unwind_dump()
48 	for (sp = PTR_ALIGN(state->orig_sp, sizeof(long)); sp;  in unwind_dump()
50 		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))  in unwind_dump()
80 static inline unsigned long *last_frame(struct unwind_state *state)  in last_frame()  argument
82 	return (unsigned long *)task_pt_regs(state->task) - 2;  in last_frame()
85 static bool is_last_frame(struct unwind_state *state)  in is_last_frame()  argument
87 	return state->bp == last_frame(state);  in is_last_frame()
96 static inline unsigned long *last_aligned_frame(struct unwind_state *state)  in last_aligned_frame()  argument
98 	return last_frame(state) - GCC_REALIGN_WORDS;  in last_aligned_frame()
101 static bool is_last_aligned_frame(struct unwind_state *state)  in is_last_aligned_frame()  argument
103 	unsigned long *last_bp = last_frame(state);  in is_last_aligned_frame()
104 	unsigned long *aligned_bp = last_aligned_frame(state);  in is_last_aligned_frame()
129 	return (state->bp == aligned_bp && *(aligned_bp + 1) == *(last_bp + 1));  in is_last_aligned_frame()
132 static bool is_last_ftrace_frame(struct unwind_state *state)  in is_last_ftrace_frame()  argument
134 	unsigned long *last_bp = last_frame(state);  in is_last_ftrace_frame()
149 	return (state->bp == last_ftrace_bp &&  in is_last_ftrace_frame()
150 		*state->bp == *(state->bp + 2) &&  in is_last_ftrace_frame()
151 		*(state->bp + 1) == *(state->bp + 4));  in is_last_ftrace_frame()
154 static bool is_last_task_frame(struct unwind_state *state)  in is_last_task_frame()  argument
156 	return is_last_frame(state) || is_last_aligned_frame(state) ||  in is_last_task_frame()
157 	       is_last_ftrace_frame(state);  in is_last_task_frame()
186 static bool update_stack_state(struct unwind_state *state,  in update_stack_state()  argument
189 	struct stack_info *info = &state->stack_info;  in update_stack_state()
195 	if (state->regs)  in update_stack_state()
196 		prev_frame_end = (void *)state->regs + sizeof(*state->regs);  in update_stack_state()
198 		prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;  in update_stack_state()
205 		state->got_irq = true;  in update_stack_state()
219 		if (get_stack_info(info->next_sp, state->task, info,  in update_stack_state()
220 				   &state->stack_mask))  in update_stack_state()
224 	if (state->orig_sp && state->stack_info.type == prev_type &&  in update_stack_state()
230 		state->regs = regs;  in update_stack_state()
231 		state->bp = NULL;  in update_stack_state()
233 		state->bp = next_bp;  in update_stack_state()
234 		state->regs = NULL;  in update_stack_state()
238 	if (state->regs && user_mode(state->regs))  in update_stack_state()
239 		state->ip = 0;  in update_stack_state()
241 		addr_p = unwind_get_return_address_ptr(state);  in update_stack_state()
242 		addr = READ_ONCE_TASK_STACK(state->task, *addr_p);  in update_stack_state()
243 		state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx,  in update_stack_state()
248 	if (!state->orig_sp)  in update_stack_state()
249 		state->orig_sp = frame;  in update_stack_state()
254 bool unwind_next_frame(struct unwind_state *state)  in unwind_next_frame()  argument
259 	if (unwind_done(state))  in unwind_next_frame()
263 	if (state->regs && user_mode(state->regs))  in unwind_next_frame()
266 	if (is_last_task_frame(state)) {  in unwind_next_frame()
267 		regs = task_pt_regs(state->task);  in unwind_next_frame()
288 		state->regs = regs;  in unwind_next_frame()
289 		state->bp = NULL;  in unwind_next_frame()
290 		state->ip = 0;  in unwind_next_frame()
295 	if (state->next_bp) {  in unwind_next_frame()
296 		next_bp = state->next_bp;  in unwind_next_frame()
297 		state->next_bp = NULL;  in unwind_next_frame()
298 	} else if (state->regs) {  in unwind_next_frame()
299 		next_bp = (unsigned long *)state->regs->bp;  in unwind_next_frame()
301 		next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp);  in unwind_next_frame()
305 	if (!update_stack_state(state, next_bp))  in unwind_next_frame()
311 	state->error = true;  in unwind_next_frame()
320 	if (state->task != current)  in unwind_next_frame()
327 	if (state->got_irq && in_entry_code(state->ip))  in unwind_next_frame()
329 	if (state->regs &&  in unwind_next_frame()
330 	    state->regs->sp >= (unsigned long)last_aligned_frame(state) &&  in unwind_next_frame()
331 	    state->regs->sp < (unsigned long)task_pt_regs(state->task))  in unwind_next_frame()
341 	if (state->task != current)  in unwind_next_frame()
344 	if (state->regs) {  in unwind_next_frame()
347 			state->regs, state->task->comm,  in unwind_next_frame()
348 			state->task->pid, next_bp);  in unwind_next_frame()
349 		unwind_dump(state);  in unwind_next_frame()
353 			state->bp, state->task->comm,  in unwind_next_frame()
354 			state->task->pid, next_bp);  in unwind_next_frame()
355 		unwind_dump(state);  in unwind_next_frame()
358 	state->stack_info.type = STACK_TYPE_UNKNOWN;  in unwind_next_frame()
363 void __unwind_start(struct unwind_state *state, struct task_struct *task,  in __unwind_start()  argument
368 	memset(state, 0, sizeof(*state));  in __unwind_start()
369 	state->task = task;  in __unwind_start()
370 	state->got_irq = (regs);  in __unwind_start()
374 		state->stack_info.type = STACK_TYPE_UNKNOWN;  in __unwind_start()
390 		state->next_bp = bp;  in __unwind_start()
395 	get_stack_info(bp, state->task, &state->stack_info,  in __unwind_start()
396 		       &state->stack_mask);  in __unwind_start()
397 	update_stack_state(state, bp);  in __unwind_start()
404 	while (!unwind_done(state) &&  in __unwind_start()
405 	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||  in __unwind_start()
406 			(state->next_bp == NULL && state->bp < first_frame)))  in __unwind_start()
407 		unwind_next_frame(state);  in __unwind_start()