1137 	struct bpf_insn *insn = env->prog->insnsi;  in check_subprogs()  local
1147 		if (insn[i].code != (BPF_JMP | BPF_CALL))  in check_subprogs()
1149 		if (insn[i].src_reg != BPF_PSEUDO_CALL)  in check_subprogs()
1155 		ret = add_subprog(env, i + insn[i].imm + 1);  in check_subprogs()
1173 		u8 code = insn[i].code;  in check_subprogs()
1179 		off = i + insn[i].off + 1;  in check_subprogs()
1259 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,  in is_reg64()  argument
1264 	code = insn->code;  in is_reg64()
1279 			if (insn->src_reg == BPF_PSEUDO_CALL)  in is_reg64()
1293 	    (class == BPF_ALU && op == BPF_END && insn->imm == 64))  in is_reg64()
1340 static bool insn_no_def(struct bpf_insn *insn)  in insn_no_def()  argument
1342 	u8 class = BPF_CLASS(insn->code);  in insn_no_def()
1349 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)  in insn_has_def32()  argument
1351 	if (insn_no_def(insn))  in insn_has_def32()
1354 	return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP);  in insn_has_def32()
1375 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;  in check_reg_arg()  local
1385 	rw64 = is_reg64(env, insn, regno, reg, t);  in check_reg_arg()
1461 	struct bpf_insn *insn = env->prog->insnsi + idx;  in backtrack_insn()  local
1462 	u8 class = BPF_CLASS(insn->code);  in backtrack_insn()
1463 	u8 opcode = BPF_OP(insn->code);  in backtrack_insn()
1464 	u8 mode = BPF_MODE(insn->code);  in backtrack_insn()
1465 	u32 dreg = 1u << insn->dst_reg;  in backtrack_insn()
1466 	u32 sreg = 1u << insn->src_reg;  in backtrack_insn()
1469 	if (insn->code == 0)  in backtrack_insn()
1474 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in backtrack_insn()
1481 			if (BPF_SRC(insn->code) == BPF_X) {  in backtrack_insn()
1498 			if (BPF_SRC(insn->code) == BPF_X) {  in backtrack_insn()
1519 		if (insn->src_reg != BPF_REG_FP)  in backtrack_insn()
1521 		if (BPF_SIZE(insn->code) != BPF_DW)  in backtrack_insn()
1528 		spi = (-insn->off - 1) / BPF_REG_SIZE;  in backtrack_insn()
1543 		if (insn->dst_reg != BPF_REG_FP)  in backtrack_insn()
1545 		if (BPF_SIZE(insn->code) != BPF_DW)  in backtrack_insn()
1547 		spi = (-insn->off - 1) / BPF_REG_SIZE;  in backtrack_insn()
1560 			if (insn->src_reg == BPF_PSEUDO_CALL)  in backtrack_insn()
2612 	struct bpf_insn *insn = env->prog->insnsi;  in check_max_stack_depth()  local
2629 		if (insn[i].code != (BPF_JMP | BPF_CALL))  in check_max_stack_depth()
2631 		if (insn[i].src_reg != BPF_PSEUDO_CALL)  in check_max_stack_depth()
2638 		i = i + insn[i].imm + 1;  in check_max_stack_depth()
2667 				  const struct bpf_insn *insn, int idx)  in get_callee_stack_depth()  argument
2669 	int start = idx + insn->imm + 1, subprog;  in get_callee_stack_depth()
2899 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)  in check_xadd()  argument
2903 	if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||  in check_xadd()
2904 	    insn->imm != 0) {  in check_xadd()
2910 	err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_xadd()
2915 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_xadd()
2919 	if (is_pointer_value(env, insn->src_reg)) {  in check_xadd()
2920 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);  in check_xadd()
2924 	if (is_ctx_reg(env, insn->dst_reg) ||  in check_xadd()
2925 	    is_pkt_reg(env, insn->dst_reg) ||  in check_xadd()
2926 	    is_flow_key_reg(env, insn->dst_reg) ||  in check_xadd()
2927 	    is_sk_reg(env, insn->dst_reg)) {  in check_xadd()
2929 			insn->dst_reg,  in check_xadd()
2930 			reg_type_str[reg_state(env, insn->dst_reg)->type]);  in check_xadd()
2935 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_xadd()
2936 			       BPF_SIZE(insn->code), BPF_READ, -1, true);  in check_xadd()
2941 	return check_mem_access(env, insn_idx, insn->dst_reg, insn->off,  in check_xadd()
2942 				BPF_SIZE(insn->code), BPF_WRITE, -1, true);  in check_xadd()
3762 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,  in check_func_call()  argument
3775 	target_insn = *insn_idx + insn->imm;  in check_func_call()
4247 				    const struct bpf_insn *insn)  in can_skip_alu_sanitation()  argument
4249 	return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;  in can_skip_alu_sanitation()
4270 			    struct bpf_insn *insn)  in sanitize_val_alu()  argument
4274 	if (can_skip_alu_sanitation(env, insn))  in sanitize_val_alu()
4281 			    struct bpf_insn *insn,  in sanitize_ptr_alu()  argument
4289 	u8 opcode = BPF_OP(insn->code);  in sanitize_ptr_alu()
4294 	if (can_skip_alu_sanitation(env, insn))  in sanitize_ptr_alu()
4338 				   struct bpf_insn *insn,  in adjust_ptr_min_max_vals()  argument
4350 	u32 dst = insn->dst_reg, src = insn->src_reg;  in adjust_ptr_min_max_vals()
4351 	u8 opcode = BPF_OP(insn->code);  in adjust_ptr_min_max_vals()
4365 	if (BPF_CLASS(insn->code) != BPF_ALU64) {  in adjust_ptr_min_max_vals()
4413 		ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);  in adjust_ptr_min_max_vals()
4468 		ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);  in adjust_ptr_min_max_vals()
4579 				      struct bpf_insn *insn,  in adjust_scalar_min_max_vals()  argument
4584 	u8 opcode = BPF_OP(insn->code);  in adjust_scalar_min_max_vals()
4588 	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;  in adjust_scalar_min_max_vals()
4589 	u32 dst = insn->dst_reg;  in adjust_scalar_min_max_vals()
4625 		ret = sanitize_val_alu(env, insn);  in adjust_scalar_min_max_vals()
4649 		ret = sanitize_val_alu(env, insn);  in adjust_scalar_min_max_vals()
4765 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
4790 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
4820 			mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
4846 		mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_scalar_min_max_vals()
4850 	if (BPF_CLASS(insn->code) != BPF_ALU64) {  in adjust_scalar_min_max_vals()
4864 				   struct bpf_insn *insn)  in adjust_reg_min_max_vals()  argument
4870 	u8 opcode = BPF_OP(insn->code);  in adjust_reg_min_max_vals()
4873 	dst_reg = &regs[insn->dst_reg];  in adjust_reg_min_max_vals()
4877 	if (BPF_SRC(insn->code) == BPF_X) {  in adjust_reg_min_max_vals()
4878 		src_reg = &regs[insn->src_reg];  in adjust_reg_min_max_vals()
4886 					mark_reg_unknown(env, regs, insn->dst_reg);  in adjust_reg_min_max_vals()
4890 					insn->dst_reg,  in adjust_reg_min_max_vals()
4898 				err = mark_chain_precision(env, insn->dst_reg);  in adjust_reg_min_max_vals()
4901 				return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
4906 			err = mark_chain_precision(env, insn->src_reg);  in adjust_reg_min_max_vals()
4909 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
4917 		__mark_reg_known(&off_reg, insn->imm);  in adjust_reg_min_max_vals()
4920 			return adjust_ptr_min_max_vals(env, insn,  in adjust_reg_min_max_vals()
4935 	return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);  in adjust_reg_min_max_vals()
4939 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_alu_op()  argument
4942 	u8 opcode = BPF_OP(insn->code);  in check_alu_op()
4947 			if (BPF_SRC(insn->code) != 0 ||  in check_alu_op()
4948 			    insn->src_reg != BPF_REG_0 ||  in check_alu_op()
4949 			    insn->off != 0 || insn->imm != 0) {  in check_alu_op()
4954 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||  in check_alu_op()
4955 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||  in check_alu_op()
4956 			    BPF_CLASS(insn->code) == BPF_ALU64) {  in check_alu_op()
4963 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
4967 		if (is_pointer_value(env, insn->dst_reg)) {  in check_alu_op()
4969 				insn->dst_reg);  in check_alu_op()
4974 		err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_alu_op()
4980 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
4981 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
4987 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
4991 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
4998 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
5002 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
5003 			struct bpf_reg_state *src_reg = regs + insn->src_reg;  in check_alu_op()
5004 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;  in check_alu_op()
5006 			if (BPF_CLASS(insn->code) == BPF_ALU64) {  in check_alu_op()
5015 				if (is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
5018 						insn->src_reg);  in check_alu_op()
5026 							 insn->dst_reg);  in check_alu_op()
5035 			mark_reg_unknown(env, regs, insn->dst_reg);  in check_alu_op()
5036 			regs[insn->dst_reg].type = SCALAR_VALUE;  in check_alu_op()
5037 			if (BPF_CLASS(insn->code) == BPF_ALU64) {  in check_alu_op()
5038 				__mark_reg_known(regs + insn->dst_reg,  in check_alu_op()
5039 						 insn->imm);  in check_alu_op()
5041 				__mark_reg_known(regs + insn->dst_reg,  in check_alu_op()
5042 						 (u32)insn->imm);  in check_alu_op()
5052 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
5053 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
5058 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_alu_op()
5062 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
5069 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_alu_op()
5074 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {  in check_alu_op()
5080 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {  in check_alu_op()
5081 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;  in check_alu_op()
5083 			if (insn->imm < 0 || insn->imm >= size) {  in check_alu_op()
5084 				verbose(env, "invalid shift %d\n", insn->imm);  in check_alu_op()
5090 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in check_alu_op()
5094 		return adjust_reg_min_max_vals(env, insn);  in check_alu_op()
5719 static bool try_match_pkt_pointers(const struct bpf_insn *insn,  in try_match_pkt_pointers()  argument
5725 	if (BPF_SRC(insn->code) != BPF_X)  in try_match_pkt_pointers()
5729 	if (BPF_CLASS(insn->code) == BPF_JMP32)  in try_match_pkt_pointers()
5732 	switch (BPF_OP(insn->code)) {  in try_match_pkt_pointers()
5817 			     struct bpf_insn *insn, int *insn_idx)  in check_cond_jmp_op()  argument
5823 	u8 opcode = BPF_OP(insn->code);  in check_cond_jmp_op()
5834 	if (BPF_SRC(insn->code) == BPF_X) {  in check_cond_jmp_op()
5835 		if (insn->imm != 0) {  in check_cond_jmp_op()
5841 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_cond_jmp_op()
5845 		if (is_pointer_value(env, insn->src_reg)) {  in check_cond_jmp_op()
5847 				insn->src_reg);  in check_cond_jmp_op()
5850 		src_reg = &regs[insn->src_reg];  in check_cond_jmp_op()
5852 		if (insn->src_reg != BPF_REG_0) {  in check_cond_jmp_op()
5859 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in check_cond_jmp_op()
5863 	dst_reg = &regs[insn->dst_reg];  in check_cond_jmp_op()
5864 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;  in check_cond_jmp_op()
5866 	if (BPF_SRC(insn->code) == BPF_K)  in check_cond_jmp_op()
5867 		pred = is_branch_taken(dst_reg, insn->imm,  in check_cond_jmp_op()
5874 		err = mark_chain_precision(env, insn->dst_reg);  in check_cond_jmp_op()
5875 		if (BPF_SRC(insn->code) == BPF_X && !err)  in check_cond_jmp_op()
5876 			err = mark_chain_precision(env, insn->src_reg);  in check_cond_jmp_op()
5882 		*insn_idx += insn->off;  in check_cond_jmp_op()
5891 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,  in check_cond_jmp_op()
5904 	if (BPF_SRC(insn->code) == BPF_X) {  in check_cond_jmp_op()
5905 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];  in check_cond_jmp_op()
5919 				reg_set_min_max(&other_branch_regs[insn->dst_reg],  in check_cond_jmp_op()
5927 				reg_set_min_max_inv(&other_branch_regs[insn->src_reg],  in check_cond_jmp_op()
5936 				reg_combine_min_max(&other_branch_regs[insn->src_reg],  in check_cond_jmp_op()
5937 						    &other_branch_regs[insn->dst_reg],  in check_cond_jmp_op()
5941 		reg_set_min_max(&other_branch_regs[insn->dst_reg],  in check_cond_jmp_op()
5942 					dst_reg, insn->imm, opcode, is_jmp32);  in check_cond_jmp_op()
5949 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
5950 	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&  in check_cond_jmp_op()
5955 		mark_ptr_or_null_regs(this_branch, insn->dst_reg,  in check_cond_jmp_op()
5957 		mark_ptr_or_null_regs(other_branch, insn->dst_reg,  in check_cond_jmp_op()
5959 	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],  in check_cond_jmp_op()
5961 		   is_pointer_value(env, insn->dst_reg)) {  in check_cond_jmp_op()
5963 			insn->dst_reg);  in check_cond_jmp_op()
5972 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()  argument
5979 	if (BPF_SIZE(insn->code) != BPF_DW) {  in check_ld_imm()
5983 	if (insn->off != 0) {  in check_ld_imm()
5988 	err = check_reg_arg(env, insn->dst_reg, DST_OP);  in check_ld_imm()
5992 	if (insn->src_reg == 0) {  in check_ld_imm()
5993 		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;  in check_ld_imm()
5995 		regs[insn->dst_reg].type = SCALAR_VALUE;  in check_ld_imm()
5996 		__mark_reg_known(&regs[insn->dst_reg], imm);  in check_ld_imm()
6001 	mark_reg_known_zero(env, regs, insn->dst_reg);  in check_ld_imm()
6002 	regs[insn->dst_reg].map_ptr = map;  in check_ld_imm()
6004 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {  in check_ld_imm()
6005 		regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;  in check_ld_imm()
6006 		regs[insn->dst_reg].off = aux->map_off;  in check_ld_imm()
6008 			regs[insn->dst_reg].id = ++env->id_gen;  in check_ld_imm()
6009 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {  in check_ld_imm()
6010 		regs[insn->dst_reg].type = CONST_PTR_TO_MAP;  in check_ld_imm()
6046 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()  argument
6050 	u8 mode = BPF_MODE(insn->code);  in check_ld_abs()
6075 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||  in check_ld_abs()
6076 	    BPF_SIZE(insn->code) == BPF_DW ||  in check_ld_abs()
6077 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {  in check_ld_abs()
6110 		err = check_reg_arg(env, insn->src_reg, SRC_OP);  in check_ld_abs()
6822 static void clean_live_states(struct bpf_verifier_env *env, int insn,  in clean_live_states()  argument
6828 	sl = *explored_state(env, insn);  in clean_live_states()
6832 		if (sl->state.insn_idx != insn ||  in clean_live_states()
7528 		struct bpf_insn *insn;  in do_check()  local
7539 		insn = &insns[env->insn_idx];  in do_check()
7540 		class = BPF_CLASS(insn->code);  in do_check()
7593 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);  in do_check()
7608 			err = check_alu_op(env, insn);  in do_check()
7618 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
7622 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);  in do_check()
7626 			src_reg_type = regs[insn->src_reg].type;  in do_check()
7631 			err = check_mem_access(env, env->insn_idx, insn->src_reg,  in do_check()
7632 					       insn->off, BPF_SIZE(insn->code),  in do_check()
7633 					       BPF_READ, insn->dst_reg, false);  in do_check()
7661 			if (BPF_MODE(insn->code) == BPF_XADD) {  in do_check()
7662 				err = check_xadd(env, env->insn_idx, insn);  in do_check()
7670 			err = check_reg_arg(env, insn->src_reg, SRC_OP);  in do_check()
7674 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
7678 			dst_reg_type = regs[insn->dst_reg].type;  in do_check()
7681 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
7682 					       insn->off, BPF_SIZE(insn->code),  in do_check()
7683 					       BPF_WRITE, insn->src_reg, false);  in do_check()
7697 			if (BPF_MODE(insn->code) != BPF_MEM ||  in do_check()
7698 			    insn->src_reg != BPF_REG_0) {  in do_check()
7703 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);  in do_check()
7707 			if (is_ctx_reg(env, insn->dst_reg)) {  in do_check()
7709 					insn->dst_reg,  in do_check()
7710 					reg_type_str[reg_state(env, insn->dst_reg)->type]);  in do_check()
7715 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
7716 					       insn->off, BPF_SIZE(insn->code),  in do_check()
7722 			u8 opcode = BPF_OP(insn->code);  in do_check()
7726 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
7727 				    insn->off != 0 ||  in do_check()
7728 				    (insn->src_reg != BPF_REG_0 &&  in do_check()
7729 				     insn->src_reg != BPF_PSEUDO_CALL) ||  in do_check()
7730 				    insn->dst_reg != BPF_REG_0 ||  in do_check()
7737 				    (insn->src_reg == BPF_PSEUDO_CALL ||  in do_check()
7738 				     insn->imm != BPF_FUNC_spin_unlock)) {  in do_check()
7742 				if (insn->src_reg == BPF_PSEUDO_CALL)  in do_check()
7743 					err = check_func_call(env, insn, &env->insn_idx);  in do_check()
7745 					err = check_helper_call(env, insn->imm, env->insn_idx);  in do_check()
7750 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
7751 				    insn->imm != 0 ||  in do_check()
7752 				    insn->src_reg != BPF_REG_0 ||  in do_check()
7753 				    insn->dst_reg != BPF_REG_0 ||  in do_check()
7759 				env->insn_idx += insn->off + 1;  in do_check()
7763 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
7764 				    insn->imm != 0 ||  in do_check()
7765 				    insn->src_reg != BPF_REG_0 ||  in do_check()
7766 				    insn->dst_reg != BPF_REG_0 ||  in do_check()
7821 				err = check_cond_jmp_op(env, insn, &env->insn_idx);  in do_check()
7826 			u8 mode = BPF_MODE(insn->code);  in do_check()
7829 				err = check_ld_abs(env, insn);  in do_check()
7834 				err = check_ld_imm(env, insn);  in do_check()
7926 	struct bpf_insn *insn = env->prog->insnsi;  in replace_map_fd_with_map_ptr()  local
7934 	for (i = 0; i < insn_cnt; i++, insn++) {  in replace_map_fd_with_map_ptr()
7935 		if (BPF_CLASS(insn->code) == BPF_LDX &&  in replace_map_fd_with_map_ptr()
7936 		    (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {  in replace_map_fd_with_map_ptr()
7941 		if (BPF_CLASS(insn->code) == BPF_STX &&  in replace_map_fd_with_map_ptr()
7942 		    ((BPF_MODE(insn->code) != BPF_MEM &&  in replace_map_fd_with_map_ptr()
7943 		      BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {  in replace_map_fd_with_map_ptr()
7948 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {  in replace_map_fd_with_map_ptr()
7954 			if (i == insn_cnt - 1 || insn[1].code != 0 ||  in replace_map_fd_with_map_ptr()
7955 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||  in replace_map_fd_with_map_ptr()
7956 			    insn[1].off != 0) {  in replace_map_fd_with_map_ptr()
7961 			if (insn[0].src_reg == 0)  in replace_map_fd_with_map_ptr()
7968 			if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD &&  in replace_map_fd_with_map_ptr()
7969 			     insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) ||  in replace_map_fd_with_map_ptr()
7970 			    (insn[0].src_reg == BPF_PSEUDO_MAP_FD &&  in replace_map_fd_with_map_ptr()
7971 			     insn[1].imm != 0)) {  in replace_map_fd_with_map_ptr()
7977 			f = fdget(insn[0].imm);  in replace_map_fd_with_map_ptr()
7981 					insn[0].imm);  in replace_map_fd_with_map_ptr()
7992 			if (insn->src_reg == BPF_PSEUDO_MAP_FD) {  in replace_map_fd_with_map_ptr()
7995 				u32 off = insn[1].imm;  in replace_map_fd_with_map_ptr()
8021 			insn[0].imm = (u32)addr;  in replace_map_fd_with_map_ptr()
8022 			insn[1].imm = addr >> 32;  in replace_map_fd_with_map_ptr()
8061 			insn++;  in replace_map_fd_with_map_ptr()
8067 		if (!bpf_opcode_in_insntable(insn->code)) {  in replace_map_fd_with_map_ptr()
8068 			verbose(env, "unknown opcode %02x\n", insn->code);  in replace_map_fd_with_map_ptr()
8100 	struct bpf_insn *insn = env->prog->insnsi;  in convert_pseudo_ld_imm64()  local
8104 	for (i = 0; i < insn_cnt; i++, insn++)  in convert_pseudo_ld_imm64()
8105 		if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))  in convert_pseudo_ld_imm64()
8106 			insn->src_reg = 0;  in convert_pseudo_ld_imm64()
8117 	struct bpf_insn *insn = new_prog->insnsi;  in adjust_insn_aux_data()  local
8125 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);  in adjust_insn_aux_data()
8139 		new_data[i].zext_dst = insn_has_def32(env, insn + i);  in adjust_insn_aux_data()
8341 	struct bpf_insn *insn = env->prog->insnsi;  in sanitize_dead_code()  local
8348 		memcpy(insn + i, &trap, sizeof(trap));  in sanitize_dead_code()
8370 	struct bpf_insn *insn = env->prog->insnsi;  in opt_hard_wire_dead_code_branches()  local
8374 	for (i = 0; i < insn_cnt; i++, insn++) {  in opt_hard_wire_dead_code_branches()
8375 		if (!insn_is_cond_jump(insn->code))  in opt_hard_wire_dead_code_branches()
8379 			ja.off = insn->off;  in opt_hard_wire_dead_code_branches()
8380 		else if (!aux_data[i + 1 + insn->off].seen)  in opt_hard_wire_dead_code_branches()
8388 		memcpy(insn, &ja, sizeof(ja));  in opt_hard_wire_dead_code_branches()
8419 	struct bpf_insn *insn = env->prog->insnsi;  in opt_remove_nops()  local
8424 		if (memcmp(&insn[i], &ja, sizeof(ja)))  in opt_remove_nops()
8454 		struct bpf_insn insn;  in opt_subreg_zext_lo32_rnd_hi32()  local
8456 		insn = insns[adj_idx];  in opt_subreg_zext_lo32_rnd_hi32()
8464 			code = insn.code;  in opt_subreg_zext_lo32_rnd_hi32()
8466 			if (insn_no_def(&insn))  in opt_subreg_zext_lo32_rnd_hi32()
8473 			if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) {  in opt_subreg_zext_lo32_rnd_hi32()
8486 			rnd_hi32_patch[0] = insn;  in opt_subreg_zext_lo32_rnd_hi32()
8488 			rnd_hi32_patch[3].dst_reg = insn.dst_reg;  in opt_subreg_zext_lo32_rnd_hi32()
8497 		zext_patch[0] = insn;  in opt_subreg_zext_lo32_rnd_hi32()
8498 		zext_patch[1].dst_reg = insn.dst_reg;  in opt_subreg_zext_lo32_rnd_hi32()
8499 		zext_patch[1].src_reg = insn.dst_reg;  in opt_subreg_zext_lo32_rnd_hi32()
8525 	struct bpf_insn insn_buf[16], *insn;  in convert_ctx_accesses()  local
8554 	insn = env->prog->insnsi + delta;  in convert_ctx_accesses()
8556 	for (i = 0; i < insn_cnt; i++, insn++) {  in convert_ctx_accesses()
8559 		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||  in convert_ctx_accesses()
8560 		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||  in convert_ctx_accesses()
8561 		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||  in convert_ctx_accesses()
8562 		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW))  in convert_ctx_accesses()
8564 		else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||  in convert_ctx_accesses()
8565 			 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||  in convert_ctx_accesses()
8566 			 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||  in convert_ctx_accesses()
8567 			 insn->code == (BPF_STX | BPF_MEM | BPF_DW))  in convert_ctx_accesses()
8586 				*insn,  in convert_ctx_accesses()
8596 			insn      = new_prog->insnsi + i + delta;  in convert_ctx_accesses()
8621 		size = BPF_LDST_BYTES(insn);  in convert_ctx_accesses()
8630 		off = insn->off;  in convert_ctx_accesses()
8645 			insn->off = off & ~(size_default - 1);  in convert_ctx_accesses()
8646 			insn->code = BPF_LDX | BPF_MEM | size_code;  in convert_ctx_accesses()
8650 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,  in convert_ctx_accesses()
8664 									insn->dst_reg,  in convert_ctx_accesses()
8666 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,  in convert_ctx_accesses()
8671 									insn->dst_reg,  in convert_ctx_accesses()
8673 				insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,  in convert_ctx_accesses()
8686 		insn      = new_prog->insnsi + i + delta;  in convert_ctx_accesses()
8696 	struct bpf_insn *insn;  in jit_subprogs()  local
8703 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
8704 		if (insn->code != (BPF_JMP | BPF_CALL) ||  in jit_subprogs()
8705 		    insn->src_reg != BPF_PSEUDO_CALL)  in jit_subprogs()
8711 		subprog = find_subprog(env, i + insn->imm + 1);  in jit_subprogs()
8714 				  i + insn->imm + 1);  in jit_subprogs()
8720 		insn->off = subprog;  in jit_subprogs()
8724 		env->insn_aux_data[i].call_imm = insn->imm;  in jit_subprogs()
8726 		insn->imm = 1;  in jit_subprogs()
8785 		insn = func[i]->insnsi;  in jit_subprogs()
8786 		for (j = 0; j < func[i]->len; j++, insn++) {  in jit_subprogs()
8787 			if (insn->code != (BPF_JMP | BPF_CALL) ||  in jit_subprogs()
8788 			    insn->src_reg != BPF_PSEUDO_CALL)  in jit_subprogs()
8790 			subprog = insn->off;  in jit_subprogs()
8791 			insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -  in jit_subprogs()
8832 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
8833 		if (insn->code != (BPF_JMP | BPF_CALL) ||  in jit_subprogs()
8834 		    insn->src_reg != BPF_PSEUDO_CALL)  in jit_subprogs()
8836 		insn->off = env->insn_aux_data[i].call_imm;  in jit_subprogs()
8837 		subprog = find_subprog(env, i + insn->off + 1);  in jit_subprogs()
8838 		insn->imm = subprog;  in jit_subprogs()
8855 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {  in jit_subprogs()
8856 		if (insn->code != (BPF_JMP | BPF_CALL) ||  in jit_subprogs()
8857 		    insn->src_reg != BPF_PSEUDO_CALL)  in jit_subprogs()
8859 		insn->off = 0;  in jit_subprogs()
8860 		insn->imm = env->insn_aux_data[i].call_imm;  in jit_subprogs()
8870 	struct bpf_insn *insn = prog->insnsi;  in fixup_call_args()  local
8884 	for (i = 0; i < prog->len; i++, insn++) {  in fixup_call_args()
8885 		if (insn->code != (BPF_JMP | BPF_CALL) ||  in fixup_call_args()
8886 		    insn->src_reg != BPF_PSEUDO_CALL)  in fixup_call_args()
8888 		depth = get_callee_stack_depth(env, insn, i);  in fixup_call_args()
8891 		bpf_patch_call_args(insn, depth);  in fixup_call_args()
8906 	struct bpf_insn *insn = prog->insnsi;  in fixup_bpf_calls()  local
8916 	for (i = 0; i < insn_cnt; i++, insn++) {  in fixup_bpf_calls()
8917 		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||  in fixup_bpf_calls()
8918 		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||  in fixup_bpf_calls()
8919 		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||  in fixup_bpf_calls()
8920 		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {  in fixup_bpf_calls()
8921 			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;  in fixup_bpf_calls()
8923 				BPF_MOV32_REG(insn->src_reg, insn->src_reg),  in fixup_bpf_calls()
8925 				BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2),  in fixup_bpf_calls()
8926 				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),  in fixup_bpf_calls()
8928 				*insn,  in fixup_bpf_calls()
8931 				BPF_MOV32_REG(insn->src_reg, insn->src_reg),  in fixup_bpf_calls()
8933 				BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1),  in fixup_bpf_calls()
8934 				*insn,  in fixup_bpf_calls()
8938 			if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||  in fixup_bpf_calls()
8939 			    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {  in fixup_bpf_calls()
8953 			insn      = new_prog->insnsi + i + delta;  in fixup_bpf_calls()
8957 		if (BPF_CLASS(insn->code) == BPF_LD &&  in fixup_bpf_calls()
8958 		    (BPF_MODE(insn->code) == BPF_ABS ||  in fixup_bpf_calls()
8959 		     BPF_MODE(insn->code) == BPF_IND)) {  in fixup_bpf_calls()
8960 			cnt = env->ops->gen_ld_abs(insn, insn_buf);  in fixup_bpf_calls()
8972 			insn      = new_prog->insnsi + i + delta;  in fixup_bpf_calls()
8976 		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||  in fixup_bpf_calls()
8977 		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {  in fixup_bpf_calls()
8994 			off_reg = issrc ? insn->src_reg : insn->dst_reg;  in fixup_bpf_calls()
9005 				insn->src_reg = BPF_REG_AX;  in fixup_bpf_calls()
9011 				insn->code = insn->code == code_add ?  in fixup_bpf_calls()
9013 			*patch++ = *insn;  in fixup_bpf_calls()
9024 			insn      = new_prog->insnsi + i + delta;  in fixup_bpf_calls()
9028 		if (insn->code != (BPF_JMP | BPF_CALL))  in fixup_bpf_calls()
9030 		if (insn->src_reg == BPF_PSEUDO_CALL)  in fixup_bpf_calls()
9033 		if (insn->imm == BPF_FUNC_get_route_realm)  in fixup_bpf_calls()
9035 		if (insn->imm == BPF_FUNC_get_prandom_u32)  in fixup_bpf_calls()
9037 		if (insn->imm == BPF_FUNC_override_return)  in fixup_bpf_calls()
9039 		if (insn->imm == BPF_FUNC_tail_call) {  in fixup_bpf_calls()
9054 			insn->imm = 0;  in fixup_bpf_calls()
9055 			insn->code = BPF_JMP | BPF_TAIL_CALL;  in fixup_bpf_calls()
9079 			insn_buf[2] = *insn;  in fixup_bpf_calls()
9087 			insn      = new_prog->insnsi + i + delta;  in fixup_bpf_calls()
9096 		    (insn->imm == BPF_FUNC_map_lookup_elem ||  in fixup_bpf_calls()
9097 		     insn->imm == BPF_FUNC_map_update_elem ||  in fixup_bpf_calls()
9098 		     insn->imm == BPF_FUNC_map_delete_elem ||  in fixup_bpf_calls()
9099 		     insn->imm == BPF_FUNC_map_push_elem   ||  in fixup_bpf_calls()
9100 		     insn->imm == BPF_FUNC_map_pop_elem    ||  in fixup_bpf_calls()
9101 		     insn->imm == BPF_FUNC_map_peek_elem)) {  in fixup_bpf_calls()
9108 			if (insn->imm == BPF_FUNC_map_lookup_elem &&  in fixup_bpf_calls()
9123 				insn      = new_prog->insnsi + i + delta;  in fixup_bpf_calls()
9142 			switch (insn->imm) {  in fixup_bpf_calls()
9144 				insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -  in fixup_bpf_calls()
9148 				insn->imm = BPF_CAST_CALL(ops->map_update_elem) -  in fixup_bpf_calls()
9152 				insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -  in fixup_bpf_calls()
9156 				insn->imm = BPF_CAST_CALL(ops->map_push_elem) -  in fixup_bpf_calls()
9160 				insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -  in fixup_bpf_calls()
9164 				insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -  in fixup_bpf_calls()
9173 		fn = env->ops->get_func_proto(insn->imm, env->prog);  in fixup_bpf_calls()
9180 				func_id_name(insn->imm), insn->imm);  in fixup_bpf_calls()
9183 		insn->imm = fn->func - __bpf_call_base;  in fixup_bpf_calls()