37 	const struct bpf_prog *prog;  in bpf_prog_run_array_cg()  local
49 	while ((prog = READ_ONCE(item->prog))) {  in bpf_prog_run_array_cg()
51 		func_ret = run_prog(prog, ctx);  in bpf_prog_run_array_cg()
209 				     struct bpf_prog *prog,  in bpf_cgroup_storages_alloc()  argument
220 		map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storages_alloc()
228 		storages[stype] = bpf_cgroup_storage_alloc(prog, stype);  in bpf_cgroup_storages_alloc()
294 			if (pl->prog) {  in cgroup_bpf_release()
295 				if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
296 					bpf_trampoline_unlink_cgroup_shim(pl->prog);  in cgroup_bpf_release()
297 				bpf_prog_put(pl->prog);  in cgroup_bpf_release()
300 				if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
301 					bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);  in cgroup_bpf_release()
345 	if (pl->prog)  in prog_list_prog()
346 		return pl->prog;  in prog_list_prog()
348 		return pl->link->link.prog;  in prog_list_prog()
434 			item->prog = prog_list_prog(pl);  in compute_effective_progs()
557 					       struct bpf_prog *prog,  in find_attach_entry()  argument
572 		if (prog && pl->prog == prog && prog != replace_prog)  in find_attach_entry()
583 			if (pl->prog == replace_prog)  in find_attach_entry()
608 			       struct bpf_prog *prog, struct bpf_prog *replace_prog,  in __cgroup_bpf_attach()  argument
616 	struct bpf_prog *new_prog = prog ? : link->link.prog;  in __cgroup_bpf_attach()
626 	if (link && (prog || replace_prog))  in __cgroup_bpf_attach()
652 	pl = find_attach_entry(progs, prog, link, replace_prog,  in __cgroup_bpf_attach()
658 				      prog ? : link->link.prog, cgrp))  in __cgroup_bpf_attach()
662 		old_prog = pl->prog;  in __cgroup_bpf_attach()
682 	pl->prog = prog;  in __cgroup_bpf_attach()
713 		pl->prog = old_prog;  in __cgroup_bpf_attach()
725 			     struct bpf_prog *prog, struct bpf_prog *replace_prog,  in cgroup_bpf_attach()  argument
733 	ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags);  in cgroup_bpf_attach()
779 		WRITE_ONCE(item->prog, link->link.prog);  in replace_effective_prog()
808 	if (link->link.prog->type != new_prog->type)  in __cgroup_bpf_replace()
820 	old_prog = xchg(&link->link.prog, new_prog);  in __cgroup_bpf_replace()
840 	if (old_prog && link->prog != old_prog) {  in cgroup_bpf_replace()
851 					       struct bpf_prog *prog,  in find_detach_entry()  argument
868 	if (!prog && !link)  in find_detach_entry()
876 		if (pl->prog == prog && pl->link == link)  in find_detach_entry()
892 static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,  in purge_effective_progs()  argument
919 				if (pl->prog == prog && pl->link == link)  in purge_effective_progs()
949 static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_detach()  argument
959 	if (prog)  in __cgroup_bpf_detach()
960 		attach_btf_id = prog->aux->attach_btf_id;  in __cgroup_bpf_detach()
962 		attach_btf_id = link->link.prog->aux->attach_btf_id;  in __cgroup_bpf_detach()
971 	if (prog && link)  in __cgroup_bpf_detach()
975 	pl = find_detach_entry(progs, prog, link, flags & BPF_F_ALLOW_MULTI);  in __cgroup_bpf_detach()
980 	old_prog = pl->prog;  in __cgroup_bpf_detach()
981 	pl->prog = NULL;  in __cgroup_bpf_detach()
986 		pl->prog = old_prog;  in __cgroup_bpf_detach()
1007 static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in cgroup_bpf_detach()  argument
1013 	ret = __cgroup_bpf_detach(cgrp, prog, NULL, type);  in cgroup_bpf_detach()
1086 			struct bpf_prog *prog;  in __cgroup_bpf_query()  local
1093 				prog = prog_list_prog(pl);  in __cgroup_bpf_query()
1094 				id = prog->aux->id;  in __cgroup_bpf_query()
1130 			   enum bpf_prog_type ptype, struct bpf_prog *prog)  in cgroup_bpf_prog_attach()  argument
1149 	ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,  in cgroup_bpf_prog_attach()
1160 	struct bpf_prog *prog;  in cgroup_bpf_prog_detach()  local
1168 	prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);  in cgroup_bpf_prog_detach()
1169 	if (IS_ERR(prog))  in cgroup_bpf_prog_detach()
1170 		prog = NULL;  in cgroup_bpf_prog_detach()
1172 	ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type);  in cgroup_bpf_prog_detach()
1173 	if (prog)  in cgroup_bpf_prog_detach()
1174 		bpf_prog_put(prog);  in cgroup_bpf_prog_detach()
1203 		bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);  in bpf_cgroup_link_release()
1273 int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)  in cgroup_bpf_link_attach()  argument
1293 		      prog);  in cgroup_bpf_link_attach()
1602 cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cgroup_dev_func_proto()  argument
1606 	func_proto = cgroup_common_func_proto(func_id, prog);  in cgroup_dev_func_proto()
1610 	func_proto = cgroup_current_func_proto(func_id, prog);  in cgroup_dev_func_proto()
1624 				       const struct bpf_prog *prog,  in cgroup_dev_is_valid_access()  argument
2150 sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in sysctl_func_proto()  argument
2154 	func_proto = cgroup_common_func_proto(func_id, prog);  in sysctl_func_proto()
2158 	func_proto = cgroup_current_func_proto(func_id, prog);  in sysctl_func_proto()
2181 				   const struct bpf_prog *prog,  in sysctl_is_valid_access()  argument
2210 				     struct bpf_prog *prog, u32 *target_size)  in sysctl_convert_ctx_access()  argument
2295 cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cg_sockopt_func_proto()  argument
2299 	func_proto = cgroup_common_func_proto(func_id, prog);  in cg_sockopt_func_proto()
2303 	func_proto = cgroup_current_func_proto(func_id, prog);  in cg_sockopt_func_proto()
2316 		if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)  in cg_sockopt_func_proto()
2320 		if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)  in cg_sockopt_func_proto()
2337 				       const struct bpf_prog *prog,  in cg_sockopt_is_valid_access()  argument
2353 			return prog->expected_attach_type ==  in cg_sockopt_is_valid_access()
2360 			return prog->expected_attach_type ==  in cg_sockopt_is_valid_access()
2388 		return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;  in cg_sockopt_is_valid_access()
2405 					 struct bpf_prog *prog,  in cg_sockopt_convert_ctx_access()  argument
2480 				   const struct bpf_prog *prog)  in cg_sockopt_get_prologue()  argument
2499 cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cgroup_common_func_proto()  argument
2505 		switch (prog->expected_attach_type) {  in cgroup_common_func_proto()
2520 		switch (prog->expected_attach_type) {  in cgroup_common_func_proto()
2541 cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cgroup_current_func_proto()  argument