1 /* Common BPF helpers to be used by all BPF programs loaded by Android */ 2 3 #include <linux/bpf.h> 4 #include <stdbool.h> 5 #include <stdint.h> 6 7 #include "bpf_map_def.h" 8 9 /****************************************************************************** 10 * WARNING: CHANGES TO THIS FILE OUTSIDE OF AOSP/MASTER ARE LIKELY TO BREAK * 11 * DEVICE COMPATIBILITY WITH MAINLINE MODULES SHIPPING EBPF CODE. * 12 * * 13 * THIS WILL LIKELY RESULT IN BRICKED DEVICES AT SOME ARBITRARY FUTURE TIME * 14 * * 15 * THAT GOES ESPECIALLY FOR THE 'SECTION' 'LICENSE' AND 'CRITICAL' MACROS * 16 * * 17 * We strongly suggest that if you need changes to bpfloader functionality * 18 * you get your changes reviewed and accepted into aosp/master. * 19 * * 20 ******************************************************************************/ 21 22 // The actual versions of the bpfloader that shipped in various Android releases 23 24 // Android P/Q/R: BpfLoader was initially part of netd, 25 // this was later split out into a standalone binary, but was unversioned. 26 27 // Android S / 12 (api level 31) - added 'tethering' mainline eBPF support 28 #define BPFLOADER_S_VERSION 2u 29 30 // Android T / 13 Beta 3 (api level 33) - added support for 'netd_shared' 31 #define BPFLOADER_T_BETA3_VERSION 13u 32 33 // v0.18 added support for shared and pindir, but still ignores selinux_content 34 // v0.19 added support for selinux_content along with the required selinux changes 35 // and should be available starting with Android T Beta 4 36 // 37 // Android T / 13 (api level 33) - support for shared/selinux_context/pindir 38 #define BPFLOADER_T_VERSION 19u 39 40 // BpfLoader v0.25+ support obj@ver.o files 41 #define BPFLOADER_OBJ_AT_VER_VERSION 25u 42 43 // Bpfloader v0.33+ supports {map,prog}.ignore_on_{eng,user,userdebug} 44 #define BPFLOADER_IGNORED_ON_VERSION 33u 45 46 /* For mainline module use, you can #define BPFLOADER_{MIN/MAX}_VER 47 * before #include "bpf_helpers.h" to change which bpfloaders will 48 * process the resulting .o file. 49 * 50 * While this will work outside of mainline too, there just is no point to 51 * using it when the .o and the bpfloader ship in sync with each other. 52 * In which case it's just best to use the default. 53 */ 54 #ifndef BPFLOADER_MIN_VER 55 #define BPFLOADER_MIN_VER COMPILE_FOR_BPFLOADER_VERSION 56 #endif 57 58 #ifndef BPFLOADER_MAX_VER 59 #define BPFLOADER_MAX_VER DEFAULT_BPFLOADER_MAX_VER 60 #endif 61 62 /* place things in different elf sections */ 63 #define SECTION(NAME) __attribute__((section(NAME), used)) 64 65 /* Must be present in every program, example usage: 66 * LICENSE("GPL"); or LICENSE("Apache 2.0"); 67 * 68 * We also take this opportunity to embed a bunch of other useful values in 69 * the resulting .o (This is to enable some limited forward compatibility 70 * with mainline module shipped ebpf programs) 71 * 72 * The bpfloader_{min/max}_ver defines the [min, max) range of bpfloader 73 * versions that should load this .o file (bpfloaders outside of this range 74 * will simply ignore/skip this *entire* .o) 75 * The [inclusive,exclusive) matches what we do for kernel ver dependencies. 76 * 77 * The size_of_bpf_{map,prog}_def allow the bpfloader to load programs where 78 * these structures have been extended with additional fields (they will of 79 * course simply be ignored then). 80 * 81 * If missing, bpfloader_{min/max}_ver default to 0/0x10000 ie. [v0.0, v1.0), 82 * while size_of_bpf_{map/prog}_def default to 32/20 which are the v0.0 sizes. 83 */ 84 #define LICENSE(NAME) \ 85 unsigned int _bpfloader_min_ver SECTION("bpfloader_min_ver") = BPFLOADER_MIN_VER; \ 86 unsigned int _bpfloader_max_ver SECTION("bpfloader_max_ver") = BPFLOADER_MAX_VER; \ 87 size_t _size_of_bpf_map_def SECTION("size_of_bpf_map_def") = sizeof(struct bpf_map_def); \ 88 size_t _size_of_bpf_prog_def SECTION("size_of_bpf_prog_def") = sizeof(struct bpf_prog_def); \ 89 char _license[] SECTION("license") = (NAME) 90 91 /* flag the resulting bpf .o file as critical to system functionality, 92 * loading all kernel version appropriate programs in it must succeed 93 * for bpfloader success 94 */ 95 #define CRITICAL(REASON) char _critical[] SECTION("critical") = (REASON) 96 97 /* 98 * Helper functions called from eBPF programs written in C. These are 99 * implemented in the kernel sources. 100 */ 101 102 #define KVER_NONE 0 103 #define KVER(a, b, c) (((a) << 24) + ((b) << 16) + (c)) 104 #define KVER_INF 0xFFFFFFFFu 105 106 /* 107 * BPFFS (ie. /sys/fs/bpf) labelling is as follows: 108 * subdirectory selinux context mainline usecase / usable by 109 * / fs_bpf no [*] core operating system (ie. platform) 110 * /loader fs_bpf_loader no, U+ (as yet unused) 111 * /net_private fs_bpf_net_private yes, T+ network_stack 112 * /net_shared fs_bpf_net_shared yes, T+ network_stack & system_server 113 * /netd_readonly fs_bpf_netd_readonly yes, T+ network_stack & system_server & r/o to netd 114 * /netd_shared fs_bpf_netd_shared yes, T+ network_stack & system_server & netd [**] 115 * /tethering fs_bpf_tethering yes, S+ network_stack 116 * /vendor fs_bpf_vendor no, T+ vendor 117 * 118 * [*] initial support for bpf was added back in P, 119 * but things worked differently back then with no bpfloader, 120 * and instead netd doing stuff by hand, 121 * bpfloader with pinning into /sys/fs/bpf was (I believe) added in Q 122 * (and was definitely there in R). 123 * 124 * [**] additionally bpf programs are accessible to netutils_wrapper 125 * for use by iptables xt_bpf extensions. 126 * 127 * See cs/p:aosp-master%20-file:prebuilts/%20file:genfs_contexts%20"genfscon%20bpf" 128 */ 129 130 /* generic functions */ 131 132 /* 133 * Type-unsafe bpf map functions - avoid if possible. 134 * 135 * Using these it is possible to pass in keys/values of the wrong type/size, 136 * or, for 'bpf_map_lookup_elem_unsafe' receive into a pointer to the wrong type. 137 * You will not get a compile time failure, and for certain types of errors you 138 * might not even get a failure from the kernel's ebpf verifier during program load, 139 * instead stuff might just not work right at runtime. 140 * 141 * Instead please use: 142 * DEFINE_BPF_MAP(foo_map, TYPE, KeyType, ValueType, num_entries) 143 * where TYPE can be something like HASH or ARRAY, and num_entries is an integer. 144 * 145 * This defines the map (hence this should not be used in a header file included 146 * from multiple locations) and provides type safe accessors: 147 * ValueType * bpf_foo_map_lookup_elem(const KeyType *) 148 * int bpf_foo_map_update_elem(const KeyType *, const ValueType *, flags) 149 * int bpf_foo_map_delete_elem(const KeyType *) 150 * 151 * This will make sure that if you change the type of a map you'll get compile 152 * errors at any spots you forget to update with the new type. 153 * 154 * Note: these all take pointers to const map because from the C/eBPF point of view 155 * the map struct is really just a readonly map definition of the in kernel object. 156 * Runtime modification of the map defining struct is meaningless, since 157 * the contents is only ever used during bpf program loading & map creation 158 * by the bpf loader, and not by the eBPF program itself. 159 */ 160 static void* (*bpf_map_lookup_elem_unsafe)(const struct bpf_map_def* map, 161 const void* key) = (void*)BPF_FUNC_map_lookup_elem; 162 static int (*bpf_map_update_elem_unsafe)(const struct bpf_map_def* map, const void* key, 163 const void* value, unsigned long long flags) = (void*) 164 BPF_FUNC_map_update_elem; 165 static int (*bpf_map_delete_elem_unsafe)(const struct bpf_map_def* map, 166 const void* key) = (void*)BPF_FUNC_map_delete_elem; 167 static int (*bpf_ringbuf_output_unsafe)(const struct bpf_map_def* ringbuf, 168 const void* data, __u64 size, __u64 flags) = (void*) 169 BPF_FUNC_ringbuf_output; 170 static void* (*bpf_ringbuf_reserve_unsafe)(const struct bpf_map_def* ringbuf, 171 __u64 size, __u64 flags) = (void*) 172 BPF_FUNC_ringbuf_reserve; 173 static void (*bpf_ringbuf_submit_unsafe)(const void* data, __u64 flags) = (void*) 174 BPF_FUNC_ringbuf_submit; 175 176 #define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ 177 struct ____btf_map_##name { \ 178 type_key key; \ 179 type_val value; \ 180 }; \ 181 struct ____btf_map_##name \ 182 __attribute__ ((section(".maps." #name), used)) \ 183 ____btf_map_##name = { } 184 185 #define BPF_ASSERT_LOADER_VERSION(min_loader, ignore_eng, ignore_user, ignore_userdebug) \ 186 _Static_assert( \ 187 (min_loader) >= BPFLOADER_IGNORED_ON_VERSION || \ 188 !((ignore_eng) || (ignore_user) || (ignore_userdebug)), \ 189 "bpfloader min version must be >= 0.33 in order to use ignored_on"); 190 191 #define DEFINE_BPF_MAP_BASE(the_map, TYPE, keysize, valuesize, num_entries, \ 192 usr, grp, md, selinux, pindir, share, minkver, \ 193 maxkver, minloader, maxloader, ignore_eng, \ 194 ignore_user, ignore_userdebug) \ 195 const struct bpf_map_def SECTION("maps") the_map = { \ 196 .type = BPF_MAP_TYPE_##TYPE, \ 197 .key_size = (keysize), \ 198 .value_size = (valuesize), \ 199 .max_entries = (num_entries), \ 200 .map_flags = 0, \ 201 .uid = (usr), \ 202 .gid = (grp), \ 203 .mode = (md), \ 204 .bpfloader_min_ver = (minloader), \ 205 .bpfloader_max_ver = (maxloader), \ 206 .min_kver = (minkver), \ 207 .max_kver = (maxkver), \ 208 .selinux_context = (selinux), \ 209 .pin_subdir = (pindir), \ 210 .shared = (share), \ 211 .ignore_on_eng = (ignore_eng), \ 212 .ignore_on_user = (ignore_user), \ 213 .ignore_on_userdebug = (ignore_userdebug), \ 214 }; \ 215 BPF_ASSERT_LOADER_VERSION(minloader, ignore_eng, ignore_user, ignore_userdebug); 216 217 // Type safe macro to declare a ring buffer and related output functions. 218 // Compatibility: 219 // * BPF ring buffers are only available kernels 5.8 and above. Any program 220 // accessing the ring buffer should set a program level min_kver >= 5.8. 221 // * The definition below sets a map min_kver of 5.8 which requires targeting 222 // a BPFLOADER_MIN_VER >= BPFLOADER_S_VERSION. 223 #define DEFINE_BPF_RINGBUF_EXT(the_map, ValueType, size_bytes, usr, grp, md, \ 224 selinux, pindir, share, min_loader, max_loader, \ 225 ignore_eng, ignore_user, ignore_userdebug) \ 226 DEFINE_BPF_MAP_BASE(the_map, RINGBUF, 0, 0, size_bytes, usr, grp, md, \ 227 selinux, pindir, share, KVER(5, 8, 0), KVER_INF, \ 228 min_loader, max_loader, ignore_eng, ignore_user, \ 229 ignore_userdebug); \ 230 static inline __always_inline __unused int bpf_##the_map##_output( \ 231 const ValueType* v) { \ 232 return bpf_ringbuf_output_unsafe(&the_map, v, sizeof(*v), 0); \ 233 } \ 234 static inline __always_inline __unused \ 235 ValueType* bpf_##the_map##_reserve() { \ 236 return bpf_ringbuf_reserve_unsafe(&the_map, sizeof(ValueType), 0); \ 237 } \ 238 static inline __always_inline __unused void bpf_##the_map##_submit( \ 239 const ValueType* v) { \ 240 bpf_ringbuf_submit_unsafe(v, 0); \ 241 } 242 243 /* There exist buggy kernels with pre-T OS, that due to 244 * kernel patch "[ALPS05162612] bpf: fix ubsan error" 245 * do not support userspace writes into non-zero index of bpf map arrays. 246 * 247 * We use this assert to prevent us from being able to define such a map. 248 */ 249 250 #ifdef THIS_BPF_PROGRAM_IS_FOR_TEST_PURPOSES_ONLY 251 #define BPF_MAP_ASSERT_OK(type, entries, mode) 252 #elif BPFLOADER_MIN_VER >= BPFLOADER_T_BETA3_VERSION 253 #define BPF_MAP_ASSERT_OK(type, entries, mode) 254 #else 255 #define BPF_MAP_ASSERT_OK(type, entries, mode) \ 256 _Static_assert(((type) != BPF_MAP_TYPE_ARRAY) || ((entries) <= 1) || !((mode) & 0222), \ 257 "Writable arrays with more than 1 element not supported on pre-T devices.") 258 #endif 259 260 /* type safe macro to declare a map and related accessor functions */ 261 #define DEFINE_BPF_MAP_EXT(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md, \ 262 selinux, pindir, share, min_loader, max_loader, ignore_eng, \ 263 ignore_user, ignore_userdebug) \ 264 DEFINE_BPF_MAP_BASE(the_map, TYPE, sizeof(KeyType), sizeof(ValueType), \ 265 num_entries, usr, grp, md, selinux, pindir, share, \ 266 KVER_NONE, KVER_INF, min_loader, max_loader, \ 267 ignore_eng, ignore_user, ignore_userdebug); \ 268 BPF_MAP_ASSERT_OK(BPF_MAP_TYPE_##TYPE, (num_entries), (md)); \ 269 BPF_ANNOTATE_KV_PAIR(the_map, KeyType, ValueType); \ 270 \ 271 static inline __always_inline __unused ValueType* bpf_##the_map##_lookup_elem( \ 272 const KeyType* k) { \ 273 return bpf_map_lookup_elem_unsafe(&the_map, k); \ 274 }; \ 275 \ 276 static inline __always_inline __unused int bpf_##the_map##_update_elem( \ 277 const KeyType* k, const ValueType* v, unsigned long long flags) { \ 278 return bpf_map_update_elem_unsafe(&the_map, k, v, flags); \ 279 }; \ 280 \ 281 static inline __always_inline __unused int bpf_##the_map##_delete_elem(const KeyType* k) { \ 282 return bpf_map_delete_elem_unsafe(&the_map, k); \ 283 }; 284 285 #ifndef DEFAULT_BPF_MAP_SELINUX_CONTEXT 286 #define DEFAULT_BPF_MAP_SELINUX_CONTEXT "" 287 #endif 288 289 #ifndef DEFAULT_BPF_MAP_PIN_SUBDIR 290 #define DEFAULT_BPF_MAP_PIN_SUBDIR "" 291 #endif 292 293 #ifndef DEFAULT_BPF_MAP_UID 294 #define DEFAULT_BPF_MAP_UID AID_ROOT 295 #elif BPFLOADER_MIN_VER < 28u 296 #error "Bpf Map UID must be left at default of AID_ROOT for BpfLoader prior to v0.28" 297 #endif 298 299 #define DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md) \ 300 DEFINE_BPF_MAP_EXT(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md, \ 301 DEFAULT_BPF_MAP_SELINUX_CONTEXT, DEFAULT_BPF_MAP_PIN_SUBDIR, false, \ 302 BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, /*ignore_on_eng*/false, \ 303 /*ignore_on_user*/false, /*ignore_on_userdebug*/false) 304 305 #define DEFINE_BPF_MAP(the_map, TYPE, KeyType, ValueType, num_entries) \ 306 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 307 DEFAULT_BPF_MAP_UID, AID_ROOT, 0600) 308 309 #define DEFINE_BPF_MAP_RO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 310 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 311 DEFAULT_BPF_MAP_UID, gid, 0440) 312 313 #define DEFINE_BPF_MAP_GWO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 314 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 315 DEFAULT_BPF_MAP_UID, gid, 0620) 316 317 #define DEFINE_BPF_MAP_GRO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 318 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 319 DEFAULT_BPF_MAP_UID, gid, 0640) 320 321 #define DEFINE_BPF_MAP_GRW(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 322 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 323 DEFAULT_BPF_MAP_UID, gid, 0660) 324 325 // LLVM eBPF builtins: they directly generate BPF_LD_ABS/BPF_LD_IND (skb may be ignored?) 326 unsigned long long load_byte(void* skb, unsigned long long off) asm("llvm.bpf.load.byte"); 327 unsigned long long load_half(void* skb, unsigned long long off) asm("llvm.bpf.load.half"); 328 unsigned long long load_word(void* skb, unsigned long long off) asm("llvm.bpf.load.word"); 329 330 static int (*bpf_probe_read)(void* dst, int size, void* unsafe_ptr) = (void*) BPF_FUNC_probe_read; 331 static int (*bpf_probe_read_str)(void* dst, int size, void* unsafe_ptr) = (void*) BPF_FUNC_probe_read_str; 332 static int (*bpf_probe_read_user)(void* dst, int size, const void* unsafe_ptr) = (void*)BPF_FUNC_probe_read_user; 333 static int (*bpf_probe_read_user_str)(void* dst, int size, const void* unsafe_ptr) = (void*) BPF_FUNC_probe_read_user_str; 334 static unsigned long long (*bpf_ktime_get_ns)(void) = (void*) BPF_FUNC_ktime_get_ns; 335 static unsigned long long (*bpf_ktime_get_boot_ns)(void) = (void*)BPF_FUNC_ktime_get_boot_ns; 336 static int (*bpf_trace_printk)(const char* fmt, int fmt_size, ...) = (void*) BPF_FUNC_trace_printk; 337 static unsigned long long (*bpf_get_current_pid_tgid)(void) = (void*) BPF_FUNC_get_current_pid_tgid; 338 static unsigned long long (*bpf_get_current_uid_gid)(void) = (void*) BPF_FUNC_get_current_uid_gid; 339 static unsigned long long (*bpf_get_smp_processor_id)(void) = (void*) BPF_FUNC_get_smp_processor_id; 340 static long (*bpf_get_stackid)(void* ctx, void* map, uint64_t flags) = (void*) BPF_FUNC_get_stackid; 341 static long (*bpf_get_current_comm)(void* buf, uint32_t buf_size) = (void*) BPF_FUNC_get_current_comm; 342 343 #define DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 344 min_loader, max_loader, opt, selinux, pindir, ignore_eng, \ 345 ignore_user, ignore_userdebug) \ 346 const struct bpf_prog_def SECTION("progs") the_prog##_def = { \ 347 .uid = (prog_uid), \ 348 .gid = (prog_gid), \ 349 .min_kver = (min_kv), \ 350 .max_kver = (max_kv), \ 351 .optional = (opt), \ 352 .bpfloader_min_ver = (min_loader), \ 353 .bpfloader_max_ver = (max_loader), \ 354 .selinux_context = (selinux), \ 355 .pin_subdir = (pindir), \ 356 .ignore_on_eng = (ignore_eng), \ 357 .ignore_on_user = (ignore_user), \ 358 .ignore_on_userdebug = (ignore_userdebug), \ 359 }; \ 360 SECTION(SECTION_NAME) \ 361 int the_prog 362 363 #ifndef DEFAULT_BPF_PROG_SELINUX_CONTEXT 364 #define DEFAULT_BPF_PROG_SELINUX_CONTEXT "" 365 #endif 366 367 #ifndef DEFAULT_BPF_PROG_PIN_SUBDIR 368 #define DEFAULT_BPF_PROG_PIN_SUBDIR "" 369 #endif 370 371 #define DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 372 opt) \ 373 DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 374 BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, opt, \ 375 DEFAULT_BPF_PROG_SELINUX_CONTEXT, DEFAULT_BPF_PROG_PIN_SUBDIR, \ 376 false, false, false) 377 378 // Programs (here used in the sense of functions/sections) marked optional are allowed to fail 379 // to load (for example due to missing kernel patches). 380 // The bpfloader will just ignore these failures and continue processing the next section. 381 // 382 // A non-optional program (function/section) failing to load causes a failure and aborts 383 // processing of the entire .o, if the .o is additionally marked critical, this will result 384 // in the entire bpfloader process terminating with a failure and not setting the bpf.progs_loaded 385 // system property. This in turn results in waitForProgsLoaded() never finishing. 386 // 387 // ie. a non-optional program in a critical .o is mandatory for kernels matching the min/max kver. 388 389 // programs requiring a kernel version >= min_kv && < max_kv 390 #define DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv) \ 391 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 392 false) 393 #define DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, \ 394 max_kv) \ 395 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, true) 396 397 // programs requiring a kernel version >= min_kv 398 #define DEFINE_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \ 399 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \ 400 false) 401 #define DEFINE_OPTIONAL_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \ 402 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \ 403 true) 404 405 // programs with no kernel version requirements 406 #define DEFINE_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \ 407 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, 0, KVER_INF, false) 408 #define DEFINE_OPTIONAL_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \ 409 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, 0, KVER_INF, true) 410