1 /* Common BPF helpers to be used by all BPF programs loaded by Android */ 2 3 #include <linux/bpf.h> 4 #include <stdbool.h> 5 #include <stdint.h> 6 7 #include "bpf_map_def.h" 8 9 /****************************************************************************** 10 * WARNING: CHANGES TO THIS FILE OUTSIDE OF AOSP/MAIN ARE LIKELY TO BREAK * 11 * DEVICE COMPATIBILITY WITH MAINLINE MODULES SHIPPING EBPF CODE. * 12 * * 13 * THIS WILL LIKELY RESULT IN BRICKED DEVICES AT SOME ARBITRARY FUTURE TIME * 14 * * 15 * THAT GOES ESPECIALLY FOR THE 'SECTION' 'LICENSE' AND 'CRITICAL' MACROS * 16 * * 17 * We strongly suggest that if you need changes to bpfloader functionality * 18 * you get your changes reviewed and accepted into aosp/master. * 19 * * 20 ******************************************************************************/ 21 22 // The actual versions of the bpfloader that shipped in various Android releases 23 24 // Android P/Q/R: BpfLoader was initially part of netd, 25 // this was later split out into a standalone binary, but was unversioned. 26 27 // Android S / 12 (api level 31) - added 'tethering' mainline eBPF support 28 #define BPFLOADER_S_VERSION 2u 29 30 // Android T / 13 (api level 33) - support for shared/selinux_context/pindir 31 #define BPFLOADER_T_VERSION 19u 32 33 // BpfLoader v0.25+ support obj@ver.o files 34 #define BPFLOADER_OBJ_AT_VER_VERSION 25u 35 36 // Bpfloader v0.33+ supports {map,prog}.ignore_on_{eng,user,userdebug} 37 #define BPFLOADER_IGNORED_ON_VERSION 33u 38 39 // Android U / 14 (api level 34) - various new program types added 40 #define BPFLOADER_U_VERSION 38u 41 42 // Android U QPR2 / 14 (api level 34) - platform only 43 // (note: the platform bpfloader in V isn't really versioned at all, 44 // as there is no need as it can only load objects compiled at the 45 // same time as itself and the rest of the platform) 46 #define BPFLOADER_U_QPR2_VERSION 41u 47 #define BPFLOADER_PLATFORM_VERSION BPFLOADER_U_QPR2_VERSION 48 49 // Android Mainline BpfLoader when running on Android S (sdk=31) 50 // Note: this value (and the following +1u's) are hardcoded in NetBpfLoad.cpp 51 #define BPFLOADER_MAINLINE_S_VERSION 42u 52 53 // Android Mainline BpfLoader when running on Android T (sdk=33) 54 #define BPFLOADER_MAINLINE_T_VERSION (BPFLOADER_MAINLINE_S_VERSION + 1u) 55 56 // Android Mainline BpfLoader when running on Android U (sdk=34) 57 #define BPFLOADER_MAINLINE_U_VERSION (BPFLOADER_MAINLINE_T_VERSION + 1u) 58 59 // Android Mainline BpfLoader when running on Android U QPR3 60 #define BPFLOADER_MAINLINE_U_QPR3_VERSION (BPFLOADER_MAINLINE_U_VERSION + 1u) 61 62 // Android Mainline BpfLoader when running on Android V (sdk=35) 63 #define BPFLOADER_MAINLINE_V_VERSION (BPFLOADER_MAINLINE_U_QPR3_VERSION + 1u) 64 65 // Android Mainline BpfLoader when running on Android 25Q2 (sdk=36) 66 #define BPFLOADER_MAINLINE_25Q2_VERSION (BPFLOADER_MAINLINE_V_VERSION + 1u) 67 68 /* For mainline module use, you can #define BPFLOADER_{MIN/MAX}_VER 69 * before #include "bpf_helpers.h" to change which bpfloaders will 70 * process the resulting .o file. 71 * 72 * While this will work outside of mainline too, there just is no point to 73 * using it when the .o and the bpfloader ship in sync with each other. 74 * In which case it's just best to use the default. 75 */ 76 #ifndef BPFLOADER_MIN_VER 77 #define BPFLOADER_MIN_VER BPFLOADER_PLATFORM_VERSION // inclusive, ie. >= 78 #endif 79 80 #ifndef BPFLOADER_MAX_VER 81 #define BPFLOADER_MAX_VER 0x10000u // exclusive, ie. < v1.0 82 #endif 83 84 /* place things in different elf sections */ 85 #define SECTION(NAME) __attribute__((section(NAME), used)) 86 87 /* Must be present in every program, example usage: 88 * LICENSE("GPL"); or LICENSE("Apache 2.0"); 89 * 90 * We also take this opportunity to embed a bunch of other useful values in 91 * the resulting .o (This is to enable some limited forward compatibility 92 * with mainline module shipped ebpf programs) 93 * 94 * The bpfloader_{min/max}_ver defines the [min, max) range of bpfloader 95 * versions that should load this .o file (bpfloaders outside of this range 96 * will simply ignore/skip this *entire* .o) 97 * The [inclusive,exclusive) matches what we do for kernel ver dependencies. 98 * 99 * The size_of_bpf_{map,prog}_def allow the bpfloader to load programs where 100 * these structures have been extended with additional fields (they will of 101 * course simply be ignored then). 102 * 103 * If missing, bpfloader_{min/max}_ver default to 0/0x10000 ie. [v0.0, v1.0), 104 * while size_of_bpf_{map/prog}_def default to 32/20 which are the v0.0 sizes. 105 * 106 * This macro also disables loading BTF map debug information, as versions 107 * of the platform bpfloader that support BTF require fork-exec of btfloader 108 * which causes a regression in boot time. 109 */ 110 #define LICENSE(NAME) \ 111 unsigned int _bpfloader_min_ver SECTION("bpfloader_min_ver") = BPFLOADER_MIN_VER; \ 112 unsigned int _bpfloader_max_ver SECTION("bpfloader_max_ver") = BPFLOADER_MAX_VER; \ 113 size_t _size_of_bpf_map_def SECTION("size_of_bpf_map_def") = sizeof(struct bpf_map_def); \ 114 size_t _size_of_bpf_prog_def SECTION("size_of_bpf_prog_def") = sizeof(struct bpf_prog_def); \ 115 unsigned _btf_min_bpfloader_ver SECTION("btf_min_bpfloader_ver") = BPFLOADER_MAINLINE_S_VERSION; \ 116 unsigned _btf_user_min_bpfloader_ver SECTION("btf_user_min_bpfloader_ver") = 0xFFFFFFFFu; \ 117 char _license[] SECTION("license") = (NAME) 118 119 /* flag the resulting bpf .o file as critical to system functionality, 120 * loading all kernel version appropriate programs in it must succeed 121 * for bpfloader success 122 */ 123 #define CRITICAL(REASON) char _critical[] SECTION("critical") = (REASON) 124 125 // Helpers for writing kernel version specific bpf programs 126 127 struct kver_uint { unsigned int kver; }; 128 #define KVER_(v) ((struct kver_uint){ .kver = (v) }) 129 #define KVER(a, b, c) KVER_(((a) << 24) + ((b) << 16) + (c)) 130 #define KVER_NONE KVER_(0) 131 #define KVER_4_9 KVER(4, 9, 0) 132 #define KVER_4_14 KVER(4, 14, 0) 133 #define KVER_4_19 KVER(4, 19, 0) 134 #define KVER_5_4 KVER(5, 4, 0) 135 #define KVER_5_10 KVER(5, 10, 0) 136 #define KVER_5_15 KVER(5, 15, 0) 137 #define KVER_6_1 KVER(6, 1, 0) 138 #define KVER_6_6 KVER(6, 6, 0) 139 #define KVER_6_12 KVER(6, 12, 0) 140 #define KVER_INF KVER_(0xFFFFFFFFu) 141 142 #define KVER_IS_AT_LEAST(kver, a, b, c) ((kver).kver >= KVER(a, b, c).kver) 143 144 // Helpers for writing sdk level specific bpf programs 145 // 146 // Note: we choose to follow sdk api level values, but there is no real need for this: 147 // These just need to be monotonically increasing. We could also use values ten or even 148 // a hundred times larger to leave room for quarters or months. We may also just use 149 // dates or something (2502 or 202506 for 25Q2) or even the mainline bpfloader version... 150 // For now this easily suffices for our use case. 151 152 struct sdk_level_uint { unsigned int sdk_level; }; 153 #define SDK_LEVEL_(v) ((struct sdk_level_uint){ .sdk_level = (v) }) 154 #define SDK_LEVEL_NONE SDK_LEVEL_(0) 155 #define SDK_LEVEL_S SDK_LEVEL_(31) // Android 12 156 #define SDK_LEVEL_Sv2 SDK_LEVEL_(32) // Android 12L 157 #define SDK_LEVEL_T SDK_LEVEL_(33) // Android 13 158 #define SDK_LEVEL_U SDK_LEVEL_(34) // Android 14 159 #define SDK_LEVEL_V SDK_LEVEL_(35) // Android 15 160 #define SDK_LEVEL_24Q3 SDK_LEVEL_V 161 #define SDK_LEVEL_25Q2 SDK_LEVEL_(36) // Android 16 162 163 #define SDK_LEVEL_IS_AT_LEAST(lvl, v) ((lvl).sdk_level >= (SDK_LEVEL_##v).sdk_level) 164 165 /* 166 * BPFFS (ie. /sys/fs/bpf) labelling is as follows: 167 * subdirectory selinux context mainline usecase / usable by 168 * / fs_bpf no [*] core operating system (ie. platform) 169 * /loader fs_bpf_loader no, U+ (as yet unused) 170 * /net_private fs_bpf_net_private yes, T+ network_stack 171 * /net_shared fs_bpf_net_shared yes, T+ network_stack & system_server 172 * /netd_readonly fs_bpf_netd_readonly yes, T+ network_stack & system_server & r/o to netd 173 * /netd_shared fs_bpf_netd_shared yes, T+ network_stack & system_server & netd [**] 174 * /tethering fs_bpf_tethering yes, S+ network_stack 175 * /vendor fs_bpf_vendor no, T+ vendor 176 * 177 * [*] initial support for bpf was added back in P, 178 * but things worked differently back then with no bpfloader, 179 * and instead netd doing stuff by hand, 180 * bpfloader with pinning into /sys/fs/bpf was (I believe) added in Q 181 * (and was definitely there in R). 182 * 183 * [**] additionally bpf programs are accessible to netutils_wrapper 184 * for use by iptables xt_bpf extensions. 185 * 186 * See cs/p:aosp-master%20-file:prebuilts/%20file:genfs_contexts%20"genfscon%20bpf" 187 */ 188 189 /* 190 * Helper functions called from eBPF programs written in C. These are 191 * implemented in the kernel sources. 192 */ 193 194 /* generic functions */ 195 196 /* 197 * Type-unsafe bpf map functions - avoid if possible. 198 * 199 * Using these it is possible to pass in keys/values of the wrong type/size, 200 * or, for 'bpf_map_lookup_elem_unsafe' receive into a pointer to the wrong type. 201 * You will not get a compile time failure, and for certain types of errors you 202 * might not even get a failure from the kernel's ebpf verifier during program load, 203 * instead stuff might just not work right at runtime. 204 * 205 * Instead please use: 206 * DEFINE_BPF_MAP(foo_map, TYPE, KeyType, ValueType, num_entries) 207 * where TYPE can be something like HASH or ARRAY, and num_entries is an integer. 208 * 209 * This defines the map (hence this should not be used in a header file included 210 * from multiple locations) and provides type safe accessors: 211 * ValueType * bpf_foo_map_lookup_elem(const KeyType *) 212 * int bpf_foo_map_update_elem(const KeyType *, const ValueType *, flags) 213 * int bpf_foo_map_delete_elem(const KeyType *) 214 * 215 * This will make sure that if you change the type of a map you'll get compile 216 * errors at any spots you forget to update with the new type. 217 * 218 * Note: these all take pointers to const map because from the C/eBPF point of view 219 * the map struct is really just a readonly map definition of the in kernel object. 220 * Runtime modification of the map defining struct is meaningless, since 221 * the contents is only ever used during bpf program loading & map creation 222 * by the bpf loader, and not by the eBPF program itself. 223 */ 224 static void* (*bpf_map_lookup_elem_unsafe)(const struct bpf_map_def* map, 225 const void* key) = (void*)BPF_FUNC_map_lookup_elem; 226 static int (*bpf_map_update_elem_unsafe)(const struct bpf_map_def* map, const void* key, 227 const void* value, unsigned long long flags) = (void*) 228 BPF_FUNC_map_update_elem; 229 static int (*bpf_map_delete_elem_unsafe)(const struct bpf_map_def* map, 230 const void* key) = (void*)BPF_FUNC_map_delete_elem; 231 static int (*bpf_ringbuf_output_unsafe)(const struct bpf_map_def* ringbuf, 232 const void* data, __u64 size, __u64 flags) = (void*) 233 BPF_FUNC_ringbuf_output; 234 static void* (*bpf_ringbuf_reserve_unsafe)(const struct bpf_map_def* ringbuf, 235 __u64 size, __u64 flags) = (void*) 236 BPF_FUNC_ringbuf_reserve; 237 static void (*bpf_ringbuf_submit_unsafe)(const void* data, __u64 flags) = (void*) 238 BPF_FUNC_ringbuf_submit; 239 240 #define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ 241 struct ____btf_map_##name { \ 242 type_key key; \ 243 type_val value; \ 244 }; \ 245 struct ____btf_map_##name \ 246 __attribute__ ((section(".maps." #name), used)) \ 247 ____btf_map_##name = { } 248 249 #define BPF_ASSERT_LOADER_VERSION(min_loader, ignore_eng, ignore_user, ignore_userdebug) \ 250 _Static_assert( \ 251 (min_loader) >= BPFLOADER_IGNORED_ON_VERSION || \ 252 !((ignore_eng).ignore_on_eng || \ 253 (ignore_user).ignore_on_user || \ 254 (ignore_userdebug).ignore_on_userdebug), \ 255 "bpfloader min version must be >= 0.33 in order to use ignored_on"); 256 257 #define ABSOLUTE(x) ((x) < 0 ? -(x) : (x)) 258 259 #define DEFAULT_BPF_MAP_FLAGS(type, num_entries, mapflags) \ 260 ( (mapflags) | \ 261 ((num_entries) < 0 ? BPF_F_NO_PREALLOC : 0) | \ 262 (type == BPF_MAP_TYPE_LPM_TRIE ? BPF_F_NO_PREALLOC : 0) \ 263 ) 264 265 #define DEFINE_BPF_MAP_BASE(the_map, TYPE, keysize, valuesize, num_entries, \ 266 usr, grp, md, selinux, pindir, share, minkver, \ 267 maxkver, minloader, maxloader, ignore_eng, \ 268 ignore_user, ignore_userdebug, mapflags) \ 269 const struct bpf_map_def SECTION("maps") the_map = { \ 270 .type = BPF_MAP_TYPE_##TYPE, \ 271 .key_size = (keysize), \ 272 .value_size = (valuesize), \ 273 .max_entries = ABSOLUTE(num_entries), \ 274 .map_flags = DEFAULT_BPF_MAP_FLAGS(BPF_MAP_TYPE_##TYPE, num_entries, mapflags), \ 275 .uid = (usr), \ 276 .gid = (grp), \ 277 .mode = (md), \ 278 .bpfloader_min_ver = (minloader), \ 279 .bpfloader_max_ver = (maxloader), \ 280 .min_kver = (minkver).kver, \ 281 .max_kver = (maxkver).kver, \ 282 .selinux_context = (selinux), \ 283 .pin_subdir = (pindir), \ 284 .shared = (share).shared, \ 285 .ignore_on_eng = (ignore_eng).ignore_on_eng, \ 286 .ignore_on_user = (ignore_user).ignore_on_user, \ 287 .ignore_on_userdebug = (ignore_userdebug).ignore_on_userdebug, \ 288 }; \ 289 BPF_ASSERT_LOADER_VERSION(minloader, ignore_eng, ignore_user, ignore_userdebug); 290 291 // Type safe macro to declare a ring buffer and related output functions. 292 // Compatibility: 293 // * BPF ring buffers are only available kernels 5.8 and above. Any program 294 // accessing the ring buffer should set a program level min_kver >= 5.10, 295 // since 5.10 is the next LTS version. 296 // * The definition below sets a map min_kver of 5.10 which requires targeting 297 // a BPFLOADER_MIN_VER >= BPFLOADER_S_VERSION. 298 #define DEFINE_BPF_RINGBUF_EXT(the_map, ValueType, size_bytes, usr, grp, md, \ 299 selinux, pindir, share, min_loader, max_loader, \ 300 ignore_eng, ignore_user, ignore_userdebug) \ 301 DEFINE_BPF_MAP_BASE(the_map, RINGBUF, 0, 0, size_bytes, usr, grp, md, \ 302 selinux, pindir, share, KVER_5_10, KVER_INF, \ 303 min_loader, max_loader, ignore_eng, ignore_user, \ 304 ignore_userdebug, 0); \ 305 \ 306 _Static_assert((size_bytes) >= 4096, "min 4 kiB ringbuffer size"); \ 307 _Static_assert((size_bytes) <= 0x10000000, "max 256 MiB ringbuffer size"); \ 308 _Static_assert(((size_bytes) & ((size_bytes) - 1)) == 0, \ 309 "ring buffer size must be a power of two"); \ 310 \ 311 static inline __always_inline __unused int bpf_##the_map##_output( \ 312 const ValueType* v) { \ 313 return bpf_ringbuf_output_unsafe(&the_map, v, sizeof(*v), 0); \ 314 } \ 315 \ 316 static inline __always_inline __unused \ 317 ValueType* bpf_##the_map##_reserve() { \ 318 return bpf_ringbuf_reserve_unsafe(&the_map, sizeof(ValueType), 0); \ 319 } \ 320 \ 321 static inline __always_inline __unused void bpf_##the_map##_submit( \ 322 const ValueType* v) { \ 323 bpf_ringbuf_submit_unsafe(v, 0); \ 324 } 325 326 #define DEFINE_BPF_RINGBUF(the_map, ValueType, size_bytes, usr, grp, md) \ 327 DEFINE_BPF_RINGBUF_EXT(the_map, ValueType, size_bytes, usr, grp, md, \ 328 DEFAULT_BPF_MAP_SELINUX_CONTEXT, DEFAULT_BPF_MAP_PIN_SUBDIR, \ 329 PRIVATE, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, \ 330 LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG) 331 332 /* There exist buggy kernels with pre-T OS, that due to 333 * kernel patch "[ALPS05162612] bpf: fix ubsan error" 334 * do not support userspace writes into non-zero index of bpf map arrays. 335 * 336 * We use this assert to prevent us from being able to define such a map. 337 */ 338 339 #ifdef THIS_BPF_PROGRAM_IS_FOR_TEST_PURPOSES_ONLY 340 #define BPF_MAP_ASSERT_OK(type, entries, mode) 341 #elif BPFLOADER_MIN_VER >= BPFLOADER_T_VERSION 342 #define BPF_MAP_ASSERT_OK(type, entries, mode) 343 #else 344 #define BPF_MAP_ASSERT_OK(type, entries, mode) \ 345 _Static_assert(((type) != BPF_MAP_TYPE_ARRAY) || ((entries) <= 1) || !((mode) & 0222), \ 346 "Writable arrays with more than 1 element not supported on pre-T devices.") 347 #endif 348 349 /* type safe macro to declare a map and related accessor functions */ 350 #define DEFINE_BPF_MAP_EXT(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md, \ 351 selinux, pindir, share, min_loader, max_loader, ignore_eng, \ 352 ignore_user, ignore_userdebug, mapFlags) \ 353 DEFINE_BPF_MAP_BASE(the_map, TYPE, sizeof(KeyType), sizeof(ValueType), \ 354 num_entries, usr, grp, md, selinux, pindir, share, \ 355 KVER_NONE, KVER_INF, min_loader, max_loader, \ 356 ignore_eng, ignore_user, ignore_userdebug, mapFlags); \ 357 BPF_MAP_ASSERT_OK(BPF_MAP_TYPE_##TYPE, (num_entries), (md)); \ 358 _Static_assert(sizeof(KeyType) < 1024, "aosp/2370288 requires < 1024 byte keys"); \ 359 _Static_assert(sizeof(ValueType) < 65536, "aosp/2370288 requires < 65536 byte values"); \ 360 BPF_ANNOTATE_KV_PAIR(the_map, KeyType, ValueType); \ 361 \ 362 static inline __always_inline __unused ValueType* bpf_##the_map##_lookup_elem( \ 363 const KeyType* k) { \ 364 return bpf_map_lookup_elem_unsafe(&the_map, k); \ 365 }; \ 366 \ 367 static inline __always_inline __unused int bpf_##the_map##_update_elem( \ 368 const KeyType* k, const ValueType* v, unsigned long long flags) { \ 369 return bpf_map_update_elem_unsafe(&the_map, k, v, flags); \ 370 }; \ 371 \ 372 static inline __always_inline __unused int bpf_##the_map##_delete_elem(const KeyType* k) { \ 373 return bpf_map_delete_elem_unsafe(&the_map, k); \ 374 }; 375 376 #ifndef DEFAULT_BPF_MAP_SELINUX_CONTEXT 377 #define DEFAULT_BPF_MAP_SELINUX_CONTEXT "" 378 #endif 379 380 #ifndef DEFAULT_BPF_MAP_PIN_SUBDIR 381 #define DEFAULT_BPF_MAP_PIN_SUBDIR "" 382 #endif 383 384 #ifndef DEFAULT_BPF_MAP_UID 385 #define DEFAULT_BPF_MAP_UID AID_ROOT 386 #elif BPFLOADER_MIN_VER < 28u 387 #error "Bpf Map UID must be left at default of AID_ROOT for BpfLoader prior to v0.28" 388 #endif 389 390 // for maps not meant to be accessed from userspace 391 #define DEFINE_BPF_MAP_KERNEL_INTERNAL(the_map, TYPE, KeyType, ValueType, num_entries) \ 392 DEFINE_BPF_MAP_EXT(the_map, TYPE, KeyType, ValueType, num_entries, AID_ROOT, AID_ROOT, \ 393 0000, "fs_bpf_loader", "", PRIVATE, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, \ 394 LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG, 0) 395 396 #define DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md) \ 397 DEFINE_BPF_MAP_EXT(the_map, TYPE, KeyType, ValueType, num_entries, usr, grp, md, \ 398 DEFAULT_BPF_MAP_SELINUX_CONTEXT, DEFAULT_BPF_MAP_PIN_SUBDIR, \ 399 PRIVATE, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, \ 400 LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG, 0) 401 402 #define DEFINE_BPF_MAP(the_map, TYPE, KeyType, ValueType, num_entries) \ 403 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 404 DEFAULT_BPF_MAP_UID, AID_ROOT, 0600) 405 406 #define DEFINE_BPF_MAP_RO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 407 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 408 DEFAULT_BPF_MAP_UID, gid, 0440) 409 410 #define DEFINE_BPF_MAP_GWO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 411 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 412 DEFAULT_BPF_MAP_UID, gid, 0620) 413 414 #define DEFINE_BPF_MAP_GRO(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 415 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 416 DEFAULT_BPF_MAP_UID, gid, 0640) 417 418 #define DEFINE_BPF_MAP_GRW(the_map, TYPE, KeyType, ValueType, num_entries, gid) \ 419 DEFINE_BPF_MAP_UGM(the_map, TYPE, KeyType, ValueType, num_entries, \ 420 DEFAULT_BPF_MAP_UID, gid, 0660) 421 422 // idea from Linux include/linux/compiler_types.h (eBPF is always a 64-bit arch) 423 #define NATIVE_WORD(t) ((sizeof(t) == 1) || (sizeof(t) == 2) || (sizeof(t) == 4) || (sizeof(t) == 8)) 424 425 // simplified from Linux include/asm-generic/rwonce.h 426 #define READ_ONCE(x) \ 427 ({ \ 428 _Static_assert(NATIVE_WORD(x), "READ_ONCE requires a native word size"); \ 429 (*(const volatile typeof(x) *)&(x)) \ 430 }) 431 432 #define WRITE_ONCE(x, value) \ 433 do { \ 434 _Static_assert(NATIVE_WORD(x), "WRITE_ONCE requires a native word size"); \ 435 *(volatile typeof(x) *)&(x) = (value); \ 436 } while (0) 437 438 // LLVM eBPF builtins: they directly generate BPF_LD_ABS/BPF_LD_IND (skb may be ignored?) 439 unsigned long long load_byte(void* skb, unsigned long long off) asm("llvm.bpf.load.byte"); 440 unsigned long long load_half(void* skb, unsigned long long off) asm("llvm.bpf.load.half"); 441 unsigned long long load_word(void* skb, unsigned long long off) asm("llvm.bpf.load.word"); 442 443 static int (*bpf_probe_read)(void* dst, int size, void* unsafe_ptr) = (void*) BPF_FUNC_probe_read; 444 static int (*bpf_probe_read_str)(void* dst, int size, void* unsafe_ptr) = (void*) BPF_FUNC_probe_read_str; 445 static int (*bpf_probe_read_user)(void* dst, int size, const void* unsafe_ptr) = (void*)BPF_FUNC_probe_read_user; 446 static int (*bpf_probe_read_user_str)(void* dst, int size, const void* unsafe_ptr) = (void*) BPF_FUNC_probe_read_user_str; 447 static unsigned long long (*bpf_ktime_get_ns)(void) = (void*) BPF_FUNC_ktime_get_ns; 448 static unsigned long long (*bpf_ktime_get_boot_ns)(void) = (void*)BPF_FUNC_ktime_get_boot_ns; 449 static unsigned long long (*bpf_get_current_pid_tgid)(void) = (void*) BPF_FUNC_get_current_pid_tgid; 450 static unsigned long long (*bpf_get_current_uid_gid)(void) = (void*) BPF_FUNC_get_current_uid_gid; 451 static unsigned long long (*bpf_get_smp_processor_id)(void) = (void*) BPF_FUNC_get_smp_processor_id; 452 static long (*bpf_get_stackid)(void* ctx, void* map, uint64_t flags) = (void*) BPF_FUNC_get_stackid; 453 static long (*bpf_get_current_comm)(void* buf, uint32_t buf_size) = (void*) BPF_FUNC_get_current_comm; 454 // bpf_sk_fullsock requires 5.1+ kernel 455 static struct bpf_sock* (*bpf_sk_fullsock)(struct bpf_sock* sk) = (void*) BPF_FUNC_sk_fullsock; 456 457 // GPL only: 458 static int (*bpf_trace_printk)(const char* fmt, int fmt_size, ...) = (void*) BPF_FUNC_trace_printk; 459 #define bpf_printf(s, n...) bpf_trace_printk(s, sizeof(s), ## n) 460 // Note: bpf only supports up to 3 arguments, log via: bpf_printf("msg %d %d %d", 1, 2, 3); 461 // and read via the blocking: sudo cat /sys/kernel/debug/tracing/trace_pipe 462 463 #define DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 464 min_loader, max_loader, opt, selinux, pindir, ignore_eng, \ 465 ignore_user, ignore_userdebug) \ 466 const struct bpf_prog_def SECTION("progs") the_prog##_def = { \ 467 .uid = (prog_uid), \ 468 .gid = (prog_gid), \ 469 .min_kver = (min_kv).kver, \ 470 .max_kver = (max_kv).kver, \ 471 .optional = (opt).optional, \ 472 .bpfloader_min_ver = (min_loader), \ 473 .bpfloader_max_ver = (max_loader), \ 474 .selinux_context = (selinux), \ 475 .pin_subdir = (pindir), \ 476 .ignore_on_eng = (ignore_eng).ignore_on_eng, \ 477 .ignore_on_user = (ignore_user).ignore_on_user, \ 478 .ignore_on_userdebug = (ignore_userdebug).ignore_on_userdebug, \ 479 }; \ 480 SECTION(SECTION_NAME) \ 481 int the_prog 482 483 #define DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 484 opt) \ 485 DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 486 BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, opt, "", "", \ 487 LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG) 488 489 // Programs (here used in the sense of functions/sections) marked optional are allowed to fail 490 // to load (for example due to missing kernel patches). 491 // The bpfloader will just ignore these failures and continue processing the next section. 492 // 493 // A non-optional program (function/section) failing to load causes a failure and aborts 494 // processing of the entire .o, if the .o is additionally marked critical, this will result 495 // in the entire bpfloader process terminating with a failure and not setting the bpf.progs_loaded 496 // system property. This in turn results in waitForProgsLoaded() never finishing. 497 // 498 // ie. a non-optional program in a critical .o is mandatory for kernels matching the min/max kver. 499 500 // programs requiring a kernel version >= min_kv && < max_kv 501 #define DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv) \ 502 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 503 MANDATORY) 504 #define DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, \ 505 max_kv) \ 506 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \ 507 OPTIONAL) 508 509 // programs requiring a kernel version >= min_kv 510 #define DEFINE_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \ 511 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \ 512 MANDATORY) 513 #define DEFINE_OPTIONAL_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \ 514 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \ 515 OPTIONAL) 516 517 // programs with no kernel version requirements 518 #define DEFINE_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \ 519 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, KVER_NONE, KVER_INF, \ 520 MANDATORY) 521 #define DEFINE_OPTIONAL_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \ 522 DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, KVER_NONE, KVER_INF, \ 523 OPTIONAL) 524