1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <linux/if.h>
18 #include <linux/ip.h>
19 #include <linux/ipv6.h>
20 #include <linux/pkt_cls.h>
21 #include <linux/tcp.h>
22
23 // bionic kernel uapi linux/udp.h header is munged...
24 #define __kernel_udphdr udphdr
25 #include <linux/udp.h>
26
27 #include "bpf_helpers.h"
28 #include "bpf_net_helpers.h"
29 #include "bpf_tethering.h"
30
31 // From kernel:include/net/ip.h
32 #define IP_DF 0x4000 // Flag: "Don't Fragment"
33
34 // ----- Helper functions for offsets to fields -----
35
36 // They all assume simple IP packets:
37 // - no VLAN ethernet tags
38 // - no IPv4 options (see IPV4_HLEN/TCP4_OFFSET/UDP4_OFFSET)
39 // - no IPv6 extension headers
40 // - no TCP options (see TCP_HLEN)
41
42 //#define ETH_HLEN sizeof(struct ethhdr)
43 #define IP4_HLEN sizeof(struct iphdr)
44 #define IP6_HLEN sizeof(struct ipv6hdr)
45 #define TCP_HLEN sizeof(struct tcphdr)
46 #define UDP_HLEN sizeof(struct udphdr)
47
48 // Offsets from beginning of L4 (TCP/UDP) header
49 #define TCP_OFFSET(field) offsetof(struct tcphdr, field)
50 #define UDP_OFFSET(field) offsetof(struct udphdr, field)
51
52 // Offsets from beginning of L3 (IPv4) header
53 #define IP4_OFFSET(field) offsetof(struct iphdr, field)
54 #define IP4_TCP_OFFSET(field) (IP4_HLEN + TCP_OFFSET(field))
55 #define IP4_UDP_OFFSET(field) (IP4_HLEN + UDP_OFFSET(field))
56
57 // Offsets from beginning of L3 (IPv6) header
58 #define IP6_OFFSET(field) offsetof(struct ipv6hdr, field)
59 #define IP6_TCP_OFFSET(field) (IP6_HLEN + TCP_OFFSET(field))
60 #define IP6_UDP_OFFSET(field) (IP6_HLEN + UDP_OFFSET(field))
61
62 // Offsets from beginning of L2 (ie. Ethernet) header (which must be present)
63 #define ETH_IP4_OFFSET(field) (ETH_HLEN + IP4_OFFSET(field))
64 #define ETH_IP4_TCP_OFFSET(field) (ETH_HLEN + IP4_TCP_OFFSET(field))
65 #define ETH_IP4_UDP_OFFSET(field) (ETH_HLEN + IP4_UDP_OFFSET(field))
66 #define ETH_IP6_OFFSET(field) (ETH_HLEN + IP6_OFFSET(field))
67 #define ETH_IP6_TCP_OFFSET(field) (ETH_HLEN + IP6_TCP_OFFSET(field))
68 #define ETH_IP6_UDP_OFFSET(field) (ETH_HLEN + IP6_UDP_OFFSET(field))
69
70 // ----- Tethering Error Counters -----
71
DEFINE_BPF_MAP_GRW(tether_error_map,ARRAY,uint32_t,uint32_t,BPF_TETHER_ERR__MAX,AID_NETWORK_STACK)72 DEFINE_BPF_MAP_GRW(tether_error_map, ARRAY, uint32_t, uint32_t, BPF_TETHER_ERR__MAX,
73 AID_NETWORK_STACK)
74
75 #define COUNT_AND_RETURN(counter, ret) do { \
76 uint32_t code = BPF_TETHER_ERR_ ## counter; \
77 uint32_t *count = bpf_tether_error_map_lookup_elem(&code); \
78 if (count) __sync_fetch_and_add(count, 1); \
79 return ret; \
80 } while(0)
81
82 #define TC_DROP(counter) COUNT_AND_RETURN(counter, TC_ACT_SHOT)
83 #define TC_PUNT(counter) COUNT_AND_RETURN(counter, TC_ACT_OK)
84
85 #define XDP_DROP(counter) COUNT_AND_RETURN(counter, XDP_DROP)
86 #define XDP_PUNT(counter) COUNT_AND_RETURN(counter, XDP_PASS)
87
88 // ----- Tethering Data Stats and Limits -----
89
90 // Tethering stats, indexed by upstream interface.
91 DEFINE_BPF_MAP_GRW(tether_stats_map, HASH, TetherStatsKey, TetherStatsValue, 16, AID_NETWORK_STACK)
92
93 // Tethering data limit, indexed by upstream interface.
94 // (tethering allowed when stats[iif].rxBytes + stats[iif].txBytes < limit[iif])
95 DEFINE_BPF_MAP_GRW(tether_limit_map, HASH, TetherLimitKey, TetherLimitValue, 16, AID_NETWORK_STACK)
96
97 // ----- IPv6 Support -----
98
99 DEFINE_BPF_MAP_GRW(tether_downstream6_map, HASH, TetherDownstream6Key, Tether6Value, 64,
100 AID_NETWORK_STACK)
101
102 DEFINE_BPF_MAP_GRW(tether_downstream64_map, HASH, TetherDownstream64Key, TetherDownstream64Value,
103 1024, AID_NETWORK_STACK)
104
105 DEFINE_BPF_MAP_GRW(tether_upstream6_map, HASH, TetherUpstream6Key, Tether6Value, 64,
106 AID_NETWORK_STACK)
107
108 static inline __always_inline int do_forward6(struct __sk_buff* skb, const bool is_ethernet,
109 const bool downstream) {
110 // Must be meta-ethernet IPv6 frame
111 if (skb->protocol != htons(ETH_P_IPV6)) return TC_ACT_OK;
112
113 // Require ethernet dst mac address to be our unicast address.
114 if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_OK;
115
116 const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
117
118 // Since the program never writes via DPA (direct packet access) auto-pull/unclone logic does
119 // not trigger and thus we need to manually make sure we can read packet headers via DPA.
120 // Note: this is a blind best effort pull, which may fail or pull less - this doesn't matter.
121 // It has to be done early cause it will invalidate any skb->data/data_end derived pointers.
122 try_make_readable(skb, l2_header_size + IP6_HLEN + TCP_HLEN);
123
124 void* data = (void*)(long)skb->data;
125 const void* data_end = (void*)(long)skb->data_end;
126 struct ethhdr* eth = is_ethernet ? data : NULL; // used iff is_ethernet
127 struct ipv6hdr* ip6 = is_ethernet ? (void*)(eth + 1) : data;
128
129 // Must have (ethernet and) ipv6 header
130 if (data + l2_header_size + sizeof(*ip6) > data_end) return TC_ACT_OK;
131
132 // Ethertype - if present - must be IPv6
133 if (is_ethernet && (eth->h_proto != htons(ETH_P_IPV6))) return TC_ACT_OK;
134
135 // IP version must be 6
136 if (ip6->version != 6) TC_PUNT(INVALID_IP_VERSION);
137
138 // Cannot decrement during forward if already zero or would be zero,
139 // Let the kernel's stack handle these cases and generate appropriate ICMP errors.
140 if (ip6->hop_limit <= 1) TC_PUNT(LOW_TTL);
141
142 // If hardware offload is running and programming flows based on conntrack entries,
143 // try not to interfere with it.
144 if (ip6->nexthdr == IPPROTO_TCP) {
145 struct tcphdr* tcph = (void*)(ip6 + 1);
146
147 // Make sure we can get at the tcp header
148 if (data + l2_header_size + sizeof(*ip6) + sizeof(*tcph) > data_end)
149 TC_PUNT(INVALID_TCP_HEADER);
150
151 // Do not offload TCP packets with any one of the SYN/FIN/RST flags
152 if (tcph->syn || tcph->fin || tcph->rst) TC_PUNT(TCP_CONTROL_PACKET);
153 }
154
155 // Protect against forwarding packets sourced from ::1 or fe80::/64 or other weirdness.
156 __be32 src32 = ip6->saddr.s6_addr32[0];
157 if (src32 != htonl(0x0064ff9b) && // 64:ff9b:/32 incl. XLAT464 WKP
158 (src32 & htonl(0xe0000000)) != htonl(0x20000000)) // 2000::/3 Global Unicast
159 TC_PUNT(NON_GLOBAL_SRC);
160
161 // Protect against forwarding packets destined to ::1 or fe80::/64 or other weirdness.
162 __be32 dst32 = ip6->daddr.s6_addr32[0];
163 if (dst32 != htonl(0x0064ff9b) && // 64:ff9b:/32 incl. XLAT464 WKP
164 (dst32 & htonl(0xe0000000)) != htonl(0x20000000)) // 2000::/3 Global Unicast
165 TC_PUNT(NON_GLOBAL_DST);
166
167 // In the upstream direction do not forward traffic within the same /64 subnet.
168 if (!downstream && (src32 == dst32) && (ip6->saddr.s6_addr32[1] == ip6->daddr.s6_addr32[1]))
169 TC_PUNT(LOCAL_SRC_DST);
170
171 TetherDownstream6Key kd = {
172 .iif = skb->ifindex,
173 .neigh6 = ip6->daddr,
174 };
175
176 TetherUpstream6Key ku = {
177 .iif = skb->ifindex,
178 };
179 if (is_ethernet) __builtin_memcpy(downstream ? kd.dstMac : ku.dstMac, eth->h_dest, ETH_ALEN);
180
181 Tether6Value* v = downstream ? bpf_tether_downstream6_map_lookup_elem(&kd)
182 : bpf_tether_upstream6_map_lookup_elem(&ku);
183
184 // If we don't find any offload information then simply let the core stack handle it...
185 if (!v) return TC_ACT_OK;
186
187 uint32_t stat_and_limit_k = downstream ? skb->ifindex : v->oif;
188
189 TetherStatsValue* stat_v = bpf_tether_stats_map_lookup_elem(&stat_and_limit_k);
190
191 // If we don't have anywhere to put stats, then abort...
192 if (!stat_v) TC_PUNT(NO_STATS_ENTRY);
193
194 uint64_t* limit_v = bpf_tether_limit_map_lookup_elem(&stat_and_limit_k);
195
196 // If we don't have a limit, then abort...
197 if (!limit_v) TC_PUNT(NO_LIMIT_ENTRY);
198
199 // Required IPv6 minimum mtu is 1280, below that not clear what we should do, abort...
200 if (v->pmtu < IPV6_MIN_MTU) TC_PUNT(BELOW_IPV6_MTU);
201
202 // Approximate handling of TCP/IPv6 overhead for incoming LRO/GRO packets: default
203 // outbound path mtu of 1500 is not necessarily correct, but worst case we simply
204 // undercount, which is still better then not accounting for this overhead at all.
205 // Note: this really shouldn't be device/path mtu at all, but rather should be
206 // derived from this particular connection's mss (ie. from gro segment size).
207 // This would require a much newer kernel with newer ebpf accessors.
208 // (This is also blindly assuming 12 bytes of tcp timestamp option in tcp header)
209 uint64_t packets = 1;
210 uint64_t bytes = skb->len;
211 if (bytes > v->pmtu) {
212 const int tcp_overhead = sizeof(struct ipv6hdr) + sizeof(struct tcphdr) + 12;
213 const int mss = v->pmtu - tcp_overhead;
214 const uint64_t payload = bytes - tcp_overhead;
215 packets = (payload + mss - 1) / mss;
216 bytes = tcp_overhead * packets + payload;
217 }
218
219 // Are we past the limit? If so, then abort...
220 // Note: will not overflow since u64 is 936 years even at 5Gbps.
221 // Do not drop here. Offload is just that, whenever we fail to handle
222 // a packet we let the core stack deal with things.
223 // (The core stack needs to handle limits correctly anyway,
224 // since we don't offload all traffic in both directions)
225 if (stat_v->rxBytes + stat_v->txBytes + bytes > *limit_v) TC_PUNT(LIMIT_REACHED);
226
227 if (!is_ethernet) {
228 // Try to inject an ethernet header, and simply return if we fail.
229 // We do this even if TX interface is RAWIP and thus does not need an ethernet header,
230 // because this is easier and the kernel will strip extraneous ethernet header.
231 if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {
232 __sync_fetch_and_add(downstream ? &stat_v->rxErrors : &stat_v->txErrors, 1);
233 TC_PUNT(CHANGE_HEAD_FAILED);
234 }
235
236 // bpf_skb_change_head() invalidates all pointers - reload them
237 data = (void*)(long)skb->data;
238 data_end = (void*)(long)skb->data_end;
239 eth = data;
240 ip6 = (void*)(eth + 1);
241
242 // I do not believe this can ever happen, but keep the verifier happy...
243 if (data + sizeof(struct ethhdr) + sizeof(*ip6) > data_end) {
244 __sync_fetch_and_add(downstream ? &stat_v->rxErrors : &stat_v->txErrors, 1);
245 TC_DROP(TOO_SHORT);
246 }
247 };
248
249 // At this point we always have an ethernet header - which will get stripped by the
250 // kernel during transmit through a rawip interface. ie. 'eth' pointer is valid.
251 // Additionally note that 'is_ethernet' and 'l2_header_size' are no longer correct.
252
253 // CHECKSUM_COMPLETE is a 16-bit one's complement sum,
254 // thus corrections for it need to be done in 16-byte chunks at even offsets.
255 // IPv6 nexthdr is at offset 6, while hop limit is at offset 7
256 uint8_t old_hl = ip6->hop_limit;
257 --ip6->hop_limit;
258 uint8_t new_hl = ip6->hop_limit;
259
260 // bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
261 // (-ENOTSUPP) if it isn't.
262 bpf_csum_update(skb, 0xFFFF - ntohs(old_hl) + ntohs(new_hl));
263
264 __sync_fetch_and_add(downstream ? &stat_v->rxPackets : &stat_v->txPackets, packets);
265 __sync_fetch_and_add(downstream ? &stat_v->rxBytes : &stat_v->txBytes, bytes);
266
267 // Overwrite any mac header with the new one
268 // For a rawip tx interface it will simply be a bunch of zeroes and later stripped.
269 *eth = v->macHeader;
270
271 // Redirect to forwarded interface.
272 //
273 // Note that bpf_redirect() cannot fail unless you pass invalid flags.
274 // The redirect actually happens after the ebpf program has already terminated,
275 // and can fail for example for mtu reasons at that point in time, but there's nothing
276 // we can do about it here.
277 return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
278 }
279
280 DEFINE_BPF_PROG("schedcls/tether_downstream6_ether", AID_ROOT, AID_NETWORK_STACK,
281 sched_cls_tether_downstream6_ether)
282 (struct __sk_buff* skb) {
283 return do_forward6(skb, /* is_ethernet */ true, /* downstream */ true);
284 }
285
286 DEFINE_BPF_PROG("schedcls/tether_upstream6_ether", AID_ROOT, AID_NETWORK_STACK,
287 sched_cls_tether_upstream6_ether)
288 (struct __sk_buff* skb) {
289 return do_forward6(skb, /* is_ethernet */ true, /* downstream */ false);
290 }
291
292 // Note: section names must be unique to prevent programs from appending to each other,
293 // so instead the bpf loader will strip everything past the final $ symbol when actually
294 // pinning the program into the filesystem.
295 //
296 // bpf_skb_change_head() is only present on 4.14+ and 2 trivial kernel patches are needed:
297 // ANDROID: net: bpf: Allow TC programs to call BPF_FUNC_skb_change_head
298 // ANDROID: net: bpf: permit redirect from ingress L3 to egress L2 devices at near max mtu
299 // (the first of those has already been upstreamed)
300 //
301 // 5.4 kernel support was only added to Android Common Kernel in R,
302 // and thus a 5.4 kernel always supports this.
303 //
304 // Hence, these mandatory (must load successfully) implementations for 5.4+ kernels:
305 DEFINE_BPF_PROG_KVER("schedcls/tether_downstream6_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
306 sched_cls_tether_downstream6_rawip_5_4, KVER(5, 4, 0))
307 (struct __sk_buff* skb) {
308 return do_forward6(skb, /* is_ethernet */ false, /* downstream */ true);
309 }
310
311 DEFINE_BPF_PROG_KVER("schedcls/tether_upstream6_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
312 sched_cls_tether_upstream6_rawip_5_4, KVER(5, 4, 0))
313 (struct __sk_buff* skb) {
314 return do_forward6(skb, /* is_ethernet */ false, /* downstream */ false);
315 }
316
317 // and these identical optional (may fail to load) implementations for [4.14..5.4) patched kernels:
318 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream6_rawip$4_14",
319 AID_ROOT, AID_NETWORK_STACK,
320 sched_cls_tether_downstream6_rawip_4_14,
321 KVER(4, 14, 0), KVER(5, 4, 0))
322 (struct __sk_buff* skb) {
323 return do_forward6(skb, /* is_ethernet */ false, /* downstream */ true);
324 }
325
326 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream6_rawip$4_14",
327 AID_ROOT, AID_NETWORK_STACK,
328 sched_cls_tether_upstream6_rawip_4_14,
329 KVER(4, 14, 0), KVER(5, 4, 0))
330 (struct __sk_buff* skb) {
331 return do_forward6(skb, /* is_ethernet */ false, /* downstream */ false);
332 }
333
334 // and define no-op stubs for [4.9,4.14) and unpatched [4.14,5.4) kernels.
335 // (if the above real 4.14+ program loaded successfully, then bpfloader will have already pinned
336 // it at the same location this one would be pinned at and will thus skip loading this stub)
337 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream6_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
338 sched_cls_tether_downstream6_rawip_stub, KVER_NONE, KVER(5, 4, 0))
339 (struct __sk_buff* skb) {
340 return TC_ACT_OK;
341 }
342
343 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream6_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
344 sched_cls_tether_upstream6_rawip_stub, KVER_NONE, KVER(5, 4, 0))
345 (struct __sk_buff* skb) {
346 return TC_ACT_OK;
347 }
348
349 // ----- IPv4 Support -----
350
351 DEFINE_BPF_MAP_GRW(tether_downstream4_map, HASH, Tether4Key, Tether4Value, 1024, AID_NETWORK_STACK)
352
353 DEFINE_BPF_MAP_GRW(tether_upstream4_map, HASH, Tether4Key, Tether4Value, 1024, AID_NETWORK_STACK)
354
do_forward4(struct __sk_buff * skb,const bool is_ethernet,const bool downstream,const bool updatetime)355 static inline __always_inline int do_forward4(struct __sk_buff* skb, const bool is_ethernet,
356 const bool downstream, const bool updatetime) {
357 // Require ethernet dst mac address to be our unicast address.
358 if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_OK;
359
360 // Must be meta-ethernet IPv4 frame
361 if (skb->protocol != htons(ETH_P_IP)) return TC_ACT_OK;
362
363 const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
364
365 // Since the program never writes via DPA (direct packet access) auto-pull/unclone logic does
366 // not trigger and thus we need to manually make sure we can read packet headers via DPA.
367 // Note: this is a blind best effort pull, which may fail or pull less - this doesn't matter.
368 // It has to be done early cause it will invalidate any skb->data/data_end derived pointers.
369 try_make_readable(skb, l2_header_size + IP4_HLEN + TCP_HLEN);
370
371 void* data = (void*)(long)skb->data;
372 const void* data_end = (void*)(long)skb->data_end;
373 struct ethhdr* eth = is_ethernet ? data : NULL; // used iff is_ethernet
374 struct iphdr* ip = is_ethernet ? (void*)(eth + 1) : data;
375
376 // Must have (ethernet and) ipv4 header
377 if (data + l2_header_size + sizeof(*ip) > data_end) return TC_ACT_OK;
378
379 // Ethertype - if present - must be IPv4
380 if (is_ethernet && (eth->h_proto != htons(ETH_P_IP))) return TC_ACT_OK;
381
382 // IP version must be 4
383 if (ip->version != 4) TC_PUNT(INVALID_IP_VERSION);
384
385 // We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
386 if (ip->ihl != 5) TC_PUNT(HAS_IP_OPTIONS);
387
388 // Calculate the IPv4 one's complement checksum of the IPv4 header.
389 __wsum sum4 = 0;
390 for (int i = 0; i < sizeof(*ip) / sizeof(__u16); ++i) {
391 sum4 += ((__u16*)ip)[i];
392 }
393 // Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
394 sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse u32 into range 1 .. 0x1FFFE
395 sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse any potential carry into u16
396 // for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
397 if (sum4 != 0xFFFF) TC_PUNT(CHECKSUM);
398
399 // Minimum IPv4 total length is the size of the header
400 if (ntohs(ip->tot_len) < sizeof(*ip)) TC_PUNT(TRUNCATED_IPV4);
401
402 // We are incapable of dealing with IPv4 fragments
403 if (ip->frag_off & ~htons(IP_DF)) TC_PUNT(IS_IP_FRAG);
404
405 // Cannot decrement during forward if already zero or would be zero,
406 // Let the kernel's stack handle these cases and generate appropriate ICMP errors.
407 if (ip->ttl <= 1) TC_PUNT(LOW_TTL);
408
409 // If we cannot update the 'last_used' field due to lack of bpf_ktime_get_boot_ns() helper,
410 // then it is not safe to offload UDP due to the small conntrack timeouts, as such,
411 // in such a situation we can only support TCP. This also has the added nice benefit of
412 // using a separate error counter, and thus making it obvious which version of the program
413 // is loaded.
414 if (!updatetime && ip->protocol != IPPROTO_TCP) TC_PUNT(NON_TCP);
415
416 // We do not support offloading anything besides IPv4 TCP and UDP, due to need for NAT,
417 // but no need to check this if !updatetime due to check immediately above.
418 if (updatetime && (ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP))
419 TC_PUNT(NON_TCP_UDP);
420
421 // We want to make sure that the compiler will, in the !updatetime case, entirely optimize
422 // out all the non-tcp logic. Also note that at this point is_udp === !is_tcp.
423 const bool is_tcp = !updatetime || (ip->protocol == IPPROTO_TCP);
424
425 // This is a bit of a hack to make things easier on the bpf verifier.
426 // (In particular I believe the Linux 4.14 kernel's verifier can get confused later on about
427 // what offsets into the packet are valid and can spuriously reject the program, this is
428 // because it fails to realize that is_tcp && !is_tcp is impossible)
429 //
430 // For both TCP & UDP we'll need to read and modify the src/dst ports, which so happen to
431 // always be in the first 4 bytes of the L4 header. Additionally for UDP we'll need access
432 // to the checksum field which is in bytes 7 and 8. While for TCP we'll need to read the
433 // TCP flags (at offset 13) and access to the checksum field (2 bytes at offset 16).
434 // As such we *always* need access to at least 8 bytes.
435 if (data + l2_header_size + sizeof(*ip) + 8 > data_end) TC_PUNT(SHORT_L4_HEADER);
436
437 struct tcphdr* tcph = is_tcp ? (void*)(ip + 1) : NULL;
438 struct udphdr* udph = is_tcp ? NULL : (void*)(ip + 1);
439
440 if (is_tcp) {
441 // Make sure we can get at the tcp header
442 if (data + l2_header_size + sizeof(*ip) + sizeof(*tcph) > data_end)
443 TC_PUNT(SHORT_TCP_HEADER);
444
445 // If hardware offload is running and programming flows based on conntrack entries, try not
446 // to interfere with it, so do not offload TCP packets with any one of the SYN/FIN/RST flags
447 if (tcph->syn || tcph->fin || tcph->rst) TC_PUNT(TCP_CONTROL_PACKET);
448 } else { // UDP
449 // Make sure we can get at the udp header
450 if (data + l2_header_size + sizeof(*ip) + sizeof(*udph) > data_end)
451 TC_PUNT(SHORT_UDP_HEADER);
452
453 // Skip handling of CHECKSUM_COMPLETE packets with udp checksum zero due to need for
454 // additional updating of skb->csum (this could be fixed up manually with more effort).
455 //
456 // Note that the in-kernel implementation of 'int64_t bpf_csum_update(skb, u32 csum)' is:
457 // if (skb->ip_summed == CHECKSUM_COMPLETE)
458 // return (skb->csum = csum_add(skb->csum, csum));
459 // else
460 // return -ENOTSUPP;
461 //
462 // So this will punt any CHECKSUM_COMPLETE packet with a zero UDP checksum,
463 // and leave all other packets unaffected (since it just at most adds zero to skb->csum).
464 //
465 // In practice this should almost never trigger because most nics do not generate
466 // CHECKSUM_COMPLETE packets on receive - especially so for nics/drivers on a phone.
467 //
468 // Additionally since we're forwarding, in most cases the value of the skb->csum field
469 // shouldn't matter (it's not used by physical nic egress).
470 //
471 // It only matters if we're ingressing through a CHECKSUM_COMPLETE capable nic
472 // and egressing through a virtual interface looping back to the kernel itself
473 // (ie. something like veth) where the CHECKSUM_COMPLETE/skb->csum can get reused
474 // on ingress.
475 //
476 // If we were in the kernel we'd simply probably call
477 // void skb_checksum_complete_unset(struct sk_buff *skb) {
478 // if (skb->ip_summed == CHECKSUM_COMPLETE) skb->ip_summed = CHECKSUM_NONE;
479 // }
480 // here instead. Perhaps there should be a bpf helper for that?
481 if (!udph->check && (bpf_csum_update(skb, 0) >= 0)) TC_PUNT(UDP_CSUM_ZERO);
482 }
483
484 Tether4Key k = {
485 .iif = skb->ifindex,
486 .l4Proto = ip->protocol,
487 .src4.s_addr = ip->saddr,
488 .dst4.s_addr = ip->daddr,
489 .srcPort = is_tcp ? tcph->source : udph->source,
490 .dstPort = is_tcp ? tcph->dest : udph->dest,
491 };
492 if (is_ethernet) __builtin_memcpy(k.dstMac, eth->h_dest, ETH_ALEN);
493
494 Tether4Value* v = downstream ? bpf_tether_downstream4_map_lookup_elem(&k)
495 : bpf_tether_upstream4_map_lookup_elem(&k);
496
497 // If we don't find any offload information then simply let the core stack handle it...
498 if (!v) return TC_ACT_OK;
499
500 uint32_t stat_and_limit_k = downstream ? skb->ifindex : v->oif;
501
502 TetherStatsValue* stat_v = bpf_tether_stats_map_lookup_elem(&stat_and_limit_k);
503
504 // If we don't have anywhere to put stats, then abort...
505 if (!stat_v) TC_PUNT(NO_STATS_ENTRY);
506
507 uint64_t* limit_v = bpf_tether_limit_map_lookup_elem(&stat_and_limit_k);
508
509 // If we don't have a limit, then abort...
510 if (!limit_v) TC_PUNT(NO_LIMIT_ENTRY);
511
512 // Required IPv4 minimum mtu is 68, below that not clear what we should do, abort...
513 if (v->pmtu < 68) TC_PUNT(BELOW_IPV4_MTU);
514
515 // Approximate handling of TCP/IPv4 overhead for incoming LRO/GRO packets: default
516 // outbound path mtu of 1500 is not necessarily correct, but worst case we simply
517 // undercount, which is still better then not accounting for this overhead at all.
518 // Note: this really shouldn't be device/path mtu at all, but rather should be
519 // derived from this particular connection's mss (ie. from gro segment size).
520 // This would require a much newer kernel with newer ebpf accessors.
521 // (This is also blindly assuming 12 bytes of tcp timestamp option in tcp header)
522 uint64_t packets = 1;
523 uint64_t bytes = skb->len;
524 if (bytes > v->pmtu) {
525 const int tcp_overhead = sizeof(struct iphdr) + sizeof(struct tcphdr) + 12;
526 const int mss = v->pmtu - tcp_overhead;
527 const uint64_t payload = bytes - tcp_overhead;
528 packets = (payload + mss - 1) / mss;
529 bytes = tcp_overhead * packets + payload;
530 }
531
532 // Are we past the limit? If so, then abort...
533 // Note: will not overflow since u64 is 936 years even at 5Gbps.
534 // Do not drop here. Offload is just that, whenever we fail to handle
535 // a packet we let the core stack deal with things.
536 // (The core stack needs to handle limits correctly anyway,
537 // since we don't offload all traffic in both directions)
538 if (stat_v->rxBytes + stat_v->txBytes + bytes > *limit_v) TC_PUNT(LIMIT_REACHED);
539
540 if (!is_ethernet) {
541 // Try to inject an ethernet header, and simply return if we fail.
542 // We do this even if TX interface is RAWIP and thus does not need an ethernet header,
543 // because this is easier and the kernel will strip extraneous ethernet header.
544 if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {
545 __sync_fetch_and_add(downstream ? &stat_v->rxErrors : &stat_v->txErrors, 1);
546 TC_PUNT(CHANGE_HEAD_FAILED);
547 }
548
549 // bpf_skb_change_head() invalidates all pointers - reload them
550 data = (void*)(long)skb->data;
551 data_end = (void*)(long)skb->data_end;
552 eth = data;
553 ip = (void*)(eth + 1);
554 tcph = is_tcp ? (void*)(ip + 1) : NULL;
555 udph = is_tcp ? NULL : (void*)(ip + 1);
556
557 // I do not believe this can ever happen, but keep the verifier happy...
558 if (data + sizeof(struct ethhdr) + sizeof(*ip) + (is_tcp ? sizeof(*tcph) : sizeof(*udph)) > data_end) {
559 __sync_fetch_and_add(downstream ? &stat_v->rxErrors : &stat_v->txErrors, 1);
560 TC_DROP(TOO_SHORT);
561 }
562 };
563
564 // At this point we always have an ethernet header - which will get stripped by the
565 // kernel during transmit through a rawip interface. ie. 'eth' pointer is valid.
566 // Additionally note that 'is_ethernet' and 'l2_header_size' are no longer correct.
567
568 // Overwrite any mac header with the new one
569 // For a rawip tx interface it will simply be a bunch of zeroes and later stripped.
570 *eth = v->macHeader;
571
572 // Decrement the IPv4 TTL, we already know it's greater than 1.
573 // u8 TTL field is followed by u8 protocol to make a u16 for ipv4 header checksum update.
574 // Since we're keeping the ipv4 checksum valid (which means the checksum of the entire
575 // ipv4 header remains 0), the overall checksum of the entire packet does not change.
576 const int sz2 = sizeof(__be16);
577 const __be16 old_ttl_proto = *(__be16 *)&ip->ttl;
578 const __be16 new_ttl_proto = old_ttl_proto - htons(0x0100);
579 bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_ttl_proto, new_ttl_proto, sz2);
580 bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(ttl), &new_ttl_proto, sz2, 0);
581
582 const int l4_offs_csum = is_tcp ? ETH_IP4_TCP_OFFSET(check) : ETH_IP4_UDP_OFFSET(check);
583 const int sz4 = sizeof(__be32);
584 // UDP 0 is special and stored as FFFF (this flag also causes a csum of 0 to be unmodified)
585 const int l4_flags = is_tcp ? 0 : BPF_F_MARK_MANGLED_0;
586 const __be32 old_daddr = k.dst4.s_addr;
587 const __be32 old_saddr = k.src4.s_addr;
588 const __be32 new_daddr = v->dst46.s6_addr32[3];
589 const __be32 new_saddr = v->src46.s6_addr32[3];
590
591 bpf_l4_csum_replace(skb, l4_offs_csum, old_daddr, new_daddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);
592 bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_daddr, new_daddr, sz4);
593 bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(daddr), &new_daddr, sz4, 0);
594
595 bpf_l4_csum_replace(skb, l4_offs_csum, old_saddr, new_saddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);
596 bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_saddr, new_saddr, sz4);
597 bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(saddr), &new_saddr, sz4, 0);
598
599 // The offsets for TCP and UDP ports: source (u16 @ L4 offset 0) & dest (u16 @ L4 offset 2) are
600 // actually the same, so the compiler should just optimize them both down to a constant.
601 bpf_l4_csum_replace(skb, l4_offs_csum, k.srcPort, v->srcPort, sz2 | l4_flags);
602 bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(source) : ETH_IP4_UDP_OFFSET(source),
603 &v->srcPort, sz2, 0);
604
605 bpf_l4_csum_replace(skb, l4_offs_csum, k.dstPort, v->dstPort, sz2 | l4_flags);
606 bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(dest) : ETH_IP4_UDP_OFFSET(dest),
607 &v->dstPort, sz2, 0);
608
609 // This requires the bpf_ktime_get_boot_ns() helper which was added in 5.8,
610 // and backported to all Android Common Kernel 4.14+ trees.
611 if (updatetime) v->last_used = bpf_ktime_get_boot_ns();
612
613 __sync_fetch_and_add(downstream ? &stat_v->rxPackets : &stat_v->txPackets, packets);
614 __sync_fetch_and_add(downstream ? &stat_v->rxBytes : &stat_v->txBytes, bytes);
615
616 // Redirect to forwarded interface.
617 //
618 // Note that bpf_redirect() cannot fail unless you pass invalid flags.
619 // The redirect actually happens after the ebpf program has already terminated,
620 // and can fail for example for mtu reasons at that point in time, but there's nothing
621 // we can do about it here.
622 return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
623 }
624
625 // Full featured (required) implementations for 5.8+ kernels (these are S+ by definition)
626
627 DEFINE_BPF_PROG_KVER("schedcls/tether_downstream4_rawip$5_8", AID_ROOT, AID_NETWORK_STACK,
628 sched_cls_tether_downstream4_rawip_5_8, KVER(5, 8, 0))
629 (struct __sk_buff* skb) {
630 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ true, /* updatetime */ true);
631 }
632
633 DEFINE_BPF_PROG_KVER("schedcls/tether_upstream4_rawip$5_8", AID_ROOT, AID_NETWORK_STACK,
634 sched_cls_tether_upstream4_rawip_5_8, KVER(5, 8, 0))
635 (struct __sk_buff* skb) {
636 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ false, /* updatetime */ true);
637 }
638
639 DEFINE_BPF_PROG_KVER("schedcls/tether_downstream4_ether$5_8", AID_ROOT, AID_NETWORK_STACK,
640 sched_cls_tether_downstream4_ether_5_8, KVER(5, 8, 0))
641 (struct __sk_buff* skb) {
642 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ true, /* updatetime */ true);
643 }
644
645 DEFINE_BPF_PROG_KVER("schedcls/tether_upstream4_ether$5_8", AID_ROOT, AID_NETWORK_STACK,
646 sched_cls_tether_upstream4_ether_5_8, KVER(5, 8, 0))
647 (struct __sk_buff* skb) {
648 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ false, /* updatetime */ true);
649 }
650
651 // Full featured (optional) implementations for 4.14-S, 4.19-S & 5.4-S kernels
652 // (optional, because we need to be able to fallback for 4.14/4.19/5.4 pre-S kernels)
653
654 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$opt",
655 AID_ROOT, AID_NETWORK_STACK,
656 sched_cls_tether_downstream4_rawip_opt,
657 KVER(4, 14, 0), KVER(5, 8, 0))
658 (struct __sk_buff* skb) {
659 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ true, /* updatetime */ true);
660 }
661
662 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$opt",
663 AID_ROOT, AID_NETWORK_STACK,
664 sched_cls_tether_upstream4_rawip_opt,
665 KVER(4, 14, 0), KVER(5, 8, 0))
666 (struct __sk_buff* skb) {
667 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ false, /* updatetime */ true);
668 }
669
670 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$opt",
671 AID_ROOT, AID_NETWORK_STACK,
672 sched_cls_tether_downstream4_ether_opt,
673 KVER(4, 14, 0), KVER(5, 8, 0))
674 (struct __sk_buff* skb) {
675 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ true, /* updatetime */ true);
676 }
677
678 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$opt",
679 AID_ROOT, AID_NETWORK_STACK,
680 sched_cls_tether_upstream4_ether_opt,
681 KVER(4, 14, 0), KVER(5, 8, 0))
682 (struct __sk_buff* skb) {
683 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ false, /* updatetime */ true);
684 }
685
686 // Partial (TCP-only: will not update 'last_used' field) implementations for 4.14+ kernels.
687 // These will be loaded only if the above optional ones failed (loading of *these* must succeed
688 // for 5.4+, since that is always an R patched kernel).
689 //
690 // [Note: as a result TCP connections will not have their conntrack timeout refreshed, however,
691 // since /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established defaults to 432000 (seconds),
692 // this in practice means they'll break only after 5 days. This seems an acceptable trade-off.
693 //
694 // Additionally kernel/tests change "net-test: add bpf_ktime_get_ns / bpf_ktime_get_boot_ns tests"
695 // which enforces and documents the required kernel cherrypicks will make it pretty unlikely that
696 // many devices upgrading to S will end up relying on these fallback programs.
697
698 // RAWIP: Required for 5.4-R kernels -- which always support bpf_skb_change_head().
699
700 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
701 sched_cls_tether_downstream4_rawip_5_4, KVER(5, 4, 0), KVER(5, 8, 0))
702 (struct __sk_buff* skb) {
703 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ true, /* updatetime */ false);
704 }
705
706 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
707 sched_cls_tether_upstream4_rawip_5_4, KVER(5, 4, 0), KVER(5, 8, 0))
708 (struct __sk_buff* skb) {
709 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ false, /* updatetime */ false);
710 }
711
712 // RAWIP: Optional for 4.14/4.19 (R) kernels -- which support bpf_skb_change_head().
713 // [Note: fallback for 4.14/4.19 (P/Q) kernels is below in stub section]
714
715 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$4_14",
716 AID_ROOT, AID_NETWORK_STACK,
717 sched_cls_tether_downstream4_rawip_4_14,
718 KVER(4, 14, 0), KVER(5, 4, 0))
719 (struct __sk_buff* skb) {
720 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ true, /* updatetime */ false);
721 }
722
723 DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$4_14",
724 AID_ROOT, AID_NETWORK_STACK,
725 sched_cls_tether_upstream4_rawip_4_14,
726 KVER(4, 14, 0), KVER(5, 4, 0))
727 (struct __sk_buff* skb) {
728 return do_forward4(skb, /* is_ethernet */ false, /* downstream */ false, /* updatetime */ false);
729 }
730
731 // ETHER: Required for 4.14-Q/R, 4.19-Q/R & 5.4-R kernels.
732
733 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$4_14", AID_ROOT, AID_NETWORK_STACK,
734 sched_cls_tether_downstream4_ether_4_14, KVER(4, 14, 0), KVER(5, 8, 0))
735 (struct __sk_buff* skb) {
736 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ true, /* updatetime */ false);
737 }
738
739 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$4_14", AID_ROOT, AID_NETWORK_STACK,
740 sched_cls_tether_upstream4_ether_4_14, KVER(4, 14, 0), KVER(5, 8, 0))
741 (struct __sk_buff* skb) {
742 return do_forward4(skb, /* is_ethernet */ true, /* downstream */ false, /* updatetime */ false);
743 }
744
745 // Placeholder (no-op) implementations for older Q kernels
746
747 // RAWIP: 4.9-P/Q, 4.14-P/Q & 4.19-Q kernels -- without bpf_skb_change_head() for tc programs
748
749 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
750 sched_cls_tether_downstream4_rawip_stub, KVER_NONE, KVER(5, 4, 0))
751 (struct __sk_buff* skb) {
752 return TC_ACT_OK;
753 }
754
755 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
756 sched_cls_tether_upstream4_rawip_stub, KVER_NONE, KVER(5, 4, 0))
757 (struct __sk_buff* skb) {
758 return TC_ACT_OK;
759 }
760
761 // ETHER: 4.9-P/Q kernel
762
763 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$stub", AID_ROOT, AID_NETWORK_STACK,
764 sched_cls_tether_downstream4_ether_stub, KVER_NONE, KVER(4, 14, 0))
765 (struct __sk_buff* skb) {
766 return TC_ACT_OK;
767 }
768
769 DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$stub", AID_ROOT, AID_NETWORK_STACK,
770 sched_cls_tether_upstream4_ether_stub, KVER_NONE, KVER(4, 14, 0))
771 (struct __sk_buff* skb) {
772 return TC_ACT_OK;
773 }
774
775 // ----- XDP Support -----
776
777 DEFINE_BPF_MAP_GRW(tether_dev_map, DEVMAP_HASH, uint32_t, uint32_t, 64, AID_NETWORK_STACK)
778
do_xdp_forward6(struct xdp_md * ctx,const bool is_ethernet,const bool downstream)779 static inline __always_inline int do_xdp_forward6(struct xdp_md *ctx, const bool is_ethernet,
780 const bool downstream) {
781 return XDP_PASS;
782 }
783
do_xdp_forward4(struct xdp_md * ctx,const bool is_ethernet,const bool downstream)784 static inline __always_inline int do_xdp_forward4(struct xdp_md *ctx, const bool is_ethernet,
785 const bool downstream) {
786 return XDP_PASS;
787 }
788
do_xdp_forward_ether(struct xdp_md * ctx,const bool downstream)789 static inline __always_inline int do_xdp_forward_ether(struct xdp_md *ctx, const bool downstream) {
790 const void* data = (void*)(long)ctx->data;
791 const void* data_end = (void*)(long)ctx->data_end;
792 const struct ethhdr* eth = data;
793
794 // Make sure we actually have an ethernet header
795 if ((void*)(eth + 1) > data_end) return XDP_PASS;
796
797 if (eth->h_proto == htons(ETH_P_IPV6))
798 return do_xdp_forward6(ctx, /* is_ethernet */ true, downstream);
799 if (eth->h_proto == htons(ETH_P_IP))
800 return do_xdp_forward4(ctx, /* is_ethernet */ true, downstream);
801
802 // Anything else we don't know how to handle...
803 return XDP_PASS;
804 }
805
do_xdp_forward_rawip(struct xdp_md * ctx,const bool downstream)806 static inline __always_inline int do_xdp_forward_rawip(struct xdp_md *ctx, const bool downstream) {
807 const void* data = (void*)(long)ctx->data;
808 const void* data_end = (void*)(long)ctx->data_end;
809
810 // The top nibble of both IPv4 and IPv6 headers is the IP version.
811 if (data_end - data < 1) return XDP_PASS;
812 const uint8_t v = (*(uint8_t*)data) >> 4;
813
814 if (v == 6) return do_xdp_forward6(ctx, /* is_ethernet */ false, downstream);
815 if (v == 4) return do_xdp_forward4(ctx, /* is_ethernet */ false, downstream);
816
817 // Anything else we don't know how to handle...
818 return XDP_PASS;
819 }
820
821 #define DEFINE_XDP_PROG(str, func) \
822 DEFINE_BPF_PROG_KVER(str, AID_ROOT, AID_NETWORK_STACK, func, KVER(5, 9, 0))(struct xdp_md *ctx)
823
824 DEFINE_XDP_PROG("xdp/tether_downstream_ether",
825 xdp_tether_downstream_ether) {
826 return do_xdp_forward_ether(ctx, /* downstream */ true);
827 }
828
829 DEFINE_XDP_PROG("xdp/tether_downstream_rawip",
830 xdp_tether_downstream_rawip) {
831 return do_xdp_forward_rawip(ctx, /* downstream */ true);
832 }
833
834 DEFINE_XDP_PROG("xdp/tether_upstream_ether",
835 xdp_tether_upstream_ether) {
836 return do_xdp_forward_ether(ctx, /* downstream */ false);
837 }
838
839 DEFINE_XDP_PROG("xdp/tether_upstream_rawip",
840 xdp_tether_upstream_rawip) {
841 return do_xdp_forward_rawip(ctx, /* downstream */ false);
842 }
843
844 LICENSE("Apache 2.0");
845 CRITICAL("tethering");
846