64 static inline __always_inline int do_forward6(struct __sk_buff* skb,
71     if (skb->protocol != htons(ETH_P_IPV6)) return TC_ACT_PIPE;
74     if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;
81     if (bpf_skb_pull_data(skb, l2_header_size + IP6_HLEN)) return TC_ACT_PIPE;
83     void* data = (void*)(long)skb->data;
84     const void* data_end = (void*)(long)skb->data_end;
105         bpf_skb_pull_data(skb, l2_header_size + IP6_HLEN + TCP_HLEN);
107         data = (void*)(long)skb->data;
108         data_end = (void*)(long)skb->data_end;
139             .iif = skb->ifindex,
144             .iif = skb->ifindex,
156     uint32_t stat_and_limit_k = stream.down ? skb->ifindex : v->oif;
179     uint64_t L3_bytes = skb->len - l2_header_size;
200         if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {
206         data = (void*)(long)skb->data;
207         data_end = (void*)(long)skb->data_end;
231     bpf_csum_update(skb, 0xFFFF - ntohs(old_hl) + ntohs(new_hl));
251 (struct __sk_buff* skb) {
252     return do_forward6(skb, ETHER, DOWNSTREAM, KVER_NONE);
257 (struct __sk_buff* skb) {
258     return do_forward6(skb, ETHER, UPSTREAM, KVER_NONE);
277 (struct __sk_buff* skb) {
278     return do_forward6(skb, RAWIP, DOWNSTREAM, KVER_4_14);
283 (struct __sk_buff* skb) {
284     return do_forward6(skb, RAWIP, UPSTREAM, KVER_4_14);
290 (__unused struct __sk_buff* skb) {
296 (__unused struct __sk_buff* skb) {
306 static inline __always_inline int do_forward4_bottom(struct __sk_buff* skb,  in do_forward4_bottom()  argument
356         if (!udph->check && (bpf_csum_update(skb, 0) >= 0)) TC_PUNT(UDP_CSUM_ZERO);  in do_forward4_bottom()
360             .iif = skb->ifindex,  in do_forward4_bottom()
375     uint32_t stat_and_limit_k = stream.down ? skb->ifindex : v->oif;  in do_forward4_bottom()
398     uint64_t L3_bytes = skb->len - l2_header_size;  in do_forward4_bottom()
419         if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {  in do_forward4_bottom()
425         data = (void*)(long)skb->data;  in do_forward4_bottom()
426         data_end = (void*)(long)skb->data_end;  in do_forward4_bottom()
454     bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_ttl_proto, new_ttl_proto, sz2);  in do_forward4_bottom()
455     bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(ttl), &new_ttl_proto, sz2, 0);  in do_forward4_bottom()
466     bpf_l4_csum_replace(skb, l4_offs_csum, old_daddr, new_daddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);  in do_forward4_bottom()
467     bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_daddr, new_daddr, sz4);  in do_forward4_bottom()
468     bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(daddr), &new_daddr, sz4, 0);  in do_forward4_bottom()
470     bpf_l4_csum_replace(skb, l4_offs_csum, old_saddr, new_saddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);  in do_forward4_bottom()
471     bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_saddr, new_saddr, sz4);  in do_forward4_bottom()
472     bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(saddr), &new_saddr, sz4, 0);  in do_forward4_bottom()
476     bpf_l4_csum_replace(skb, l4_offs_csum, k.srcPort, v->srcPort, sz2 | l4_flags);  in do_forward4_bottom()
477     bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(source) : ETH_IP4_UDP_OFFSET(source),  in do_forward4_bottom()
480     bpf_l4_csum_replace(skb, l4_offs_csum, k.dstPort, v->dstPort, sz2 | l4_flags);  in do_forward4_bottom()
481     bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(dest) : ETH_IP4_UDP_OFFSET(dest),  in do_forward4_bottom()
500 static inline __always_inline int do_forward4(struct __sk_buff* skb,  in do_forward4()  argument
508     if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;  in do_forward4()
511     if (skb->protocol != htons(ETH_P_IP)) return TC_ACT_PIPE;  in do_forward4()
519     try_make_writable(skb, l2_header_size + IP4_HLEN + TCP_HLEN);  in do_forward4()
521     void* data = (void*)(long)skb->data;  in do_forward4()
522     const void* data_end = (void*)(long)skb->data_end;  in do_forward4()
593       return do_forward4_bottom(skb, l2_header_size, data, data_end, eth, ip,  in do_forward4()
596       return do_forward4_bottom(skb, l2_header_size, data, data_end, eth, ip,  in do_forward4()
605 (struct __sk_buff* skb) {
606     return do_forward4(skb, RAWIP, DOWNSTREAM, UPDATETIME, KVER_5_10);
611 (struct __sk_buff* skb) {
612     return do_forward4(skb, RAWIP, UPSTREAM, UPDATETIME, KVER_5_10);
617 (struct __sk_buff* skb) {
618     return do_forward4(skb, ETHER, DOWNSTREAM, UPDATETIME, KVER_5_10);
623 (struct __sk_buff* skb) {
624     return do_forward4(skb, ETHER, UPSTREAM, UPDATETIME, KVER_5_10);
634 (struct __sk_buff* skb) {
635     return do_forward4(skb, RAWIP, DOWNSTREAM, UPDATETIME, KVER_4_14);
642 (struct __sk_buff* skb) {
643     return do_forward4(skb, RAWIP, UPSTREAM, UPDATETIME, KVER_4_14);
650 (struct __sk_buff* skb) {
651     return do_forward4(skb, ETHER, DOWNSTREAM, UPDATETIME, KVER_4_14);
658 (struct __sk_buff* skb) {
659     return do_forward4(skb, ETHER, UPSTREAM, UPDATETIME, KVER_4_14);
678 (struct __sk_buff* skb) {
679     return do_forward4(skb, RAWIP, DOWNSTREAM, NO_UPDATETIME, KVER_5_4);
684 (struct __sk_buff* skb) {
685     return do_forward4(skb, RAWIP, UPSTREAM, NO_UPDATETIME, KVER_5_4);
695 (struct __sk_buff* skb) {
696     return do_forward4(skb, RAWIP, DOWNSTREAM, NO_UPDATETIME, KVER_4_14);
703 (struct __sk_buff* skb) {
704     return do_forward4(skb, RAWIP, UPSTREAM, NO_UPDATETIME, KVER_4_14);
711 (struct __sk_buff* skb) {
712     return do_forward4(skb, ETHER, DOWNSTREAM, NO_UPDATETIME, KVER_4_14);
717 (struct __sk_buff* skb) {
718     return do_forward4(skb, ETHER, UPSTREAM, NO_UPDATETIME, KVER_4_14);
727 (__unused struct __sk_buff* skb) {
733 (__unused struct __sk_buff* skb) {
741 (__unused struct __sk_buff* skb) {
747 (__unused struct __sk_buff* skb) {