1From 8dd0a15e60cfee7e7f1be1ea051d0e09031f8fdd Mon Sep 17 00:00:00 2001 2From: jiangheng <jiangheng12@huawei.com> 3Date: Tue, 29 Mar 2022 21:33:17 +0800 4Subject: [PATCH] refactor event and add HW checksum offload 5 6--- 7 src/api/api_msg.c | 9 ++++ 8 src/api/posix_api.c | 2 + 9 src/api/sockets.c | 4 +- 10 src/core/ipv4/icmp.c | 13 +++++ 11 src/core/ipv4/ip4.c | 24 ++++++++- 12 src/core/ipv4/ip4_frag.c | 23 +++++++++ 13 src/core/pbuf.c | 9 +++- 14 src/core/tcp_in.c | 17 +++++++ 15 src/core/tcp_out.c | 72 +++++++++++++++++++++++++- 16 src/include/dpdk_cksum.h | 107 +++++++++++++++++++++++++++++++++++++++ 17 src/include/lwip/pbuf.h | 12 ++++- 18 src/include/lwipopts.h | 30 ++++++++--- 19 src/include/lwipsock.h | 18 +++---- 20 src/netif/ethernet.c | 8 +++ 21 14 files changed, 322 insertions(+), 26 deletions(-) 22 create mode 100644 src/include/dpdk_cksum.h 23 24diff --git a/src/api/api_msg.c b/src/api/api_msg.c 25index 3072dd9..672f022 100644 26--- a/src/api/api_msg.c 27+++ b/src/api/api_msg.c 28@@ -57,6 +57,7 @@ 29 #if USE_LIBOS 30 #include "lwip/sockets.h" 31 #include "lwipsock.h" 32+#include "posix_api.h" 33 #endif 34 35 #include <string.h> 36@@ -1758,7 +1759,15 @@ lwip_netconn_do_writemore(struct netconn *conn WRITE_DELAYED_PARAM) 37 } else { 38 write_more = 0; 39 } 40+#if USE_LIBOS 41+ /* vector->ptr is private arg sock */ 42+ LWIP_UNUSED_ARG(dataptr); 43+ write_more = 0; 44+ err = tcp_write(conn->pcb.tcp, conn->current_msg->msg.w.vector->ptr, len, apiflags); 45+ conn->current_msg->msg.w.len = len; 46+#else 47 err = tcp_write(conn->pcb.tcp, dataptr, len, apiflags); 48+#endif 49 if (err == ERR_OK) { 50 conn->current_msg->msg.w.offset += len; 51 conn->current_msg->msg.w.vector_off += len; 52diff --git a/src/api/posix_api.c b/src/api/posix_api.c 53index bce07f5..3f85bad 100644 54--- a/src/api/posix_api.c 55+++ b/src/api/posix_api.c 56@@ -42,6 +42,7 @@ 57 58 #include "lwip/err.h" 59 #include "lwipsock.h" 60+#include "posix_api.h" 61 62 posix_api_t *posix_api; 63 posix_api_t posix_api_val; 64@@ -64,6 +65,7 @@ void posix_api_fork(void) 65 posix_api->get_socket = chld_get_socket; 66 } 67 68+ 69 int posix_api_init(void) 70 { 71 /* the symbol we use here won't be NULL, so we don't need dlerror() 72diff --git a/src/api/sockets.c b/src/api/sockets.c 73index 21de5d9..3d94454 100644 74--- a/src/api/sockets.c 75+++ b/src/api/sockets.c 76@@ -65,6 +65,7 @@ 77 #if USE_LIBOS 78 #include <stdarg.h> 79 #include "lwipsock.h" 80+#include "posix_api.h" 81 #endif 82 83 #include <string.h> 84@@ -2682,9 +2683,6 @@ event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len) 85 check_waiters = 0; 86 } 87 sock->sendevent = 1; 88-#if USE_LIBOS 89- add_epoll_event(conn, EPOLLOUT); 90-#endif 91 break; 92 case NETCONN_EVT_SENDMINUS: 93 sock->sendevent = 0; 94diff --git a/src/core/ipv4/icmp.c b/src/core/ipv4/icmp.c 95index 59b493a..c58ae25 100644 96--- a/src/core/ipv4/icmp.c 97+++ b/src/core/ipv4/icmp.c 98@@ -51,6 +51,10 @@ 99 100 #include <string.h> 101 102+#if USE_LIBOS && CHECKSUM_GEN_IP_HW 103+#include "dpdk_cksum.h" 104+#endif 105+ 106 #ifdef LWIP_HOOK_FILENAME 107 #include LWIP_HOOK_FILENAME 108 #endif 109@@ -236,7 +240,16 @@ icmp_input(struct pbuf *p, struct netif *inp) 110 IPH_CHKSUM_SET(iphdr, 0); 111 #if CHECKSUM_GEN_IP 112 IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_GEN_IP) { 113+#if CHECKSUM_GEN_IP_HW 114+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { 115+ iph_cksum_set(p, hlen, 1); 116+ } else { 117+ iph_cksum_set(p, hlen, 0); 118+ IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, hlen)); 119+ } 120+#else 121 IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, hlen)); 122+#endif 123 } 124 #endif /* CHECKSUM_GEN_IP */ 125 126diff --git a/src/core/ipv4/ip4.c b/src/core/ipv4/ip4.c 127index c83afbe..1334cdc 100644 128--- a/src/core/ipv4/ip4.c 129+++ b/src/core/ipv4/ip4.c 130@@ -59,6 +59,10 @@ 131 132 #include <string.h> 133 134+#if USE_LIBOS && (CHECKSUM_CHECK_IP_HW || CHECKSUM_GEN_IP_HW) 135+#include "dpdk_cksum.h" 136+#endif 137+ 138 #ifdef LWIP_HOOK_FILENAME 139 #include LWIP_HOOK_FILENAME 140 #endif 141@@ -503,8 +507,17 @@ ip4_input(struct pbuf *p, struct netif *inp) 142 /* verify checksum */ 143 #if CHECKSUM_CHECK_IP 144 IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_IP) { 145+#if CHECKSUM_CHECK_IP_HW 146+ u64_t ret; 147+ if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_IPV4_CKSUM) { 148+ ret = is_cksum_ipbad(p); 149+ } else { 150+ ret = (u64_t)inet_chksum(iphdr, iphdr_hlen); 151+ } 152+ if (ret != 0) { 153+#else 154 if (inet_chksum(iphdr, iphdr_hlen) != 0) { 155- 156+#endif 157 LWIP_DEBUGF(IP_DEBUG | LWIP_DBG_LEVEL_SERIOUS, 158 ("Checksum (0x%"X16_F") failed, IP packet dropped.\n", inet_chksum(iphdr, iphdr_hlen))); 159 ip4_debug_print(p); 160@@ -972,7 +985,16 @@ ip4_output_if_opt_src(struct pbuf *p, const ip4_addr_t *src, const ip4_addr_t *d 161 IPH_CHKSUM_SET(iphdr, 0); 162 #if CHECKSUM_GEN_IP 163 IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) { 164+#if CHECKSUM_GEN_IP_HW 165+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { 166+ iph_cksum_set(p, ip_hlen, 1); 167+ } else { 168+ iph_cksum_set(p, ip_hlen, 0); 169+ IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, ip_hlen)); 170+ } 171+#else 172 IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, ip_hlen)); 173+#endif 174 } 175 #endif /* CHECKSUM_GEN_IP */ 176 #endif /* CHECKSUM_GEN_IP_INLINE */ 177diff --git a/src/core/ipv4/ip4_frag.c b/src/core/ipv4/ip4_frag.c 178index a445530..17a4ccd 100644 179--- a/src/core/ipv4/ip4_frag.c 180+++ b/src/core/ipv4/ip4_frag.c 181@@ -51,6 +51,10 @@ 182 183 #include <string.h> 184 185+#if USE_LIBOS && CHECKSUM_GEN_IP_HW 186+#include "dpdk_cksum.h" 187+#endif 188+ 189 #if IP_REASSEMBLY 190 /** 191 * The IP reassembly code currently has the following limitations: 192@@ -632,8 +636,17 @@ ip4_reass(struct pbuf *p) 193 /* @todo: do we need to set/calculate the correct checksum? */ 194 #if CHECKSUM_GEN_IP 195 IF__NETIF_CHECKSUM_ENABLED(ip_current_input_netif(), NETIF_CHECKSUM_GEN_IP) { 196+#if CHECKSUM_GEN_IP_HW 197+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { 198+ iph_cksum_set(p, IP_HLEN, 1); 199+ } else { 200+ iph_cksum_set(p, IP_HLEN, 0); 201 IPH_CHKSUM_SET(fraghdr, inet_chksum(fraghdr, IP_HLEN)); 202 } 203+#else 204+ IPH_CHKSUM_SET(fraghdr, inet_chksum(fraghdr, IP_HLEN)); 205+#endif 206+ } 207 #endif /* CHECKSUM_GEN_IP */ 208 209 p = ipr->p; 210@@ -862,8 +875,18 @@ ip4_frag(struct pbuf *p, struct netif *netif, const ip4_addr_t *dest) 211 IPH_CHKSUM_SET(iphdr, 0); 212 #if CHECKSUM_GEN_IP 213 IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_IP) { 214+#if CHECKSUM_GEN_IP_HW 215+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_IPV4_CKSUM) { 216+ iph_cksum_set(p, IP_HLEN, 1); 217+ } else { 218+ iph_cksum_set(p, IP_HLEN, 0); 219 IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, IP_HLEN)); 220 } 221+ 222+#else 223+ IPH_CHKSUM_SET(iphdr, inet_chksum(iphdr, IP_HLEN)); 224+#endif 225+ } 226 #endif /* CHECKSUM_GEN_IP */ 227 228 /* No need for separate header pbuf - we allowed room for it in rambuf 229diff --git a/src/core/pbuf.c b/src/core/pbuf.c 230index cd6b558..247681d 100644 231--- a/src/core/pbuf.c 232+++ b/src/core/pbuf.c 233@@ -282,7 +282,7 @@ pbuf_alloc(pbuf_layer layer, u16_t length, pbuf_type type) 234 235 /* If pbuf is to be allocated in RAM, allocate memory for it. */ 236 #if USE_LIBOS 237- /* alloc mbuf to reduce copy in sending */ 238+ /* alloc mbuf avoid send copy */ 239 p = lwip_alloc_pbuf(layer, length, type); 240 #else 241 p = (struct pbuf *)mem_malloc(alloc_len); 242@@ -1019,6 +1019,13 @@ pbuf_copy_partial_pbuf(struct pbuf *p_to, const struct pbuf *p_from, u16_t copy_ 243 /* current p_from does not fit into current p_to */ 244 len_calc = p_to->len - offset_to; 245 } 246+ 247+#if USE_LIBOS && (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW) 248+ p_to->l2_len = p_from->l2_len; 249+ p_to->l3_len = p_from->l3_len; 250+ p_to->ol_flags = p_from->ol_flags; 251+#endif 252+ 253 len = (u16_t)LWIP_MIN(copy_len, len_calc); 254 MEMCPY((u8_t *)p_to->payload + offset_to, (u8_t *)p_from->payload + offset_from, len); 255 offset_to += len; 256diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c 257index 0d3a2f1..b1bbe00 100644 258--- a/src/core/tcp_in.c 259+++ b/src/core/tcp_in.c 260@@ -64,6 +64,10 @@ 261 262 #include <string.h> 263 264+#if USE_LIBOS && CHECKSUM_CHECK_TCP_HW 265+#include <dpdk_cksum.h> 266+#endif /* CHECKSUM_CHECK_TCP_HW */ 267+ 268 #ifdef LWIP_HOOK_FILENAME 269 #include LWIP_HOOK_FILENAME 270 #endif 271@@ -172,11 +176,24 @@ tcp_input(struct pbuf *p, struct netif *inp) 272 #if CHECKSUM_CHECK_TCP 273 IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_TCP) { 274 /* Verify TCP checksum. */ 275+#if CHECKSUM_CHECK_TCP_HW 276+ u64_t ret; 277+ if (get_eth_params_rx_ol() & DEV_RX_OFFLOAD_TCP_CKSUM) { 278+ ret = is_cksum_tcpbad(p); 279+ } else { 280+ ret = (u64_t)ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, 281+ ip_current_src_addr(), ip_current_dest_addr()); 282+ 283+ } 284+ if (ret != 0) { 285+ LWIP_DEBUGF(TCP_INPUT_DEBUG, ("tcp_input: packet discarded due to failing checksum\n")); 286+#else 287 u16_t chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, 288 ip_current_src_addr(), ip_current_dest_addr()); 289 if (chksum != 0) { 290 LWIP_DEBUGF(TCP_INPUT_DEBUG, ("tcp_input: packet discarded due to failing checksum 0x%04"X16_F"\n", 291 chksum)); 292+#endif 293 tcp_debug_print(tcphdr); 294 TCP_STATS_INC(tcp.chkerr); 295 goto dropped; 296diff --git a/src/core/tcp_out.c b/src/core/tcp_out.c 297index b99974d..1b0af8d 100644 298--- a/src/core/tcp_out.c 299+++ b/src/core/tcp_out.c 300@@ -80,6 +80,13 @@ 301 302 #include <string.h> 303 304+#if USE_LIBOS 305+#include "lwipsock.h" 306+#if CHECKSUM_GEN_TCP_HW 307+#include "dpdk_cksum.h" 308+#endif 309+#endif 310+ 311 #ifdef LWIP_HOOK_FILENAME 312 #include LWIP_HOOK_FILENAME 313 #endif 314@@ -660,8 +667,11 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags) 315 pbuf_cat(p/*header*/, p2/*data*/); 316 } 317 #else /* USE_LIBOS */ 318- p = (struct pbuf *)arg; 319- seglen = p->len; 320+ p = write_lwip_data((struct lwip_sock *)arg, len - pos, &apiflags); 321+ if (p == NULL) { 322+ break; 323+ } 324+ seglen = p->tot_len; 325 #endif /* USE_LIBOS */ 326 327 queuelen += pbuf_clen(p); 328@@ -789,8 +799,13 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags) 329 /* 330 * Finally update the pcb state. 331 */ 332+#if USE_LIBOS 333+ pcb->snd_lbb += pos; 334+ pcb->snd_buf -= pos; 335+#else 336 pcb->snd_lbb += len; 337 pcb->snd_buf -= len; 338+#endif 339 pcb->snd_queuelen = queuelen; 340 341 LWIP_DEBUGF(TCP_QLEN_DEBUG, ("tcp_write: %"S16_F" (after enqueued)\n", 342@@ -1584,6 +1599,11 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif 343 344 #if CHECKSUM_GEN_TCP 345 IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) { 346+#if CHECKSUM_GEN_TCP_HW 347+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) { 348+ tcph_cksum_set(seg->p, TCP_HLEN); 349+ seg->tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP,seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); 350+ } else { 351 #if TCP_CHECKSUM_ON_COPY 352 u32_t acc; 353 #if TCP_CHECKSUM_ON_COPY_SANITY_CHECK 354@@ -1618,6 +1638,44 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif 355 seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP, 356 seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); 357 #endif /* TCP_CHECKSUM_ON_COPY */ 358+ 359+ } 360+#else 361+#if TCP_CHECKSUM_ON_COPY 362+ u32_t acc; 363+#if TCP_CHECKSUM_ON_COPY_SANITY_CHECK 364+ u16_t chksum_slow = ip_chksum_pseudo(seg->p, IP_PROTO_TCP, 365+ seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); 366+#endif /* TCP_CHECKSUM_ON_COPY_SANITY_CHECK */ 367+ if ((seg->flags & TF_SEG_DATA_CHECKSUMMED) == 0) { 368+ LWIP_ASSERT("data included but not checksummed", 369+ seg->p->tot_len == TCPH_HDRLEN_BYTES(seg->tcphdr)); 370+ } 371+ 372+ /* rebuild TCP header checksum (TCP header changes for retransmissions!) */ 373+ acc = ip_chksum_pseudo_partial(seg->p, IP_PROTO_TCP, 374+ seg->p->tot_len, TCPH_HDRLEN_BYTES(seg->tcphdr), &pcb->local_ip, &pcb->remote_ip); 375+ /* add payload checksum */ 376+ if (seg->chksum_swapped) { 377+ seg_chksum_was_swapped = 1; 378+ seg->chksum = SWAP_BYTES_IN_WORD(seg->chksum); 379+ seg->chksum_swapped = 0; 380+ } 381+ acc = (u16_t)~acc + seg->chksum; 382+ seg->tcphdr->chksum = (u16_t)~FOLD_U32T(acc); 383+#if TCP_CHECKSUM_ON_COPY_SANITY_CHECK 384+ if (chksum_slow != seg->tcphdr->chksum) { 385+ TCP_CHECKSUM_ON_COPY_SANITY_CHECK_FAIL( 386+ ("tcp_output_segment: calculated checksum is %"X16_F" instead of %"X16_F"\n", 387+ seg->tcphdr->chksum, chksum_slow)); 388+ seg->tcphdr->chksum = chksum_slow; 389+ } 390+#endif /* TCP_CHECKSUM_ON_COPY_SANITY_CHECK */ 391+#else /* TCP_CHECKSUM_ON_COPY */ 392+ seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP, 393+ seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip); 394+#endif /* TCP_CHECKSUM_ON_COPY */ 395+#endif /* CHECKSUM_GEN_TCP_HW */ 396 } 397 #endif /* CHECKSUM_GEN_TCP */ 398 TCP_STATS_INC(tcp.xmit); 399@@ -1959,8 +2017,18 @@ tcp_output_control_segment(const struct tcp_pcb *pcb, struct pbuf *p, 400 #if CHECKSUM_GEN_TCP 401 IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) { 402 struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload; 403+#if CHECKSUM_GEN_TCP_HW 404+ if (get_eth_params_tx_ol() & DEV_TX_OFFLOAD_TCP_CKSUM) { 405+ tcph_cksum_set(p, TCP_HLEN); 406+ tcphdr->chksum = ip_chksum_pseudo_offload(IP_PROTO_TCP, p->tot_len, src, dst); 407+ } else { 408+ tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, 409+ src, dst); 410+ } 411+#else 412 tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len, 413 src, dst); 414+#endif 415 } 416 #endif 417 if (pcb != NULL) { 418diff --git a/src/include/dpdk_cksum.h b/src/include/dpdk_cksum.h 419new file mode 100644 420index 0000000..e57be4d 421--- /dev/null 422+++ b/src/include/dpdk_cksum.h 423@@ -0,0 +1,107 @@ 424+/* 425+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science. 426+ * All rights reserved. 427+ * 428+ * Redistribution and use in source and binary forms, with or without modification, 429+ * are permitted provided that the following conditions are met: 430+ * 431+ * 1. Redistributions of source code must retain the above copyright notice, 432+ * this list of conditions and the following disclaimer. 433+ * 2. Redistributions in binary form must reproduce the above copyright notice, 434+ * this list of conditions and the following disclaimer in the documentation 435+ * and/or other materials provided with the distribution. 436+ * 3. The name of the author may not be used to endorse or promote products 437+ * derived from this software without specific prior written permission. 438+ * 439+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 440+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 441+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 442+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 443+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 444+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 445+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 446+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 447+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 448+ * OF SUCH DAMAGE. 449+ * 450+ * This file is part of the lwIP TCP/IP stack. 451+ * 452+ * Author: Huawei Technologies 453+ * 454+ */ 455+ 456+#ifndef __DPDK_CKSUM_H__ 457+#define __DPDK_CKSUM_H__ 458+ 459+#include "lwipopts.h" 460+#if USE_LIBOS 461+#include <rte_ethdev.h> 462+ 463+#if CHECKSUM_OFFLOAD_ALL 464+#include <rte_mbuf_core.h> 465+#include "lwip/pbuf.h" 466+#endif 467+ 468+extern uint64_t get_eth_params_rx_ol(void); 469+extern uint64_t get_eth_params_tx_ol(void); 470+#if CHECKSUM_CHECK_IP_HW 471+// for ip4_input 472+static inline u64_t is_cksum_ipbad(struct pbuf *p) { 473+ return p->ol_flags & (RTE_MBUF_F_RX_IP_CKSUM_BAD); 474+} 475+#endif /* CHECKSUM_CHECK_IP_HW */ 476+ 477+#if CHECKSUM_CHECK_TCP_HW 478+// for tcp_input 479+static inline u64_t is_cksum_tcpbad(struct pbuf *p) { 480+ return p->ol_flags & (RTE_MBUF_F_RX_L4_CKSUM_BAD); 481+} 482+#endif /* CHECKSUM_CHECK_TCP_HW */ 483+ 484+#if CHECKSUM_GEN_IP_HW 485+static inline void ethh_cksum_set(struct pbuf *p, u16_t len) { 486+ p->l2_len = len; 487+} 488+ 489+// replaces IPH_CHKSUM_SET 490+static inline void iph_cksum_set(struct pbuf *p, u16_t len, bool do_ipcksum) { 491+ p->ol_flags |= RTE_MBUF_F_TX_IPV4; 492+ if (do_ipcksum) { 493+ p->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM; 494+ } 495+ p->l3_len = len; 496+} 497+#endif /* CHECKSUM_GEN_IP_HW */ 498+ 499+// replace ip_chksum_pseudo 500+#if CHECKSUM_GEN_TCP_HW 501+#include <rte_ip.h> 502+ 503+static inline void tcph_cksum_set(struct pbuf *p, u16_t len) { 504+ (void)len; 505+ p->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM; 506+} 507+ 508+static inline u16_t ip_chksum_pseudo_offload(u8_t proto, u16_t proto_len, 509+ const ip_addr_t *src, const ip_addr_t *dst) 510+{ 511+ struct ipv4_psd_header { 512+ uint32_t src_addr; /* IP address of source host. */ 513+ uint32_t dst_addr; /* IP address of destination host. */ 514+ uint8_t zero; /* zero. */ 515+ uint8_t proto; /* L4 protocol type. */ 516+ uint16_t len; /* L4 length. */ 517+ } psd_hdr; 518+ 519+ psd_hdr.src_addr = ip4_addr_get_u32(src); 520+ psd_hdr.dst_addr = ip4_addr_get_u32(dst); 521+ psd_hdr.proto = proto; 522+ psd_hdr.len = lwip_htons(proto_len); 523+ psd_hdr.zero = 0; 524+ 525+ return rte_raw_cksum(&psd_hdr, sizeof(psd_hdr)); 526+} 527+#endif /* CHECKSUM_GEN_TCP_HW */ 528+ 529+#endif /* USE_LIBOS */ 530+#endif /* __DPDK_CKSUM_H__ */ 531diff --git a/src/include/lwip/pbuf.h b/src/include/lwip/pbuf.h 532index 3894574..87cd960 100644 533--- a/src/include/lwip/pbuf.h 534+++ b/src/include/lwip/pbuf.h 535@@ -220,6 +220,15 @@ struct pbuf { 536 /** For incoming packets, this contains the input netif's index */ 537 u8_t if_idx; 538 539+#if USE_LIBOS && CHECKSUM_OFFLOAD_ALL 540+ /** checksum offload ol_flags */ 541+ u64_t ol_flags; 542+ /** checksum offload l2_len */ 543+ u64_t l2_len:7; 544+ /** checksum offload l3_len */ 545+ u64_t l3_len:9; 546+#endif /* USE_LIBOS CHECKSUM_OFFLOAD_SWITCH */ 547+ 548 /** In case the user needs to store data custom data on a pbuf */ 549 LWIP_PBUF_CUSTOM_DATA 550 }; 551@@ -271,9 +280,8 @@ void pbuf_free_ooseq(void); 552 553 /* Initializes the pbuf module. This call is empty for now, but may not be in future. */ 554 #define pbuf_init() 555- 556 #if USE_LIBOS 557-struct pbuf *lwip_alloc_pbuf(pbuf_layer l, u16_t length, pbuf_type type); 558+struct pbuf *lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type); 559 #endif 560 struct pbuf *pbuf_alloc(pbuf_layer l, u16_t length, pbuf_type type); 561 struct pbuf *pbuf_alloc_reference(void *payload, u16_t length, pbuf_type type); 562diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h 563index e0364a2..df587c0 100644 564--- a/src/include/lwipopts.h 565+++ b/src/include/lwipopts.h 566@@ -129,14 +129,6 @@ 567 568 #define LWIP_STATS_DISPLAY 1 569 570-#define CHECKSUM_GEN_IP 1 /* master switch */ 571- 572-#define CHECKSUM_GEN_TCP 1 /* master switch */ 573- 574-#define CHECKSUM_CHECK_IP 1 /* master switch */ 575- 576-#define CHECKSUM_CHECK_TCP 1 /* master switch */ 577- 578 #define LWIP_TIMEVAL_PRIVATE 0 579 580 #define USE_LIBOS 1 581@@ -177,6 +169,28 @@ 582 583 #define ARP_TABLE_SIZE 512 584 585+/* --------------------------------------- 586+ * ------- NIC offloads -------- 587+ * --------------------------------------- 588+ */ 589+#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 /* checksum ability check before checksum*/ 590+ 591+// rx cksum 592+#define CHECKSUM_CHECK_IP 1 /* master switch */ 593+#define CHECKSUM_CHECK_TCP 1 /* master switch */ 594+// tx cksum 595+#define CHECKSUM_GEN_IP 1 /* master switch */ 596+#define CHECKSUM_GEN_TCP 1 /* master switch */ 597+ 598+// rx offload cksum 599+#define CHECKSUM_CHECK_IP_HW (1 && CHECKSUM_CHECK_IP) /* hardware switch */ 600+#define CHECKSUM_CHECK_TCP_HW (1 && CHECKSUM_CHECK_TCP) /* hardware switch */ 601+// tx offload cksum 602+#define CHECKSUM_GEN_IP_HW (1 && CHECKSUM_GEN_IP) /* hardware switch */ 603+#define CHECKSUM_GEN_TCP_HW (1 && CHECKSUM_GEN_TCP) /* hardware switch */ 604+ 605+#define CHECKSUM_OFFLOAD_ALL (CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW || CHECKSUM_CHECK_IP_HW || CHECKSUM_CHECK_TCP_HW) 606+ 607 #if USE_LIBOS 608 #define PER_THREAD __thread 609 #else 610diff --git a/src/include/lwipsock.h b/src/include/lwipsock.h 611index 36bcaed..eec4e8e 100644 612--- a/src/include/lwipsock.h 613+++ b/src/include/lwipsock.h 614@@ -36,7 +36,6 @@ 615 #include "lwip/opt.h" 616 #include "lwip/api.h" 617 618-#include "posix_api.h" 619 #include "eventpoll.h" 620 621 /* move some definitions to the lwipsock.h for libnet to use, and 622@@ -62,7 +61,8 @@ union lwip_sock_lastdata { 623 624 #if USE_LIBOS 625 struct protocol_stack; 626-struct weakup_poll; 627+struct wakeup_poll; 628+struct rte_ring; 629 #endif 630 /** Contains all internal pointers and states used for a socket */ 631 struct lwip_sock { 632@@ -93,16 +93,16 @@ struct lwip_sock { 633 634 #if USE_LIBOS 635 uint32_t epoll_events; /* registered events */ 636- uint32_t events; /* available events */ 637- volatile bool have_event; /* avoid recurring events */ 638- volatile bool have_rpc_send; /* avoid recurring rpc_send */ 639+ volatile uint32_t events; /* available events */ 640 epoll_data_t ep_data; 641- struct weakup_poll *weakup; 642+ struct wakeup_poll *wakeup; 643 struct protocol_stack *stack; 644- void *recv_ring; 645+ struct rte_ring *recv_ring; 646+ struct rte_ring *recv_wait_free; 647 struct pbuf *recv_lastdata; /* unread data in one pbuf */ 648 struct pbuf *send_lastdata; /* unread data in one pbuf */ 649- void *send_ring; 650+ struct rte_ring *send_ring; 651+ struct rte_ring *send_idle_ring; 652 int32_t recv_flags; 653 int32_t send_flags; 654 bool wait_close; 655@@ -112,7 +112,6 @@ struct lwip_sock { 656 struct list_node listen_list; 657 struct list_node recv_list; 658 struct list_node event_list; 659- struct list_node wakeup_list; 660 struct list_node send_list; 661 int32_t nextfd; /* listenfd list */ 662 #endif 663@@ -160,6 +159,7 @@ get_socket_without_errno(int s) 664 665 extern void add_recv_list(int32_t fd); 666 extern ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags); 667+extern struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags); 668 extern void gazelle_clean_sock(int32_t fd); 669 extern void gazelle_init_sock(int32_t fd); 670 #endif /* USE_LIBOS */ 671diff --git a/src/netif/ethernet.c b/src/netif/ethernet.c 672index dd171e2..ab976a8 100644 673--- a/src/netif/ethernet.c 674+++ b/src/netif/ethernet.c 675@@ -56,6 +56,10 @@ 676 #include "netif/ppp/pppoe.h" 677 #endif /* PPPOE_SUPPORT */ 678 679+#if USE_LIBOS && (CHECKSUM_GEN_TCP_HW || CHECKSUM_GEN_IP_HW) 680+#include "dpdk_cksum.h" 681+#endif 682+ 683 #ifdef LWIP_HOOK_FILENAME 684 #include LWIP_HOOK_FILENAME 685 #endif 686@@ -308,6 +312,10 @@ ethernet_output(struct netif * netif, struct pbuf * p, 687 LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, 688 ("ethernet_output: sending packet %p\n", (void *)p)); 689 690+#if CHECKSUM_GEN_IP_HW || CHECKSUM_GEN_TCP_HW 691+ ethh_cksum_set(p, sizeof(*ethhdr)); 692+#endif 693+ 694 /* send the packet */ 695 return netif->linkoutput(netif, p); 696 697-- 6982.23.0 699