1From 28f8ba80cd733e14e0540c414a18134b3c3fcc94 Mon Sep 17 00:00:00 2001 2From: FanBin <fanbin12@huawei.com> 3Date: Wed, 15 Feb 2023 10:09:39 +0800 4Subject: [PATCH] lwip reuse ip port 5 6--- 7 src/core/tcp.c | 40 +++++++++++++++++++++++++++++--- 8 src/core/tcp_in.c | 32 +++++++++++++++++++++++++ 9 src/include/lwip/api.h | 4 ++++ 10 src/include/lwip/priv/tcp_priv.h | 19 +++++++++++++++ 11 src/include/lwip/tcp.h | 8 +++++++ 12 src/include/lwipopts.h | 4 ++++ 13 6 files changed, 104 insertions(+), 3 deletions(-) 14 15diff --git a/src/core/tcp.c b/src/core/tcp.c 16index f75d214..3171c5e 100644 17--- a/src/core/tcp.c 18+++ b/src/core/tcp.c 19@@ -111,6 +111,7 @@ 20 #include "lwip/ip6.h" 21 #include "lwip/ip6_addr.h" 22 #include "lwip/nd6.h" 23+#include "lwip/api.h" 24 25 #include <string.h> 26 #include <pthread.h> 27@@ -772,6 +773,9 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port) 28 /* Check if the address already is in use (on all lists) */ 29 for (i = 0; i < max_pcb_list; i++) { 30 for (cpcb = *tcp_pcb_lists[i]; cpcb != NULL; cpcb = cpcb->next) { 31+#if REUSE_IPPORT 32+ continue; 33+#else 34 if (cpcb->local_port == port) { 35 #if SO_REUSE 36 /* Omit checking for the same port if both pcbs have REUSEADDR set. 37@@ -790,6 +794,7 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port) 38 } 39 } 40 } 41+#endif /* REUSE_IPORT */ 42 } 43 } 44 } 45@@ -921,7 +926,18 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) 46 res = ERR_ALREADY; 47 goto done; 48 } 49-#if SO_REUSE 50+ 51+#if REUSE_IPPORT 52+ struct tcp_pcb_listen *first_same_port_pcb = NULL; 53+ for (lpcb = tcp_listen_pcbs.listen_pcbs; lpcb != NULL; lpcb = lpcb->next) { 54+ if ((lpcb->local_port == pcb->local_port) && 55+ ip_addr_cmp(&lpcb->local_ip, &pcb->local_ip)) { 56+ /* this address/port is already used */ 57+ first_same_port_pcb = lpcb; 58+ break; 59+ } 60+ } 61+#else 62 if (ip_get_option(pcb, SOF_REUSEADDR)) { 63 /* Since SOF_REUSEADDR allows reusing a local address before the pcb's usage 64 is declared (listen-/connection-pcb), we have to make sure now that 65@@ -936,7 +952,7 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) 66 } 67 } 68 } 69-#endif /* SO_REUSE */ 70+#endif /* REUSE_IPPORT */ 71 72 #if USE_LIBOS 73 vdev_reg_done(REG_RING_TCP_LISTEN, pcb); 74@@ -955,6 +971,16 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) 75 lpcb->netif_idx = pcb->netif_idx; 76 lpcb->ttl = pcb->ttl; 77 lpcb->tos = pcb->tos; 78+ 79+#if REUSE_IPPORT 80+ lpcb->connect_num = 0; 81+ lpcb->next_same_port_pcb = NULL; 82+ 83+ struct netconn* conn = pcb->callback_arg; 84+ lpcb->socket_fd = conn->socket; 85+ lpcb->master_lpcb = conn->is_master_fd; 86+#endif 87+ 88 #if LWIP_IPV4 && LWIP_IPV6 89 IP_SET_TYPE_VAL(lpcb->remote_ip, pcb->local_ip.type); 90 #endif /* LWIP_IPV4 && LWIP_IPV6 */ 91@@ -979,7 +1005,15 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err) 92 lpcb->accepts_pending = 0; 93 tcp_backlog_set(lpcb, backlog); 94 #endif /* TCP_LISTEN_BACKLOG */ 95- TCP_REG(&tcp_listen_pcbs.pcbs, (struct tcp_pcb *)lpcb); 96+ 97+#if REUSE_IPPORT 98+ if (first_same_port_pcb != NULL) { 99+ TCP_REG_SAMEPORT((struct tcp_pcb_listen *)first_same_port_pcb, (struct tcp_pcb_listen *)lpcb); 100+ } else 101+#endif 102+ { 103+ TCP_REG(&tcp_listen_pcbs.pcbs, (struct tcp_pcb *)lpcb); 104+ } 105 res = ERR_OK; 106 done: 107 if (err != NULL) { 108diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c 109index 35ec6d9..9f5c34a 100644 110--- a/src/core/tcp_in.c 111+++ b/src/core/tcp_in.c 112@@ -356,6 +356,9 @@ tcp_input(struct pbuf *p, struct netif *inp) 113 } 114 } 115 116+#if REUSE_IPPORT 117+ struct tcp_pcb_listen *min_cnts_lpcb = NULL; 118+#endif 119 /* Finally, if we still did not get a match, we check all PCBs that 120 are LISTENing for incoming connections. */ 121 prev = NULL; 122@@ -379,6 +382,30 @@ tcp_input(struct pbuf *p, struct netif *inp) 123 } else if (IP_ADDR_PCB_VERSION_MATCH_EXACT(lpcb, ip_current_dest_addr())) { 124 if (ip_addr_cmp(&lpcb->local_ip, ip_current_dest_addr())) { 125 /* found an exact match */ 126+#if REUSE_IPPORT 127+ // check master fd 128+ struct tcp_pcb_listen *tmp_lpcb = lpcb; 129+ u8_t have_master_fd = 0; 130+ while (tmp_lpcb != NULL) { 131+ if (tmp_lpcb->master_lpcb) { 132+ have_master_fd = 1; 133+ } 134+ tmp_lpcb = tmp_lpcb->next_same_port_pcb; 135+ } 136+ 137+ tmp_lpcb = lpcb; 138+ min_cnts_lpcb = lpcb; 139+ u16_t min_conn_num = MAX_CONN_NUM_PER_THREAD; 140+ while (tmp_lpcb != NULL) { 141+ if (!have_master_fd || tmp_lpcb->master_lpcb) { 142+ if (tmp_lpcb->connect_num < min_conn_num) { 143+ min_cnts_lpcb = tmp_lpcb; 144+ min_conn_num = tmp_lpcb->connect_num; 145+ } 146+ } 147+ tmp_lpcb = tmp_lpcb->next_same_port_pcb; 148+ } 149+#endif 150 break; 151 } else if (ip_addr_isany(&lpcb->local_ip)) { 152 /* found an ANY-match */ 153@@ -428,7 +455,12 @@ tcp_input(struct pbuf *p, struct netif *inp) 154 tcphdr_opt1len, tcphdr_opt2, p) == ERR_OK) 155 #endif 156 { 157+#if REUSE_IPPORT 158+ tcp_listen_input(min_cnts_lpcb); 159+ min_cnts_lpcb->connect_num++; 160+#else 161 tcp_listen_input(lpcb); 162+#endif 163 } 164 pbuf_free(p); 165 return; 166diff --git a/src/include/lwip/api.h b/src/include/lwip/api.h 167index 6dec8c0..430a7a0 100644 168--- a/src/include/lwip/api.h 169+++ b/src/include/lwip/api.h 170@@ -318,6 +318,10 @@ struct netconn { 171 #endif /* LWIP_TCP */ 172 /** A callback function that is informed about events for this netconn */ 173 netconn_callback callback; 174+ 175+#if REUSE_IPPORT 176+ u8_t is_master_fd; 177+#endif 178 }; 179 180 /** This vector type is passed to @ref netconn_write_vectors_partly to send 181diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h 182index b242428..97f799e 100644 183--- a/src/include/lwip/priv/tcp_priv.h 184+++ b/src/include/lwip/priv/tcp_priv.h 185@@ -353,6 +353,15 @@ static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pc 186 qtuple.dst_ip = pcb->remote_ip.addr; 187 qtuple.dst_port = lwip_htons(pcb->remote_port); 188 189+#if REUSE_IPPORT 190+ if (reg_type == REG_RING_TCP_CONNECT_CLOSE) { 191+ struct tcp_pcb_listen* lpcb = pcb->listener; 192+ if (lpcb != NULL) { 193+ lpcb->connect_num--; 194+ } 195+ } 196+#endif 197+ 198 return vdev_reg_xmit(reg_type, &qtuple); 199 } 200 static inline void vdev_unreg_done(const struct tcp_pcb *pcb) 201@@ -473,6 +482,16 @@ static inline void vdev_unreg_done(const struct tcp_pcb *pcb) 202 tcp_timer_needed(); \ 203 } while (0) 204 205+#define TCP_REG_SAMEPORT(first_pcb, lpcb) \ 206+ do { \ 207+ struct tcp_pcb_listen *tmp_pcb = first_pcb; \ 208+ while (tmp_pcb->next_same_port_pcb != NULL) { \ 209+ tmp_pcb = tmp_pcb->next_same_port_pcb; \ 210+ }; \ 211+ tmp_pcb->next_same_port_pcb = lpcb; \ 212+ tcp_timer_needed(); \ 213+ } while (0) 214+ 215 #define TCP_RMV_HASH(pcbs, npcb) \ 216 do { \ 217 hlist_del_init(&(npcb)->tcp_node); \ 218diff --git a/src/include/lwip/tcp.h b/src/include/lwip/tcp.h 219index 0b65b01..312320b 100644 220--- a/src/include/lwip/tcp.h 221+++ b/src/include/lwip/tcp.h 222@@ -252,6 +252,14 @@ struct tcp_pcb_listen { 223 u8_t backlog; 224 u8_t accepts_pending; 225 #endif /* TCP_LISTEN_BACKLOG */ 226+ 227+#if REUSE_IPPORT 228+ struct tcp_pcb_listen* next_same_port_pcb; 229+ u16_t connect_num; 230+ int socket_fd; 231+ u8_t master_lpcb; 232+#endif 233+ 234 }; 235 236 237diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h 238index fedded9..be58ec3 100644 239--- a/src/include/lwipopts.h 240+++ b/src/include/lwipopts.h 241@@ -143,6 +143,10 @@ 242 243 #define USE_LIBOS_ZC_RING 0 244 245+#define REUSE_IPPORT 1 246+ 247+#define MAX_CONN_NUM_PER_THREAD 65535 248+ 249 #define SO_REUSE 1 250 251 #define SIOCSHIWAT 1 252-- 2532.33.0 254 255