• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1From 388525230f809bfa61fe31921b54ebfb6aae57ec Mon Sep 17 00:00:00 2001
2From: jiangheng <jiangheng12@huawei.com>
3Date: Fri, 31 Dec 2021 17:32:49 +0800
4Subject: [PATCH] adapt lstack
5
6---
7 src/Makefile                         |   5 +-
8 src/api/api_lib.c                    |   2 +
9 src/api/api_msg.c                    |  46 +++
10 src/api/dir.mk                       |   2 +-
11 src/api/perf.c                       | 182 ++++++++++++
12 src/api/posix_api.c                  | 156 ++++++++++
13 src/api/sockets.c                    | 558 ++++++++++++++++++++++++++++++++++-
14 src/api/sys_arch.c                   | 379 ++++++++++++++++++++++++
15 src/api/tcpip.c                      |  34 ++-
16 src/core/dir.mk                      |   8 +-
17 src/core/init.c                      |   4 +-
18 src/core/ip.c                        |   2 +-
19 src/core/ipv4/ip4.c                  |  14 +
20 src/core/ipv6/ip6.c                  |  10 +
21 src/core/mem.c                       |   6 +-
22 src/core/memp.c                      |   4 +
23 src/core/netif.c                     |   8 +-
24 src/core/pbuf.c                      |   4 +
25 src/core/stats.c                     |  13 +-
26 src/core/tcp.c                       | 196 +++++++++++-
27 src/core/tcp_in.c                    | 101 ++++++-
28 src/core/tcp_out.c                   |  25 +-
29 src/core/timeouts.c                  |  18 +-
30 src/core/udp.c                       |  15 +
31 src/include/arch/cc.h                |  80 ++++-
32 src/include/arch/perf.h              | 155 ++++++++++
33 src/include/arch/sys_arch.h          |  92 +++++-
34 src/include/eventpoll.h              |  72 +++++
35 src/include/hlist.h                  | 233 +++++++++++++++
36 src/include/list.h                   | 110 +++++++
37 src/include/lwip/api.h               |  35 +++
38 src/include/lwip/debug.h             |   1 +
39 src/include/lwip/def.h               |  15 +
40 src/include/lwip/ip.h                |   8 +-
41 src/include/lwip/memp.h              |  17 ++
42 src/include/lwip/netif.h             |   4 +-
43 src/include/lwip/opt.h               |  62 +++-
44 src/include/lwip/priv/memp_std.h     |   7 +
45 src/include/lwip/priv/sockets_priv.h |  49 +--
46 src/include/lwip/priv/tcp_priv.h     | 162 +++++++++-
47 src/include/lwip/prot/ip4.h          |  15 +
48 src/include/lwip/sockets.h           |  67 ++++-
49 src/include/lwip/stats.h             |   4 +-
50 src/include/lwip/tcp.h               |  94 +++++-
51 src/include/lwip/tcpip.h             |   2 +-
52 src/include/lwip/timeouts.h          |   4 +
53 src/include/lwiplog.h                |  81 +++++
54 src/include/lwipopts.h               | 253 ++++++++++++----
55 src/include/lwipsock.h               | 155 ++++++++++
56 src/include/memp_def.h               |  66 +++++
57 src/include/posix_api.h              |  88 ++++++
58 src/include/reg_sock.h               |  62 ++++
59 src/netif/dir.mk                     |   2 +-
60 53 files changed, 3581 insertions(+), 206 deletions(-)
61 create mode 100644 src/api/perf.c
62 create mode 100644 src/api/posix_api.c
63 create mode 100644 src/api/sys_arch.c
64 create mode 100644 src/include/arch/perf.h
65 create mode 100644 src/include/eventpoll.h
66 create mode 100644 src/include/hlist.h
67 create mode 100644 src/include/list.h
68 create mode 100644 src/include/lwiplog.h
69 create mode 100644 src/include/lwipsock.h
70 create mode 100644 src/include/memp_def.h
71 create mode 100644 src/include/posix_api.h
72 create mode 100644 src/include/reg_sock.h
73
74diff --git a/src/Makefile b/src/Makefile
75index 3ecf8d2..1676a71 100644
76--- a/src/Makefile
77+++ b/src/Makefile
78@@ -2,7 +2,7 @@ LWIP_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
79 ROOT_DIR := $(dir $(abspath $(LWIP_DIR)))
80
81 LWIP_INC = $(LWIP_DIR)/include
82-#DPDK_INCLUDE_FILE ?= /usr/include/dpdk
83+DPDK_INCLUDE_FILE ?= /usr/include/dpdk
84
85 SEC_FLAGS = -fstack-protector-strong -Werror -Wall -Wl,-z,relro,-z,now -Wl,-z,noexecstack -Wtrampolines -fPIC
86
87@@ -10,7 +10,8 @@ CC = gcc
88 AR = ar
89 OPTIMIZATION = -O3
90 INC = -I$(LWIP_DIR) \
91-      -I$(LWIP_INC)
92+      -I$(LWIP_INC) \
93+      -I$(DPDK_INCLUDE_FILE)
94
95 CFLAGS = -g $(OPTIMIZATION) $(INC) $(SEC_FLAGS)
96 ARFLAGS = crDP
97diff --git a/src/api/api_lib.c b/src/api/api_lib.c
98index ffa14d6..ba9f3c5 100644
99--- a/src/api/api_lib.c
100+++ b/src/api/api_lib.c
101@@ -1061,7 +1061,9 @@ netconn_write_vectors_partly(struct netconn *conn, struct netvector *vectors, u1
102   /* For locking the core: this _can_ be delayed on low memory/low send buffer,
103      but if it is, this is done inside api_msg.c:do_write(), so we can use the
104      non-blocking version here. */
105+  PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_DATA_SEND);
106   err = netconn_apimsg(lwip_netconn_do_write, &API_MSG_VAR_REF(msg));
107+  PERF_STOP_INCREASE_COUNT("lwip_netconn_do_write", PERF_LAYER_TCP);
108   if (err == ERR_OK) {
109     if (bytes_written != NULL) {
110       *bytes_written = API_MSG_VAR_REF(msg).msg.w.offset;
111diff --git a/src/api/api_msg.c b/src/api/api_msg.c
112index 3f08e03..d5a738f 100644
113--- a/src/api/api_msg.c
114+++ b/src/api/api_msg.c
115@@ -54,6 +54,11 @@
116 #include "lwip/mld6.h"
117 #include "lwip/priv/tcpip_priv.h"
118
119+#if USE_LIBOS
120+#include "lwip/sockets.h"
121+#include "lwipsock.h"
122+#endif
123+
124 #include <string.h>
125
126 /* netconns are polled once per second (e.g. continue write on memory error) */
127@@ -452,6 +457,14 @@ err_tcp(void *arg, err_t err)
128   old_state = conn->state;
129   conn->state = NETCONN_NONE;
130
131+#if USE_LIBOS
132+  if (CONN_TYPE_IS_HOST(conn)) {
133+    LWIP_DEBUGF(API_MSG_DEBUG,
134+        ("linux localhost connection already success, ignore lwip err_tcp fd=%d\n", conn->socket));
135+    return;
136+  }
137+#endif /* USE_LIBOS */
138+
139   SYS_ARCH_UNPROTECT(lev);
140
141   /* Notify the user layer about a connection error. Used to signal select. */
142@@ -595,6 +608,10 @@ accept_function(void *arg, struct tcp_pcb *newpcb, err_t err)
143     API_EVENT(conn, NETCONN_EVT_RCVPLUS, 0);
144   }
145
146+#if USE_LIBOS
147+  LWIP_DEBUGF(API_MSG_DEBUG, ("libos incoming connection established\n"));
148+  SET_CONN_TYPE_LIBOS(newconn);
149+#endif
150   return ERR_OK;
151 }
152 #endif /* LWIP_TCP */
153@@ -1315,6 +1332,31 @@ lwip_netconn_do_connected(void *arg, struct tcp_pcb *pcb, err_t err)
154     return ERR_VAL;
155   }
156
157+#if USE_LIBOS
158+  if (CONN_TYPE_IS_HOST(conn)) {
159+    LWIP_DEBUGF(API_MSG_DEBUG,
160+        ("libos outgoing connection abort fd=%d\n", conn->socket));
161+    return ERR_ABRT;
162+  }
163+
164+  LWIP_DEBUGF(API_MSG_DEBUG, ("libos outgoing connection established\n"));
165+  if (CONN_TYPE_HAS_INPRG(conn) && CONN_TYPE_HAS_HOST(conn)) {
166+    int s = conn->socket;
167+    struct lwip_sock *sock = get_socket_without_errno(s);
168+
169+    if (!!sock && !!sock->epoll_data) {
170+      struct epoll_event ee = {0};
171+      ee.data.fd = s;
172+      ee.events |= EPOLLIN | EPOLLOUT | EPOLLERR;
173+      posix_api->epoll_ctl_fn(sock->epoll_data->fd, EPOLL_CTL_DEL, s, &ee);
174+      posix_api->shutdown_fn(s, SHUT_RDWR);
175+      LWIP_DEBUGF(API_MSG_DEBUG,
176+          ("linux outgoing connection abort fd=%d\n", s));
177+    }
178+  }
179+  SET_CONN_TYPE_LIBOS(conn);
180+#endif
181+
182   LWIP_ASSERT("conn->state == NETCONN_CONNECT", conn->state == NETCONN_CONNECT);
183   LWIP_ASSERT("(conn->current_msg != NULL) || conn->in_non_blocking_connect",
184               (conn->current_msg != NULL) || IN_NONBLOCKING_CONNECT(conn));
185@@ -1338,6 +1380,7 @@ lwip_netconn_do_connected(void *arg, struct tcp_pcb *pcb, err_t err)
186   if (was_blocking) {
187     sys_sem_signal(op_completed_sem);
188   }
189+
190   return ERR_OK;
191 }
192 #endif /* LWIP_TCP */
193@@ -1372,6 +1415,7 @@ lwip_netconn_do_connect(void *m)
194 #endif /* LWIP_UDP */
195 #if LWIP_TCP
196       case NETCONN_TCP:
197+        PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_SEND);
198         /* Prevent connect while doing any other action. */
199         if (msg->conn->state == NETCONN_CONNECT) {
200           err = ERR_ALREADY;
201@@ -1389,6 +1433,7 @@ lwip_netconn_do_connect(void *m)
202               err = ERR_INPROGRESS;
203             } else {
204               msg->conn->current_msg = msg;
205+              PERF_STOP_INCREASE_COUNT("lwip_netconn_do_connect", PERF_LAYER_TCP);
206               /* sys_sem_signal() is called from lwip_netconn_do_connected (or err_tcp()),
207                  when the connection is established! */
208 #if LWIP_TCPIP_CORE_LOCKING
209@@ -1402,6 +1447,7 @@ lwip_netconn_do_connect(void *m)
210             }
211           }
212         }
213+        PERF_STOP_INCREASE_COUNT("lwip_netconn_do_connect", PERF_LAYER_TCP);
214         break;
215 #endif /* LWIP_TCP */
216       default:
217diff --git a/src/api/dir.mk b/src/api/dir.mk
218index 72142ab..afbf863 100644
219--- a/src/api/dir.mk
220+++ b/src/api/dir.mk
221@@ -1,3 +1,3 @@
222-SRC = api_lib.c api_msg.c err.c netbuf.c netdb.c netifapi.c sockets.c tcpip.c
223+SRC = api_lib.c api_msg.c err.c netbuf.c netdb.c netifapi.c sockets.c tcpip.c perf.c posix_api.c sys_arch.c
224
225 $(eval $(call register_dir, api, $(SRC)))
226diff --git a/src/api/perf.c b/src/api/perf.c
227new file mode 100644
228index 0000000..1c2a273
229--- /dev/null
230+++ b/src/api/perf.c
231@@ -0,0 +1,182 @@
232+/*
233+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
234+ * All rights reserved.
235+ *
236+ * Redistribution and use in source and binary forms, with or without modification,
237+ * are permitted provided that the following conditions are met:
238+ *
239+ * 1. Redistributions of source code must retain the above copyright notice,
240+ *    this list of conditions and the following disclaimer.
241+ * 2. Redistributions in binary form must reproduce the above copyright notice,
242+ *    this list of conditions and the following disclaimer in the documentation
243+ *    and/or other materials provided with the distribution.
244+ * 3. The name of the author may not be used to endorse or promote products
245+ *    derived from this software without specific prior written permission.
246+ *
247+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
248+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
249+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
250+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
251+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
252+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
253+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
254+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
255+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
256+ * OF SUCH DAMAGE.
257+ *
258+ * This file is part of the lwIP TCP/IP stack.
259+ *
260+ * Author: Huawei Technologies
261+ *
262+ */
263+
264+#include "arch/perf.h"
265+
266+#include <signal.h>
267+
268+#include <lwip/stats.h>
269+
270+#if LWIP_RECORD_PERF
271+
272+#define SIG_FUNC_NUM      3
273+
274+#define SIG_STATS_DISPLAY 38
275+#define SIG_PERF_BEGIN    39
276+#define SIG_PERF_END      40
277+
278+typedef void (*pSignalFunc) (int);
279+static void signal_stats_display(int s);
280+static void signal_perf_begin(int s);
281+static void signal_perf_end(int s);
282+
283+uint32_t g_record_perf;
284+__thread uint64_t g_timeTaken[PERF_POINT_END];
285+__thread int g_perfPoint[PERF_LAYER_END];
286+__thread struct timespec tvStart[PERF_LAYER_END];
287+volatile uint64_t g_perfMaxtime[PERF_POINT_END];
288+volatile uint64_t g_astPacketCnt[PERF_POINT_END];
289+volatile uint64_t g_astPacketProcTime[PERF_POINT_END];
290+
291+char *g_ppLayerName[PERF_POINT_END] = {
292+    "IP_RECV",
293+    "TCP_DATA_RECV",
294+    "UDP_PARTIAL",
295+    "TCP_SYN_RECV",
296+    "TCP_SYN_ACK_SEND",
297+    "TCP_ACK_RECV",
298+    "TCP_SYN_SEND",
299+    "TCP_SYN_ACK_RECV",
300+    "TCP_ACK_SEND",
301+    "TCP_DATA_SEND",
302+    "IP_SEND"
303+};
304+
305+static int gsig_arr[SIG_FUNC_NUM] = {
306+    SIG_STATS_DISPLAY,
307+    SIG_PERF_BEGIN,
308+    SIG_PERF_END
309+};
310+
311+static pSignalFunc g_Funcs[SIG_FUNC_NUM] = {
312+    signal_stats_display,
313+    signal_perf_begin,
314+    signal_perf_end,
315+};
316+
317+static void print_perf_data_and_reset()
318+{
319+    int i;
320+    printf("\n********* PERF DATA START*************\n");
321+    for (i = 0; i < PERF_POINT_END; i++) {
322+        printf("%-20s Total:     PacketProcTime: %-15"PRIu64", Maxtime: %-15"PRIu64", packetCnt: %-15"PRIu64"\n",
323+            g_ppLayerName[i], __sync_fetch_and_or(&g_astPacketProcTime[i], 0),
324+            __sync_fetch_and_or(&g_perfMaxtime[i], 0),
325+            __sync_fetch_and_or(&g_astPacketCnt[i], 0));
326+
327+        if (__sync_fetch_and_or(&g_astPacketProcTime[i], 0) && __sync_fetch_and_or(&g_astPacketCnt[i], 0)) {
328+        printf("%-20s Average:   PacketProcTime: %-15lf, MaxTime: %-15"PRIu64"\n", g_ppLayerName[i],
329+            (double)__sync_fetch_and_or(&g_astPacketProcTime[i], 0) / (double)__sync_fetch_and_or(&g_astPacketCnt[i], 0),
330+            __sync_or_and_fetch(&g_perfMaxtime[i], 0));
331+        }
332+
333+        __sync_fetch_and_and (&g_astPacketProcTime[i], 0);
334+        __sync_fetch_and_and (&g_astPacketCnt[i], 0);
335+        __sync_fetch_and_and (&g_perfMaxtime[i], 0);
336+    }
337+    printf("\n********* PERF DATA END*************\n");
338+}
339+
340+static void signal_stats_display(int s)
341+{
342+    struct sigaction s_test;
343+    printf("Received signal %d, stats display.\n", s);
344+    stats_display();
345+    s_test.sa_handler = (void *) signal_stats_display;
346+    if (sigemptyset(&s_test.sa_mask) != 0) {
347+        printf("sigemptyset failed.\n");
348+    }
349+    s_test.sa_flags = SA_RESETHAND;
350+    if (sigaction(s, &s_test, NULL) != 0) {
351+        printf("Could not register %d signal handler.\n", s);
352+    }
353+}
354+
355+static void signal_perf_begin(int s)
356+{
357+    struct sigaction s_test;
358+    printf("Received signal %d, perf_begin.\n", s);
359+    g_record_perf = 1;
360+    s_test.sa_handler = (void *) signal_perf_begin;
361+    if (sigemptyset(&s_test.sa_mask) != 0) {
362+        printf("sigemptyset failed.\n");
363+    }
364+    s_test.sa_flags = SA_RESETHAND;
365+    if (sigaction(s, &s_test, NULL) != 0) {
366+        printf("Could not register %d signal handler.\n", s);
367+    }
368+}
369+
370+static void signal_perf_end(int s)
371+{
372+    struct sigaction s_test;
373+    printf("Received signal %d, perf_end\n", s);
374+    g_record_perf = 0;
375+    print_perf_data_and_reset();
376+    s_test.sa_handler = (void *) signal_perf_end;
377+    if (sigemptyset(&s_test.sa_mask) != 0) {
378+        printf("sigemptyset failed.\n");
379+    }
380+    s_test.sa_flags = SA_RESETHAND;
381+    if (sigaction(s, &s_test, NULL) != 0) {
382+        printf("Could not register %d signal handler.\n", s);
383+    }
384+}
385+
386+int check_layer_point(int layer, int point)
387+{
388+    if (point == g_perfPoint[layer]) {
389+        return 1;
390+    }
391+    return 0;
392+}
393+
394+int perf_init(void)
395+{
396+    int i;
397+    struct sigaction s_test;
398+    for (i = 0; i < SIG_FUNC_NUM; i++) {
399+        s_test.sa_handler = (void *) g_Funcs[i];
400+        if (sigemptyset(&s_test.sa_mask) != 0) {
401+            printf("sigemptyset failed.\n");
402+            return 1;
403+        }
404+
405+        s_test.sa_flags = SA_RESETHAND;
406+        if (sigaction(gsig_arr[i], &s_test, NULL) != 0) {
407+            printf("Could not register %d signal handler.\n", gsig_arr[i]);
408+            return 1;
409+        }
410+    }
411+    return 0;
412+}
413+#endif
414diff --git a/src/api/posix_api.c b/src/api/posix_api.c
415new file mode 100644
416index 0000000..a917cea
417--- /dev/null
418+++ b/src/api/posix_api.c
419@@ -0,0 +1,156 @@
420+/*
421+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
422+ * All rights reserved.
423+ *
424+ * Redistribution and use in source and binary forms, with or without modification,
425+ * are permitted provided that the following conditions are met:
426+ *
427+ * 1. Redistributions of source code must retain the above copyright notice,
428+ *    this list of conditions and the following disclaimer.
429+ * 2. Redistributions in binary form must reproduce the above copyright notice,
430+ *    this list of conditions and the following disclaimer in the documentation
431+ *    and/or other materials provided with the distribution.
432+ * 3. The name of the author may not be used to endorse or promote products
433+ *    derived from this software without specific prior written permission.
434+ *
435+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
436+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
437+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
438+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
439+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
440+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
441+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
442+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
443+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
444+ * OF SUCH DAMAGE.
445+ *
446+ * This file is part of the lwIP TCP/IP stack.
447+ *
448+ * Author: Huawei Technologies
449+ *
450+ */
451+
452+#define _GNU_SOURCE
453+#include <dlfcn.h>
454+#include <fcntl.h>
455+#include <sys/epoll.h>
456+#include <sys/socket.h>
457+#include <sys/stat.h>
458+#include <unistd.h>
459+
460+#include <rte_log.h>
461+
462+#include "lwip/err.h"
463+#include "lwipsock.h"
464+
465+posix_api_t *posix_api;
466+posix_api_t posix_api_val;
467+
468+static int chld_is_epfd(int fd)
469+{
470+    return 0;
471+}
472+
473+static struct lwip_sock *chld_get_socket(int fd)
474+{
475+    return NULL;
476+}
477+
478+void posix_api_fork(void)
479+{
480+    /* lstack helper api */
481+    posix_api->is_chld = 1;
482+    posix_api->is_epfd = chld_is_epfd;
483+    posix_api->get_socket = chld_get_socket;
484+}
485+
486+static int chose_dlsym_handle(void *__restrict* khandle)
487+{
488+    void *dlhandle;
489+    int (*gazelle_epoll_create)(int size);
490+    dlhandle = dlopen ("liblstack.so", RTLD_LAZY);
491+    if (dlhandle == NULL) {
492+        return ERR_IF;
493+    }
494+
495+    gazelle_epoll_create = dlsym(dlhandle, "epoll_create");
496+    if (gazelle_epoll_create == NULL) {
497+        return ERR_MEM;
498+    }
499+
500+    dlclose(dlhandle);
501+
502+    *khandle = RTLD_NEXT;
503+    if (dlsym(*khandle, "epoll_create") == gazelle_epoll_create) {
504+        RTE_LOG(ERR, EAL, "posix api use RTLD_DEFAULT\n");
505+        *khandle = RTLD_DEFAULT;
506+    } else {
507+        RTE_LOG(ERR, EAL, "posix api use RTLD_NEXT\n");
508+    }
509+
510+    return ERR_OK;
511+}
512+
513+int posix_api_init(void)
514+{
515+/* the symbol we use here won't be NULL, so we don't need dlerror()
516+ to test error */
517+#define CHECK_DLSYM_RET_RETURN(ret) do {  \
518+        if ((ret) == NULL)                   \
519+            goto err_out;                 \
520+    } while (0)
521+
522+    posix_api = &posix_api_val;
523+
524+    void *__restrict handle;
525+    int ret = chose_dlsym_handle(&handle);
526+    if (ret != ERR_OK) {
527+        return ret;
528+    }
529+
530+    /* glibc standard api */
531+    CHECK_DLSYM_RET_RETURN(posix_api->socket_fn = dlsym(handle, "socket"));
532+    CHECK_DLSYM_RET_RETURN(posix_api->accept_fn = dlsym(handle, "accept"));
533+    CHECK_DLSYM_RET_RETURN(posix_api->accept4_fn = dlsym(handle, "accept4"));
534+    CHECK_DLSYM_RET_RETURN(posix_api->bind_fn = dlsym(handle, "bind"));
535+    CHECK_DLSYM_RET_RETURN(posix_api->listen_fn = dlsym(handle, "listen"));
536+    CHECK_DLSYM_RET_RETURN(posix_api->connect_fn = dlsym(handle, "connect"));
537+    CHECK_DLSYM_RET_RETURN(posix_api->setsockopt_fn = dlsym(handle, "setsockopt"));
538+    CHECK_DLSYM_RET_RETURN(posix_api->getsockopt_fn = dlsym(handle, "getsockopt"));
539+    CHECK_DLSYM_RET_RETURN(posix_api->getpeername_fn = dlsym(handle, "getpeername"));
540+    CHECK_DLSYM_RET_RETURN(posix_api->getsockname_fn = dlsym(handle, "getsockname"));
541+    CHECK_DLSYM_RET_RETURN(posix_api->shutdown_fn = dlsym(handle, "shutdown"));
542+    CHECK_DLSYM_RET_RETURN(posix_api->close_fn = dlsym(handle, "close"));
543+    CHECK_DLSYM_RET_RETURN(posix_api->read_fn = dlsym(handle, "read"));
544+    CHECK_DLSYM_RET_RETURN(posix_api->write_fn = dlsym(handle, "write"));
545+    CHECK_DLSYM_RET_RETURN(posix_api->recv_fn = dlsym(handle, "recv"));
546+    CHECK_DLSYM_RET_RETURN(posix_api->send_fn = dlsym(handle, "send"));
547+    CHECK_DLSYM_RET_RETURN(posix_api->recv_msg = dlsym(handle, "recvmsg"));
548+    CHECK_DLSYM_RET_RETURN(posix_api->send_msg = dlsym(handle, "sendmsg"));
549+    CHECK_DLSYM_RET_RETURN(posix_api->recv_from = dlsym(handle, "recvfrom"));
550+    CHECK_DLSYM_RET_RETURN(posix_api->send_to = dlsym(handle, "sendto"));
551+    CHECK_DLSYM_RET_RETURN(posix_api->fcntl_fn = dlsym(handle, "fcntl"));
552+    CHECK_DLSYM_RET_RETURN(posix_api->fcntl64_fn = dlsym(handle, "fcntl64"));
553+    CHECK_DLSYM_RET_RETURN(posix_api->pipe_fn = dlsym(handle, "pipe"));
554+    CHECK_DLSYM_RET_RETURN(posix_api->epoll_create_fn = dlsym(handle, "epoll_create"));
555+    CHECK_DLSYM_RET_RETURN(posix_api->epoll_ctl_fn = dlsym(handle, "epoll_ctl"));
556+    CHECK_DLSYM_RET_RETURN(posix_api->epoll_wait_fn = dlsym(handle, "epoll_wait"));
557+    CHECK_DLSYM_RET_RETURN(posix_api->fork_fn = dlsym(handle, "fork"));
558+    CHECK_DLSYM_RET_RETURN(posix_api->eventfd_fn = dlsym(handle, "eventfd"));
559+    CHECK_DLSYM_RET_RETURN(posix_api->sigaction_fn = dlsym(handle, "sigaction"));
560+    CHECK_DLSYM_RET_RETURN(posix_api->poll_fn = dlsym(handle, "poll"));
561+    CHECK_DLSYM_RET_RETURN(posix_api->ioctl_fn = dlsym(handle, "ioctl"));
562+
563+    /* lstack helper api */
564+    posix_api->get_socket = get_socket;
565+    posix_api->is_epfd = lwip_is_epfd;
566+    posix_api->epoll_close_fn = lwip_epoll_close;
567+
568+    /* support fork */
569+    posix_api->is_chld = 0;
570+    return ERR_OK;
571+
572+err_out:
573+    return ERR_MEM;
574+#undef CHECK_DLSYM_RET_RETURN
575+}
576diff --git a/src/api/sockets.c b/src/api/sockets.c
577index 7852635..3262c1b 100644
578--- a/src/api/sockets.c
579+++ b/src/api/sockets.c
580@@ -62,6 +62,11 @@
581 #include <stdarg.h>
582 #endif
583
584+#if USE_LIBOS
585+#include <stdarg.h>
586+#include "lwipsock.h"
587+#endif
588+
589 #include <string.h>
590
591 #ifdef LWIP_HOOK_FILENAME
592@@ -85,13 +90,29 @@
593 #define API_SELECT_CB_VAR_ALLOC(name, retblock)   API_VAR_ALLOC_EXT(struct lwip_select_cb, MEMP_SELECT_CB, name, retblock)
594 #define API_SELECT_CB_VAR_FREE(name)              API_VAR_FREE(MEMP_SELECT_CB, name)
595
596+#if USE_LIBOS
597+enum KERNEL_LWIP_PATH {
598+  PATH_KERNEL = 0,
599+  PATH_LWIP,
600+  PATH_ERR,
601+};
602+#endif
603+
604 #if LWIP_IPV4
605+#if USE_LIBOS
606+#define IP4ADDR_PORT_TO_SOCKADDR(sin, ipaddr, port) do { \
607+      (sin)->sin_family = AF_INET; \
608+      (sin)->sin_port = lwip_htons((port)); \
609+      inet_addr_from_ip4addr(&(sin)->sin_addr, ipaddr); \
610+      memset((sin)->sin_zero, 0, SIN_ZERO_LEN); }while(0)
611+#else
612 #define IP4ADDR_PORT_TO_SOCKADDR(sin, ipaddr, port) do { \
613       (sin)->sin_len = sizeof(struct sockaddr_in); \
614       (sin)->sin_family = AF_INET; \
615       (sin)->sin_port = lwip_htons((port)); \
616       inet_addr_from_ip4addr(&(sin)->sin_addr, ipaddr); \
617       memset((sin)->sin_zero, 0, SIN_ZERO_LEN); }while(0)
618+#endif /* USE_LIBOS */
619 #define SOCKADDR4_TO_IP4ADDR_PORT(sin, ipaddr, port) do { \
620     inet_addr_to_ip4addr(ip_2_ip4(ipaddr), &((sin)->sin_addr)); \
621     (port) = lwip_ntohs((sin)->sin_port); }while(0)
622@@ -257,7 +278,12 @@ static void lwip_socket_drop_registered_mld6_memberships(int s);
623 #endif /* LWIP_IPV6_MLD */
624
625 /** The global array of available sockets */
626+#if USE_LIBOS
627+uint32_t sockets_num;
628+struct lwip_sock *sockets;
629+#else
630 static struct lwip_sock sockets[NUM_SOCKETS];
631+#endif /* USE_LIBOS */
632
633 #if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
634 #if LWIP_TCPIP_CORE_LOCKING
635@@ -285,7 +311,7 @@ static struct lwip_select_cb *select_cb_list;
636
637 /* Forward declaration of some functions */
638 #if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
639-static void event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len);
640+void event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len);
641 #define DEFAULT_SOCKET_EVENTCB event_callback
642 static void select_check_waiters(int s, int has_recvevent, int has_sendevent, int has_errevent);
643 #else
644@@ -411,7 +437,13 @@ static struct lwip_sock *
645 tryget_socket_unconn_nouse(int fd)
646 {
647   int s = fd - LWIP_SOCKET_OFFSET;
648-  if ((s < 0) || (s >= NUM_SOCKETS)) {
649+
650+#if USE_LIBOS
651+  if ((s < 0) || (s >= sockets_num))
652+#else
653+  if ((s < 0) || (s >= NUM_SOCKETS))
654+#endif /* USE_LIBOS */
655+  {
656     LWIP_DEBUGF(SOCKETS_DEBUG, ("tryget_socket_unconn(%d): invalid\n", fd));
657     return NULL;
658   }
659@@ -475,8 +507,13 @@ tryget_socket(int fd)
660  * @param fd externally used socket index
661  * @return struct lwip_sock for the socket or NULL if not found
662  */
663+#if USE_LIBOS
664+struct lwip_sock *
665+get_socket(int fd)
666+#else
667 static struct lwip_sock *
668 get_socket(int fd)
669+#endif /* USE_LIBOS */
670 {
671   struct lwip_sock *sock = tryget_socket(fd);
672   if (!sock) {
673@@ -489,6 +526,24 @@ get_socket(int fd)
674   return sock;
675 }
676
677+#if USE_LIBOS
678+/**
679+ * Map a externally used socket index to the internal socket representation.
680+ *
681+ * @param s externally used socket index
682+ * @return struct lwip_sock for the socket or NULL if not found without
683+ * checking.
684+ */
685+struct lwip_sock *
686+get_socket_by_fd(int fd)
687+{
688+  if ((fd < LWIP_SOCKET_OFFSET) || (fd >= sockets_num + LWIP_SOCKET_OFFSET)) {
689+    return NULL;
690+  }
691+  return &sockets[fd - LWIP_SOCKET_OFFSET];
692+}
693+#endif /* USE_LIBOS */
694+
695 /**
696  * Allocate a new socket for a given netconn.
697  *
698@@ -504,6 +559,62 @@ alloc_socket(struct netconn *newconn, int accepted)
699   SYS_ARCH_DECL_PROTECT(lev);
700   LWIP_UNUSED_ARG(accepted);
701
702+#if USE_LIBOS
703+  int type, protocol = 0, domain = AF_INET;
704+  switch (NETCONNTYPE_GROUP(newconn->type)) {
705+    case NETCONN_RAW:
706+      type = SOCK_RAW;
707+      break;
708+    case NETCONN_UDPLITE:
709+    case NETCONN_UDP:
710+      type = SOCK_DGRAM;
711+      break;
712+    case NETCONN_TCP:
713+      type = SOCK_STREAM;
714+      break;
715+    default:
716+      type = -1;
717+      break;
718+  }
719+
720+  SYS_ARCH_PROTECT(lev);
721+  i = posix_api->socket_fn(domain, type, protocol);
722+  if (i == -1) {
723+    goto err;
724+  }
725+
726+  if ((i < LWIP_SOCKET_OFFSET) || (i >= sockets_num + LWIP_SOCKET_OFFSET)) {
727+    goto err;
728+  }
729+
730+  if (!sockets[i].conn && (sockets[i].select_waiting == 0)) {
731+    /*initialize state as NETCONN_HOST | NETCONN_LIBOS,
732+     *if connection accepted and alloc_socket called, it can be only NETCONN_LIBOS*/
733+    if (accepted)
734+      SET_CONN_TYPE_LIBOS(newconn);
735+    else
736+      SET_CONN_TYPE_LIBOS_OR_HOST(newconn);
737+    sockets[i].conn       = newconn;
738+    /* The socket is not yet known to anyone, so no need to protect
739+       after having marked it as used. */
740+    SYS_ARCH_UNPROTECT(lev);
741+    sockets[i].lastdata.pbuf = NULL;
742+    sockets[i].rcvevent   = 0;
743+    /* TCP sendbuf is empty, but the socket is not yet writable until connected
744+     * (unless it has been created by accept()). */
745+    sockets[i].sendevent  = (NETCONNTYPE_GROUP(newconn->type) == NETCONN_TCP ? (accepted != 0) : 1);
746+    sockets[i].errevent   = 0;
747+    sockets[i].epoll_data = NULL;
748+    init_list_node_null(&sockets[i].list);
749+    return i + LWIP_SOCKET_OFFSET;
750+  }
751+
752+err:
753+  posix_api->close_fn(i);
754+  SYS_ARCH_UNPROTECT(lev);
755+  return -1;
756+#else /* USE_LIBOS */
757+
758   /* allocate a new socket identifier */
759   for (i = 0; i < NUM_SOCKETS; ++i) {
760     /* Protect socket array */
761@@ -535,6 +646,8 @@ alloc_socket(struct netconn *newconn, int accepted)
762     SYS_ARCH_UNPROTECT(lev);
763   }
764   return -1;
765+
766+#endif /* USE_LIBOS */
767 }
768
769 /** Free a socket (under lock)
770@@ -629,10 +742,43 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
771   SYS_ARCH_DECL_PROTECT(lev);
772
773   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_accept(%d)...\n", s));
774+#if USE_LIBOS
775+  int sys_errno = 0;
776+
777+  sock = posix_api->get_socket(s);
778+  /*AF_UNIX case*/
779+  if (!sock) {
780+    if (rearm_accept_fd(s) < 0) {
781+      LWIP_DEBUGF(SOCKETS_DEBUG,
782+          ("failed to rearm accept fd=%d errno=%d\n", s, errno));
783+    }
784+    return posix_api->accept_fn(s, addr, addrlen);
785+  }
786+
787+  /*for AF_INET, we may try both linux and lwip*/
788+  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
789+    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
790+    set_errno(EINVAL);
791+    return -1;
792+  }
793+
794+  if (rearm_accept_fd(s) < 0) {
795+    LWIP_DEBUGF(SOCKETS_DEBUG,
796+        ("failed to rearm accept fd=%d errno=%d\n", s, errno));
797+  }
798+
799+  /* raise accept syscall in palce */
800+  newsock = posix_api->accept_fn(s, addr, addrlen);
801+  if (newsock >= 0) {
802+    return newsock;
803+  }
804+  sys_errno = errno;
805+#else
806   sock = get_socket(s);
807   if (!sock) {
808     return -1;
809   }
810+#endif
811
812   /* wait for a new connection */
813   err = netconn_accept(sock->conn, &newconn);
814@@ -646,6 +792,9 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
815       sock_set_errno(sock, err_to_errno(err));
816     }
817     done_socket(sock);
818+#if USE_LIBOS
819+    set_errno(sys_errno);
820+#endif /* USE_LIBOS */
821     return -1;
822   }
823   LWIP_ASSERT("newconn != NULL", newconn != NULL);
824@@ -657,7 +806,11 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
825     done_socket(sock);
826     return -1;
827   }
828+#if USE_LIBOS
829+  LWIP_ASSERT("invalid socket index", (newsock >= LWIP_SOCKET_OFFSET) && (newsock < sockets_num + LWIP_SOCKET_OFFSET));
830+#else
831   LWIP_ASSERT("invalid socket index", (newsock >= LWIP_SOCKET_OFFSET) && (newsock < NUM_SOCKETS + LWIP_SOCKET_OFFSET));
832+#endif /* USE_LIBOS */
833   nsock = &sockets[newsock - LWIP_SOCKET_OFFSET];
834
835   /* See event_callback: If data comes in right away after an accept, even
836@@ -695,9 +848,11 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
837     }
838
839     IPADDR_PORT_TO_SOCKADDR(&tempaddr, &naddr, port);
840+#if !USE_LIBOS
841     if (*addrlen > tempaddr.sa.sa_len) {
842       *addrlen = tempaddr.sa.sa_len;
843     }
844+#endif /* USE_LIBOS */
845     MEMCPY(addr, &tempaddr, *addrlen);
846
847     LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_accept(%d) returning new sock=%d addr=", s, newsock));
848@@ -720,11 +875,24 @@ lwip_bind(int s, const struct sockaddr *name, socklen_t namelen)
849   ip_addr_t local_addr;
850   u16_t local_port;
851   err_t err;
852-
853+#if USE_LIBOS
854+  sock = posix_api->get_socket(s);
855+  /*AF_UNIX case*/
856+  if (!sock) {
857+    return posix_api->bind_fn(s, name, namelen);
858+  }
859+  /*for AF_INET, we may try both linux and lwip*/
860+  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
861+    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
862+    set_errno(EINVAL);
863+    return -1;
864+  }
865+#else
866   sock = get_socket(s);
867   if (!sock) {
868     return -1;
869   }
870+#endif
871
872   if (!SOCK_ADDR_TYPE_MATCH(name, sock)) {
873     /* sockaddr does not match socket type (IPv4/IPv6) */
874@@ -744,6 +912,18 @@ lwip_bind(int s, const struct sockaddr *name, socklen_t namelen)
875   ip_addr_debug_print_val(SOCKETS_DEBUG, local_addr);
876   LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F")\n", local_port));
877
878+#if USE_LIBOS
879+  /* Supports kernel NIC IP address. */
880+  int ret = posix_api->bind_fn(s, name, namelen);
881+  if (ret < 0) {
882+    LWIP_DEBUGF(SOCKETS_DEBUG, ("bind syscall failed\n"));
883+    /* bind must succeed on both linux and libos */
884+    if (!is_host_ipv4(local_addr.addr)) {
885+      return ret;
886+    }
887+  }
888+#endif /* USE_LIBOS */
889+
890 #if LWIP_IPV4 && LWIP_IPV6
891   /* Dual-stack: Unmap IPv4 mapped IPv6 addresses */
892   if (IP_IS_V6_VAL(local_addr) && ip6_addr_isipv4mappedipv6(ip_2_ip6(&local_addr))) {
893@@ -776,10 +956,29 @@ lwip_close(int s)
894
895   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_close(%d)\n", s));
896
897+#if USE_LIBOS
898+  int ret;
899+  if (posix_api->is_epfd(s)) {
900+    return posix_api->epoll_close_fn(s);
901+  }
902+
903+  ret = posix_api->close_fn(s);
904+  if (ret < 0)
905+    return ret;
906+  if (posix_api->is_chld == 0)
907+    clean_host_fd(s);
908+
909+  sock = posix_api->get_socket(s);
910+  /*AF_UNIX case*/
911+  if (!sock) {
912+    return ret;
913+  }
914+#else
915   sock = get_socket(s);
916   if (!sock) {
917     return -1;
918   }
919+#endif /* USE_LIBOS */
920
921   if (sock->conn != NULL) {
922     is_tcp = NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP;
923@@ -803,6 +1002,13 @@ lwip_close(int s)
924     return -1;
925   }
926
927+#if USE_LIBOS
928+  sock->epoll = LIBOS_EPOLLNONE;
929+  sock->events = 0;
930+  sock->epoll_data = NULL;
931+  list_del_node_null(&sock->list);
932+#endif
933+
934   free_socket(sock, is_tcp);
935   set_errno(0);
936   return 0;
937@@ -814,10 +1020,28 @@ lwip_connect(int s, const struct sockaddr *name, socklen_t namelen)
938   struct lwip_sock *sock;
939   err_t err;
940
941+#if USE_LIBOS
942+  int ret;
943+
944+  sock = posix_api->get_socket(s);
945+  if (!sock) {
946+    return posix_api->connect_fn(s, name, namelen);
947+  }
948+
949+  /* raise connect syscall in place */
950+  ADD_CONN_TYPE_INPRG(sock->conn);
951+  ret = posix_api->connect_fn(s, name, namelen);
952+  if (!ret) {
953+    SET_CONN_TYPE_HOST(sock->conn);
954+    LWIP_DEBUGF(SOCKETS_DEBUG, ("linux connect succeed fd=%d\n", s));
955+    return ret;
956+  }
957+#else
958   sock = get_socket(s);
959   if (!sock) {
960     return -1;
961   }
962+#endif
963
964   if (!SOCK_ADDR_TYPE_MATCH_OR_UNSPEC(name, sock)) {
965     /* sockaddr does not match socket type (IPv4/IPv6) */
966@@ -862,6 +1086,11 @@ lwip_connect(int s, const struct sockaddr *name, socklen_t namelen)
967     return -1;
968   }
969
970+#if USE_LIBOS
971+  LWIP_DEBUGF(SOCKETS_DEBUG, ("libos connect succeed fd=%d\n",s));
972+  SET_CONN_TYPE_LIBOS(sock->conn);
973+#endif /* USE_LIBOS */
974+
975   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_connect(%d) succeeded\n", s));
976   sock_set_errno(sock, 0);
977   done_socket(sock);
978@@ -884,10 +1113,29 @@ lwip_listen(int s, int backlog)
979
980   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_listen(%d, backlog=%d)\n", s, backlog));
981
982+#if USE_LIBOS
983+  int ret;
984+
985+  sock = posix_api->get_socket(s);
986+  /*AF_UNIX case*/
987+  if (!sock) {
988+    return posix_api->listen_fn(s, backlog);
989+  }
990+  /*for AF_INET, we may try both linux and lwip*/
991+  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
992+    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
993+    set_errno(EADDRINUSE);
994+    return -1;
995+  }
996+
997+  if ((ret = posix_api->listen_fn(s, backlog)) == -1)
998+    return ret;
999+#else
1000   sock = get_socket(s);
1001   if (!sock) {
1002     return -1;
1003   }
1004+#endif
1005
1006   /* limit the "backlog" parameter to fit in an u8_t */
1007   backlog = LWIP_MIN(LWIP_MAX(backlog, 0), 0xff);
1008@@ -919,6 +1167,9 @@ static ssize_t
1009 lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
1010 {
1011   u8_t apiflags = NETCONN_NOAUTORCVD;
1012+#if USE_LIBOS
1013+  apiflags = 0;
1014+#endif
1015   ssize_t recvd = 0;
1016   ssize_t recv_left = (len <= SSIZE_MAX) ? (ssize_t)len : SSIZE_MAX;
1017
1018@@ -938,6 +1189,13 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
1019     /* Check if there is data left from the last recv operation. */
1020     if (sock->lastdata.pbuf) {
1021       p = sock->lastdata.pbuf;
1022+#if USE_LIBOS
1023+      if ((flags & MSG_PEEK) == 0) {
1024+        if ((NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP)) {
1025+          del_epoll_event(sock->conn, EPOLLIN);
1026+        }
1027+      }
1028+#endif
1029     } else {
1030       /* No data was left from the previous operation, so we try to get
1031          some from the network. */
1032@@ -1008,10 +1266,22 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
1033     /* @todo: do we need to support peeking more than one pbuf? */
1034   } while ((recv_left > 0) && !(flags & MSG_PEEK));
1035 lwip_recv_tcp_done:
1036-  if ((recvd > 0) && !(flags & MSG_PEEK)) {
1037-    /* ensure window update after copying all data */
1038-    netconn_tcp_recvd(sock->conn, (size_t)recvd);
1039+#if USE_LIBOS
1040+  if (apiflags & NETCONN_NOAUTORCVD)
1041+#endif
1042+  {
1043+    if ((recvd > 0) && !(flags & MSG_PEEK)) {
1044+      /* ensure window update after copying all data */
1045+      netconn_tcp_recvd(sock->conn, (size_t)recvd);
1046+    }
1047   }
1048+#if USE_LIBOS
1049+  if ((flags & MSG_PEEK) == 0) {
1050+    if (((NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP)) && sock->lastdata.pbuf) {
1051+      add_epoll_event(sock->conn, EPOLLIN);
1052+    }
1053+  }
1054+#endif
1055   sock_set_errno(sock, 0);
1056   return recvd;
1057 }
1058@@ -1040,11 +1310,13 @@ lwip_sock_make_addr(struct netconn *conn, ip_addr_t *fromaddr, u16_t port,
1059 #endif /* LWIP_IPV4 && LWIP_IPV6 */
1060
1061   IPADDR_PORT_TO_SOCKADDR(&saddr, fromaddr, port);
1062+#if !USE_LIBOS
1063   if (*fromlen < saddr.sa.sa_len) {
1064     truncated = 1;
1065   } else if (*fromlen > saddr.sa.sa_len) {
1066     *fromlen = saddr.sa.sa_len;
1067   }
1068+#endif
1069   MEMCPY(from, &saddr, *fromlen);
1070   return truncated;
1071 }
1072@@ -1194,6 +1466,43 @@ lwip_recvfrom_udp_raw(struct lwip_sock *sock, int flags, struct msghdr *msg, u16
1073   return ERR_OK;
1074 }
1075
1076+#if USE_LIBOS
1077+static inline enum KERNEL_LWIP_PATH select_path(int s)
1078+{
1079+  struct lwip_sock *sock;
1080+
1081+  sock = posix_api->get_socket(s);
1082+  /*AF_UNIX case*/
1083+  if (!sock) {
1084+    if (rearm_host_fd(s) < 0) {
1085+      LWIP_DEBUGF(SOCKETS_DEBUG, ("failed to rearm fd=%d errno=%d\n", s, errno));
1086+    }
1087+    return PATH_KERNEL;
1088+  }
1089+
1090+  if (CONN_TYPE_HAS_INPRG(sock->conn)) {
1091+    set_errno(EWOULDBLOCK);
1092+    return PATH_ERR;
1093+  }
1094+
1095+  /*for AF_INET, we can try erther linux or lwip*/
1096+  if (CONN_TYPE_IS_HOST(sock->conn)) {
1097+    if (rearm_host_fd(s) < 0) {
1098+      LWIP_DEBUGF(SOCKETS_DEBUG, ("failed to rearm read fd=%d errno=%d\n", s, errno));
1099+    }
1100+    return PATH_KERNEL;
1101+  }
1102+
1103+  if (!CONN_TYPE_IS_LIBOS(sock->conn)) {
1104+    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type is not libos bit type=%x", netconn_type(sock->conn)));
1105+    set_errno(EINVAL);
1106+    return PATH_ERR;
1107+  }
1108+
1109+  return PATH_LWIP;
1110+}
1111+#endif
1112+
1113 ssize_t
1114 lwip_recvfrom(int s, void *mem, size_t len, int flags,
1115               struct sockaddr *from, socklen_t *fromlen)
1116@@ -1201,6 +1510,15 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
1117   struct lwip_sock *sock;
1118   ssize_t ret;
1119
1120+#if USE_LIBOS
1121+  enum KERNEL_LWIP_PATH path = select_path(s);
1122+  if (path == PATH_ERR) {
1123+    return -1;
1124+  } else if (path == PATH_KERNEL) {
1125+    return posix_api->recv_from(s, mem, len, flags, from, fromlen);
1126+  }
1127+#endif
1128+
1129   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, %p, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags));
1130   sock = get_socket(s);
1131   if (!sock) {
1132@@ -1250,6 +1568,14 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
1133 ssize_t
1134 lwip_read(int s, void *mem, size_t len)
1135 {
1136+#if USE_LIBOS
1137+  enum KERNEL_LWIP_PATH path = select_path(s);
1138+  if (path == PATH_ERR) {
1139+    return -1;
1140+  } else if (path == PATH_KERNEL) {
1141+    return posix_api->read_fn(s, mem, len);
1142+  }
1143+#endif
1144   return lwip_recvfrom(s, mem, len, 0, NULL, NULL);
1145 }
1146
1147@@ -1283,6 +1609,15 @@ lwip_recvmsg(int s, struct msghdr *message, int flags)
1148   int i;
1149   ssize_t buflen;
1150
1151+#if USE_LIBOS
1152+  enum KERNEL_LWIP_PATH path = select_path(s);
1153+  if (path == PATH_ERR) {
1154+    return -1;
1155+  } else if (path == PATH_KERNEL) {
1156+    return posix_api->recv_msg(s, message, flags);
1157+  }
1158+#endif
1159+
1160   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=%p, flags=0x%x)\n", s, (void *)message, flags));
1161   LWIP_ERROR("lwip_recvmsg: invalid message pointer", message != NULL, return ERR_ARG;);
1162   LWIP_ERROR("lwip_recvmsg: unsupported flags", (flags & ~(MSG_PEEK|MSG_DONTWAIT)) == 0,
1163@@ -1427,6 +1762,15 @@ lwip_sendmsg(int s, const struct msghdr *msg, int flags)
1164 #endif
1165   err_t err = ERR_OK;
1166
1167+#if USE_LIBOS
1168+  enum KERNEL_LWIP_PATH path = select_path(s);
1169+  if (path == PATH_ERR) {
1170+    return -1;
1171+  } else if (path == PATH_KERNEL) {
1172+    return posix_api->send_msg(s, msg, flags);
1173+  }
1174+#endif
1175+
1176   sock = get_socket(s);
1177   if (!sock) {
1178     return -1;
1179@@ -1436,10 +1780,10 @@ lwip_sendmsg(int s, const struct msghdr *msg, int flags)
1180              sock_set_errno(sock, err_to_errno(ERR_ARG)); done_socket(sock); return -1;);
1181   LWIP_ERROR("lwip_sendmsg: invalid msghdr iov", msg->msg_iov != NULL,
1182              sock_set_errno(sock, err_to_errno(ERR_ARG)); done_socket(sock); return -1;);
1183-  LWIP_ERROR("lwip_sendmsg: maximum iovs exceeded", (msg->msg_iovlen > 0) && (msg->msg_iovlen <= IOV_MAX),
1184-             sock_set_errno(sock, EMSGSIZE); done_socket(sock); return -1;);
1185-  LWIP_ERROR("lwip_sendmsg: unsupported flags", (flags & ~(MSG_DONTWAIT | MSG_MORE)) == 0,
1186-             sock_set_errno(sock, EOPNOTSUPP); done_socket(sock); return -1;);
1187+  //LWIP_ERROR("lwip_sendmsg: maximum iovs exceeded", (msg->msg_iovlen > 0) && (msg->msg_iovlen <= IOV_MAX),
1188+  //           sock_set_errno(sock, EMSGSIZE); done_socket(sock); return -1;);
1189+  //LWIP_ERROR("lwip_sendmsg: unsupported flags", (flags & ~(MSG_DONTWAIT | MSG_MORE)) == 0,
1190+  //           sock_set_errno(sock, EOPNOTSUPP); done_socket(sock); return -1;);
1191
1192   LWIP_UNUSED_ARG(msg->msg_control);
1193   LWIP_UNUSED_ARG(msg->msg_controllen);
1194@@ -1590,6 +1934,15 @@ lwip_sendto(int s, const void *data, size_t size, int flags,
1195   u16_t remote_port;
1196   struct netbuf buf;
1197
1198+#if USE_LIBOS
1199+  enum KERNEL_LWIP_PATH path = select_path(s);
1200+  if (path == PATH_ERR) {
1201+    return -1;
1202+  } else if (path == PATH_KERNEL) {
1203+    return posix_api->send_to(s, data, size, flags, to, tolen);
1204+  }
1205+#endif
1206+
1207   sock = get_socket(s);
1208   if (!sock) {
1209     return -1;
1210@@ -1688,6 +2041,11 @@ lwip_socket(int domain, int type, int protocol)
1211
1212   LWIP_UNUSED_ARG(domain); /* @todo: check this */
1213
1214+#if USE_LIBOS
1215+  if ((domain != AF_INET && domain != AF_UNSPEC) || posix_api->is_chld)
1216+    return posix_api->socket_fn(domain, type, protocol);
1217+#endif
1218+
1219   /* create a netconn */
1220   switch (type) {
1221     case SOCK_RAW:
1222@@ -1744,6 +2102,14 @@ lwip_socket(int domain, int type, int protocol)
1223 ssize_t
1224 lwip_write(int s, const void *data, size_t size)
1225 {
1226+#if USE_LIBOS
1227+  enum KERNEL_LWIP_PATH path = select_path(s);
1228+  if (path == PATH_ERR) {
1229+    return -1;
1230+  } else if (path == PATH_KERNEL) {
1231+    return posix_api->write_fn(s, data, size);
1232+  }
1233+#endif
1234   return lwip_send(s, data, size, 0);
1235 }
1236
1237@@ -2479,7 +2845,7 @@ lwip_poll_should_wake(const struct lwip_select_cb *scb, int fd, int has_recveven
1238  *   NETCONN_EVT_ERROR
1239  * This requirement will be asserted in select_check_waiters()
1240  */
1241-static void
1242+void
1243 event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len)
1244 {
1245   int s, check_waiters;
1246@@ -2528,23 +2894,38 @@ event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len)
1247       if (sock->rcvevent > 1) {
1248         check_waiters = 0;
1249       }
1250+#if USE_LIBOS
1251+      add_epoll_event(conn, EPOLLIN);
1252+#endif
1253       break;
1254     case NETCONN_EVT_RCVMINUS:
1255       sock->rcvevent--;
1256       check_waiters = 0;
1257+#if USE_LIBOS
1258+      del_epoll_event(conn, EPOLLIN);
1259+#endif
1260       break;
1261     case NETCONN_EVT_SENDPLUS:
1262       if (sock->sendevent) {
1263         check_waiters = 0;
1264       }
1265       sock->sendevent = 1;
1266+#if USE_LIBOS
1267+      add_epoll_event(conn, EPOLLOUT);
1268+#endif
1269       break;
1270     case NETCONN_EVT_SENDMINUS:
1271       sock->sendevent = 0;
1272       check_waiters = 0;
1273+#if USE_LIBOS
1274+      del_epoll_event(conn, EPOLLOUT);
1275+#endif
1276       break;
1277     case NETCONN_EVT_ERROR:
1278       sock->errevent = 1;
1279+#if USE_LIBOS
1280+      add_epoll_event(conn, EPOLLERR);
1281+#endif
1282       break;
1283     default:
1284       LWIP_ASSERT("unknown event", 0);
1285@@ -2739,9 +3120,11 @@ lwip_getaddrname(int s, struct sockaddr *name, socklen_t *namelen, u8_t local)
1286   ip_addr_debug_print_val(SOCKETS_DEBUG, naddr);
1287   LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F")\n", port));
1288
1289+#if !USE_LIBOS
1290   if (*namelen > saddr.sa.sa_len) {
1291     *namelen = saddr.sa.sa_len;
1292   }
1293+#endif
1294   MEMCPY(name, &saddr, *namelen);
1295
1296   sock_set_errno(sock, 0);
1297@@ -2752,12 +3135,41 @@ lwip_getaddrname(int s, struct sockaddr *name, socklen_t *namelen, u8_t local)
1298 int
1299 lwip_getpeername(int s, struct sockaddr *name, socklen_t *namelen)
1300 {
1301+#if USE_LIBOS
1302+  struct lwip_sock *sock;
1303+
1304+  sock = posix_api->get_socket(s);
1305+  if (!sock) {
1306+    return posix_api->getpeername_fn(s, name, namelen);
1307+  }
1308+  /*for AF_INET, if has only host type bit, just call linux api,
1309+   *if has libos and host type bits, it's a not connected fd, call
1310+   *linux api and return -1(errno == ENOTCONN) is also ok*/
1311+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1312+    return posix_api->getpeername_fn(s, name, namelen);
1313+  }
1314+#endif
1315+
1316   return lwip_getaddrname(s, name, namelen, 0);
1317 }
1318
1319 int
1320 lwip_getsockname(int s, struct sockaddr *name, socklen_t *namelen)
1321 {
1322+#if USE_LIBOS
1323+  struct lwip_sock *sock;
1324+
1325+  sock = posix_api->get_socket(s);
1326+  if (!sock) {
1327+    return posix_api->getsockname_fn(s, name, namelen);
1328+  }
1329+  /*for AF_INET, if has only host type bit, just call linux api,
1330+   *if has libos and host type bits, also call linux api*/
1331+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1332+    return posix_api->getsockname_fn(s, name, namelen);
1333+  }
1334+#endif
1335+
1336   return lwip_getaddrname(s, name, namelen, 1);
1337 }
1338
1339@@ -2765,15 +3177,28 @@ int
1340 lwip_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen)
1341 {
1342   int err;
1343-  struct lwip_sock *sock = get_socket(s);
1344 #if !LWIP_TCPIP_CORE_LOCKING
1345   err_t cberr;
1346   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
1347 #endif /* !LWIP_TCPIP_CORE_LOCKING */
1348
1349+#if USE_LIBOS
1350+  struct lwip_sock *sock = posix_api->get_socket(s);
1351+
1352+  if (!sock) {
1353+    return posix_api->getsockopt_fn(s, level, optname, optval, optlen);
1354+  }
1355+  /*for AF_INET, we return linux result? */
1356+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1357+    return posix_api->getsockopt_fn(s, level, optname, optval, optlen);
1358+  }
1359+#else
1360+  struct lwip_sock *sock = get_socket(s);
1361+
1362   if (!sock) {
1363     return -1;
1364   }
1365+#endif /* USE_LIBOS */
1366
1367   if ((NULL == optval) || (NULL == optlen)) {
1368     sock_set_errno(sock, EFAULT);
1369@@ -3211,15 +3636,30 @@ int
1370 lwip_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen)
1371 {
1372   int err = 0;
1373-  struct lwip_sock *sock = get_socket(s);
1374 #if !LWIP_TCPIP_CORE_LOCKING
1375   err_t cberr;
1376   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
1377 #endif /* !LWIP_TCPIP_CORE_LOCKING */
1378
1379+#if USE_LIBOS
1380+  struct lwip_sock *sock = posix_api->get_socket(s);
1381+
1382+  if (!sock) {
1383+    return posix_api->setsockopt_fn(s, level, optname, optval, optlen);
1384+  }
1385+  /*for AF_INET, we may try both linux and lwip*/
1386+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1387+    if (posix_api->setsockopt_fn(s, level, optname, optval, optlen) < 0) {
1388+      return -1;
1389+    }
1390+  }
1391+#else
1392+  struct lwip_sock *sock = get_socket(s);
1393+
1394   if (!sock) {
1395     return -1;
1396   }
1397+#endif /* USE_LIBOS */
1398
1399   if (NULL == optval) {
1400     sock_set_errno(sock, EFAULT);
1401@@ -3333,6 +3773,7 @@ lwip_setsockopt_impl(int s, int level, int optname, const void *optval, socklen_
1402         case SO_KEEPALIVE:
1403 #if SO_REUSE
1404         case SO_REUSEADDR:
1405+        case SO_REUSEPORT:
1406 #endif /* SO_REUSE */
1407           if ((optname == SO_BROADCAST) &&
1408               (NETCONNTYPE_GROUP(sock->conn->type) != NETCONN_UDP)) {
1409@@ -3745,6 +4186,29 @@ lwip_setsockopt_impl(int s, int level, int optname, const void *optval, socklen_
1410   return err;
1411 }
1412
1413+#if USE_LIBOS
1414+int
1415+lwip_ioctl(int s, long cmd, ...)
1416+{
1417+  struct lwip_sock *sock = posix_api->get_socket(s);
1418+  u8_t val;
1419+
1420+  int ret = -1;
1421+  void *argp;
1422+  va_list ap;
1423+
1424+  va_start(ap, cmd);
1425+  argp = va_arg(ap, void *);
1426+  va_end(ap);
1427+
1428+  if (!sock) {
1429+    return posix_api->ioctl_fn(s, cmd, argp);
1430+  }
1431+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1432+    if ((ret = posix_api->ioctl_fn(s, cmd, argp)) == -1)
1433+      return ret;
1434+  }
1435+#else
1436 int
1437 lwip_ioctl(int s, long cmd, void *argp)
1438 {
1439@@ -3757,6 +4221,7 @@ lwip_ioctl(int s, long cmd, void *argp)
1440   if (!sock) {
1441     return -1;
1442   }
1443+#endif /* USE_LIBOS */
1444
1445   switch (cmd) {
1446 #if LWIP_SO_RCVBUF || LWIP_FIONREAD_LINUXMODE
1447@@ -3839,6 +4304,26 @@ lwip_ioctl(int s, long cmd, void *argp)
1448  * the flag O_NONBLOCK is implemented for F_SETFL.
1449  */
1450 int
1451+#if USE_LIBOS
1452+lwip_fcntl(int s, int cmd, ...)
1453+{
1454+  struct lwip_sock *sock = posix_api->get_socket(s);
1455+  int val, ret = -1;
1456+  int op_mode = 0;
1457+  va_list ap;
1458+
1459+  va_start(ap, cmd);
1460+  val = va_arg(ap, int);
1461+  va_end(ap);
1462+
1463+  if (!sock) {
1464+    return posix_api->fcntl_fn(s, cmd, val);
1465+  }
1466+  if (CONN_TYPE_HAS_HOST(sock->conn)) {
1467+    if ((ret = posix_api->fcntl_fn(s, cmd, val)) == -1)
1468+      return ret;
1469+  }
1470+#else /* USE_LIBOS */
1471 lwip_fcntl(int s, int cmd, int val)
1472 {
1473   struct lwip_sock *sock = get_socket(s);
1474@@ -3848,6 +4333,7 @@ lwip_fcntl(int s, int cmd, int val)
1475   if (!sock) {
1476     return -1;
1477   }
1478+#endif /* USE_LIBOS */
1479
1480   switch (cmd) {
1481     case F_GETFL:
1482@@ -4163,4 +4649,50 @@ lwip_socket_drop_registered_mld6_memberships(int s)
1483 }
1484 #endif /* LWIP_IPV6_MLD */
1485
1486+#if USE_LIBOS
1487+void lwip_sock_init(void)
1488+{
1489+  if (sockets_num == 0) {
1490+    sockets_num = NUM_SOCKETS;
1491+    sockets = calloc(sockets_num, sizeof(struct lwip_sock));
1492+    LWIP_ASSERT("sockets != NULL", sockets != NULL);
1493+    memset(sockets, 0, sockets_num * sizeof(struct lwip_sock));
1494+  }
1495+  return;
1496+}
1497+
1498+//modify from lwip_close
1499+void lwip_exit(void)
1500+{
1501+  int i, is_tcp;
1502+  struct lwip_sock *sock;
1503+
1504+  if (memp_pools[MEMP_SYS_MBOX] == NULL) {
1505+    return;
1506+  }
1507+
1508+  for (i = 0; i < sockets_num; i++) {
1509+    sock = &sockets[i];
1510+    if (!sock->conn)
1511+      continue;
1512+#if LWIP_IGMP
1513+    /* drop all possibly joined IGMP memberships */
1514+    lwip_socket_drop_registered_memberships(i);
1515+#endif /* LWIP_IGMP */
1516+    /*
1517+     * process is exiting, call netconn_delete to
1518+     * close tcp connection, and ignore the return value
1519+     */
1520+    is_tcp = NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP;
1521+    netconn_delete(sock->conn);
1522+    free_socket(sock, is_tcp);
1523+  }
1524+
1525+  free(sockets);
1526+  sockets = NULL;
1527+  sockets_num = 0;
1528+}
1529+
1530+#endif /* USE_LIBOS */
1531+
1532 #endif /* LWIP_SOCKET */
1533diff --git a/src/api/sys_arch.c b/src/api/sys_arch.c
1534new file mode 100644
1535index 0000000..55561b1
1536--- /dev/null
1537+++ b/src/api/sys_arch.c
1538@@ -0,0 +1,379 @@
1539+/*
1540+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
1541+ * All rights reserved.
1542+ *
1543+ * Redistribution and use in source and binary forms, with or without modification,
1544+ * are permitted provided that the following conditions are met:
1545+ *
1546+ * 1. Redistributions of source code must retain the above copyright notice,
1547+ *    this list of conditions and the following disclaimer.
1548+ * 2. Redistributions in binary form must reproduce the above copyright notice,
1549+ *    this list of conditions and the following disclaimer in the documentation
1550+ *    and/or other materials provided with the distribution.
1551+ * 3. The name of the author may not be used to endorse or promote products
1552+ *    derived from this software without specific prior written permission.
1553+ *
1554+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
1555+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
1556+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
1557+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
1558+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
1559+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1560+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1561+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
1562+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
1563+ * OF SUCH DAMAGE.
1564+ *
1565+ * This file is part of the lwIP TCP/IP stack.
1566+ *
1567+ * Author: Huawei Technologies
1568+ *
1569+ */
1570+
1571+#define _GNU_SOURCE
1572+#include <pthread.h>
1573+#include <string.h>
1574+#include <time.h>
1575+#include <unistd.h>
1576+
1577+#include <rte_memzone.h>
1578+#include <rte_ring.h>
1579+
1580+#include "lwip/err.h"
1581+#include "lwip/mem.h"
1582+#include "lwip/memp.h"
1583+#include "lwip/opt.h"
1584+#include "lwip/sys.h"
1585+#include "lwip/timeouts.h"
1586+#include "arch/sys_arch.h"
1587+
1588+struct sys_mutex {
1589+    volatile unsigned int m;
1590+};
1591+
1592+struct sys_mutex lstack_mutex;
1593+
1594+struct sys_sem lstack_sem;
1595+
1596+#define MAX_THREAD_NAME 64
1597+#define MBOX_NAME_PREFIX  "_mbox_0x"
1598+#define MAX_MBOX_NAME_LEN (sizeof(MBOX_NAME_PREFIX) + 32) // log(UINT64_MAX) < 32
1599+
1600+struct sys_thread {
1601+    struct sys_thread *next;
1602+    char name[MAX_THREAD_NAME];
1603+    lwip_thread_fn fn;
1604+    void *arg;
1605+    int stacksize;
1606+    int prio;
1607+    pthread_t tid;
1608+};
1609+
1610+
1611+struct sys_mem_stats {
1612+    uint32_t tot_len;
1613+};
1614+
1615+static PER_THREAD struct sys_mem_stats hugepage_stats;
1616+
1617+static PER_THREAD uint64_t cycles_per_ms __attribute__((aligned(64)));
1618+static PER_THREAD uint64_t sys_start_ms __attribute__((aligned(64)));
1619+
1620+/*
1621+ * Mailbox
1622+ * */
1623+static int mbox_wait_func(void)
1624+{
1625+#if LWIP_TIMERS
1626+    sys_timer_run();
1627+#endif /* LWIP_TIMER */
1628+    return eth_dev_poll();
1629+}
1630+
1631+err_t sys_mbox_new(struct sys_mbox **mb, int size)
1632+{
1633+    int ret;
1634+    struct sys_mbox *mbox;
1635+
1636+    mbox = (struct sys_mbox *)memp_malloc(MEMP_SYS_MBOX);
1637+    if (mbox == NULL) {
1638+        return ERR_MEM;
1639+    }
1640+
1641+    mbox->flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
1642+
1643+    ret = snprintf(mbox->name, sizeof(mbox->name), MBOX_NAME_PREFIX"%"PRIXPTR, (uintptr_t)mbox);
1644+    if (ret < 0) {
1645+        memp_free(MEMP_SYS_MBOX, mbox);
1646+        return ERR_VAL;
1647+    }
1648+
1649+    mbox->size = size;
1650+    mbox->socket_id = rte_socket_id();
1651+    mbox->ring = rte_ring_create(mbox->name, mbox->size, mbox->socket_id, mbox->flags);
1652+    if (!mbox->ring) {
1653+		RTE_LOG(ERR, EAL, "cannot create rte_ring for mbox\n");
1654+        memp_free(MEMP_SYS_MBOX, mbox);
1655+        return ERR_MEM;
1656+    }
1657+    mbox->wait_fn = mbox_wait_func;
1658+    *mb = mbox;
1659+
1660+    return ERR_OK;
1661+}
1662+
1663+void sys_mbox_free(struct sys_mbox **mb)
1664+{
1665+    struct sys_mbox *mbox = *mb;
1666+    rte_ring_free(mbox->ring);
1667+    memp_free(MEMP_SYS_MBOX, mbox);
1668+}
1669+
1670+err_t sys_mbox_trypost(struct sys_mbox **mb, void *msg)
1671+{
1672+    unsigned int n;
1673+    struct sys_mbox *mbox = *mb;
1674+
1675+    n = rte_ring_sp_enqueue_bulk(mbox->ring, &msg, 1, NULL);
1676+    if (!n)
1677+        return ERR_BUF;
1678+    return ERR_OK;
1679+}
1680+
1681+void sys_mbox_post(struct sys_mbox **mb, void *msg)
1682+{
1683+    struct sys_mbox *mbox = *mb;
1684+
1685+    /* NOTE:  sys_mbox_post is used on mbox defined in src/api/tcpip.c.
1686+    * If the ring size of mbox is greater than MEMP_NUM_TCPIP_MSG_API,
1687+    * enqueue failure will never happen.
1688+    * */
1689+    if (!rte_ring_sp_enqueue_bulk(mbox->ring, &msg, 1, NULL)) {
1690+        LWIP_ASSERT("It is failed to post msg into mbox", 0);
1691+    }
1692+}
1693+
1694+err_t sys_mbox_trypost_fromisr(sys_mbox_t *q, void *msg)
1695+{
1696+    return sys_mbox_trypost(q, msg);
1697+}
1698+
1699+uint32_t sys_arch_mbox_tryfetch(struct sys_mbox **mb, void **msg)
1700+{
1701+    unsigned int n;
1702+    struct sys_mbox *mbox = *mb;
1703+
1704+    n = rte_ring_sc_dequeue_bulk(mbox->ring, msg, 1, NULL);
1705+    if (!n) {
1706+        *msg = NULL;
1707+        return SYS_MBOX_EMPTY;
1708+    }
1709+
1710+    return 0;
1711+}
1712+
1713+uint32_t sys_arch_mbox_fetch(struct sys_mbox **mb, void **msg, uint32_t timeout)
1714+{
1715+    unsigned int n;
1716+    uint32_t poll_ts = 0;
1717+    uint32_t time_needed = 0;
1718+    struct sys_mbox *mbox = *mb;
1719+
1720+    n = rte_ring_sc_dequeue_bulk(mbox->ring, msg, 1, NULL);
1721+
1722+    if (timeout > 0)
1723+        poll_ts = sys_now();
1724+
1725+    while (!n) {
1726+        if (timeout > 0) {
1727+            time_needed = sys_now() - poll_ts;
1728+            if (time_needed >= timeout) {
1729+                return SYS_ARCH_TIMEOUT;
1730+            }
1731+        }
1732+
1733+        (void)mbox->wait_fn();
1734+
1735+        n = rte_ring_sc_dequeue_bulk(mbox->ring, msg, 1, NULL);
1736+    }
1737+
1738+    return time_needed;
1739+}
1740+
1741+int sys_mbox_empty(struct sys_mbox *mb)
1742+{
1743+    return rte_ring_count(mb->ring) == 0;
1744+}
1745+
1746+/*
1747+ * Threads
1748+ * */
1749+sys_thread_t sys_thread_new(const char *name, lwip_thread_fn function, void *arg, int stacksize, int prio)
1750+{
1751+    int err;
1752+    pthread_t tid;
1753+    struct sys_thread *thread;
1754+
1755+    thread = (struct sys_thread *)malloc(sizeof(struct sys_thread));
1756+    if (thread == NULL) {
1757+        LWIP_DEBUGF(SYS_DEBUG, ("sys_thread_new: malloc sys_thread failed\n"));
1758+        rte_exit(EXIT_FAILURE, "malloc sys_thread failed\n");
1759+    }
1760+
1761+    err = pthread_create(&tid, NULL, (void*(*)(void *))function, arg);
1762+    if (err > 0) {
1763+        LWIP_DEBUGF(SYS_DEBUG, ("sys_thread_new: pthread_create failed\n"));
1764+        rte_exit(EXIT_FAILURE, "pthread_create failed\n");
1765+    }
1766+
1767+    err = pthread_setname_np(tid, name);
1768+    if (err > 0) {
1769+        LWIP_DEBUGF(SYS_DEBUG, ("sys_thread_new: pthread_setname_np failed\n"));
1770+    }
1771+    thread->tid = tid;
1772+    thread->stacksize = stacksize;
1773+    thread->prio = prio;
1774+
1775+    return thread;
1776+}
1777+
1778+/*
1779+ * Semaphore
1780+ * */
1781+err_t sys_sem_new(struct sys_sem **sem, uint8_t count)
1782+{
1783+    *sem = (struct sys_sem *)memp_malloc(MEMP_SYS_SEM);
1784+    if ((*sem) == NULL) {
1785+        return ERR_MEM;
1786+    }
1787+    (*sem)->c = 0;
1788+    (*sem)->wait_fn = mbox_wait_func;
1789+    return ERR_OK;
1790+}
1791+
1792+void sys_sem_signal(struct sys_sem **s)
1793+{
1794+    struct sys_sem *sem = NULL;
1795+    LWIP_ASSERT("invalid sem", (s != NULL) && (*s != NULL));
1796+    sem = *s;
1797+    ++(sem->c);
1798+}
1799+
1800+static uint32_t cond_wait(struct sys_sem *sem, uint32_t timeout)
1801+{
1802+    uint32_t used_ms = 0;
1803+    uint32_t poll_ts;
1804+
1805+    if (timeout == 0) {
1806+        (void)sem->wait_fn();
1807+        return 0;
1808+    }
1809+
1810+    poll_ts = sys_now();
1811+
1812+    while (used_ms < timeout) {
1813+        if (sem->c > 0)
1814+            return timeout - used_ms;
1815+
1816+        (void)sem->wait_fn();
1817+        used_ms = sys_now() - poll_ts;
1818+    }
1819+
1820+    return SYS_ARCH_TIMEOUT;
1821+}
1822+
1823+uint32_t sys_arch_sem_wait(struct sys_sem **s, uint32_t timeout)
1824+{
1825+    uint32_t time_needed = 0;
1826+    struct sys_sem *sem = NULL;
1827+    LWIP_ASSERT("invalid sem", (s != NULL) && (*s != NULL));
1828+    sem = *s;
1829+
1830+    while (sem->c <= 0) {
1831+        if (timeout > 0) {
1832+            time_needed = cond_wait(sem, timeout);
1833+
1834+            if (time_needed == SYS_ARCH_TIMEOUT) {
1835+                return SYS_ARCH_TIMEOUT;
1836+            }
1837+        } else {
1838+            cond_wait(sem, 0);
1839+        }
1840+    }
1841+
1842+    sem->c--;
1843+    return time_needed;
1844+}
1845+
1846+void sys_sem_free(struct sys_sem **s)
1847+{
1848+    if ((s != NULL) && (*s != SYS_SEM_NULL))
1849+        memp_free(MEMP_SYS_SEM, *s);
1850+}
1851+
1852+/*
1853+ * Mutex
1854+ * */
1855+err_t sys_mutex_new(struct sys_mutex **mutex)
1856+{
1857+    return ERR_OK;
1858+}
1859+
1860+void sys_mutex_lock(struct sys_mutex **mutex)
1861+{
1862+}
1863+
1864+void sys_mutex_unlock(struct sys_mutex **mutex)
1865+{
1866+}
1867+
1868+void sys_mutex_free(struct sys_mutex **mutex)
1869+{
1870+}
1871+
1872+/* Timer from DPDK */
1873+void sys_calibrate_tsc(void)
1874+{
1875+#define MS_PER_SEC  1E3
1876+    uint64_t freq = rte_get_tsc_hz();
1877+
1878+    cycles_per_ms = (freq + MS_PER_SEC - 1) / MS_PER_SEC;
1879+    sys_start_ms = rte_rdtsc() / cycles_per_ms;
1880+}
1881+
1882+uint32_t sys_now(void)
1883+{
1884+    uint64_t cur_ms = rte_rdtsc() / cycles_per_ms;
1885+    return (uint32_t)(cur_ms - sys_start_ms);
1886+}
1887+
1888+/*
1889+ * Critical section
1890+ * */
1891+sys_prot_t sys_arch_protect(void)
1892+{
1893+    return 0;
1894+}
1895+
1896+void sys_arch_unprotect(sys_prot_t pval)
1897+{
1898+}
1899+
1900+/*
1901+ * Hugepage memory manager
1902+ * */
1903+uint8_t *sys_hugepage_malloc(const char *name, uint32_t size)
1904+{
1905+    const struct rte_memzone *mz;
1906+
1907+    mz = rte_memzone_reserve(name, size, rte_socket_id(), 0);
1908+    if (mz == NULL) {
1909+        rte_exit(EXIT_FAILURE, "failed to reserver memory for mempool[%s]\n", name);
1910+        return NULL;
1911+    }
1912+
1913+    memset(mz->addr, 0, mz->len);
1914+    hugepage_stats.tot_len += mz->len;
1915+
1916+    return (uint8_t*)mz->addr;
1917+}
1918diff --git a/src/api/tcpip.c b/src/api/tcpip.c
1919index a7e312a..d3d0b55 100644
1920--- a/src/api/tcpip.c
1921+++ b/src/api/tcpip.c
1922@@ -56,13 +56,13 @@
1923 #define TCPIP_MSG_VAR_FREE(name)    API_VAR_FREE(MEMP_TCPIP_MSG_API, name)
1924
1925 /* global variables */
1926-static tcpip_init_done_fn tcpip_init_done;
1927-static void *tcpip_init_done_arg;
1928-static sys_mbox_t tcpip_mbox;
1929+static PER_THREAD tcpip_init_done_fn tcpip_init_done;
1930+static PER_THREAD void *tcpip_init_done_arg;
1931+static PER_THREAD sys_mbox_t tcpip_mbox;
1932
1933 #if LWIP_TCPIP_CORE_LOCKING
1934 /** The global semaphore to lock the stack. */
1935-sys_mutex_t lock_tcpip_core;
1936+PER_THREAD sys_mutex_t lock_tcpip_core;
1937 #endif /* LWIP_TCPIP_CORE_LOCKING */
1938
1939 static void tcpip_thread_handle_msg(struct tcpip_msg *msg);
1940@@ -123,8 +123,13 @@ again:
1941  *
1942  * @param arg unused argument
1943  */
1944+#if USE_LIBOS
1945+__attribute__((unused)) static void
1946+tcpip_thread(void *arg)
1947+#else
1948 static void
1949 tcpip_thread(void *arg)
1950+#endif /* USE_LIBOS */
1951 {
1952   struct tcpip_msg *msg;
1953   LWIP_UNUSED_ARG(arg);
1954@@ -242,6 +247,9 @@ tcpip_inpkt(struct pbuf *p, struct netif *inp, netif_input_fn input_fn)
1955 #if LWIP_TCPIP_CORE_LOCKING_INPUT
1956   err_t ret;
1957   LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_inpkt: PACKET %p/%p\n", (void *)p, (void *)inp));
1958+#if USE_LIBOS && LWIP_TIMERS
1959+  sys_timer_run();
1960+#endif
1961   LOCK_TCPIP_CORE();
1962   ret = input_fn(p, inp);
1963   UNLOCK_TCPIP_CORE();
1964@@ -321,6 +329,9 @@ tcpip_callback(tcpip_callback_fn function, void *ctx)
1965   msg->msg.cb.function = function;
1966   msg->msg.cb.ctx = ctx;
1967
1968+#if USE_LIBOS && LWIP_TIMER
1969+  sys_timer_run();
1970+#endif
1971   sys_mbox_post(&tcpip_mbox, msg);
1972   return ERR_OK;
1973 }
1974@@ -357,6 +368,9 @@ tcpip_try_callback(tcpip_callback_fn function, void *ctx)
1975   msg->msg.cb.function = function;
1976   msg->msg.cb.ctx = ctx;
1977
1978+#if USE_LIBOS && LWIP_TIMER
1979+  sys_timer_run();
1980+#endif
1981   if (sys_mbox_trypost(&tcpip_mbox, msg) != ERR_OK) {
1982     memp_free(MEMP_TCPIP_MSG_API, msg);
1983     return ERR_MEM;
1984@@ -438,6 +452,9 @@ tcpip_send_msg_wait_sem(tcpip_callback_fn fn, void *apimsg, sys_sem_t *sem)
1985 {
1986 #if LWIP_TCPIP_CORE_LOCKING
1987   LWIP_UNUSED_ARG(sem);
1988+#if USE_LIBOS && LWIP_TIMERS
1989+  sys_timer_run();
1990+#endif
1991   LOCK_TCPIP_CORE();
1992   fn(apimsg);
1993   UNLOCK_TCPIP_CORE();
1994@@ -475,6 +492,9 @@ tcpip_api_call(tcpip_api_call_fn fn, struct tcpip_api_call_data *call)
1995 #if LWIP_TCPIP_CORE_LOCKING
1996   err_t err;
1997   LOCK_TCPIP_CORE();
1998+#if USE_LIBOS && LWIP_TIMERS
1999+  sys_timer_run();
2000+#endif
2001   err = fn(call);
2002   UNLOCK_TCPIP_CORE();
2003   return err;
2004@@ -537,6 +557,10 @@ tcpip_callbackmsg_new(tcpip_callback_fn function, void *ctx)
2005   msg->type = TCPIP_MSG_CALLBACK_STATIC;
2006   msg->msg.cb.function = function;
2007   msg->msg.cb.ctx = ctx;
2008+
2009+#if USE_LIBOS && LWIP_TIMER
2010+  sys_timer_run();
2011+#endif
2012   return (struct tcpip_callback_msg *)msg;
2013 }
2014
2015@@ -614,7 +638,9 @@ tcpip_init(tcpip_init_done_fn initfunc, void *arg)
2016   }
2017 #endif /* LWIP_TCPIP_CORE_LOCKING */
2018
2019+#if !USE_LIBOS
2020   sys_thread_new(TCPIP_THREAD_NAME, tcpip_thread, NULL, TCPIP_THREAD_STACKSIZE, TCPIP_THREAD_PRIO);
2021+#endif
2022 }
2023
2024 /**
2025diff --git a/src/core/dir.mk b/src/core/dir.mk
2026index e5a055b..ebc01a5 100644
2027--- a/src/core/dir.mk
2028+++ b/src/core/dir.mk
2029@@ -1,6 +1,6 @@
2030-SRC = inet_chksum.c init.c ip.c mem.c memp.c netif.c pbuf.c \
2031-	raw.c stats.c tcp.c tcp_in.c tcp_out.c timeouts.c udp.c \
2032-	ipv4/etharp.c ipv4/icmp.c ipv4/ip4_addr.c ipv4/ip4.c \
2033-	ipv4/ip4_frag.c
2034+SRC = def.c inet_chksum.c init.c ip.c mem.c memp.c netif.c pbuf.c \
2035+	  raw.c tcp.c tcp_in.c tcp_out.c timeouts.c udp.c stats.c\
2036+	  ipv4/icmp.c ipv4/ip4_addr.c ipv4/ip4_frag.c ipv4/etharp.c \
2037+	  ipv4/ip4.c
2038
2039 $(eval $(call register_dir, core, $(SRC)))
2040diff --git a/src/core/init.c b/src/core/init.c
2041index 3620e1d..60e1c68 100644
2042--- a/src/core/init.c
2043+++ b/src/core/init.c
2044@@ -343,9 +343,7 @@ lwip_init(void)
2045
2046   /* Modules initialization */
2047   stats_init();
2048-#if !NO_SYS
2049-  sys_init();
2050-#endif /* !NO_SYS */
2051+
2052   mem_init();
2053   memp_init();
2054   pbuf_init();
2055diff --git a/src/core/ip.c b/src/core/ip.c
2056index 18514cf..0d39d2d 100644
2057--- a/src/core/ip.c
2058+++ b/src/core/ip.c
2059@@ -61,7 +61,7 @@
2060 #include "lwip/ip.h"
2061
2062 /** Global data for both IPv4 and IPv6 */
2063-struct ip_globals ip_data;
2064+PER_THREAD struct ip_globals ip_data;
2065
2066 #if LWIP_IPV4 && LWIP_IPV6
2067
2068diff --git a/src/core/ipv4/ip4.c b/src/core/ipv4/ip4.c
2069index 26c26a9..c83afbe 100644
2070--- a/src/core/ipv4/ip4.c
2071+++ b/src/core/ipv4/ip4.c
2072@@ -282,7 +282,9 @@ ip4_forward(struct pbuf *p, struct ip_hdr *iphdr, struct netif *inp)
2073 {
2074   struct netif *netif;
2075
2076+#ifndef LWIP_PERF
2077   PERF_START;
2078+#endif
2079   LWIP_UNUSED_ARG(inp);
2080
2081   if (!ip4_canforward(p)) {
2082@@ -344,7 +346,9 @@ ip4_forward(struct pbuf *p, struct ip_hdr *iphdr, struct netif *inp)
2083   MIB2_STATS_INC(mib2.ipforwdatagrams);
2084   IP_STATS_INC(ip.xmit);
2085
2086+#ifndef LWIP_PERF
2087   PERF_STOP("ip4_forward");
2088+#endif
2089   /* don't fragment if interface has mtu set to 0 [loopif] */
2090   if (netif->mtu && (p->tot_len > netif->mtu)) {
2091     if ((IPH_OFFSET(iphdr) & PP_NTOHS(IP_DF)) == 0) {
2092@@ -438,6 +442,8 @@ ip4_input(struct pbuf *p, struct netif *inp)
2093
2094   LWIP_ASSERT_CORE_LOCKED();
2095
2096+  PERF_START(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2097+
2098   IP_STATS_INC(ip.recv);
2099   MIB2_STATS_INC(mib2.ipinreceives);
2100
2101@@ -700,13 +706,19 @@ ip4_input(struct pbuf *p, struct netif *inp)
2102       case IP_PROTO_UDPLITE:
2103 #endif /* LWIP_UDPLITE */
2104         MIB2_STATS_INC(mib2.ipindelivers);
2105+        PERF_PAUSE(PERF_LAYER_IP);
2106         udp_input(p, inp);
2107+        PERF_RESUME(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2108         break;
2109 #endif /* LWIP_UDP */
2110 #if LWIP_TCP
2111       case IP_PROTO_TCP:
2112         MIB2_STATS_INC(mib2.ipindelivers);
2113+        PERF_PAUSE(PERF_LAYER_IP);
2114+        PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_RECV);
2115         tcp_input(p, inp);
2116+        PERF_STOP_INCREASE_COUNT("tcp_input", PERF_LAYER_TCP);
2117+        PERF_RESUME(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2118         break;
2119 #endif /* LWIP_TCP */
2120 #if LWIP_ICMP
2121@@ -755,6 +767,8 @@ ip4_input(struct pbuf *p, struct netif *inp)
2122   ip4_addr_set_any(ip4_current_src_addr());
2123   ip4_addr_set_any(ip4_current_dest_addr());
2124
2125+  PERF_STOP_INCREASE_COUNT("ip4_input", PERF_LAYER_IP);
2126+
2127   return ERR_OK;
2128 }
2129
2130diff --git a/src/core/ipv6/ip6.c b/src/core/ipv6/ip6.c
2131index 060d5f3..9d904ec 100644
2132--- a/src/core/ipv6/ip6.c
2133+++ b/src/core/ipv6/ip6.c
2134@@ -522,6 +522,8 @@ ip6_input(struct pbuf *p, struct netif *inp)
2135
2136   LWIP_ASSERT_CORE_LOCKED();
2137
2138+  PERF_START(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2139+
2140   IP6_STATS_INC(ip6.recv);
2141
2142   /* identify the IP header */
2143@@ -1069,12 +1071,18 @@ options_done:
2144 #if LWIP_UDPLITE
2145     case IP6_NEXTH_UDPLITE:
2146 #endif /* LWIP_UDPLITE */
2147+      PERF_PAUSE(PERF_LAYER_IP);
2148       udp_input(p, inp);
2149+      PERF_RESUME(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2150       break;
2151 #endif /* LWIP_UDP */
2152 #if LWIP_TCP
2153     case IP6_NEXTH_TCP:
2154+      PERF_PAUSE(PERF_LAYER_IP);
2155+      PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_RECV);
2156       tcp_input(p, inp);
2157+      PERF_STOP_INCREASE_COUNT("tcp_input", PERF_LAYER_TCP);
2158+      PERF_RESUME(PERF_LAYER_IP, PERF_POINT_IP_RECV);
2159       break;
2160 #endif /* LWIP_TCP */
2161 #if LWIP_ICMP6
2162@@ -1115,6 +1123,8 @@ ip6_input_cleanup:
2163   ip6_addr_set_zero(ip6_current_src_addr());
2164   ip6_addr_set_zero(ip6_current_dest_addr());
2165
2166+  PERF_STOP_INCREASE_COUNT("ip6_input", PERF_LAYER_IP);
2167+
2168   return ERR_OK;
2169 }
2170
2171diff --git a/src/core/mem.c b/src/core/mem.c
2172index 315fb3c..84b3fcc 100644
2173--- a/src/core/mem.c
2174+++ b/src/core/mem.c
2175@@ -381,9 +381,9 @@ LWIP_DECLARE_MEMORY_ALIGNED(ram_heap, MEM_SIZE_ALIGNED + (2U * SIZEOF_STRUCT_MEM
2176 #endif /* LWIP_RAM_HEAP_POINTER */
2177
2178 /** pointer to the heap (ram_heap): for alignment, ram is now a pointer instead of an array */
2179-static u8_t *ram;
2180+static PER_THREAD u8_t *ram;
2181 /** the last entry, always unused! */
2182-static struct mem *ram_end;
2183+static PER_THREAD struct mem *ram_end;
2184
2185 /** concurrent access protection */
2186 #if !NO_SYS
2187@@ -418,7 +418,7 @@ static volatile u8_t mem_free_count;
2188 #endif /* LWIP_ALLOW_MEM_FREE_FROM_OTHER_CONTEXT */
2189
2190 /** pointer to the lowest free block, this is used for faster search */
2191-static struct mem * LWIP_MEM_LFREE_VOLATILE lfree;
2192+static PER_THREAD struct mem * LWIP_MEM_LFREE_VOLATILE lfree;
2193
2194 #if MEM_SANITY_CHECK
2195 static void mem_sanity(void);
2196diff --git a/src/core/memp.c b/src/core/memp.c
2197index 352ce5a..454ba32 100644
2198--- a/src/core/memp.c
2199+++ b/src/core/memp.c
2200@@ -78,10 +78,14 @@
2201 #define LWIP_MEMPOOL(name,num,size,desc) LWIP_MEMPOOL_DECLARE(name,num,size,desc)
2202 #include "lwip/priv/memp_std.h"
2203
2204+#if USE_LIBOS
2205+PER_THREAD struct memp_desc* memp_pools[MEMP_MAX] = {NULL};
2206+#else
2207 const struct memp_desc *const memp_pools[MEMP_MAX] = {
2208 #define LWIP_MEMPOOL(name,num,size,desc) &memp_ ## name,
2209 #include "lwip/priv/memp_std.h"
2210 };
2211+#endif /* USE_LIBOS */
2212
2213 #ifdef LWIP_HOOK_FILENAME
2214 #include LWIP_HOOK_FILENAME
2215diff --git a/src/core/netif.c b/src/core/netif.c
2216index 088b50e..70392cb 100644
2217--- a/src/core/netif.c
2218+++ b/src/core/netif.c
2219@@ -107,12 +107,12 @@ static netif_ext_callback_t *ext_callback;
2220 #endif
2221
2222 #if !LWIP_SINGLE_NETIF
2223-struct netif *netif_list;
2224+PER_THREAD struct netif *netif_list;
2225 #endif /* !LWIP_SINGLE_NETIF */
2226-struct netif *netif_default;
2227+PER_THREAD struct netif *netif_default;
2228
2229 #define netif_index_to_num(index)   ((index) - 1)
2230-static u8_t netif_num;
2231+static PER_THREAD u8_t netif_num;
2232
2233 #if LWIP_NUM_NETIF_CLIENT_DATA > 0
2234 static u8_t netif_client_id;
2235@@ -138,7 +138,7 @@ static err_t netif_loop_output_ipv6(struct netif *netif, struct pbuf *p, const i
2236 #endif
2237
2238
2239-static struct netif loop_netif;
2240+static PER_THREAD struct netif loop_netif;
2241
2242 /**
2243  * Initialize a lwip network interface structure for a loopback interface
2244diff --git a/src/core/pbuf.c b/src/core/pbuf.c
2245index 7638dfd..27afc28 100644
2246--- a/src/core/pbuf.c
2247+++ b/src/core/pbuf.c
2248@@ -737,7 +737,9 @@ pbuf_free(struct pbuf *p)
2249   }
2250   LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_free(%p)\n", (void *)p));
2251
2252+#ifndef LWIP_PERF
2253   PERF_START;
2254+#endif
2255
2256   count = 0;
2257   /* de-allocate all consecutive pbufs from the head of the chain that
2258@@ -794,7 +796,9 @@ pbuf_free(struct pbuf *p)
2259       p = NULL;
2260     }
2261   }
2262+#ifndef LWIP_PERF
2263   PERF_STOP("pbuf_free");
2264+#endif
2265   /* return number of de-allocated pbufs */
2266   return count;
2267 }
2268diff --git a/src/core/stats.c b/src/core/stats.c
2269index 34e9b27..f7e0604 100644
2270--- a/src/core/stats.c
2271+++ b/src/core/stats.c
2272@@ -47,7 +47,7 @@
2273
2274 #include <string.h>
2275
2276-struct stats_ lwip_stats;
2277+PER_THREAD struct stats_ lwip_stats;
2278
2279 void
2280 stats_init(void)
2281@@ -59,6 +59,17 @@ stats_init(void)
2282 #endif /* LWIP_DEBUG */
2283 }
2284
2285+int get_mib2_stats(char *buf)
2286+{
2287+  int len = 0;
2288+#if MIB2_STATS
2289+  len = (long)&((struct stats_mib2 *)0)->udpindatagrams;
2290+  /* we just need the ip&tcp, others not needed. */
2291+  memcpy(buf, &lwip_stats.mib2, len);
2292+#endif
2293+  return len;
2294+}
2295+
2296 #if LWIP_STATS_DISPLAY
2297 void
2298 stats_display_proto(struct stats_proto *proto, const char *name)
2299diff --git a/src/core/tcp.c b/src/core/tcp.c
2300index 371db2b..9e75810 100644
2301--- a/src/core/tcp.c
2302+++ b/src/core/tcp.c
2303@@ -113,6 +113,7 @@
2304 #include "lwip/nd6.h"
2305
2306 #include <string.h>
2307+#include <pthread.h>
2308
2309 #ifdef LWIP_HOOK_FILENAME
2310 #include LWIP_HOOK_FILENAME
2311@@ -157,36 +158,50 @@ static const char *const tcp_state_str[] = {
2312
2313 /* last local TCP port */
2314 static u16_t tcp_port = TCP_LOCAL_PORT_RANGE_START;
2315+static pthread_mutex_t g_tcp_port_mutex = PTHREAD_MUTEX_INITIALIZER;
2316
2317 /* Incremented every coarse grained timer shot (typically every 500 ms). */
2318-u32_t tcp_ticks;
2319-static const u8_t tcp_backoff[13] =
2320+PER_THREAD u32_t tcp_ticks;
2321+static PER_THREAD const u8_t tcp_backoff[13] =
2322 { 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7};
2323 /* Times per slowtmr hits */
2324-static const u8_t tcp_persist_backoff[7] = { 3, 6, 12, 24, 48, 96, 120 };
2325+static PER_THREAD const u8_t tcp_persist_backoff[7] = { 3, 6, 12, 24, 48, 96, 120 };
2326
2327 /* The TCP PCB lists. */
2328
2329 /** List of all TCP PCBs bound but not yet (connected || listening) */
2330-struct tcp_pcb *tcp_bound_pcbs;
2331+PER_THREAD struct tcp_pcb *tcp_bound_pcbs;
2332 /** List of all TCP PCBs in LISTEN state */
2333-union tcp_listen_pcbs_t tcp_listen_pcbs;
2334+PER_THREAD union tcp_listen_pcbs_t tcp_listen_pcbs;
2335 /** List of all TCP PCBs that are in a state in which
2336  * they accept or send data. */
2337-struct tcp_pcb *tcp_active_pcbs;
2338+PER_THREAD struct tcp_pcb *tcp_active_pcbs;
2339 /** List of all TCP PCBs in TIME-WAIT state */
2340-struct tcp_pcb *tcp_tw_pcbs;
2341+PER_THREAD struct tcp_pcb *tcp_tw_pcbs;
2342
2343 /** An array with all (non-temporary) PCB lists, mainly used for smaller code size */
2344-struct tcp_pcb **const tcp_pcb_lists[] = {&tcp_listen_pcbs.pcbs, &tcp_bound_pcbs,
2345-         &tcp_active_pcbs, &tcp_tw_pcbs
2346-};
2347+PER_THREAD struct tcp_pcb ** tcp_pcb_lists[NUM_TCP_PCB_LISTS] = {NULL, NULL, NULL, NULL};
2348+
2349+#if TCP_PCB_HASH
2350+#define INIT_TCP_HTABLE(ht_ptr) \
2351+  do { \
2352+    int _i; \
2353+    (ht_ptr)->size = TCP_HTABLE_SIZE; \
2354+    for (_i = 0; _i < TCP_HTABLE_SIZE; ++_i) { \
2355+      if (sys_mutex_new(&(ht_ptr)->array[_i].mutex) != ERR_OK) \
2356+        LWIP_ASSERT("failed to create ht->array[].mutex", 0);\
2357+      INIT_HLIST_HEAD(&(ht_ptr)->array[_i].chain); \
2358+    }\
2359+  } while (0)
2360+
2361+PER_THREAD struct tcp_hash_table  *tcp_active_htable; /* key: lport/fport/lip/fip */
2362+#endif
2363
2364-u8_t tcp_active_pcbs_changed;
2365+PER_THREAD u8_t tcp_active_pcbs_changed;
2366
2367 /** Timer counter to handle calling slow-timer from tcp_tmr() */
2368-static u8_t tcp_timer;
2369-static u8_t tcp_timer_ctr;
2370+static PER_THREAD u8_t tcp_timer;
2371+static PER_THREAD u8_t tcp_timer_ctr;
2372 static u16_t tcp_new_port(void);
2373
2374 static err_t tcp_close_shutdown_fin(struct tcp_pcb *pcb);
2375@@ -200,9 +215,20 @@ static void tcp_ext_arg_invoke_callbacks_destroyed(struct tcp_pcb_ext_args *ext_
2376 void
2377 tcp_init(void)
2378 {
2379+  tcp_pcb_lists[0] = &tcp_listen_pcbs.pcbs;
2380+  tcp_pcb_lists[1] = &tcp_bound_pcbs;
2381+  tcp_pcb_lists[2] = &tcp_active_pcbs;
2382+  tcp_pcb_lists[3] = &tcp_tw_pcbs;
2383+
2384 #ifdef LWIP_RAND
2385   tcp_port = TCP_ENSURE_LOCAL_PORT_RANGE(LWIP_RAND());
2386 #endif /* LWIP_RAND */
2387+
2388+#if TCP_PCB_HASH
2389+  tcp_active_htable = (struct tcp_hash_table*)mem_malloc(sizeof(struct tcp_hash_table));
2390+  LWIP_ASSERT("malloc tcp_active_htable mem failed.", tcp_active_htable != NULL);
2391+  INIT_TCP_HTABLE(tcp_active_htable);
2392+#endif
2393 }
2394
2395 /** Free a tcp pcb */
2396@@ -361,6 +387,9 @@ tcp_close_shutdown(struct tcp_pcb *pcb, u8_t rst_on_unacked_data)
2397               pcb->local_port, pcb->remote_port);
2398
2399       tcp_pcb_purge(pcb);
2400+#if TCP_PCB_HASH
2401+      TCP_RMV_ACTIVE_HASH(pcb);
2402+#endif
2403       TCP_RMV_ACTIVE(pcb);
2404       /* Deallocate the pcb since we already sent a RST for it */
2405       if (tcp_input_pcb == pcb) {
2406@@ -395,6 +424,9 @@ tcp_close_shutdown(struct tcp_pcb *pcb, u8_t rst_on_unacked_data)
2407       tcp_free_listen(pcb);
2408       break;
2409     case SYN_SENT:
2410+#if TCP_PCB_HASH
2411+      TCP_PCB_REMOVE_ACTIVE_HASH(pcb);
2412+#endif
2413       TCP_PCB_REMOVE_ACTIVE(pcb);
2414       tcp_free(pcb);
2415       MIB2_STATS_INC(mib2.tcpattemptfails);
2416@@ -494,6 +526,7 @@ tcp_close(struct tcp_pcb *pcb)
2417     /* Set a flag not to receive any more data... */
2418     tcp_set_flags(pcb, TF_RXCLOSED);
2419   }
2420+
2421   /* ... and close */
2422   return tcp_close_shutdown(pcb, 1);
2423 }
2424@@ -599,6 +632,9 @@ tcp_abandon(struct tcp_pcb *pcb, int reset)
2425     } else {
2426       send_rst = reset;
2427       local_port = pcb->local_port;
2428+#if TCP_PCB_HASH
2429+      TCP_PCB_REMOVE_ACTIVE_HASH(pcb);
2430+#endif
2431       TCP_PCB_REMOVE_ACTIVE(pcb);
2432     }
2433     if (pcb->unacked != NULL) {
2434@@ -880,6 +916,11 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
2435     }
2436   }
2437 #endif /* SO_REUSE */
2438+
2439+#if USE_LIBOS
2440+  vdev_reg_done(REG_RING_TCP_LISTEN, pcb);
2441+#endif
2442+
2443   lpcb = (struct tcp_pcb_listen *)memp_malloc(MEMP_TCP_PCB_LISTEN);
2444   if (lpcb == NULL) {
2445     res = ERR_MEM;
2446@@ -1015,6 +1056,7 @@ tcp_new_port(void)
2447   u16_t n = 0;
2448   struct tcp_pcb *pcb;
2449
2450+  pthread_mutex_lock(&g_tcp_port_mutex);
2451 again:
2452   tcp_port++;
2453   if (tcp_port == TCP_LOCAL_PORT_RANGE_END) {
2454@@ -1032,6 +1074,8 @@ again:
2455       }
2456     }
2457   }
2458+  pthread_mutex_unlock(&g_tcp_port_mutex);
2459+
2460   return tcp_port;
2461 }
2462
2463@@ -1142,6 +1186,10 @@ tcp_connect(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port,
2464 #endif /* SO_REUSE */
2465   }
2466
2467+#if USE_LIBOS
2468+  vdev_reg_done(REG_RING_TCP_CONNECT, pcb);
2469+#endif
2470+
2471   iss = tcp_next_iss(pcb);
2472   pcb->rcv_nxt = 0;
2473   pcb->snd_nxt = iss;
2474@@ -1174,6 +1222,9 @@ tcp_connect(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port,
2475     if (old_local_port != 0) {
2476       TCP_RMV(&tcp_bound_pcbs, pcb);
2477     }
2478+#if TCP_PCB_HASH
2479+    TCP_REG_ACTIVE_HASH(pcb);
2480+#endif
2481     TCP_REG_ACTIVE(pcb);
2482     MIB2_STATS_INC(mib2.tcpactiveopens);
2483
2484@@ -1389,11 +1440,26 @@ tcp_slowtmr_start:
2485       if (prev != NULL) {
2486         LWIP_ASSERT("tcp_slowtmr: middle tcp != tcp_active_pcbs", pcb != tcp_active_pcbs);
2487         prev->next = pcb->next;
2488+#if USE_LIBOS
2489+        if (pcb->next)
2490+          pcb->next->prev = prev;
2491+        //dont set next NULL, it will be used below
2492+        pcb->prev = NULL;
2493+#endif
2494       } else {
2495         /* This PCB was the first. */
2496         LWIP_ASSERT("tcp_slowtmr: first pcb == tcp_active_pcbs", tcp_active_pcbs == pcb);
2497         tcp_active_pcbs = pcb->next;
2498+#if USE_LIBOS
2499+        if (pcb->next)
2500+          pcb->next->prev = NULL;
2501+        //dont set next NULL, it will be used below
2502+        pcb->prev = NULL;
2503+#endif
2504       }
2505+#if TCP_PCB_HASH
2506+      TCP_RMV_ACTIVE_HASH(pcb);
2507+#endif
2508
2509       if (pcb_reset) {
2510         tcp_rst(pcb, pcb->snd_nxt, pcb->rcv_nxt, &pcb->local_ip, &pcb->remote_ip,
2511@@ -1404,6 +1470,9 @@ tcp_slowtmr_start:
2512       last_state = pcb->state;
2513       pcb2 = pcb;
2514       pcb = pcb->next;
2515+#if USE_LIBOS
2516+      pcb2->next = NULL;
2517+#endif
2518       tcp_free(pcb2);
2519
2520       tcp_active_pcbs_changed = 0;
2521@@ -1455,13 +1524,28 @@ tcp_slowtmr_start:
2522       if (prev != NULL) {
2523         LWIP_ASSERT("tcp_slowtmr: middle tcp != tcp_tw_pcbs", pcb != tcp_tw_pcbs);
2524         prev->next = pcb->next;
2525+#if USE_LIBOS
2526+        if (pcb->next)
2527+          pcb->next->prev = prev;
2528+        //dont set next NULL, it will be used below
2529+        pcb->prev = NULL;
2530+#endif
2531       } else {
2532         /* This PCB was the first. */
2533         LWIP_ASSERT("tcp_slowtmr: first pcb == tcp_tw_pcbs", tcp_tw_pcbs == pcb);
2534         tcp_tw_pcbs = pcb->next;
2535+#if USE_LIBOS
2536+        if (pcb->next)
2537+          pcb->next->prev = NULL;
2538+        //dont set next NULL, it will be used below
2539+        pcb->prev = NULL;
2540+#endif
2541       }
2542       pcb2 = pcb;
2543       pcb = pcb->next;
2544+#if USE_LIBOS
2545+      pcb2->next = NULL;
2546+#endif
2547       tcp_free(pcb2);
2548     } else {
2549       prev = pcb;
2550@@ -2210,6 +2294,14 @@ tcp_pcb_remove(struct tcp_pcb **pcblist, struct tcp_pcb *pcb)
2551   LWIP_ASSERT("tcp_pcb_remove: tcp_pcbs_sane()", tcp_pcbs_sane());
2552 }
2553
2554+#if TCP_PCB_HASH
2555+void
2556+tcp_pcb_remove_hash(struct tcp_hash_table *htb, struct tcp_pcb *pcb)
2557+{
2558+  TCP_RMV_HASH(htb, pcb);
2559+}
2560+#endif /* TCP_PCB_HASH */
2561+
2562 /**
2563  * Calculates a new initial sequence number for new connections.
2564  *
2565@@ -2384,6 +2476,84 @@ tcp_tcp_get_tcp_addrinfo(struct tcp_pcb *pcb, int local, ip_addr_t *addr, u16_t
2566   return ERR_VAL;
2567 }
2568
2569+uint32_t tcp_get_conn_num(void)
2570+{
2571+    struct tcp_pcb *pcb = NULL;
2572+    struct tcp_pcb_listen *pcbl = NULL;
2573+    uint32_t conn_num = 0;
2574+
2575+    for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) {
2576+        conn_num++;
2577+    }
2578+
2579+    for (pcbl = tcp_listen_pcbs.listen_pcbs; pcbl != NULL; pcbl = pcbl->next) {
2580+        conn_num++;
2581+    }
2582+
2583+    for (pcb = tcp_tw_pcbs; pcb != NULL; pcb = pcb->next) {
2584+        conn_num++;
2585+    }
2586+
2587+    return conn_num;
2588+}
2589+
2590+void tcp_get_conn(char *buf, int32_t len, uint32_t *conn_num)
2591+{
2592+  int tmp_len = 0;
2593+  char *tmp_buf = buf;
2594+  struct tcp_pcb_dp tdp;
2595+  struct tcp_pcb *pcb = NULL;
2596+  struct tcp_pcb_listen *pcbl = NULL;
2597+
2598+#define COPY_TDP(b, l) \
2599+  do { \
2600+    if (l + sizeof(tdp) <= len) { \
2601+	  memcpy(b, &tdp, sizeof(tdp)); \
2602+	  b += sizeof(tdp); \
2603+	  l += sizeof(tdp); \
2604+      *conn_num += 1; \
2605+    } else \
2606+      return; \
2607+  } while(0);
2608+
2609+  *conn_num = 0;
2610+
2611+  for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) {
2612+    tdp.state = ACTIVE_LIST;
2613+    tdp.lip = pcb->local_ip.addr;
2614+    tdp.rip = pcb->remote_ip.addr;
2615+    tdp.l_port = pcb->local_port;
2616+    tdp.r_port = pcb->remote_port;
2617+    tdp.s_next = pcb->snd_queuelen;
2618+    /* lwip not cache rcv buf. Set it to 0. */
2619+    tdp.r_next = 0;
2620+    tdp.tcp_sub_state = pcb->state;
2621+    COPY_TDP(tmp_buf, tmp_len);
2622+  }
2623+
2624+  for (pcbl = tcp_listen_pcbs.listen_pcbs; pcbl != NULL; pcbl = pcbl->next) {
2625+    tdp.state = LISTEN_LIST;
2626+    tdp.lip = pcbl->local_ip.addr;
2627+    tdp.rip = pcbl->remote_ip.addr;
2628+    tdp.l_port = pcbl->local_port;
2629+    tdp.tcp_sub_state = pcbl->state;
2630+    COPY_TDP(tmp_buf, tmp_len);
2631+  }
2632+
2633+  for (pcb = tcp_tw_pcbs; pcb != NULL; pcb = pcb->next) {
2634+    tdp.state = TIME_WAIT_LIST;
2635+    tdp.lip = pcb->local_ip.addr;
2636+    tdp.rip = pcb->remote_ip.addr;
2637+    tdp.l_port = pcb->local_port;
2638+    tdp.r_port = pcb->remote_port;
2639+    tdp.s_next = pcb->snd_queuelen;
2640+    /* lwip not cache rcv buf. Set it to 0. */
2641+    tdp.r_next = 0;
2642+    tdp.tcp_sub_state = pcb->state;
2643+    COPY_TDP(tmp_buf, tmp_len);
2644+  }
2645+}
2646+
2647 #if TCP_QUEUE_OOSEQ
2648 /* Free all ooseq pbufs (and possibly reset SACK state) */
2649 void
2650diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c
2651index 2202e38..2b4c160 100644
2652--- a/src/core/tcp_in.c
2653+++ b/src/core/tcp_in.c
2654@@ -71,21 +71,22 @@
2655 /* These variables are global to all functions involved in the input
2656    processing of TCP segments. They are set by the tcp_input()
2657    function. */
2658-static struct tcp_seg inseg;
2659-static struct tcp_hdr *tcphdr;
2660-static u16_t tcphdr_optlen;
2661-static u16_t tcphdr_opt1len;
2662-static u8_t *tcphdr_opt2;
2663-static u16_t tcp_optidx;
2664-static u32_t seqno, ackno;
2665-static tcpwnd_size_t recv_acked;
2666-static u16_t tcplen;
2667-static u8_t flags;
2668-
2669-static u8_t recv_flags;
2670-static struct pbuf *recv_data;
2671-
2672-struct tcp_pcb *tcp_input_pcb;
2673+static PER_THREAD struct tcp_seg inseg;
2674+static PER_THREAD struct tcp_hdr *tcphdr;
2675+static PER_THREAD u16_t tcphdr_optlen;
2676+static PER_THREAD u16_t tcphdr_opt1len;
2677+static PER_THREAD u8_t *tcphdr_opt2;
2678+static PER_THREAD u16_t tcp_optidx;
2679+static PER_THREAD u32_t seqno;
2680+static PER_THREAD u32_t ackno;
2681+static PER_THREAD tcpwnd_size_t recv_acked;
2682+static PER_THREAD u16_t tcplen;
2683+static PER_THREAD u8_t flags;
2684+
2685+static PER_THREAD u8_t recv_flags;
2686+static PER_THREAD struct pbuf *recv_data;
2687+
2688+PER_THREAD struct tcp_pcb *tcp_input_pcb;
2689
2690 /* Forward declarations. */
2691 static err_t tcp_process(struct tcp_pcb *pcb);
2692@@ -126,11 +127,20 @@ tcp_input(struct pbuf *p, struct netif *inp)
2693   u8_t hdrlen_bytes;
2694   err_t err;
2695
2696+#if TCP_PCB_HASH
2697+  u32_t idx;
2698+  struct hlist_head *head;
2699+  struct hlist_node *node;
2700+  pcb = NULL;
2701+#endif
2702+
2703   LWIP_UNUSED_ARG(inp);
2704   LWIP_ASSERT_CORE_LOCKED();
2705   LWIP_ASSERT("tcp_input: invalid pbuf", p != NULL);
2706
2707+#ifndef LWIP_PERF
2708   PERF_START;
2709+#endif
2710
2711   TCP_STATS_INC(tcp.recv);
2712   MIB2_STATS_INC(mib2.tcpinsegs);
2713@@ -247,7 +257,15 @@ tcp_input(struct pbuf *p, struct netif *inp)
2714      for an active connection. */
2715   prev = NULL;
2716
2717+#if TCP_PCB_HASH
2718+  idx = TUPLE4_HASH_FN( ip_current_dest_addr()->addr, tcphdr->dest,
2719+                      ip_current_src_addr()->addr, tcphdr->src) &
2720+         (tcp_active_htable->size - 1);
2721+  head = &tcp_active_htable->array[idx].chain;
2722+  tcppcb_hlist_for_each(pcb, node, head) {
2723+#else
2724   for (pcb = tcp_active_pcbs; pcb != NULL; pcb = pcb->next) {
2725+#endif
2726     LWIP_ASSERT("tcp_input: active pcb->state != CLOSED", pcb->state != CLOSED);
2727     LWIP_ASSERT("tcp_input: active pcb->state != TIME-WAIT", pcb->state != TIME_WAIT);
2728     LWIP_ASSERT("tcp_input: active pcb->state != LISTEN", pcb->state != LISTEN);
2729@@ -263,6 +281,7 @@ tcp_input(struct pbuf *p, struct netif *inp)
2730         pcb->local_port == tcphdr->dest &&
2731         ip_addr_cmp(&pcb->remote_ip, ip_current_src_addr()) &&
2732         ip_addr_cmp(&pcb->local_ip, ip_current_dest_addr())) {
2733+#if !TCP_PCB_HASH
2734       /* Move this PCB to the front of the list so that subsequent
2735          lookups will be faster (we exploit locality in TCP segment
2736          arrivals). */
2737@@ -275,9 +294,14 @@ tcp_input(struct pbuf *p, struct netif *inp)
2738         TCP_STATS_INC(tcp.cachehit);
2739       }
2740       LWIP_ASSERT("tcp_input: pcb->next != pcb (after cache)", pcb->next != pcb);
2741+#endif
2742       break;
2743     }
2744+#if TCP_PCB_HASH
2745+    pcb = NULL;
2746+#else
2747     prev = pcb;
2748+#endif
2749   }
2750
2751   if (pcb == NULL) {
2752@@ -363,8 +387,15 @@ tcp_input(struct pbuf *p, struct netif *inp)
2753          arrivals). */
2754       if (prev != NULL) {
2755         ((struct tcp_pcb_listen *)prev)->next = lpcb->next;
2756+#if USE_LIBOS
2757+        if (lpcb->next)
2758+          lpcb->next->prev = (struct tcp_pcb_listen *)prev;
2759+#endif
2760         /* our successor is the remainder of the listening list */
2761         lpcb->next = tcp_listen_pcbs.listen_pcbs;
2762+#if USE_LIBOS
2763+        lpcb->prev = NULL;
2764+#endif
2765         /* put this listening pcb at the head of the listening list */
2766         tcp_listen_pcbs.listen_pcbs = lpcb;
2767       } else {
2768@@ -445,6 +476,9 @@ tcp_input(struct pbuf *p, struct netif *inp)
2769            application that the connection is dead before we
2770            deallocate the PCB. */
2771         TCP_EVENT_ERR(pcb->state, pcb->errf, pcb->callback_arg, ERR_RST);
2772+#if TCP_PCB_HASH
2773+        tcp_pcb_remove_hash(tcp_active_htable, pcb);
2774+#endif
2775         tcp_pcb_remove(&tcp_active_pcbs, pcb);
2776         tcp_free(pcb);
2777       } else {
2778@@ -550,7 +584,19 @@ tcp_input(struct pbuf *p, struct netif *inp)
2779           goto aborted;
2780         }
2781         /* Try to send something out. */
2782+#if LWIP_RECORD_PERF
2783+        if (check_layer_point(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_ACK_RECV)) {
2784+          PERF_PAUSE(PERF_LAYER_TCP);
2785+          PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_ACK_SEND);
2786+        }
2787+#endif
2788         tcp_output(pcb);
2789+#if LWIP_RECORD_PERF
2790+        if (check_layer_point(PERF_LAYER_TCP, PERF_POINT_TCP_ACK_SEND)) {
2791+          PERF_STOP_INCREASE_COUNT("tcp_in", PERF_LAYER_TCP);
2792+          PERF_RESUME(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_ACK_RECV);
2793+        }
2794+#endif
2795 #if TCP_INPUT_DEBUG
2796 #if TCP_DEBUG
2797         tcp_debug_print_state(pcb->state);
2798@@ -583,7 +629,9 @@ aborted:
2799   }
2800
2801   LWIP_ASSERT("tcp_input: tcp_pcbs_sane()", tcp_pcbs_sane());
2802+#ifndef LWIP_PERF
2803   PERF_STOP("tcp_input");
2804+#endif
2805   return;
2806 dropped:
2807   TCP_STATS_INC(tcp.drop);
2808@@ -610,6 +658,9 @@ tcp_input_delayed_close(struct tcp_pcb *pcb)
2809           ensure the application doesn't continue using the PCB. */
2810       TCP_EVENT_ERR(pcb->state, pcb->errf, pcb->callback_arg, ERR_CLSD);
2811     }
2812+#if TCP_PCB_HASH
2813+    tcp_pcb_remove_hash(tcp_active_htable, pcb);
2814+#endif
2815     tcp_pcb_remove(&tcp_active_pcbs, pcb);
2816     tcp_free(pcb);
2817     return 1;
2818@@ -649,6 +700,7 @@ tcp_listen_input(struct tcp_pcb_listen *pcb)
2819     tcp_rst((const struct tcp_pcb *)pcb, ackno, seqno + tcplen, ip_current_dest_addr(),
2820             ip_current_src_addr(), tcphdr->dest, tcphdr->src);
2821   } else if (flags & TCP_SYN) {
2822+    PERF_UPDATE_POINT(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_RECV);
2823     LWIP_DEBUGF(TCP_DEBUG, ("TCP connection request %"U16_F" -> %"U16_F".\n", tcphdr->src, tcphdr->dest));
2824 #if TCP_LISTEN_BACKLOG
2825     if (pcb->accepts_pending >= pcb->backlog) {
2826@@ -695,6 +747,9 @@ tcp_listen_input(struct tcp_pcb_listen *pcb)
2827     npcb->netif_idx = pcb->netif_idx;
2828     /* Register the new PCB so that we can begin receiving segments
2829        for it. */
2830+#if TCP_PCB_HASH
2831+    TCP_REG_ACTIVE_HASH(npcb);
2832+#endif
2833     TCP_REG_ACTIVE(npcb);
2834
2835     /* Parse any options in the SYN. */
2836@@ -715,13 +770,18 @@ tcp_listen_input(struct tcp_pcb_listen *pcb)
2837     }
2838 #endif
2839
2840+    PERF_PAUSE(PERF_LAYER_TCP);
2841+    PERF_START(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_ACK_SEND);
2842     /* Send a SYN|ACK together with the MSS option. */
2843     rc = tcp_enqueue_flags(npcb, TCP_SYN | TCP_ACK);
2844     if (rc != ERR_OK) {
2845       tcp_abandon(npcb, 0);
2846+      PERF_RESUME(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_RECV);
2847       return;
2848     }
2849     tcp_output(npcb);
2850+    PERF_STOP_INCREASE_COUNT("tcp_output", PERF_LAYER_TCP);
2851+    PERF_RESUME(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_RECV);
2852   }
2853   return;
2854 }
2855@@ -858,6 +918,7 @@ tcp_process(struct tcp_pcb *pcb)
2856       /* received SYN ACK with expected sequence number? */
2857       if ((flags & TCP_ACK) && (flags & TCP_SYN)
2858           && (ackno == pcb->lastack + 1)) {
2859+        PERF_UPDATE_POINT(PERF_LAYER_TCP, PERF_POINT_TCP_SYN_ACK_RECV);
2860         pcb->rcv_nxt = seqno + 1;
2861         pcb->rcv_ann_right_edge = pcb->rcv_nxt;
2862         pcb->lastack = ackno;
2863@@ -925,6 +986,7 @@ tcp_process(struct tcp_pcb *pcb)
2864         /* expected ACK number? */
2865         if (TCP_SEQ_BETWEEN(ackno, pcb->lastack + 1, pcb->snd_nxt)) {
2866           pcb->state = ESTABLISHED;
2867+          PERF_UPDATE_POINT(PERF_LAYER_TCP, PERF_POINT_TCP_ACK_RECV);
2868           LWIP_DEBUGF(TCP_DEBUG, ("TCP connection established %"U16_F" -> %"U16_F".\n", inseg.tcphdr->src, inseg.tcphdr->dest));
2869 #if LWIP_CALLBACK_API || TCP_LISTEN_BACKLOG
2870           if (pcb->listener == NULL) {
2871@@ -995,6 +1057,9 @@ tcp_process(struct tcp_pcb *pcb)
2872                       ("TCP connection closed: FIN_WAIT_1 %"U16_F" -> %"U16_F".\n", inseg.tcphdr->src, inseg.tcphdr->dest));
2873           tcp_ack_now(pcb);
2874           tcp_pcb_purge(pcb);
2875+#if TCP_PCB_HASH
2876+          TCP_RMV_ACTIVE_HASH(pcb);
2877+#endif
2878           TCP_RMV_ACTIVE(pcb);
2879           pcb->state = TIME_WAIT;
2880           TCP_REG(&tcp_tw_pcbs, pcb);
2881@@ -1013,6 +1078,9 @@ tcp_process(struct tcp_pcb *pcb)
2882         LWIP_DEBUGF(TCP_DEBUG, ("TCP connection closed: FIN_WAIT_2 %"U16_F" -> %"U16_F".\n", inseg.tcphdr->src, inseg.tcphdr->dest));
2883         tcp_ack_now(pcb);
2884         tcp_pcb_purge(pcb);
2885+#if TCP_PCB_HASH
2886+        TCP_RMV_ACTIVE_HASH(pcb);
2887+#endif
2888         TCP_RMV_ACTIVE(pcb);
2889         pcb->state = TIME_WAIT;
2890         TCP_REG(&tcp_tw_pcbs, pcb);
2891@@ -1023,6 +1091,9 @@ tcp_process(struct tcp_pcb *pcb)
2892       if ((flags & TCP_ACK) && ackno == pcb->snd_nxt && pcb->unsent == NULL) {
2893         LWIP_DEBUGF(TCP_DEBUG, ("TCP connection closed: CLOSING %"U16_F" -> %"U16_F".\n", inseg.tcphdr->src, inseg.tcphdr->dest));
2894         tcp_pcb_purge(pcb);
2895+#if TCP_PCB_HASH
2896+        TCP_RMV_ACTIVE_HASH(pcb);
2897+#endif
2898         TCP_RMV_ACTIVE(pcb);
2899         pcb->state = TIME_WAIT;
2900         TCP_REG(&tcp_tw_pcbs, pcb);
2901diff --git a/src/core/tcp_out.c b/src/core/tcp_out.c
2902index 8149d39..dac498e 100644
2903--- a/src/core/tcp_out.c
2904+++ b/src/core/tcp_out.c
2905@@ -271,7 +271,7 @@ tcp_pbuf_prealloc(pbuf_layer layer, u16_t length, u16_t max_length,
2906   return p;
2907 }
2908 #else /* TCP_OVERSIZE */
2909-#define tcp_pbuf_prealloc(layer, length, mx, os, pcb, api, fst) pbuf_alloc((layer), (length), PBUF_RAM)
2910+#define tcp_pbuf_prealloc(layer, length, mx, os, pcb, api, fst) pbuf_alloc((layer), (length), PBUF_POOL)
2911 #endif /* TCP_OVERSIZE */
2912
2913 #if TCP_CHECKSUM_ON_COPY
2914@@ -640,7 +640,7 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags)
2915       ((struct pbuf_rom *)p2)->payload = (const u8_t *)arg + pos;
2916
2917       /* Second, allocate a pbuf for the headers. */
2918-      if ((p = pbuf_alloc(PBUF_TRANSPORT, optlen, PBUF_RAM)) == NULL) {
2919+      if ((p = pbuf_alloc(PBUF_TRANSPORT, optlen, PBUF_POOL)) == NULL) {
2920         /* If allocation fails, we have to deallocate the data pbuf as
2921          * well. */
2922         pbuf_free(p2);
2923@@ -1458,6 +1458,11 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
2924   err_t err;
2925   u16_t len;
2926   u32_t *opts;
2927+
2928+#if LWIP_RECORD_PERF
2929+  int tmpPoint;
2930+#endif
2931+
2932 #if TCP_CHECKSUM_ON_COPY
2933   int seg_chksum_was_swapped = 0;
2934 #endif
2935@@ -1604,6 +1609,9 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
2936 #endif /* CHECKSUM_GEN_TCP */
2937   TCP_STATS_INC(tcp.xmit);
2938
2939+  PERF_PAUSE_RETURN_POINT(PERF_LAYER_TCP, tmpPoint);
2940+  PERF_START(PERF_LAYER_IP, PERF_POINT_IP_SEND);
2941+
2942   NETIF_SET_HINTS(netif, &(pcb->netif_hints));
2943   err = ip_output_if(seg->p, &pcb->local_ip, &pcb->remote_ip, pcb->ttl,
2944                      pcb->tos, IP_PROTO_TCP, netif);
2945@@ -1618,6 +1626,9 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
2946   }
2947 #endif
2948
2949+  PERF_STOP_INCREASE_COUNT("ip_out", PERF_LAYER_IP);
2950+  PERF_RESUME(PERF_LAYER_TCP, tmpPoint);
2951+
2952   return err;
2953 }
2954
2955@@ -2024,6 +2035,10 @@ tcp_send_empty_ack(struct tcp_pcb *pcb)
2956   u8_t optlen, optflags = 0;
2957   u8_t num_sacks = 0;
2958
2959+#if LWIP_RECORD_PERF
2960+  int tmpPoint;
2961+#endif
2962+
2963   LWIP_ASSERT("tcp_send_empty_ack: invalid pcb", pcb != NULL);
2964
2965 #if LWIP_TCP_TIMESTAMPS
2966@@ -2040,6 +2055,9 @@ tcp_send_empty_ack(struct tcp_pcb *pcb)
2967   }
2968 #endif
2969
2970+  PERF_PAUSE_RETURN_POINT(PERF_LAYER_TCP, tmpPoint);
2971+  PERF_START(PERF_LAYER_IP, PERF_POINT_IP_SEND);
2972+
2973   p = tcp_output_alloc_header(pcb, optlen, 0, lwip_htonl(pcb->snd_nxt));
2974   if (p == NULL) {
2975     /* let tcp_fasttmr retry sending this ACK */
2976@@ -2064,6 +2082,9 @@ tcp_send_empty_ack(struct tcp_pcb *pcb)
2977     tcp_clear_flags(pcb, TF_ACK_DELAY | TF_ACK_NOW);
2978   }
2979
2980+  PERF_STOP_INCREASE_COUNT("ip_out", PERF_LAYER_IP);
2981+  PERF_RESUME(PERF_LAYER_TCP, tmpPoint);
2982+
2983   return err;
2984 }
2985
2986diff --git a/src/core/timeouts.c b/src/core/timeouts.c
2987index f37acfe..0542a32 100644
2988--- a/src/core/timeouts.c
2989+++ b/src/core/timeouts.c
2990@@ -119,9 +119,9 @@ const int lwip_num_cyclic_timers = LWIP_ARRAYSIZE(lwip_cyclic_timers);
2991 #if LWIP_TIMERS && !LWIP_TIMERS_CUSTOM
2992
2993 /** The one and only timeout list */
2994-static struct sys_timeo *next_timeout;
2995+static PER_THREAD struct sys_timeo *next_timeout;
2996
2997-static u32_t current_timeout_due_time;
2998+static PER_THREAD u32_t current_timeout_due_time;
2999
3000 #if LWIP_TESTMODE
3001 struct sys_timeo**
3002@@ -133,7 +133,7 @@ sys_timeouts_get_next_timeout(void)
3003
3004 #if LWIP_TCP
3005 /** global variable that shows if the tcp timer is currently scheduled or not */
3006-static int tcpip_tcp_timer_active;
3007+static PER_THREAD int tcpip_tcp_timer_active;
3008
3009 /**
3010  * Timer callback function that calls tcp_tmr() and reschedules itself.
3011@@ -442,6 +442,18 @@ sys_timeouts_sleeptime(void)
3012   }
3013 }
3014
3015+#if USE_LIBOS
3016+void sys_timer_run(void)
3017+{
3018+  u32_t sleeptime;
3019+
3020+  sleeptime = sys_timeouts_sleeptime();
3021+  if (sleeptime == 0) {
3022+    sys_check_timeouts();
3023+  }
3024+}
3025+#endif /* USE_LIBOS */
3026+
3027 #else /* LWIP_TIMERS && !LWIP_TIMERS_CUSTOM */
3028 /* Satisfy the TCP code which calls this function */
3029 void
3030diff --git a/src/core/udp.c b/src/core/udp.c
3031index 0b609d3..a5f76b9 100644
3032--- a/src/core/udp.c
3033+++ b/src/core/udp.c
3034@@ -207,7 +207,11 @@ udp_input(struct pbuf *p, struct netif *inp)
3035   LWIP_ASSERT("udp_input: invalid pbuf", p != NULL);
3036   LWIP_ASSERT("udp_input: invalid netif", inp != NULL);
3037
3038+#if LWIP_RECORD_PERF
3039+  PERF_START(PERF_LAYER_UDP, PERF_POINT_UDP);
3040+#else
3041   PERF_START;
3042+#endif
3043
3044   UDP_STATS_INC(udp.recv);
3045
3046@@ -428,7 +432,12 @@ udp_input(struct pbuf *p, struct netif *inp)
3047     pbuf_free(p);
3048   }
3049 end:
3050+#if LWIP_RECORD_PERF
3051+  PERF_STOP_INCREASE_COUNT("udp_input", PERF_LAYER_UDP);
3052+#else
3053   PERF_STOP("udp_input");
3054+#endif
3055+
3056   return;
3057 #if CHECKSUM_CHECK_UDP
3058 chkerr:
3059@@ -438,7 +447,13 @@ chkerr:
3060   UDP_STATS_INC(udp.drop);
3061   MIB2_STATS_INC(mib2.udpinerrors);
3062   pbuf_free(p);
3063+
3064+#if LWIP_RECORD_PERF
3065+  PERF_STOP_INCREASE_COUNT("udp_input", PERF_LAYER_UDP);
3066+#else
3067   PERF_STOP("udp_input");
3068+#endif
3069+
3070 #endif /* CHECKSUM_CHECK_UDP */
3071 }
3072
3073diff --git a/src/include/arch/cc.h b/src/include/arch/cc.h
3074index 52b76f9..33c24b4 100644
3075--- a/src/include/arch/cc.h
3076+++ b/src/include/arch/cc.h
3077@@ -1,7 +1,81 @@
3078-#ifndef LWIP_CC_H
3079-#define LWIP_CC_H
3080+/*
3081+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3082+ * All rights reserved.
3083+ *
3084+ * Redistribution and use in source and binary forms, with or without modification,
3085+ * are permitted provided that the following conditions are met:
3086+ *
3087+ * 1. Redistributions of source code must retain the above copyright notice,
3088+ *    this list of conditions and the following disclaimer.
3089+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3090+ *    this list of conditions and the following disclaimer in the documentation
3091+ *    and/or other materials provided with the distribution.
3092+ * 3. The name of the author may not be used to endorse or promote products
3093+ *    derived from this software without specific prior written permission.
3094+ *
3095+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3096+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3097+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3098+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3099+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3100+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3101+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3102+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3103+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3104+ * OF SUCH DAMAGE.
3105+ *
3106+ * This file is part of the lwIP TCP/IP stack.
3107+ *
3108+ * Author: Huawei Technologies
3109+ *
3110+ */
3111
3112+#ifndef LWIP_ARCH_CC_H
3113+#define LWIP_ARCH_CC_H
3114
3115+#include <stdint.h>
3116+#include <stdlib.h>
3117+#include <sys/time.h>
3118+#include <sys/types.h>
3119
3120-#endif /* LWIP_CC_H */
3121+#include "lwiplog.h"
3122
3123+#define LWIP_NOASSERT
3124+
3125+#define LWIP_ERRNO_STDINCLUDE 1
3126+#define MEMP_MEMORY_BASE_PLACEHOLDER 0
3127+#define MEMZONE_NAMESIZE 32
3128+
3129+#define LWIP_RAND()     ((uint32_t)rand())
3130+
3131+extern uint8_t *sys_hugepage_malloc(const char *name, uint32_t size);
3132+
3133+#define LWIP_DECLARE_MEMP_BASE_ALIGNED(name, __size)\
3134+PER_THREAD uint8_t *memp_memory_##name##_base; \
3135+void alloc_memp_##name##_base(void) \
3136+{ \
3137+    memp_ ## name.desc = memp_desc_ ## name; \
3138+    memp_ ## name.stats = &memp_stat ## name; \
3139+    memp_ ## name.size = memp_size ## name; \
3140+    memp_ ## name.num = memp_num ## name; \
3141+    memp_ ## name.tab = &memp_tab_ ## name; \
3142+    memp_pools[MEMP_##name] = &memp_ ## name; \
3143+    \
3144+    char mpname[MEMZONE_NAMESIZE] = {0}; \
3145+    snprintf(mpname, MEMZONE_NAMESIZE, "%ld_%s", gettid(), #name); \
3146+    memp_memory_##name##_base = \
3147+        sys_hugepage_malloc(mpname, LWIP_MEM_ALIGN_BUFFER(__size)); \
3148+    memp_pools[MEMP_##name]->base = memp_memory_##name##_base; \
3149+}
3150+
3151+#define LWIP_DECLARE_MEMORY_ALIGNED(variable_name, size) \
3152+PER_THREAD uint8_t *variable_name; \
3153+void alloc_memory_##variable_name(void) \
3154+{ \
3155+    char mpname[MEMZONE_NAMESIZE] = {0}; \
3156+    snprintf(mpname, MEMZONE_NAMESIZE, "%ld_%s", gettid(), #variable_name); \
3157+    (variable_name) = \
3158+        sys_hugepage_malloc(mpname, LWIP_MEM_ALIGN_BUFFER(size)); \
3159+}
3160+
3161+#endif /* LWIP_ARCH_CC_H */
3162diff --git a/src/include/arch/perf.h b/src/include/arch/perf.h
3163new file mode 100644
3164index 0000000..e505da7
3165--- /dev/null
3166+++ b/src/include/arch/perf.h
3167@@ -0,0 +1,155 @@
3168+/*
3169+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3170+ * All rights reserved.
3171+ *
3172+ * Redistribution and use in source and binary forms, with or without modification,
3173+ * are permitted provided that the following conditions are met:
3174+ *
3175+ * 1. Redistributions of source code must retain the above copyright notice,
3176+ *    this list of conditions and the following disclaimer.
3177+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3178+ *    this list of conditions and the following disclaimer in the documentation
3179+ *    and/or other materials provided with the distribution.
3180+ * 3. The name of the author may not be used to endorse or promote products
3181+ *    derived from this software without specific prior written permission.
3182+ *
3183+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3184+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3185+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3186+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3187+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3188+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3189+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3190+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3191+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3192+ * OF SUCH DAMAGE.
3193+ *
3194+ * This file is part of the lwIP TCP/IP stack.
3195+ *
3196+ * Author: Huawei Technologies
3197+ *
3198+ */
3199+
3200+#ifndef LWIP_ARCH_PERF_H
3201+#define LWIP_ARCH_PERF_H
3202+
3203+#include <time.h>
3204+
3205+#include "lwip/debug.h"
3206+
3207+#if LWIP_RECORD_PERF
3208+enum PERF_POINT {
3209+    PERF_POINT_IP_RECV,
3210+    PERF_POINT_TCP_RECV,
3211+    PERF_POINT_UDP,
3212+    PERF_POINT_TCP_SYN_RECV,
3213+    PERF_POINT_TCP_SYN_ACK_SEND,
3214+    PERF_POINT_TCP_ACK_RECV,
3215+    PERF_POINT_TCP_SYN_SEND,
3216+    PERF_POINT_TCP_SYN_ACK_RECV,
3217+    PERF_POINT_TCP_ACK_SEND,
3218+    PERF_POINT_TCP_DATA_SEND,
3219+    PERF_POINT_IP_SEND,
3220+    PERF_POINT_END
3221+};
3222+
3223+enum PERF_LAYER {
3224+    PERF_LAYER_IP,
3225+    PERF_LAYER_TCP,
3226+    PERF_LAYER_UDP,
3227+    PERF_LAYER_END
3228+};
3229+
3230+extern uint32_t g_record_perf;
3231+
3232+extern __thread uint64_t g_timeTaken[PERF_POINT_END];
3233+extern __thread int g_perfPoint[PERF_LAYER_END];
3234+extern __thread struct timespec tvStart[PERF_LAYER_END];
3235+
3236+extern char *g_ppLayerName[PERF_POINT_END];
3237+extern volatile uint64_t g_perfMaxtime[PERF_POINT_END];
3238+extern volatile uint64_t g_astPacketCnt[PERF_POINT_END];
3239+extern volatile uint64_t g_astPacketProcTime[PERF_POINT_END];
3240+
3241+#define PERF_START(layer, point) do {\
3242+    g_perfPoint[(layer)] = (point);\
3243+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("set point %d:%s\n", layer, g_ppLayerName[g_perfPoint[(layer)]]));\
3244+    clock_gettime(CLOCK_MONOTONIC, &tvStart[(layer)]);\
3245+    g_timeTaken[(point)] = 0;\
3246+} while (0)
3247+
3248+#define PERF_UPDATE_POINT(layer, point) do {\
3249+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("old point %d:%s\n", layer, g_ppLayerName[g_perfPoint[(layer)]]));\
3250+    g_timeTaken[(point)] = g_timeTaken[g_perfPoint[(layer)]];\
3251+    g_timeTaken[g_perfPoint[(layer)]] = 0;\
3252+    g_perfPoint[(layer)] = (point);\
3253+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("new point %d:%s\n", layer, g_ppLayerName[g_perfPoint[(layer)]]));\
3254+} while (0)
3255+
3256+#define PERF_PAUSE(layer) do {\
3257+    struct timespec tvEnd;\
3258+    clock_gettime(CLOCK_MONOTONIC, &tvEnd);\
3259+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("perf pause layer%d\n", layer));\
3260+    g_timeTaken[g_perfPoint[(layer)]] += ((tvEnd.tv_sec - tvStart[(layer)].tv_sec) \
3261+            * (1000000000UL) + (tvEnd.tv_nsec - tvStart[(layer)].tv_nsec));\
3262+} while (0)
3263+
3264+#define PERF_PAUSE_RETURN_POINT(layer, pause_point) do {\
3265+    struct timespec tvEnd;\
3266+    clock_gettime(CLOCK_MONOTONIC, &tvEnd);\
3267+    g_timeTaken[g_perfPoint[(layer)]] += ((tvEnd.tv_sec - tvStart[(layer)].tv_sec) \
3268+            * (1000000000UL) + (tvEnd.tv_nsec - tvStart[(layer)].tv_nsec));\
3269+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("perf pause point %d:%s\n", layer, g_ppLayerName[g_perfPoint[(layer)]]));\
3270+    (pause_point) = g_perfPoint[(layer)];\
3271+} while (0)
3272+
3273+
3274+#define PERF_RESUME(layer, point) do {\
3275+    LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("perf resule point %d:%s\n", layer, g_ppLayerName[point]));\
3276+    clock_gettime(CLOCK_MONOTONIC, &tvStart[(layer)]);\
3277+    g_perfPoint[(layer)] = (point);\
3278+} while (0)
3279+
3280+
3281+/* x is a prompt */
3282+#define PERF_STOP_INCREASE_COUNT(x, layer) do {\
3283+    if (g_record_perf)\
3284+    {\
3285+        struct timespec tvEnd;\
3286+        int i = 2;\
3287+        uint32_t oldValue = 0;\
3288+        clock_gettime(CLOCK_MONOTONIC, &tvEnd);\
3289+        g_timeTaken[g_perfPoint[(layer)]] += ((tvEnd.tv_sec - tvStart[(layer)].tv_sec) \
3290+                * (1000000000UL) + (tvEnd.tv_nsec - tvStart[(layer)].tv_nsec));\
3291+        while (i && !oldValue)\
3292+        {\
3293+            oldValue = __sync_or_and_fetch(&g_perfMaxtime[g_perfPoint[(layer)]], 0);\
3294+            if (oldValue >= g_timeTaken[g_perfPoint[(layer)]])\
3295+            {\
3296+                break;\
3297+            }\
3298+            oldValue = __sync_val_compare_and_swap(&g_perfMaxtime[g_perfPoint[(layer)]],\
3299+                    oldValue, g_timeTaken[g_perfPoint[(layer)]]);\
3300+            i--;\
3301+        }\
3302+        __sync_fetch_and_add(&g_astPacketCnt[g_perfPoint[(layer)]], 1);\
3303+        __sync_fetch_and_add(&g_astPacketProcTime[g_perfPoint[(layer)]], g_timeTaken[g_perfPoint[(layer)]]);\
3304+        LWIP_DEBUGF(PERF_OUTPUT_DEBUG, ("Time for %s is: %ld\n",\
3305+                g_ppLayerName[g_perfPoint[(layer)]], g_timeTaken[g_perfPoint[(layer)]]));\
3306+    }\
3307+} while (0)
3308+
3309+
3310+int check_layer_point(int layer, int point);
3311+int perf_init();
3312+
3313+#else
3314+#define PERF_START(layer, point)         do {  } while (0)
3315+#define PERF_UPDATE_POINT(layer, point)  do {  } while (0)
3316+#define PERF_PAUSE(layer)                do {  } while (0)
3317+#define PERF_PAUSE_RETURN_POINT(layer, pause_point) do {  } while (0)
3318+#define PERF_RESUME(layer, point)           do {  } while (0)
3319+#define PERF_STOP_INCREASE_COUNT(x, layer)  do {  } while (0)
3320+#endif
3321+
3322+#endif /* LWIP_ARCH_PERF_H */
3323diff --git a/src/include/arch/sys_arch.h b/src/include/arch/sys_arch.h
3324index 3f555ee..b8a0d28 100644
3325--- a/src/include/arch/sys_arch.h
3326+++ b/src/include/arch/sys_arch.h
3327@@ -1,7 +1,93 @@
3328-#ifndef LWIP_SYS_ARCH_H
3329-#define LWIP_SYS_ARCH_H
3330+/*
3331+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3332+ * All rights reserved.
3333+ *
3334+ * Redistribution and use in source and binary forms, with or without modification,
3335+ * are permitted provided that the following conditions are met:
3336+ *
3337+ * 1. Redistributions of source code must retain the above copyright notice,
3338+ *    this list of conditions and the following disclaimer.
3339+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3340+ *    this list of conditions and the following disclaimer in the documentation
3341+ *    and/or other materials provided with the distribution.
3342+ * 3. The name of the author may not be used to endorse or promote products
3343+ *    derived from this software without specific prior written permission.
3344+ *
3345+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3346+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3347+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3348+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3349+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3350+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3351+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3352+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3353+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3354+ * OF SUCH DAMAGE.
3355+ *
3356+ * This file is part of the lwIP TCP/IP stack.
3357+ *
3358+ * Author: Huawei Technologies
3359+ *
3360+ */
3361
3362+#ifndef LWIP_ARCH_SYS_ARCH_H
3363+#define LWIP_ARCH_SYS_ARCH_H
3364
3365+#include <rte_cycles.h>
3366+#include <rte_debug.h>
3367
3368-#endif /* LWIP_SYS_ARCH_H */
3369+#define SYS_MBOX_NULL NULL
3370+#define SYS_SEM_NULL  NULL
3371+typedef uint32_t sys_prot_t;
3372
3373+struct sys_sem {
3374+    volatile unsigned int c;
3375+    int (*wait_fn)(void);
3376+};
3377+
3378+#define MBOX_NAME_LEN 64
3379+struct sys_mbox {
3380+    struct rte_ring *ring;
3381+    char name[MBOX_NAME_LEN];
3382+    int size;
3383+    int socket_id;
3384+    unsigned flags;
3385+    int (*wait_fn)(void);
3386+};
3387+
3388+typedef struct sys_sem *sys_sem_t;
3389+#define sys_sem_valid(sem)             (((sem) != NULL) && (*(sem) != NULL))
3390+#define sys_sem_valid_val(sem)         ((sem) != NULL)
3391+#define sys_sem_set_invalid(sem)       do { if ((sem) != NULL) { *(sem) = NULL; }} while(0)
3392+#define sys_sem_set_invalid_val(sem)   do { (sem) = NULL; } while(0)
3393+
3394+struct sys_mutex;
3395+typedef struct sys_mutex *sys_mutex_t;
3396+#define sys_mutex_valid(mutex)         sys_sem_valid(mutex)
3397+#define sys_mutex_set_invalid(mutex)   sys_sem_set_invalid(mutex)
3398+
3399+typedef struct sys_mbox *sys_mbox_t;
3400+#define sys_mbox_valid(mbox)           sys_sem_valid(mbox)
3401+#define sys_mbox_valid_val(mbox)       sys_sem_valid_val(mbox)
3402+#define sys_mbox_set_invalid(mbox)     sys_sem_set_invalid(mbox)
3403+#define sys_mbox_set_invalid_val(mbox) sys_sem_set_invalid_val(mbox)
3404+int sys_mbox_empty(struct sys_mbox *);
3405+
3406+struct sys_thread;
3407+typedef struct sys_thread *sys_thread_t;
3408+
3409+extern int eth_dev_poll(void);
3410+
3411+void sys_calibrate_tsc(void);
3412+uint32_t sys_now(void);
3413+__attribute__((always_inline)) inline int update_timeout(int timeout, uint32_t poll_ts)
3414+{
3415+    uint32_t used_ms = sys_now() - poll_ts;
3416+    if (timeout > 0 && used_ms < timeout) {
3417+        return timeout;
3418+    } else {
3419+        return 0;
3420+    }
3421+}
3422+
3423+#endif /* LWIP_ARCH_SYS_ARCH_H */
3424diff --git a/src/include/eventpoll.h b/src/include/eventpoll.h
3425new file mode 100644
3426index 0000000..01f8d64
3427--- /dev/null
3428+++ b/src/include/eventpoll.h
3429@@ -0,0 +1,72 @@
3430+/*
3431+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3432+ * All rights reserved.
3433+ *
3434+ * Redistribution and use in source and binary forms, with or without modification,
3435+ * are permitted provided that the following conditions are met:
3436+ *
3437+ * 1. Redistributions of source code must retain the above copyright notice,
3438+ *    this list of conditions and the following disclaimer.
3439+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3440+ *    this list of conditions and the following disclaimer in the documentation
3441+ *    and/or other materials provided with the distribution.
3442+ * 3. The name of the author may not be used to endorse or promote products
3443+ *    derived from this software without specific prior written permission.
3444+ *
3445+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3446+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3447+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3448+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3449+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3450+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3451+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3452+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3453+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3454+ * OF SUCH DAMAGE.
3455+ *
3456+ * This file is part of the lwIP TCP/IP stack.
3457+ *
3458+ * Author: Huawei Technologies
3459+ *
3460+ */
3461+
3462+#ifndef __EVENTPOLL_H__
3463+#define __EVENTPOLL_H__
3464+
3465+#include <sys/epoll.h>
3466+
3467+#include "lwip/api.h"
3468+#include "list.h"
3469+
3470+#define MAX_EPOLLFDS 32
3471+
3472+#define LIBOS_EPOLLNONE (0x0)
3473+#define LIBOS_BADEP     (NULL)
3474+
3475+struct event_queue {
3476+    struct list_node events;
3477+    /* total number of sockets have events */
3478+    int num_events;
3479+};
3480+
3481+struct event_array {
3482+    sys_mbox_t mbox;
3483+    volatile int num_events;
3484+    struct epoll_event events[0];
3485+};
3486+
3487+struct libos_epoll {
3488+    struct event_queue *libos_queue;
3489+    struct event_array *host_queue;
3490+    int num_hostfds;
3491+    int hints;
3492+    int fd;  /* self fd */
3493+    int efd; /* eventfd */
3494+};
3495+
3496+extern int add_epoll_event(struct netconn*, uint32_t);
3497+extern int del_epoll_event(struct netconn*, uint32_t);
3498+extern int lwip_epoll_close(int);
3499+extern int lwip_is_epfd(int);
3500+
3501+#endif /* __EVENTPOLL_H__ */
3502diff --git a/src/include/hlist.h b/src/include/hlist.h
3503new file mode 100644
3504index 0000000..7059488
3505--- /dev/null
3506+++ b/src/include/hlist.h
3507@@ -0,0 +1,233 @@
3508+/*
3509+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3510+ * All rights reserved.
3511+ *
3512+ * Redistribution and use in source and binary forms, with or without modification,
3513+ * are permitted provided that the following conditions are met:
3514+ *
3515+ * 1. Redistributions of source code must retain the above copyright notice,
3516+ *    this list of conditions and the following disclaimer.
3517+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3518+ *    this list of conditions and the following disclaimer in the documentation
3519+ *    and/or other materials provided with the distribution.
3520+ * 3. The name of the author may not be used to endorse or promote products
3521+ *    derived from this software without specific prior written permission.
3522+ *
3523+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3524+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3525+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3526+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3527+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3528+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3529+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3530+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3531+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3532+ * OF SUCH DAMAGE.
3533+ *
3534+ * This file is part of the lwIP TCP/IP stack.
3535+ *
3536+ * Author: Huawei Technologies
3537+ *
3538+ */
3539+
3540+#ifndef __HLIST_H__
3541+#define __HLIST_H__
3542+
3543+#include "list.h"
3544+
3545+//#if TCP_PCB_HASH
3546+struct hlist_node {
3547+    /**
3548+     * @pprev: point the previous node's next pointer
3549+     */
3550+    struct hlist_node *next;
3551+    struct hlist_node **pprev;
3552+};
3553+
3554+struct hlist_head {
3555+    struct hlist_node *first;
3556+};
3557+
3558+struct hlist_tail {
3559+    struct hlist_node *end;
3560+};
3561+
3562+struct hlist_ctl {
3563+    struct hlist_head head;
3564+    struct hlist_tail tail;
3565+};
3566+
3567+#define INIT_HLIST_CTRL(ptr) {(ptr)->head.first = NULL; (ptr)->tail.end = NULL;}
3568+#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
3569+#define INIT_HLIST_NODE(ptr) {(ptr)->next = NULL; (ptr)->pprev = NULL;}
3570+#define hlist_entry(ptr, type, member) \
3571+    container_of(ptr, type, member)
3572+
3573+/**
3574+ * hlist_for_each_entry    - iterate over list of given type
3575+ * @tpos:    the type * to use as a loop cursor.
3576+ * @pos:    the &struct hlist_node to use as a loop cursor.
3577+ * @head:    the head for your list.
3578+ * @member:    the name of the hlist_node within the struct.
3579+ */
3580+#define hlist_for_each_entry(tpos, pos, head, member) \
3581+    for (pos = (head)->first; \
3582+        pos && ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
3583+        pos = (pos)->next)
3584+
3585+/**
3586+ * next must be != NULL
3587+ * add n node before next node
3588+ *
3589+ * @n: new node
3590+ * @next: node in the hlist
3591+ */
3592+static inline void hlist_add_before(struct hlist_node *n, struct hlist_node *next)
3593+{
3594+    n->pprev = next->pprev;
3595+    n->next = next;
3596+    next->pprev = &n->next;
3597+    *(n->pprev) = n;
3598+}
3599+
3600+static inline int hlist_empty(const struct hlist_head *h)
3601+{
3602+    return !h->first;
3603+}
3604+
3605+static inline int hlist_unhashed(const struct hlist_node *h)
3606+{
3607+    return !h->pprev;
3608+}
3609+
3610+static inline void hlist_del_init(struct hlist_node *n)
3611+{
3612+    struct hlist_node *next = n->next;
3613+    struct hlist_node **pprev = n->pprev;
3614+
3615+    if (pprev == NULL) {
3616+        return;
3617+    }
3618+
3619+    *pprev = next;
3620+    if (next != NULL) {
3621+        next->pprev = pprev;
3622+    }
3623+
3624+    n->next = NULL;
3625+    n->pprev = NULL;
3626+}
3627+
3628+static inline void hlist_ctl_del(struct hlist_ctl *ctl, struct hlist_node *n)
3629+{
3630+    if (ctl->head.first == ctl->tail.end) {
3631+        ctl->head.first = NULL;
3632+        ctl->tail.end = NULL;
3633+        return;
3634+    }
3635+
3636+    if (ctl->tail.end == n) {
3637+        ctl->tail.end = (struct hlist_node *)n->pprev;
3638+    }
3639+
3640+    hlist_del_init(n);
3641+}
3642+
3643+static inline struct hlist_node *hlist_pop_tail(struct hlist_ctl *ctl)
3644+{
3645+    if (hlist_empty(&ctl->head)) {
3646+        return NULL;
3647+    }
3648+
3649+    if (ctl->head.first == ctl->tail.end) {
3650+        struct hlist_node *ret = ctl->tail.end;
3651+        ctl->tail.end = NULL;
3652+        ctl->head.first = NULL;
3653+        return ret;
3654+    }
3655+
3656+    struct hlist_node *temp = ctl->tail.end;
3657+
3658+    struct hlist_node **ptailPrev = ctl->tail.end->pprev;
3659+    *ptailPrev = NULL;
3660+
3661+    ctl->tail.end = (struct hlist_node *)ptailPrev;
3662+    temp->pprev = NULL;
3663+    return temp;
3664+}
3665+
3666+static inline void hlist_add_after(struct hlist_node *n, struct hlist_node *next)
3667+{
3668+    next->next = n->next;
3669+    n->next = next;
3670+    next->pprev = &n->next;
3671+    if (next->next) {
3672+        next->next->pprev = &next->next;
3673+    }
3674+}
3675+
3676+static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
3677+{
3678+    struct hlist_node *first = h->first;
3679+
3680+    n->next = first;
3681+    if (first != NULL) {
3682+        first->pprev = &n->next;
3683+    }
3684+
3685+    h->first = n;
3686+    n->pprev = &h->first;
3687+}
3688+
3689+static inline struct hlist_node *hlist_pop_head(struct hlist_ctl *ctl)
3690+{
3691+    if (hlist_empty(&ctl->head)) {
3692+        return NULL;
3693+    }
3694+
3695+    struct hlist_node *temp = ctl->head.first;
3696+    hlist_ctl_del(ctl, temp);
3697+    return temp;
3698+}
3699+
3700+static inline void hlist_ctl_add_tail(struct hlist_ctl *ctl, struct hlist_node *node)
3701+{
3702+    if (hlist_empty(&ctl->head)) {
3703+        hlist_add_head(node, &ctl->head);
3704+        ctl->tail.end = ctl->head.first;
3705+        return;
3706+    }
3707+
3708+    ctl->tail.end->next = node;
3709+
3710+    node->pprev = &(ctl->tail.end->next);
3711+    node->next = NULL;
3712+    ctl->tail.end = node;
3713+}
3714+
3715+static inline void hlist_ctl_add_head(struct hlist_node *node, struct hlist_ctl *ctl)
3716+{
3717+    hlist_add_head(node, &ctl->head);
3718+    if (ctl->tail.end == NULL) {
3719+        ctl->tail.end = ctl->head.first;
3720+    }
3721+}
3722+
3723+static inline void hlist_ctl_add_before(struct hlist_node *n, struct hlist_node *next, struct hlist_ctl *ctl)
3724+{
3725+    hlist_add_before(n, next);
3726+    if (next == ctl->head.first) {
3727+        ctl->head.first = n;
3728+    }
3729+}
3730+
3731+static inline void hlist_ctl_add_after(struct hlist_node *n, struct hlist_node *next, struct hlist_ctl *ctl)
3732+{
3733+    hlist_add_after(n, next);
3734+    if (n == ctl->tail.end) {
3735+        ctl->tail.end = next;
3736+    }
3737+}
3738+//#endif /* TCP_PCB_HASH */
3739+
3740+#endif /* __HLIST_H__ */
3741diff --git a/src/include/list.h b/src/include/list.h
3742new file mode 100644
3743index 0000000..11f94c2
3744--- /dev/null
3745+++ b/src/include/list.h
3746@@ -0,0 +1,110 @@
3747+/*
3748+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
3749+ * All rights reserved.
3750+ *
3751+ * Redistribution and use in source and binary forms, with or without modification,
3752+ * are permitted provided that the following conditions are met:
3753+ *
3754+ * 1. Redistributions of source code must retain the above copyright notice,
3755+ *    this list of conditions and the following disclaimer.
3756+ * 2. Redistributions in binary form must reproduce the above copyright notice,
3757+ *    this list of conditions and the following disclaimer in the documentation
3758+ *    and/or other materials provided with the distribution.
3759+ * 3. The name of the author may not be used to endorse or promote products
3760+ *    derived from this software without specific prior written permission.
3761+ *
3762+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
3763+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
3764+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
3765+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
3766+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
3767+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
3768+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
3769+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
3770+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
3771+ * OF SUCH DAMAGE.
3772+ *
3773+ * This file is part of the lwIP TCP/IP stack.
3774+ *
3775+ * Author: Huawei Technologies
3776+ *
3777+ */
3778+
3779+#ifndef __LIST_H__
3780+#define __LIST_H__
3781+
3782+#ifndef NULL
3783+#ifdef __cplusplus
3784+#define NULL 0
3785+#else
3786+#define NULL ((void *)0)
3787+#endif
3788+#endif
3789+
3790+struct list_node {
3791+    struct list_node *prev;
3792+    struct list_node *next;
3793+};
3794+
3795+static inline void init_list_node_null(struct list_node *n)
3796+{
3797+    n->prev = NULL;
3798+    n->next = NULL;
3799+}
3800+
3801+static inline void init_list_node(struct list_node *n)
3802+{
3803+    n->prev = n;
3804+    n->next = n;
3805+}
3806+
3807+static inline void list_add_node(struct list_node *h, struct list_node *n)
3808+{
3809+    n->next = h;
3810+    n->prev = h->prev;
3811+    h->prev->next = n;
3812+    h->prev = n;
3813+}
3814+
3815+static inline void list_del_node(struct list_node *n)
3816+{
3817+    struct list_node *prev = n->prev;
3818+    struct list_node *next = n->next;
3819+    next->prev = prev;
3820+    prev->next = next;
3821+}
3822+
3823+static inline void list_del_node_init(struct list_node *n)
3824+{
3825+    list_del_node(n);
3826+    init_list_node(n);
3827+}
3828+
3829+static inline void list_del_node_null(struct list_node *n)
3830+{
3831+    if ((n->next) && (n->prev)) {
3832+        list_del_node(n);
3833+    }
3834+    init_list_node_null(n);
3835+}
3836+
3837+static inline int list_is_null(const struct list_node *n)
3838+{
3839+    return (n->prev == NULL) && (n->next == NULL);
3840+}
3841+
3842+static inline int list_is_empty(const struct list_node *h)
3843+{
3844+    return h == h->next;
3845+}
3846+
3847+#define list_for_each_safe(pos, n, head) \
3848+    for (pos = (head)->next, n = (pos)->next; pos != (head); pos = n, n = (pos)->next)
3849+
3850+#ifndef container_of
3851+#define container_of(ptr, type, member) ({ \
3852+    typeof( ((type *)0)->member ) *__mptr = (ptr); \
3853+    (type *)((char *)__mptr - offsetof(type,member));})
3854+#endif /* container_of */
3855+
3856+#endif /* __LIST_H__ */
3857diff --git a/src/include/lwip/api.h b/src/include/lwip/api.h
3858index c2afaf2..6dec8c0 100644
3859--- a/src/include/lwip/api.h
3860+++ b/src/include/lwip/api.h
3861@@ -140,8 +140,43 @@ enum netconn_type {
3862   /** Raw connection IPv6 (dual-stack by default, unless you call @ref netconn_set_ipv6only) */
3863   , NETCONN_RAW_IPV6    = NETCONN_RAW | NETCONN_TYPE_IPV6 /* 0x48 */
3864 #endif /* LWIP_IPV6 */
3865+
3866+#if USE_LIBOS
3867+  /*here must bigger than 0xff, because (type & 0xff) is for lwip inner use*/
3868+  , NETCONN_LIBOS = 0x100
3869+  , NETCONN_HOST = 0x200
3870+  , NETCONN_INPRG = 0x400
3871+  , NETCONN_STACK = NETCONN_LIBOS | NETCONN_HOST | NETCONN_INPRG
3872+#endif /* USE_LIBOS */
3873 };
3874
3875+#ifdef USE_LIBOS
3876+#define SET_CONN_TYPE_LIBOS_OR_HOST(conn) do { \
3877+  conn->type &= ~(NETCONN_STACK); \
3878+  conn->type |= (NETCONN_LIBOS | NETCONN_HOST); } while (0)
3879+#define SET_CONN_TYPE_LIBOS(conn) do { \
3880+  conn->type &= ~(NETCONN_STACK); \
3881+  conn->type |= NETCONN_LIBOS; } while (0)
3882+#define SET_CONN_TYPE_HOST(conn) do { \
3883+  conn->type &= ~(NETCONN_STACK); \
3884+  conn->type |= NETCONN_HOST; } while (0)
3885+#define ADD_CONN_TYPE_INPRG(conn) do { \
3886+  conn->type |= NETCONN_INPRG; } while(0)
3887+#define CONN_TYPE_HAS_LIBOS_AND_HOST(conn) ((conn->type & (NETCONN_LIBOS | NETCONN_HOST)) == (NETCONN_LIBOS | NETCONN_HOST))
3888+#define CONN_TYPE_HAS_LIBOS(conn) (conn->type & NETCONN_LIBOS)
3889+#define CONN_TYPE_HAS_HOST(conn) (conn->type & NETCONN_HOST)
3890+#define CONN_TYPE_HAS_INPRG(conn) (!!(conn->type & NETCONN_INPRG))
3891+#define CONN_TYPE_IS_LIBOS(conn)  (!!(NETCONN_LIBOS == (conn->type & NETCONN_STACK)))
3892+#define CONN_TYPE_IS_HOST(conn)   (!!(NETCONN_HOST == (conn->type & NETCONN_STACK)))
3893+#else
3894+#define SET_CONN_TYPE_LIBOS_OR_HOST(conn) do {} while (0)
3895+#define SET_CONN_TYPE_LIBOS(conn) do {} while (0)
3896+#define SET_CONN_TYPE_HOST(conn) do {} while (0)
3897+#define CONN_TYPE_HAS_LIBOS_AND_HOST(conn) (0)
3898+#define CONN_TYPE_HAS_LIBOS(conn) (0)
3899+#define CONN_TYPE_HAS_HOST(conn) (0)
3900+#endif /* USE_LIBOS */
3901+
3902 /** Current state of the netconn. Non-TCP netconns are always
3903  * in state NETCONN_NONE! */
3904 enum netconn_state {
3905diff --git a/src/include/lwip/debug.h b/src/include/lwip/debug.h
3906index 579fd24..f47cbfe 100644
3907--- a/src/include/lwip/debug.h
3908+++ b/src/include/lwip/debug.h
3909@@ -145,6 +145,7 @@
3910                                    ((debug) & LWIP_DBG_ON) && \
3911                                    ((debug) & LWIP_DBG_TYPES_ON) && \
3912                                    ((s16_t)((debug) & LWIP_DBG_MASK_LEVEL) >= LWIP_DBG_MIN_LEVEL)) { \
3913+                                 LWIP_PLATFORM_LOG(debug, STRIP_BRACES(ESC_ARGS message)); \
3914                                  LWIP_PLATFORM_DIAG(message); \
3915                                  if ((debug) & LWIP_DBG_HALT) { \
3916                                    while(1); \
3917diff --git a/src/include/lwip/def.h b/src/include/lwip/def.h
3918index dfb266d..fea7187 100644
3919--- a/src/include/lwip/def.h
3920+++ b/src/include/lwip/def.h
3921@@ -116,6 +116,21 @@ u32_t lwip_htonl(u32_t x);
3922
3923 /* Provide usual function names as macros for users, but this can be turned off */
3924 #ifndef LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS
3925+
3926+/* avoid conflicts with netinet/in.h */
3927+#ifdef htons
3928+#undef htons
3929+#endif
3930+#ifdef ntohs
3931+#undef ntohs
3932+#endif
3933+#ifdef htonl
3934+#undef htonl
3935+#endif
3936+#ifdef ntohl
3937+#undef ntohl
3938+#endif
3939+
3940 #define htons(x) lwip_htons(x)
3941 #define ntohs(x) lwip_ntohs(x)
3942 #define htonl(x) lwip_htonl(x)
3943diff --git a/src/include/lwip/ip.h b/src/include/lwip/ip.h
3944index 653c3b2..d560f6b 100644
3945--- a/src/include/lwip/ip.h
3946+++ b/src/include/lwip/ip.h
3947@@ -96,9 +96,15 @@ struct ip_pcb {
3948 /*
3949  * Option flags per-socket. These are the same like SO_XXX in sockets.h
3950  */
3951+#if USE_LIBOS
3952+#define SOF_REUSEADDR     0x02U /* allow local address reuse */
3953+#define SOF_KEEPALIVE     0x09U /* keep connections alive */
3954+#define SOF_BROADCAST     0x06U /* permit to send and to receive broadcast messages (see IP_SOF_BROADCAST option) */
3955+#else
3956 #define SOF_REUSEADDR     0x04U  /* allow local address reuse */
3957 #define SOF_KEEPALIVE     0x08U  /* keep connections alive */
3958 #define SOF_BROADCAST     0x20U  /* permit to send and to receive broadcast messages (see IP_SOF_BROADCAST option) */
3959+#endif /* USE_LIBOS */
3960
3961 /* These flags are inherited (e.g. from a listen-pcb to a connection-pcb): */
3962 #define SOF_INHERITED   (SOF_REUSEADDR|SOF_KEEPALIVE)
3963@@ -125,7 +131,7 @@ struct ip_globals
3964   /** Destination IP address of current_header */
3965   ip_addr_t current_iphdr_dest;
3966 };
3967-extern struct ip_globals ip_data;
3968+extern PER_THREAD struct ip_globals ip_data;
3969
3970
3971 /** Get the interface that accepted the current packet.
3972diff --git a/src/include/lwip/memp.h b/src/include/lwip/memp.h
3973index 1630b26..64d8f31 100644
3974--- a/src/include/lwip/memp.h
3975+++ b/src/include/lwip/memp.h
3976@@ -58,7 +58,11 @@ typedef enum {
3977 #include "lwip/priv/memp_priv.h"
3978 #include "lwip/stats.h"
3979
3980+#if USE_LIBOS
3981+extern PER_THREAD struct memp_desc* memp_pools[MEMP_MAX];
3982+#else
3983 extern const struct memp_desc* const memp_pools[MEMP_MAX];
3984+#endif /* USE_LIBOS */
3985
3986 /**
3987  * @ingroup mempool
3988@@ -92,6 +96,18 @@ extern const struct memp_desc* const memp_pools[MEMP_MAX];
3989  * To relocate a pool, declare it as extern in cc.h. Example for GCC:
3990  *   extern u8_t \_\_attribute\_\_((section(".onchip_mem"))) memp_memory_my_private_pool_base[];
3991  */
3992+#if USE_LIBOS
3993+#define LWIP_MEMPOOL_DECLARE(name,num,size,desc) \
3994+    PER_THREAD struct memp_desc memp_ ## name = {0}; \
3995+    PER_THREAD char memp_desc_ ## name[] = desc; \
3996+    PER_THREAD struct stats_mem memp_stat ## name = {0}; \
3997+    PER_THREAD u16_t memp_size ## name = size; \
3998+    PER_THREAD u16_t memp_num ## name = num;   \
3999+    PER_THREAD struct memp *memp_tab_ ## name = NULL; \
4000+    LWIP_DECLARE_MEMP_BASE_ALIGNED(name, ((num) * (MEMP_SIZE + MEMP_ALIGN_SIZE(size))));
4001+
4002+#else /* USE_LIBOS */
4003+
4004 #define LWIP_MEMPOOL_DECLARE(name,num,size,desc) \
4005   LWIP_DECLARE_MEMORY_ALIGNED(memp_memory_ ## name ## _base, ((num) * (MEMP_SIZE + MEMP_ALIGN_SIZE(size)))); \
4006     \
4007@@ -108,6 +124,7 @@ extern const struct memp_desc* const memp_pools[MEMP_MAX];
4008     &memp_tab_ ## name \
4009   };
4010
4011+#endif /* USE_LIBOS */
4012 #endif /* MEMP_MEM_MALLOC */
4013
4014 /**
4015diff --git a/src/include/lwip/netif.h b/src/include/lwip/netif.h
4016index 9a16ded..057c51f 100644
4017--- a/src/include/lwip/netif.h
4018+++ b/src/include/lwip/netif.h
4019@@ -406,11 +406,11 @@ struct netif {
4020 #define NETIF_FOREACH(netif) if (((netif) = netif_default) != NULL)
4021 #else /* LWIP_SINGLE_NETIF */
4022 /** The list of network interfaces. */
4023-extern struct netif *netif_list;
4024+extern PER_THREAD struct netif *netif_list;
4025 #define NETIF_FOREACH(netif) for ((netif) = netif_list; (netif) != NULL; (netif) = (netif)->next)
4026 #endif /* LWIP_SINGLE_NETIF */
4027 /** The default network interface. */
4028-extern struct netif *netif_default;
4029+extern PER_THREAD struct netif *netif_default;
4030
4031 void netif_init(void);
4032
4033diff --git a/src/include/lwip/opt.h b/src/include/lwip/opt.h
4034index d8c82d1..8294cdd 100644
4035--- a/src/include/lwip/opt.h
4036+++ b/src/include/lwip/opt.h
4037@@ -533,6 +533,22 @@
4038 #endif
4039
4040 /**
4041+ * MEMP_NUM_SYS_SEM: the number of struct sys_sems.
4042+ * (only needed if you use the sequential API, like api_lib.c)
4043+ */
4044+#if !defined MEMP_NUM_SYS_SEM || defined __DOXYGEN__
4045+#define MEMP_NUM_SYS_SEM                128
4046+#endif
4047+
4048+/**
4049+ * MEMP_NUM_SYS_MBOX: the number of struct sys_sems.
4050+ * (only needed if you use the sequential API, like api_lib.c)
4051+ */
4052+#if !defined MEMP_NUM_SYS_MBOX || defined __DOXYGEN__
4053+#define MEMP_NUM_SYS_MBOX                128
4054+#endif
4055+
4056+/**
4057  * MEMP_NUM_SELECT_CB: the number of struct lwip_select_cb.
4058  * (Only needed if you have LWIP_MPU_COMPATIBLE==1 and use the socket API.
4059  * In that case, you need one per thread calling lwip_select.)
4060@@ -2232,7 +2248,7 @@
4061  * MIB2_STATS==1: Stats for SNMP MIB2.
4062  */
4063 #if !defined MIB2_STATS || defined __DOXYGEN__
4064-#define MIB2_STATS                      0
4065+#define MIB2_STATS                      1
4066 #endif
4067
4068 #else
4069@@ -3422,6 +3438,10 @@
4070 #define TCP_OUTPUT_DEBUG                LWIP_DBG_OFF
4071 #endif
4072
4073+#ifndef PERF_OUTPUT_DEBUG
4074+ #define PERF_OUTPUT_DEBUG              LWIP_DBG_OFF
4075+#endif
4076+
4077 /**
4078  * TCP_RST_DEBUG: Enable debugging for TCP with the RST message.
4079  */
4080@@ -3502,6 +3522,46 @@
4081 #define LWIP_TESTMODE                   0
4082 #endif
4083
4084+/**
4085+ * EPOLL_DEBUG: Enable debugging in epoll.c.
4086+ */
4087+#if !defined EPOLL_DEBUG || defined __DOXYGEN__ && USE_LIBOS
4088+#define EPOLL_DEBUG                    LWIP_DBG_OFF
4089+#endif
4090+/**
4091+ * @}
4092+ */
4093+
4094+/**
4095+ * ETHDEV_DEBUG: Enable debugging in ethdev.c.
4096+ */
4097+#if !defined ETHDEV_DEBUG || defined __DOXYGEN__ && USE_LIBOS
4098+#define ETHDEV_DEBUG                    LWIP_DBG_OFF
4099+#endif
4100+/**
4101+ * @}
4102+ */
4103+
4104+/**
4105+ * ETHDEV_DEBUG: Enable debugging in ethdev.c.
4106+ */
4107+#if !defined SYSCALL_DEBUG || defined __DOXYGEN__ && USE_LIBOS
4108+#define SYSCALL_DEBUG                    LWIP_DBG_OFF
4109+#endif
4110+/**
4111+ * @}
4112+ */
4113+
4114+/**
4115+ * CONTROL_DEBUG: Enable debugging in control_plane.c.
4116+ */
4117+#if !defined CONTROL_DEBUG || defined __DOXYGEN__ && USE_LIBOS
4118+#define CONTROL_DEBUG                    LWIP_DBG_ON
4119+#endif
4120+/**
4121+ * @}
4122+ */
4123+
4124 /*
4125    --------------------------------------------------
4126    ---------- Performance tracking options ----------
4127diff --git a/src/include/lwip/priv/memp_std.h b/src/include/lwip/priv/memp_std.h
4128index 669ad4d..395ac0c 100644
4129--- a/src/include/lwip/priv/memp_std.h
4130+++ b/src/include/lwip/priv/memp_std.h
4131@@ -122,6 +122,13 @@ LWIP_MEMPOOL(MLD6_GROUP,     MEMP_NUM_MLD6_GROUP,      sizeof(struct mld_group),
4132 #endif /* LWIP_IPV6 && LWIP_IPV6_MLD */
4133
4134
4135+#if USE_LIBOS
4136+#if !LWIP_NETCONN_SEM_PER_THREAD
4137+LWIP_MEMPOOL(SYS_SEM,     MEMP_NUM_SYS_SEM,      sizeof(struct sys_sem),     "SYS_SEM")
4138+#endif
4139+
4140+LWIP_MEMPOOL(SYS_MBOX,     MEMP_NUM_SYS_MBOX,      sizeof(struct sys_mbox),     "SYS_MBOX")
4141+#endif /* USE_LIBOS */
4142 /*
4143  * A list of pools of pbuf's used by LWIP.
4144  *
4145diff --git a/src/include/lwip/priv/sockets_priv.h b/src/include/lwip/priv/sockets_priv.h
4146index d8f9904..7268a17 100644
4147--- a/src/include/lwip/priv/sockets_priv.h
4148+++ b/src/include/lwip/priv/sockets_priv.h
4149@@ -45,56 +45,17 @@
4150 #include "lwip/sockets.h"
4151 #include "lwip/sys.h"
4152
4153+/* move some definitions to the lwipsock.h for libnet to use, and
4154+ * at the same time avoid conflict between lwip/sockets.h and sys/socket.h
4155+ */
4156+#include "lwipsock.h"
4157+
4158 #ifdef __cplusplus
4159 extern "C" {
4160 #endif
4161
4162 #define NUM_SOCKETS MEMP_NUM_NETCONN
4163
4164-/** This is overridable for the rare case where more than 255 threads
4165- * select on the same socket...
4166- */
4167-#ifndef SELWAIT_T
4168-#define SELWAIT_T u8_t
4169-#endif
4170-
4171-union lwip_sock_lastdata {
4172-  struct netbuf *netbuf;
4173-  struct pbuf *pbuf;
4174-};
4175-
4176-/** Contains all internal pointers and states used for a socket */
4177-struct lwip_sock {
4178-  /** sockets currently are built on netconns, each socket has one netconn */
4179-  struct netconn *conn;
4180-  /** data that was left from the previous read */
4181-  union lwip_sock_lastdata lastdata;
4182-#if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
4183-  /** number of times data was received, set by event_callback(),
4184-      tested by the receive and select functions */
4185-  s16_t rcvevent;
4186-  /** number of times data was ACKed (free send buffer), set by event_callback(),
4187-      tested by select */
4188-  u16_t sendevent;
4189-  /** error happened for this socket, set by event_callback(), tested by select */
4190-  u16_t errevent;
4191-  /** counter of how many threads are waiting for this socket using select */
4192-  SELWAIT_T select_waiting;
4193-#endif /* LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL */
4194-#if LWIP_NETCONN_FULLDUPLEX
4195-  /* counter of how many threads are using a struct lwip_sock (not the 'int') */
4196-  u8_t fd_used;
4197-  /* status of pending close/delete actions */
4198-  u8_t fd_free_pending;
4199-#define LWIP_SOCK_FD_FREE_TCP  1
4200-#define LWIP_SOCK_FD_FREE_FREE 2
4201-#endif
4202-};
4203-
4204-#ifndef set_errno
4205-#define set_errno(err) do { if (err) { errno = (err); } } while(0)
4206-#endif
4207-
4208 #if !LWIP_TCPIP_CORE_LOCKING
4209 /** Maximum optlen used by setsockopt/getsockopt */
4210 #define LWIP_SETGETSOCKOPT_MAXOPTLEN LWIP_MAX(16, sizeof(struct ifreq))
4211diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h
4212index 72f9126..192edc4 100644
4213--- a/src/include/lwip/priv/tcp_priv.h
4214+++ b/src/include/lwip/priv/tcp_priv.h
4215@@ -323,25 +323,42 @@ struct tcp_seg {
4216 #endif /* LWIP_WND_SCALE */
4217
4218 /* Global variables: */
4219-extern struct tcp_pcb *tcp_input_pcb;
4220-extern u32_t tcp_ticks;
4221-extern u8_t tcp_active_pcbs_changed;
4222+extern PER_THREAD struct tcp_pcb *tcp_input_pcb;
4223+extern PER_THREAD u32_t tcp_ticks;
4224+extern PER_THREAD u8_t tcp_active_pcbs_changed;
4225
4226 /* The TCP PCB lists. */
4227 union tcp_listen_pcbs_t { /* List of all TCP PCBs in LISTEN state. */
4228   struct tcp_pcb_listen *listen_pcbs;
4229   struct tcp_pcb *pcbs;
4230 };
4231-extern struct tcp_pcb *tcp_bound_pcbs;
4232-extern union tcp_listen_pcbs_t tcp_listen_pcbs;
4233-extern struct tcp_pcb *tcp_active_pcbs;  /* List of all TCP PCBs that are in a
4234+extern PER_THREAD struct tcp_pcb *tcp_bound_pcbs;
4235+extern PER_THREAD union tcp_listen_pcbs_t tcp_listen_pcbs;
4236+extern PER_THREAD struct tcp_pcb *tcp_active_pcbs;  /* List of all TCP PCBs that are in a
4237               state in which they accept or send
4238               data. */
4239-extern struct tcp_pcb *tcp_tw_pcbs;      /* List of all TCP PCBs in TIME-WAIT. */
4240+extern PER_THREAD struct tcp_pcb *tcp_tw_pcbs;      /* List of all TCP PCBs in TIME-WAIT. */
4241
4242 #define NUM_TCP_PCB_LISTS_NO_TIME_WAIT  3
4243 #define NUM_TCP_PCB_LISTS               4
4244-extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS];
4245+extern PER_THREAD struct tcp_pcb ** tcp_pcb_lists[NUM_TCP_PCB_LISTS];
4246+
4247+#if USE_LIBOS
4248+#include "reg_sock.h"
4249+static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pcb *pcb)
4250+{
4251+  LWIP_ASSERT("Invalid parameter", pcb != NULL);
4252+
4253+  struct libnet_quintuple qtuple;
4254+  qtuple.protocol   = 0;
4255+  qtuple.src_ip     = pcb->local_ip.addr;
4256+  qtuple.src_port   = lwip_htons(pcb->local_port);
4257+  qtuple.dst_ip     = pcb->remote_ip.addr;
4258+  qtuple.dst_port   = lwip_htons(pcb->remote_port);
4259+
4260+  return vdev_reg_xmit(reg_type, &qtuple);
4261+}
4262+#endif
4263
4264 /* Axioms about the above lists:
4265    1) Every TCP PCB that is not CLOSED is in one of the lists.
4266@@ -355,6 +372,54 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS];
4267 #define TCP_DEBUG_PCB_LISTS 0
4268 #endif
4269 #if TCP_DEBUG_PCB_LISTS
4270+#if USE_LIBOS
4271+#define TCP_REG(pcbs, npcb) do {\
4272+                            struct tcp_pcb *tcp_tmp_pcb; \
4273+                            LWIP_DEBUGF(TCP_DEBUG, ("TCP_REG %p local port %d\n", (npcb), (npcb)->local_port)); \
4274+                            for (tcp_tmp_pcb = *(pcbs); \
4275+          tcp_tmp_pcb != NULL; \
4276+        tcp_tmp_pcb = tcp_tmp_pcb->next) { \
4277+                                LWIP_ASSERT("TCP_REG: already registered\n", tcp_tmp_pcb != (npcb)); \
4278+                            } \
4279+                            LWIP_ASSERT("TCP_REG: pcb->state != CLOSED", ((pcbs) == &tcp_bound_pcbs) || ((npcb)->state != CLOSED)); \
4280+                            if (*pcbs)                                     \
4281+                               (*pcbs)->prev = npcb;                       \
4282+                            (npcb)->prev = NULL;                           \
4283+                            (npcb)->next = *(pcbs); \
4284+                            LWIP_ASSERT("TCP_REG: npcb->next != npcb", (npcb)->next != (npcb)); \
4285+                            *(pcbs) = (npcb); \
4286+                            LWIP_ASSERT("TCP_RMV: tcp_pcbs sane", tcp_pcbs_sane()); \
4287+              tcp_timer_needed(); \
4288+                            } while(0)
4289+#define TCP_RMV(pcbs, npcb) do { \
4290+                            if (pcb->state == LISTEN)                         \
4291+                              vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, npcb); \
4292+                            else                                              \
4293+                              vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, npcb);\
4294+                            struct tcp_pcb *tcp_tmp_pcb; \
4295+                            LWIP_ASSERT("TCP_RMV: pcbs != NULL", *(pcbs) != NULL); \
4296+                            LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removing %p from %p\n", (npcb), *(pcbs))); \
4297+                            if(*(pcbs) == (npcb)) { \
4298+                               *(pcbs) = (*pcbs)->next; \
4299+                               if (*pcbs)                                   \
4300+                                  (*pcbs)->prev = NULL;                     \
4301+                            } else { \
4302+                               struct tcp_pcb *prev, *next;                 \
4303+                               prev = npcb->prev;                           \
4304+                               next = npcb->next;                           \
4305+                               if (prev)                                    \
4306+                                  prev->next = next;                        \
4307+                               if (next)                                    \
4308+                                  next->prev = prev;                        \
4309+                               } \
4310+                            } \
4311+                            (npcb)->prev = NULL; \
4312+                            (npcb)->next = NULL; \
4313+                            LWIP_ASSERT("TCP_RMV: tcp_pcbs sane", tcp_pcbs_sane()); \
4314+                            LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removed %p from %p\n", (npcb), *(pcbs))); \
4315+                            } while(0)
4316+
4317+#else /* USE_LIBOS */
4318 #define TCP_REG(pcbs, npcb) do {\
4319                             struct tcp_pcb *tcp_tmp_pcb; \
4320                             LWIP_DEBUGF(TCP_DEBUG, ("TCP_REG %p local port %"U16_F"\n", (void *)(npcb), (npcb)->local_port)); \
4321@@ -387,8 +452,65 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS];
4322                             LWIP_DEBUGF(TCP_DEBUG, ("TCP_RMV: removed %p from %p\n", (void *)(npcb), (void *)(*(pcbs)))); \
4323                             } while(0)
4324
4325+#endif /* USE_LIBOS */
4326 #else /* LWIP_DEBUG */
4327
4328+#if TCP_PCB_HASH
4329+#define TCP_REG_HASH(pcbs, npcb)                   \
4330+  do {                                             \
4331+    u32_t idx;                                     \
4332+    struct hlist_head *hd;                         \
4333+    struct tcp_hash_table *htb = pcbs;             \
4334+    idx = TUPLE4_HASH_FN((npcb)->local_ip.addr, (npcb)->local_port,  \
4335+                       (npcb)->remote_ip.addr, (npcb)->remote_port) & \
4336+                       (htb->size - 1);            \
4337+    hd = &htb->array[idx].chain;                   \
4338+    hlist_add_head(&(npcb)->tcp_node, hd);         \
4339+    tcp_timer_needed();                            \
4340+  } while (0)
4341+
4342+#define TCP_RMV_HASH(pcbs, npcb)                   \
4343+  do {                                             \
4344+    hlist_del_init(&(npcb)->tcp_node);             \
4345+  } while (0)
4346+#endif /* TCP_PCB_HASH */
4347+
4348+#if USE_LIBOS
4349+#define TCP_REG(pcbs, npcb)                        \
4350+  do {                                             \
4351+    if (*pcbs)                                     \
4352+      (*pcbs)->prev = npcb;                        \
4353+    (npcb)->prev = NULL;                           \
4354+    (npcb)->next = *pcbs;                          \
4355+    *(pcbs) = (npcb);                              \
4356+    tcp_timer_needed();                            \
4357+  } while (0)
4358+
4359+#define TCP_RMV(pcbs, npcb)                        \
4360+  do {                                             \
4361+    if (pcb->state == LISTEN)                         \
4362+      vdev_reg_done(REG_RING_TCP_LISTEN_CLOSE, npcb); \
4363+    else                                              \
4364+      vdev_reg_done(REG_RING_TCP_CONNECT_CLOSE, npcb);\
4365+    if(*(pcbs) == (npcb)) {                        \
4366+      (*(pcbs)) = (*pcbs)->next;                   \
4367+      if (*pcbs)                                   \
4368+        (*pcbs)->prev = NULL;                      \
4369+    }                                              \
4370+    else {                                         \
4371+      struct tcp_pcb *prev, *next;                 \
4372+      prev = npcb->prev;                           \
4373+      next = npcb->next;                           \
4374+      if (prev)                                    \
4375+        prev->next = next;                         \
4376+      if (next)                                    \
4377+        next->prev = prev;                         \
4378+    }                                              \
4379+    (npcb)->prev = NULL;                           \
4380+    (npcb)->next = NULL;                           \
4381+  } while(0)
4382+
4383+#else /* USE_LIBOS */
4384 #define TCP_REG(pcbs, npcb)                        \
4385   do {                                             \
4386     (npcb)->next = *pcbs;                          \
4387@@ -415,8 +537,32 @@ extern struct tcp_pcb ** const tcp_pcb_lists[NUM_TCP_PCB_LISTS];
4388     (npcb)->next = NULL;                           \
4389   } while(0)
4390
4391+#endif /* USE_LIBOS */
4392 #endif /* LWIP_DEBUG */
4393
4394+
4395+#if TCP_PCB_HASH
4396+#define TCP_REG_ACTIVE_HASH(npcb)                  \
4397+  do {                                             \
4398+    TCP_REG_HASH(tcp_active_htable, npcb);         \
4399+    tcp_active_pcbs_changed = 1;                   \
4400+  } while (0)
4401+
4402+#define TCP_RMV_ACTIVE_HASH(npcb)                  \
4403+  do {                                             \
4404+    TCP_RMV_HASH(tcp_active_htable, npcb);         \
4405+    tcp_active_pcbs_changed = 1;                   \
4406+  } while (0)
4407+
4408+#define TCP_PCB_REMOVE_ACTIVE_HASH(pcb)            \
4409+  do {                                             \
4410+    tcp_pcb_remove_hash(tcp_active_htable, pcb);   \
4411+    tcp_active_pcbs_changed = 1;                   \
4412+  } while (0)
4413+
4414+void tcp_pcb_remove_hash(struct tcp_hash_table *htb, struct tcp_pcb *pcb);
4415+#endif /* TCP_PCB_HASH */
4416+
4417 #define TCP_REG_ACTIVE(npcb)                       \
4418   do {                                             \
4419     TCP_REG(&tcp_active_pcbs, npcb);               \
4420diff --git a/src/include/lwip/prot/ip4.h b/src/include/lwip/prot/ip4.h
4421index 9347461..c9ad89c 100644
4422--- a/src/include/lwip/prot/ip4.h
4423+++ b/src/include/lwip/prot/ip4.h
4424@@ -81,6 +81,21 @@ struct ip_hdr {
4425   PACK_STRUCT_FIELD(u16_t _id);
4426   /* fragment offset field */
4427   PACK_STRUCT_FIELD(u16_t _offset);
4428+
4429+/* avoid conflicts with netinet/ip.h */
4430+#ifdef IP_RF
4431+#undef IP_RF
4432+#endif
4433+#ifdef IP_DF
4434+#undef IP_DF
4435+#endif
4436+#ifdef IP_MF
4437+#undef IP_MF
4438+#endif
4439+#ifdef IP_OFFMASK
4440+#undef IP_OFFMASK
4441+#endif
4442+
4443 #define IP_RF 0x8000U        /* reserved fragment flag */
4444 #define IP_DF 0x4000U        /* don't fragment flag */
4445 #define IP_MF 0x2000U        /* more fragments flag */
4446diff --git a/src/include/lwip/sockets.h b/src/include/lwip/sockets.h
4447index d70d36c..345e26c 100644
4448--- a/src/include/lwip/sockets.h
4449+++ b/src/include/lwip/sockets.h
4450@@ -57,6 +57,11 @@ extern "C" {
4451
4452 /* If your port already typedef's sa_family_t, define SA_FAMILY_T_DEFINED
4453    to prevent this code from redefining it. */
4454+#if USE_LIBOS
4455+#define SA_FAMILY_T_DEFINED
4456+  typedef u16_t sa_family_t;
4457+#endif
4458+
4459 #if !defined(sa_family_t) && !defined(SA_FAMILY_T_DEFINED)
4460 typedef u8_t sa_family_t;
4461 #endif
4462@@ -69,7 +74,9 @@ typedef u16_t in_port_t;
4463 #if LWIP_IPV4
4464 /* members are in network byte order */
4465 struct sockaddr_in {
4466+#if !USE_LIBOS
4467   u8_t            sin_len;
4468+#endif
4469   sa_family_t     sin_family;
4470   in_port_t       sin_port;
4471   struct in_addr  sin_addr;
4472@@ -90,7 +97,9 @@ struct sockaddr_in6 {
4473 #endif /* LWIP_IPV6 */
4474
4475 struct sockaddr {
4476+#if !USE_LIBOS
4477   u8_t        sa_len;
4478+#endif
4479   sa_family_t sa_family;
4480   char        sa_data[14];
4481 };
4482@@ -189,6 +198,9 @@ struct ifreq {
4483 #define SOCK_DGRAM      2
4484 #define SOCK_RAW        3
4485
4486+#if USE_LIBOS
4487+#include <asm/socket.h>
4488+#else
4489 /*
4490  * Option flags per-socket. These must match the SOF_ flags in ip.h (checked in init.c)
4491  */
4492@@ -221,6 +233,12 @@ struct ifreq {
4493 #define SO_BINDTODEVICE 0x100b /* bind to device */
4494
4495 /*
4496+ * Level number for (get/set)sockopt() to apply to socket itself.
4497+ */
4498+#define  SOL_SOCKET  0xfff    /* options for socket level */
4499+#endif /* USE_LIBOS */
4500+
4501+/*
4502  * Structure used for manipulating linger option.
4503  */
4504 struct linger {
4505@@ -228,11 +246,6 @@ struct linger {
4506   int l_linger;               /* linger time in seconds */
4507 };
4508
4509-/*
4510- * Level number for (get/set)sockopt() to apply to socket itself.
4511- */
4512-#define  SOL_SOCKET  0xfff    /* options for socket level */
4513-
4514
4515 #define AF_UNSPEC       0
4516 #define AF_INET         2
4517@@ -276,11 +289,20 @@ struct linger {
4518 /*
4519  * Options for level IPPROTO_TCP
4520  */
4521+#if USE_LIBOS
4522+/* come from netinet/tcp.h */
4523+#define TCP_NODELAY    0x01    /* don't delay send to coalesce packets */
4524+#define TCP_KEEPALIVE  0x24    /* send KEEPALIVE probes when idle for pcb->keep_idle milliseconds */
4525+#define TCP_KEEPIDLE   0x04    /* set pcb->keep_idle  - Same as TCP_KEEPALIVE, but use seconds for get/setsockopt */
4526+#define TCP_KEEPINTVL  0x05    /* set pcb->keep_intvl - Use seconds for get/setsockopt */
4527+#define TCP_KEEPCNT    0x06    /* set pcb->keep_cnt   - Use number of probes sent for get/setsockopt */
4528+#else /* USE_LIBOS */
4529 #define TCP_NODELAY    0x01    /* don't delay send to coalesce packets */
4530 #define TCP_KEEPALIVE  0x02    /* send KEEPALIVE probes when idle for pcb->keep_idle milliseconds */
4531 #define TCP_KEEPIDLE   0x03    /* set pcb->keep_idle  - Same as TCP_KEEPALIVE, but use seconds for get/setsockopt */
4532 #define TCP_KEEPINTVL  0x04    /* set pcb->keep_intvl - Use seconds for get/setsockopt */
4533 #define TCP_KEEPCNT    0x05    /* set pcb->keep_cnt   - Use number of probes sent for get/setsockopt */
4534+#endif /* USE_LIBOS */
4535 #endif /* LWIP_TCP */
4536
4537 #if LWIP_IPV6
4538@@ -483,12 +505,30 @@ typedef struct fd_set
4539   unsigned char fd_bits [(FD_SETSIZE+7)/8];
4540 } fd_set;
4541
4542-#elif FD_SETSIZE < (LWIP_SOCKET_OFFSET + MEMP_NUM_NETCONN)
4543+#elif FD_SETSIZE < (LWIP_SOCKET_OFFSET + MEMP_NUM_NETCONN) && !USE_LIBOS
4544 #error "external FD_SETSIZE too small for number of sockets"
4545 #else
4546 #define LWIP_SELECT_MAXNFDS FD_SETSIZE
4547 #endif /* FD_SET */
4548
4549+#if USE_LIBOS
4550+#if !defined(POLLIN) && !defined(POLLOUT)
4551+/* come from bits/poll.h */
4552+#define POLLIN     0x001
4553+#define POLLOUT    0x004
4554+#define POLLERR    0x008
4555+#define POLLNVAL   0x020
4556+/* Below values are unimplemented */
4557+#define POLLRDNORM 0x040
4558+#define POLLRDBAND 0x080
4559+#define POLLPRI    0x002
4560+#define POLLWRNORM 0x100
4561+#define POLLWRBAND 0x200
4562+#define POLLHUP    0x010
4563+#endif
4564+#endif /* USE_LIBOS */
4565+
4566+#if LWIP_SOCKET_POLL
4567 /* poll-related defines and types */
4568 /* @todo: find a better way to guard the definition of these defines and types if already defined */
4569 #if !defined(POLLIN) && !defined(POLLOUT)
4570@@ -511,6 +551,7 @@ struct pollfd
4571   short revents;
4572 };
4573 #endif
4574+#endif /* LWIP_SOCKET_POLL */
4575
4576 /** LWIP_TIMEVAL_PRIVATE: if you want to use the struct timeval provided
4577  * by your system, set this to 0 and include <sys/time.h> in cc.h */
4578@@ -603,8 +644,15 @@ int lwip_select(int maxfdp1, fd_set *readset, fd_set *writeset, fd_set *exceptse
4579 #if LWIP_SOCKET_POLL
4580 int lwip_poll(struct pollfd *fds, nfds_t nfds, int timeout);
4581 #endif
4582+
4583+#if USE_LIBOS
4584+int lwip_ioctl(int s, long cmd, ...);
4585+int lwip_fcntl(int s, int cmd, ...);
4586+#else
4587 int lwip_ioctl(int s, long cmd, void *argp);
4588 int lwip_fcntl(int s, int cmd, int val);
4589+#endif /* USE_LIBOS */
4590+
4591 const char *lwip_inet_ntop(int af, const void *src, char *dst, socklen_t size);
4592 int lwip_inet_pton(int af, const char *src, void *dst);
4593
4594@@ -670,10 +718,17 @@ int lwip_inet_pton(int af, const char *src, void *dst);
4595 #define writev(s,iov,iovcnt)                      lwip_writev(s,iov,iovcnt)
4596 /** @ingroup socket */
4597 #define close(s)                                  lwip_close(s)
4598+
4599+#if USE_LIBOS
4600+#define fcntl(s,cmd...)                           lwip_fcntl(s,cmd)
4601+#define ioctl(s,cmd...)                           lwip_ioctl(s,cmd)
4602+#else
4603 /** @ingroup socket */
4604 #define fcntl(s,cmd,val)                          lwip_fcntl(s,cmd,val)
4605 /** @ingroup socket */
4606 #define ioctl(s,cmd,argp)                         lwip_ioctl(s,cmd,argp)
4607+#endif /* USE_LIBOS */
4608+
4609 #endif /* LWIP_POSIX_SOCKETS_IO_NAMES */
4610 #endif /* LWIP_COMPAT_SOCKETS != 2 */
4611
4612diff --git a/src/include/lwip/stats.h b/src/include/lwip/stats.h
4613index b570dba..4470531 100644
4614--- a/src/include/lwip/stats.h
4615+++ b/src/include/lwip/stats.h
4616@@ -301,7 +301,7 @@ struct stats_ {
4617 };
4618
4619 /** Global variable containing lwIP internal statistics. Add this to your debugger's watchlist. */
4620-extern struct stats_ lwip_stats;
4621+extern PER_THREAD struct stats_ lwip_stats;
4622
4623 /** Init statistics */
4624 void stats_init(void);
4625@@ -467,6 +467,8 @@ void stats_init(void);
4626 #define MIB2_STATS_INC(x)
4627 #endif
4628
4629+int get_mib2_stats(char *buf);
4630+
4631 /* Display of statistics */
4632 #if LWIP_STATS_DISPLAY
4633 void stats_display(void);
4634diff --git a/src/include/lwip/tcp.h b/src/include/lwip/tcp.h
4635index daf7599..4f86b46 100644
4636--- a/src/include/lwip/tcp.h
4637+++ b/src/include/lwip/tcp.h
4638@@ -51,6 +51,11 @@
4639 #include "lwip/ip6.h"
4640 #include "lwip/ip6_addr.h"
4641
4642+#if TCP_PCB_HASH
4643+#include "lwip/sys.h"
4644+#include "hlist.h"
4645+#endif
4646+
4647 #ifdef __cplusplus
4648 extern "C" {
4649 #endif
4650@@ -209,15 +214,27 @@ typedef u16_t tcpflags_t;
4651 /**
4652  * members common to struct tcp_pcb and struct tcp_listen_pcb
4653  */
4654+#if USE_LIBOS
4655 #define TCP_PCB_COMMON(type) \
4656   type *next; /* for the linked list */ \
4657+  type *prev; /* for the linked list */ \
4658   void *callback_arg; \
4659   TCP_PCB_EXTARGS \
4660   enum tcp_state state; /* TCP state */ \
4661   u8_t prio; \
4662   /* ports are in host byte order */ \
4663   u16_t local_port
4664-
4665+
4666+#else /* USE_LIBOS */
4667+#define TCP_PCB_COMMON(type) \
4668+  type *next; /* for the linked list */ \
4669+  void *callback_arg; \
4670+  TCP_PCB_EXTARGS \
4671+  enum tcp_state state; /* TCP state */ \
4672+  u8_t prio; \
4673+  /* ports are in host byte order */ \
4674+  u16_t local_port
4675+#endif /* USE_LIBOS */
4676
4677 /** the TCP protocol control block for listening pcbs */
4678 struct tcp_pcb_listen {
4679@@ -244,6 +261,9 @@ struct tcp_pcb {
4680   IP_PCB;
4681 /** protocol specific PCB members */
4682   TCP_PCB_COMMON(struct tcp_pcb);
4683+#if TCP_PCB_HASH
4684+  struct hlist_node tcp_node;
4685+#endif
4686
4687   /* ports are in host byte order */
4688   u16_t remote_port;
4689@@ -388,6 +408,58 @@ struct tcp_pcb {
4690 #endif
4691 };
4692
4693+#if TCP_PCB_HASH
4694+#define TCP_HTABLE_SIZE MEMP_NUM_NETCONN*12
4695+
4696+struct tcp_hashbucket
4697+{
4698+  sys_mutex_t mutex;
4699+  struct hlist_head chain;
4700+};
4701+
4702+struct tcp_hash_table
4703+{
4704+  u32_t size;
4705+  struct tcp_hashbucket array[TCP_HTABLE_SIZE];
4706+};
4707+
4708+extern PER_THREAD struct tcp_hash_table *tcp_active_htable; /* key: lport/fport/lip/fip */
4709+
4710+#define JHASH_INITVAL		0xdeadbeef
4711+
4712+static inline unsigned int rol32(unsigned int word, unsigned int shift)
4713+{
4714+  return (word << shift) | (word >> (32 - shift));
4715+}
4716+
4717+#define __jhash_final(a, b, c)			\
4718+{						\
4719+  c ^= b; c -= rol32(b, 14);		\
4720+  a ^= c; a -= rol32(c, 11);		\
4721+  b ^= a; b -= rol32(a, 25);		\
4722+  c ^= b; c -= rol32(b, 16);		\
4723+  a ^= c; a -= rol32(c, 4);		\
4724+  b ^= a; b -= rol32(a, 14);		\
4725+  c ^= b; c -= rol32(b, 24);		\
4726+}
4727+
4728+static inline unsigned int jhash_3words(unsigned int a, unsigned int b, unsigned int c)
4729+{
4730+  a += JHASH_INITVAL;
4731+  b += JHASH_INITVAL;;
4732+
4733+  __jhash_final(a, b, c);
4734+
4735+  return c;
4736+}
4737+
4738+#define TUPLE4_HASH_FN(laddr, lport, faddr, fport)	 jhash_3words(laddr, faddr,lport|(fport<<16))
4739+
4740+#define tcppcb_hlist_for_each(tcppcb, node, list) \
4741+  hlist_for_each_entry(tcppcb, node, list, tcp_node)
4742+
4743+#endif /* TCP_PCB_HASH */
4744+
4745 #if LWIP_EVENT_API
4746
4747 enum lwip_event {
4748@@ -481,6 +553,26 @@ err_t            tcp_tcp_get_tcp_addrinfo(struct tcp_pcb *pcb, int local, ip_add
4749
4750 #define tcp_dbg_get_tcp_state(pcb) ((pcb)->state)
4751
4752+enum tcp_list_state {
4753+  ACTIVE_LIST,
4754+  LISTEN_LIST,
4755+  TIME_WAIT_LIST,
4756+};
4757+
4758+struct tcp_pcb_dp {
4759+  uint32_t state;
4760+  uint32_t lip;
4761+  uint32_t rip;
4762+  uint16_t l_port;
4763+  uint16_t r_port;
4764+  uint32_t r_next;
4765+  uint32_t s_next;
4766+  uint32_t tcp_sub_state;
4767+};
4768+
4769+void tcp_get_conn(char *buf, int32_t len, uint32_t *conn_num);
4770+uint32_t tcp_get_conn_num(void);
4771+
4772 /* for compatibility with older implementation */
4773 #define tcp_new_ip6() tcp_new_ip_type(IPADDR_TYPE_V6)
4774
4775diff --git a/src/include/lwip/tcpip.h b/src/include/lwip/tcpip.h
4776index 0b8880a..d2c2440 100644
4777--- a/src/include/lwip/tcpip.h
4778+++ b/src/include/lwip/tcpip.h
4779@@ -51,7 +51,7 @@ extern "C" {
4780
4781 #if LWIP_TCPIP_CORE_LOCKING
4782 /** The global semaphore to lock the stack. */
4783-extern sys_mutex_t lock_tcpip_core;
4784+extern PER_THREAD sys_mutex_t lock_tcpip_core;
4785 #if !defined LOCK_TCPIP_CORE || defined __DOXYGEN__
4786 /** Lock lwIP core mutex (needs @ref LWIP_TCPIP_CORE_LOCKING 1) */
4787 #define LOCK_TCPIP_CORE()     sys_mutex_lock(&lock_tcpip_core)
4788diff --git a/src/include/lwip/timeouts.h b/src/include/lwip/timeouts.h
4789index b601f9e..b451554 100644
4790--- a/src/include/lwip/timeouts.h
4791+++ b/src/include/lwip/timeouts.h
4792@@ -119,6 +119,10 @@ struct sys_timeo** sys_timeouts_get_next_timeout(void);
4793 void lwip_cyclic_timer(void *arg);
4794 #endif
4795
4796+#if USE_LIBOS
4797+void sys_timer_run(void);
4798+#endif /* USE_LIBOS */
4799+
4800 #endif /* LWIP_TIMERS */
4801
4802 #ifdef __cplusplus
4803diff --git a/src/include/lwiplog.h b/src/include/lwiplog.h
4804new file mode 100644
4805index 0000000..363e516
4806--- /dev/null
4807+++ b/src/include/lwiplog.h
4808@@ -0,0 +1,81 @@
4809+/*
4810+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
4811+ * All rights reserved.
4812+ *
4813+ * Redistribution and use in source and binary forms, with or without modification,
4814+ * are permitted provided that the following conditions are met:
4815+ *
4816+ * 1. Redistributions of source code must retain the above copyright notice,
4817+ *    this list of conditions and the following disclaimer.
4818+ * 2. Redistributions in binary form must reproduce the above copyright notice,
4819+ *    this list of conditions and the following disclaimer in the documentation
4820+ *    and/or other materials provided with the distribution.
4821+ * 3. The name of the author may not be used to endorse or promote products
4822+ *    derived from this software without specific prior written permission.
4823+ *
4824+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
4825+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
4826+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
4827+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
4828+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
4829+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
4830+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
4831+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
4832+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
4833+ * OF SUCH DAMAGE.
4834+ *
4835+ * This file is part of the lwIP TCP/IP stack.
4836+ *
4837+ * Author: Huawei Technologies
4838+ *
4839+ */
4840+
4841+#ifndef __LWIPLOG_H__
4842+#define __LWIPLOG_H__
4843+
4844+#include <stdio.h>
4845+#include <sys/syscall.h>
4846+
4847+#include <rte_log.h>
4848+
4849+#include "lwipopts.h"
4850+
4851+#define gettid()    syscall(__NR_gettid)
4852+
4853+#if USE_DPDK_LOG
4854+
4855+#define LWIP_LOG_WARN   LWIP_DBG_LEVEL_WARNING
4856+#define LWIP_LOG_ERROR  LWIP_DBG_LEVEL_SERIOUS
4857+#define LWIP_LOG_FATAL  LWIP_DBG_LEVEL_SEVERE
4858+
4859+#define LWIP_PLATFORM_LOG(level, fmt, ...) \
4860+do { \
4861+    if ((level) & LWIP_LOG_FATAL) { \
4862+        RTE_LOG(ERR, EAL, fmt, ##__VA_ARGS__); \
4863+        abort();                             \
4864+    } else if ((level) & LWIP_LOG_ERROR) { \
4865+        RTE_LOG(ERR, EAL, fmt, ##__VA_ARGS__); \
4866+    } else if ((level) & LWIP_LOG_WARN) { \
4867+        RTE_LOG(WARNING, EAL, fmt, ##__VA_ARGS__); \
4868+    } else { \
4869+        RTE_LOG(INFO, EAL, fmt, ##__VA_ARGS__); \
4870+    } \
4871+} while(0)
4872+
4873+
4874+#define LWIP_PLATFORM_DIAG(x)
4875+
4876+#define ESC_ARGS(...) __VA_ARGS__
4877+#define STRIP_BRACES(args) args
4878+
4879+#define LWIP_PLATFORM_ASSERT(x) \
4880+do { LWIP_PLATFORM_LOG(LWIP_LOG_FATAL, "Assertion \"%s\" failed at line %d in %s\n", \
4881+     x, __LINE__, __FILE__); abort();} while(0)
4882+
4883+#else
4884+
4885+#define LWIP_PLATFORM_LOG(debug, message)
4886+
4887+#endif /* USE_DPDK_LOG */
4888+
4889+#endif /* __LWIPLOG_H__ */
4890diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h
4891index 4ab26f2..8893a5f 100644
4892--- a/src/include/lwipopts.h
4893+++ b/src/include/lwipopts.h
4894@@ -1,8 +1,8 @@
4895 /*
4896- * Copyright (c) 2001-2003 Swedish Institute of Computer Science.
4897- * All rights reserved.
4898- *
4899- * Redistribution and use in source and binary forms, with or without modification,
4900+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
4901+ * All rights reserved.
4902+ *
4903+ * Redistribution and use in source and binary forms, with or without modification,
4904  * are permitted provided that the following conditions are met:
4905  *
4906  * 1. Redistributions of source code must retain the above copyright notice,
4907@@ -11,70 +11,193 @@
4908  *    this list of conditions and the following disclaimer in the documentation
4909  *    and/or other materials provided with the distribution.
4910  * 3. The name of the author may not be used to endorse or promote products
4911- *    derived from this software without specific prior written permission.
4912+ *    derived from this software without specific prior written permission.
4913  *
4914- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
4915- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
4916- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
4917- * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
4918- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
4919- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
4920- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
4921- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
4922- * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
4923+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
4924+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
4925+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
4926+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
4927+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
4928+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
4929+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
4930+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
4931+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
4932  * OF SUCH DAMAGE.
4933  *
4934  * This file is part of the lwIP TCP/IP stack.
4935- *
4936- * Author: Simon Goldschmidt
4937+ *
4938+ * Author: Huawei Technologies
4939  *
4940  */
4941-#ifndef LWIP_HDR_LWIPOPTS_H__
4942-#define LWIP_HDR_LWIPOPTS_H__
4943-
4944-/* Prevent having to link sys_arch.c (we don't test the API layers in unit tests) */
4945-#define NO_SYS                          1
4946-#define LWIP_NETCONN                    0
4947-#define LWIP_SOCKET                     0
4948-#define SYS_LIGHTWEIGHT_PROT            0
4949-
4950-#define LWIP_IPV6                       1
4951-#define IPV6_FRAG_COPYHEADER            1
4952-#define LWIP_IPV6_DUP_DETECT_ATTEMPTS   0
4953-
4954-/* Enable some protocols to test them */
4955-#define LWIP_DHCP                       1
4956-#define LWIP_AUTOIP                     1
4957-
4958-#define LWIP_IGMP                       1
4959-#define LWIP_DNS                        1
4960-
4961-#define LWIP_ALTCP                      1
4962-
4963-/* Turn off checksum verification of fuzzed data */
4964-#define CHECKSUM_CHECK_IP               0
4965-#define CHECKSUM_CHECK_UDP              0
4966-#define CHECKSUM_CHECK_TCP              0
4967-#define CHECKSUM_CHECK_ICMP             0
4968-#define CHECKSUM_CHECK_ICMP6            0
4969-
4970-/* Minimal changes to opt.h required for tcp unit tests: */
4971-#define MEM_SIZE                        16000
4972-#define TCP_SND_QUEUELEN                40
4973-#define MEMP_NUM_TCP_SEG                TCP_SND_QUEUELEN
4974-#define TCP_OVERSIZE                    1
4975-#define TCP_SND_BUF                     (12 * TCP_MSS)
4976-#define TCP_WND                         (10 * TCP_MSS)
4977-#define LWIP_WND_SCALE                  1
4978-#define TCP_RCV_SCALE                   2
4979-#define PBUF_POOL_SIZE                  400 /* pbuf tests need ~200KByte */
4980-
4981-/* Minimal changes to opt.h required for etharp unit tests: */
4982-#define ETHARP_SUPPORT_STATIC_ENTRIES   1
4983-
4984-#define LWIP_NUM_NETIF_CLIENT_DATA      1
4985-#define LWIP_SNMP                       1
4986-#define MIB2_STATS                      1
4987-#define LWIP_MDNS_RESPONDER             1
4988-
4989-#endif /* LWIP_HDR_LWIPOPTS_H__ */
4990+
4991+#ifndef __LWIPOPTS_H__
4992+#define __LWIPOPTS_H__
4993+
4994+#define LWIP_TCPIP_CORE_LOCKING 1
4995+
4996+#define LWIP_NETCONN_SEM_PER_THREAD 0
4997+
4998+#define LWIP_TCP 1
4999+
5000+#define LWIP_SO_SENTIMEO 0
5001+
5002+#define LIP_SO_LINGER 0
5003+
5004+#define MEMP_USE_CUSTOM_POOLS 0
5005+#define MEM_USE_POOLS 0
5006+
5007+#define PER_TCP_PCB_BUFFER (16 * 128)
5008+
5009+#define MAX_CLIENTS (20000)
5010+
5011+#define RESERVED_CLIENTS (2000)
5012+
5013+#define MEMP_NUM_TCP_PCB (MAX_CLIENTS + RESERVED_CLIENTS)
5014+
5015+/* we use PBUF_POOL instead of PBUF_RAM in tcp_write, so reduce PBUF_RAM size,
5016+ * and do NOT let PBUF_POOL_BUFSIZE less then TCP_MSS
5017+*/
5018+#define MEM_SIZE (((PER_TCP_PCB_BUFFER + 128) * MEMP_NUM_TCP_SEG) >> 2)
5019+
5020+#define MEMP_NUM_TCP_PCB_LISTEN 3000
5021+
5022+#define MEMP_NUM_TCP_SEG (128 * 128 * 2)
5023+
5024+#define MEMP_NUM_NETCONN (MAX_CLIENTS + RESERVED_CLIENTS)
5025+
5026+#define MEMP_NUM_SYS_SEM (MAX_CLIENTS + RESERVED_CLIENTS)
5027+
5028+#define MEMP_NUM_SYS_MBOX (MAX_CLIENTS + RESERVED_CLIENTS)
5029+
5030+#define PBUF_POOL_SIZE (MAX_CLIENTS * 2)
5031+
5032+#define MEMP_MEM_MALLOC 0
5033+
5034+#define LWIP_ARP 1
5035+
5036+#define ETHARP_SUPPORT_STATIC_ENTRIES 1
5037+
5038+#define LWIP_IPV4 1
5039+
5040+#define IP_FORWARD 0
5041+
5042+#define IP_REASSEMBLY 1
5043+
5044+#define LWIP_UDP 0
5045+
5046+#define LWIP_TCP 1
5047+
5048+#define IP_HLEN 20
5049+
5050+#define TCP_HLEN 20
5051+
5052+#define FRAME_MTU 1500
5053+
5054+#define TCP_MSS (FRAME_MTU - IP_HLEN - TCP_HLEN)
5055+
5056+#define TCP_WND (40 * TCP_MSS)
5057+
5058+#define TCP_SND_BUF (5 * TCP_MSS)
5059+
5060+#define TCP_SND_QUEUELEN (8191)
5061+
5062+#define TCP_SNDLOWAT (TCP_SND_BUF / 5)
5063+
5064+#define TCP_SNDQUEUELOWAT (TCP_SND_QUEUELEN / 5)
5065+
5066+#define TCP_LISTEN_BACKLOG 1
5067+
5068+#define TCP_DEFAULT_LISTEN_BACKLOG 0xff
5069+
5070+#define TCP_OVERSIZE 0
5071+
5072+#define LWIP_NETIF_API 1
5073+
5074+#define DEFAULT_TCP_RECVMBOX_SIZE 128
5075+
5076+#define DEFAULT_ACCEPTMBOX_SIZE 1024
5077+
5078+#define LWIP_NETCONN 1
5079+
5080+#define LWIP_TCPIP_TIMEOUT 0
5081+
5082+#define LWIP_SOCKET 1
5083+
5084+#define LWIP_TCP_KEEPALIVE 1
5085+
5086+#define LWIP_STATS 1
5087+
5088+#define LWIP_STATS_DISPLAY 1
5089+
5090+#define CHECKSUM_GEN_IP 1 /*  master switch */
5091+
5092+#define CHECKSUM_GEN_TCP 1 /* master switch */
5093+
5094+#define CHECKSUM_CHECK_IP 1 /*  master switch */
5095+
5096+#define CHECKSUM_CHECK_TCP 1 /*  master switch */
5097+
5098+#define LWIP_TIMEVAL_PRIVATE 0
5099+
5100+#define USE_LIBOS 1
5101+
5102+#define LWIP_DEBUG 1
5103+
5104+#define LWIP_PERF 1
5105+
5106+#define LWIP_RECORD_PERF 0
5107+
5108+#define LWIP_SOCKET_POLL   0
5109+
5110+#define USE_LIBOS_ZC_RING 0
5111+
5112+#define SO_REUSE 1
5113+
5114+#define SIOCSHIWAT 1
5115+
5116+#define O_NONBLOCK 04000 /* same as define in bits/fcntl-linux.h */
5117+
5118+#define O_NDELAY O_NONBLOCK
5119+
5120+#define FIONBIO 0x5421 /* same as define in asm-generic/ioctls.h */
5121+
5122+#define LWIP_SUPPORT_CUSTOM_PBUF 1
5123+
5124+#define MEM_LIBC_MALLOC 0
5125+
5126+#define LWIP_TIMERS 1
5127+
5128+#define TCPIP_MBOX_SIZE (MEMP_NUM_TCPIP_MSG_API)
5129+
5130+#define TCP_PCB_HASH 1
5131+
5132+#define USE_DPDK_LOG 1
5133+
5134+#define LWIP_EPOOL_WAIT_MAX_EVENTS 30
5135+
5136+#define ARP_TABLE_SIZE 512
5137+
5138+/*
5139+   ---------------------------------------
5140+   ------- Syscall thread options --------
5141+   ---------------------------------------
5142+*/
5143+#define USE_SYSCALL_THREAD 1
5144+
5145+#define MAX_BLOCKING_ACCEPT_FD (100)
5146+
5147+#define MAX_BLOCKING_CONNECT_FD (100)
5148+
5149+#define MAX_BLOCKING_EPOLL_FD (100)
5150+
5151+#define MAX_SYSCALL_EVENTS (MAX_BLOCKING_ACCEPT_FD + MAX_BLOCKING_CONNECT_FD + MAX_BLOCKING_EPOLL_FD)
5152+
5153+#define MAX_HOST_FD (MAX_CLIENTS + RESERVED_CLIENTS)
5154+
5155+#if USE_LIBOS
5156+#define PER_THREAD __thread
5157+#else
5158+#define PER_THREAD
5159+#endif
5160+
5161+#endif /* __LWIPOPTS_H__ */
5162diff --git a/src/include/lwipsock.h b/src/include/lwipsock.h
5163new file mode 100644
5164index 0000000..dbc67b9
5165--- /dev/null
5166+++ b/src/include/lwipsock.h
5167@@ -0,0 +1,155 @@
5168+/*
5169+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
5170+ * All rights reserved.
5171+ *
5172+ * Redistribution and use in source and binary forms, with or without modification,
5173+ * are permitted provided that the following conditions are met:
5174+ *
5175+ * 1. Redistributions of source code must retain the above copyright notice,
5176+ *    this list of conditions and the following disclaimer.
5177+ * 2. Redistributions in binary form must reproduce the above copyright notice,
5178+ *    this list of conditions and the following disclaimer in the documentation
5179+ *    and/or other materials provided with the distribution.
5180+ * 3. The name of the author may not be used to endorse or promote products
5181+ *    derived from this software without specific prior written permission.
5182+ *
5183+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
5184+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
5185+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
5186+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
5187+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
5188+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
5189+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
5190+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
5191+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
5192+ * OF SUCH DAMAGE.
5193+ *
5194+ * This file is part of the lwIP TCP/IP stack.
5195+ *
5196+ * Author: Huawei Technologies
5197+ *
5198+ */
5199+
5200+#ifndef __LWIPSOCK_H__
5201+#define __LWIPSOCK_H__
5202+
5203+#include "lwip/opt.h"
5204+#include "lwip/api.h"
5205+
5206+#include "posix_api.h"
5207+#include "eventpoll.h"
5208+
5209+/* move some definitions to the lwipsock.h for libnet to use, and
5210+ * at the same time avoid conflict between lwip/sockets.h and sys/socket.h
5211+ */
5212+
5213+/* --------------------------------------------------
5214+ * the following definition is copied from lwip/priv/tcpip_priv.h
5215+ * --------------------------------------------------
5216+ */
5217+
5218+/** This is overridable for the rare case where more than 255 threads
5219+ * select on the same socket...
5220+ */
5221+#ifndef SELWAIT_T
5222+#define SELWAIT_T u8_t
5223+#endif
5224+
5225+union lwip_sock_lastdata {
5226+  struct netbuf *netbuf;
5227+  struct pbuf *pbuf;
5228+};
5229+
5230+/** Contains all internal pointers and states used for a socket */
5231+struct lwip_sock {
5232+  /** sockets currently are built on netconns, each socket has one netconn */
5233+  struct netconn *conn;
5234+  /** data that was left from the previous read */
5235+  union lwip_sock_lastdata lastdata;
5236+#if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
5237+  /** number of times data was received, set by event_callback(),
5238+      tested by the receive and select functions */
5239+  s16_t rcvevent;
5240+  /** number of times data was ACKed (free send buffer), set by event_callback(),
5241+      tested by select */
5242+  u16_t sendevent;
5243+  /** error happened for this socket, set by event_callback(), tested by select */
5244+  u16_t errevent;
5245+  /** counter of how many threads are waiting for this socket using select */
5246+  SELWAIT_T select_waiting;
5247+#endif /* LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL */
5248+#if LWIP_NETCONN_FULLDUPLEX
5249+  /* counter of how many threads are using a struct lwip_sock (not the 'int') */
5250+  u8_t fd_used;
5251+  /* status of pending close/delete actions */
5252+  u8_t fd_free_pending;
5253+#define LWIP_SOCK_FD_FREE_TCP  1
5254+#define LWIP_SOCK_FD_FREE_FREE 2
5255+#endif
5256+
5257+#if USE_LIBOS
5258+  struct list_node list;
5259+  /* registered events */
5260+  uint32_t epoll;
5261+  /* available events */
5262+  uint32_t events;
5263+  epoll_data_t ep_data;
5264+  /* libos_epoll pointer in use */
5265+  struct libos_epoll *epoll_data;
5266+#endif
5267+};
5268+
5269+#ifndef set_errno
5270+#define set_errno(err) do { if (err) { errno = (err); } } while(0)
5271+#endif
5272+
5273+
5274+/* --------------------------------------------------
5275+ * --------------- LIBNET references ----------------
5276+ * --------------------------------------------------
5277+ */
5278+#if USE_LIBOS
5279+extern uint32_t sockets_num;
5280+extern struct lwip_sock *sockets;
5281+/**
5282+ * Map a externally used socket index to the internal socket representation.
5283+ *
5284+ * @param s externally used socket index
5285+ * @return struct lwip_sock for the socket or NULL if not found
5286+ */
5287+static inline struct lwip_sock *
5288+get_socket_without_errno(int s)
5289+{
5290+  struct lwip_sock *sock = NULL;
5291+
5292+  s -= LWIP_SOCKET_OFFSET;
5293+
5294+  if ((s < 0) || (s >= sockets_num)) {
5295+    LWIP_DEBUGF(SOCKETS_DEBUG, ("get_socket(%d): invalid\n", s + LWIP_SOCKET_OFFSET));
5296+    return NULL;
5297+  }
5298+
5299+  sock = &sockets[s];
5300+
5301+  if (!sock->conn) {
5302+    LWIP_DEBUGF(SOCKETS_DEBUG, ("get_socket(%d): not active\n", s + LWIP_SOCKET_OFFSET));
5303+    return NULL;
5304+  }
5305+
5306+  return sock;
5307+}
5308+#endif /* USE_LIBOS */
5309+
5310+struct lwip_sock *get_socket(int s);
5311+struct lwip_sock *get_socket_by_fd(int s);
5312+void lwip_sock_init(void);
5313+void lwip_exit(void);
5314+
5315+extern int is_host_ipv4(uint32_t ipv4);
5316+extern int rearm_host_fd(int fd);
5317+extern int rearm_accept_fd(int fd);
5318+extern void unarm_host_fd(int fd);
5319+extern void clean_host_fd(int fd);
5320+extern int arm_host_fd(struct libos_epoll *ep, int op, int fd, struct epoll_event *event);
5321+
5322+#endif /* __LWIPSOCK_H__ */
5323diff --git a/src/include/memp_def.h b/src/include/memp_def.h
5324new file mode 100644
5325index 0000000..082f685
5326--- /dev/null
5327+++ b/src/include/memp_def.h
5328@@ -0,0 +1,66 @@
5329+/*
5330+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
5331+ * All rights reserved.
5332+ *
5333+ * Redistribution and use in source and binary forms, with or without modification,
5334+ * are permitted provided that the following conditions are met:
5335+ *
5336+ * 1. Redistributions of source code must retain the above copyright notice,
5337+ *    this list of conditions and the following disclaimer.
5338+ * 2. Redistributions in binary form must reproduce the above copyright notice,
5339+ *    this list of conditions and the following disclaimer in the documentation
5340+ *    and/or other materials provided with the distribution.
5341+ * 3. The name of the author may not be used to endorse or promote products
5342+ *    derived from this software without specific prior written permission.
5343+ *
5344+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
5345+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
5346+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
5347+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
5348+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
5349+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
5350+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
5351+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
5352+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
5353+ * OF SUCH DAMAGE.
5354+ *
5355+ * This file is part of the lwIP TCP/IP stack.
5356+ *
5357+ * Author: Huawei Technologies
5358+ *
5359+ */
5360+
5361+#ifndef __MEMP_DEF_H__
5362+#define __MEMP_DEF_H__
5363+
5364+#include "lwip/opt.h"
5365+#include "arch/cc.h"
5366+
5367+#define LWIP_MEMPOOL_BASE_DECLARE(name) \
5368+    extern void alloc_memp_##name##_base(void);
5369+
5370+#define LWIP_MEM_MEMORY_DECLARE(name) \
5371+    extern void alloc_memory_##name(void);
5372+
5373+#define LWIP_MEMPOOL_BASE_INIT(name) \
5374+    alloc_memp_##name##_base();
5375+
5376+#define LWIP_MEM_MEMORY_INIT(name) \
5377+    alloc_memory_##name();
5378+
5379+#define LWIP_MEMPOOL(name, num, size, desc) LWIP_MEMPOOL_BASE_DECLARE(name)
5380+#include <lwip/priv/memp_std.h>
5381+#undef LWIP_MEMPOOL
5382+
5383+static inline void hugepage_init(void)
5384+{
5385+#define LWIP_MEMPOOL(name,num,size,desc)  LWIP_MEMPOOL_BASE_INIT(name)
5386+#include "lwip/priv/memp_std.h"
5387+
5388+#if !MEM_LIBC_MALLOC
5389+	LWIP_MEM_MEMORY_DECLARE(ram_heap)
5390+    LWIP_MEM_MEMORY_INIT(ram_heap)
5391+#endif /* MEM_LIBC_MALLOC */
5392+}
5393+
5394+#endif /* __MEMP_DEF_H__ */
5395diff --git a/src/include/posix_api.h b/src/include/posix_api.h
5396new file mode 100644
5397index 0000000..8aa8516
5398--- /dev/null
5399+++ b/src/include/posix_api.h
5400@@ -0,0 +1,88 @@
5401+/*
5402+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
5403+ * All rights reserved.
5404+ *
5405+ * Redistribution and use in source and binary forms, with or without modification,
5406+ * are permitted provided that the following conditions are met:
5407+ *
5408+ * 1. Redistributions of source code must retain the above copyright notice,
5409+ *    this list of conditions and the following disclaimer.
5410+ * 2. Redistributions in binary form must reproduce the above copyright notice,
5411+ *    this list of conditions and the following disclaimer in the documentation
5412+ *    and/or other materials provided with the distribution.
5413+ * 3. The name of the author may not be used to endorse or promote products
5414+ *    derived from this software without specific prior written permission.
5415+ *
5416+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
5417+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
5418+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
5419+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
5420+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
5421+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
5422+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
5423+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
5424+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
5425+ * OF SUCH DAMAGE.
5426+ *
5427+ * This file is part of the lwIP TCP/IP stack.
5428+ *
5429+ * Author: Huawei Technologies
5430+ *
5431+ */
5432+
5433+#ifndef __POSIX_API_H__
5434+#define __POSIX_API_H__
5435+
5436+#include <signal.h>
5437+#include <poll.h>
5438+#include <sys/epoll.h>
5439+#include <sys/eventfd.h>
5440+
5441+typedef struct {
5442+    void *handle;
5443+    int (*socket_fn)(int domain, int type, int protocol);
5444+    int (*accept_fn)(int s, struct sockaddr*, socklen_t*);
5445+    int (*accept4_fn)(int s, struct sockaddr *addr, socklen_t *addrlen, int flags);
5446+    int (*bind_fn)(int s, const struct sockaddr*, socklen_t);
5447+    int (*listen_fn)(int s, int backlog);
5448+    int (*connect_fn)(int s, const struct sockaddr *name, socklen_t namelen);
5449+    int (*getpeername_fn)(int s, struct sockaddr *name, socklen_t *namelen);
5450+    int (*getsockname_fn)(int s, struct sockaddr *name, socklen_t *namelen);
5451+    int (*setsockopt_fn)(int s, int level, int optname, const void *optval, socklen_t optlen);
5452+    int (*getsockopt_fn)(int s, int level, int optname, void *optval, socklen_t *optlen);
5453+    int (*shutdown_fn)(int s, int how);
5454+    int (*close_fn)(int fd);
5455+    pid_t (*fork_fn)(void);
5456+    ssize_t (*read_fn)(int fd, void *mem, size_t len);
5457+    ssize_t (*write_fn)(int fd, const void *data, size_t len);
5458+    ssize_t (*recv_fn)(int sockfd, void *buf, size_t len, int flags);
5459+    ssize_t (*send_fn)(int sockfd, const void *buf, size_t len, int flags);
5460+    ssize_t (*recv_msg)(int sockfd, const struct msghdr *msg, int flags);
5461+    ssize_t (*send_msg)(int sockfd, const struct msghdr *msg, int flags);
5462+    ssize_t (*recv_from)(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen);
5463+    ssize_t (*send_to)(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
5464+        socklen_t addrlen);
5465+    int (*fcntl_fn)(int fd, int cmd, ...);
5466+    int (*fcntl64_fn)(int fd, int cmd, ...);
5467+    int (*pipe_fn)(int pipefd[2]);
5468+    int (*epoll_create_fn)(int size);
5469+    int (*epoll_ctl_fn)(int epfd, int op, int fd, struct epoll_event *event);
5470+    int (*epoll_wait_fn)(int epfd, struct epoll_event *events, int maxevents, int timeout);
5471+    int (*epoll_close_fn)(int epfd);
5472+    int (*eventfd_fn)(unsigned int initval, int flags);
5473+    int (*is_epfd)(int fd);
5474+    struct lwip_sock* (*get_socket)(int fd);
5475+    int (*sigaction_fn)(int signum, const struct sigaction *act, struct sigaction *oldact);
5476+    int (*poll_fn)(struct pollfd *fds, nfds_t nfds, int timeout);
5477+    int (*ioctl_fn)(int fd, int cmd, ...);
5478+
5479+    int is_chld;
5480+} posix_api_t;
5481+
5482+posix_api_t *posix_api;
5483+
5484+int posix_api_init(void);
5485+void posix_api_free(void);
5486+void posix_api_fork(void);
5487+
5488+#endif /* __POSIX_API_H__ */
5489diff --git a/src/include/reg_sock.h b/src/include/reg_sock.h
5490new file mode 100644
5491index 0000000..76d4c48
5492--- /dev/null
5493+++ b/src/include/reg_sock.h
5494@@ -0,0 +1,62 @@
5495+/*
5496+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
5497+ * All rights reserved.
5498+ *
5499+ * Redistribution and use in source and binary forms, with or without modification,
5500+ * are permitted provided that the following conditions are met:
5501+ *
5502+ * 1. Redistributions of source code must retain the above copyright notice,
5503+ *    this list of conditions and the following disclaimer.
5504+ * 2. Redistributions in binary form must reproduce the above copyright notice,
5505+ *    this list of conditions and the following disclaimer in the documentation
5506+ *    and/or other materials provided with the distribution.
5507+ * 3. The name of the author may not be used to endorse or promote products
5508+ *    derived from this software without specific prior written permission.
5509+ *
5510+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
5511+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
5512+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
5513+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
5514+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
5515+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
5516+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
5517+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
5518+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
5519+ * OF SUCH DAMAGE.
5520+ *
5521+ * This file is part of the lwIP TCP/IP stack.
5522+ *
5523+ * Author: Huawei Technologies
5524+ *
5525+ */
5526+
5527+#ifndef __REG_SOCK_H__
5528+#define __REG_SOCK_H__
5529+
5530+enum reg_ring_type {
5531+    REG_RING_TCP_LISTEN = 0,
5532+    REG_RING_TCP_LISTEN_CLOSE,
5533+    REG_RING_TCP_CONNECT,
5534+    REG_RING_TCP_CONNECT_CLOSE,
5535+    RING_REG_MAX,
5536+};
5537+
5538+struct libnet_quintuple {
5539+	uint32_t protocol;
5540+    /* net byte order */
5541+    uint16_t src_port;
5542+    uint16_t dst_port;
5543+    uint32_t src_ip;
5544+    uint32_t dst_ip;
5545+};
5546+
5547+struct reg_ring_msg {
5548+    enum reg_ring_type type;
5549+
5550+    uint32_t tid;
5551+    struct libnet_quintuple qtuple;
5552+};
5553+
5554+extern int vdev_reg_xmit(enum reg_ring_type type, struct libnet_quintuple *qtuple);
5555+
5556+#endif /* __REG_SOCK_H__ */
5557\ No newline at end of file
5558diff --git a/src/netif/dir.mk b/src/netif/dir.mk
5559index 233c79a..f585d5e 100644
5560--- a/src/netif/dir.mk
5561+++ b/src/netif/dir.mk
5562@@ -1,3 +1,3 @@
5563-SRC = ethernet.c
5564+SRC = ethernet.c
5565
5566 $(eval $(call register_dir, netif, $(SRC)))
5567--
55681.8.3.1
5569
5570