From 1f0f3742019e2fa62ba1669c5a880fb63a3fee12 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng12@huawei.com>
Date: Thu, 24 Feb 2022 20:08:46 +0800
Subject: [PATCH] lstack support mysql mode

---
 src/api/api_msg.c                |  26 +--
 src/api/posix_api.c              |   5 +-
 src/api/sockets.c                | 350 ++-----------------------------
 src/api/sys_arch.c               |  12 +-
 src/core/tcp_out.c               |  13 ++
 src/include/eventpoll.h          |   6 +-
 src/include/lwip/priv/tcp_priv.h |   2 +-
 src/include/lwip/sockets.h       |   2 +-
 src/include/lwipsock.h           |  29 ++-
 src/include/posix_api.h          |   2 +-
 src/include/reg_sock.h           |   8 +-
 11 files changed, 85 insertions(+), 370 deletions(-)

diff --git a/src/api/api_msg.c b/src/api/api_msg.c
index d8d6339..0f3a5b6 100644
--- a/src/api/api_msg.c
+++ b/src/api/api_msg.c
@@ -345,6 +345,12 @@ recv_tcp(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err)
 #endif /* LWIP_SO_RCVBUF */
     /* Register event with callback */
     API_EVENT(conn, NETCONN_EVT_RCVPLUS, len);
+#if USE_LIBOS
+  if (conn->state == NETCONN_WRITE || conn->state == NETCONN_CLOSE ||
+      conn->state == NETCONN_CONNECT) {
+    add_recv_list(conn->socket);
+  }
+#endif
   }
 
   return ERR_OK;
@@ -460,14 +466,6 @@ err_tcp(void *arg, err_t err)
   old_state = conn->state;
   conn->state = NETCONN_NONE;
 
-#if USE_LIBOS
-  if (CONN_TYPE_IS_HOST(conn)) {
-    LWIP_DEBUGF(API_MSG_DEBUG, 
-        ("linux localhost connection already success, ignore lwip err_tcp fd=%d\n", conn->socket));
-    return;
-  }
-#endif /* USE_LIBOS */
-
   SYS_ARCH_UNPROTECT(lev);
 
   /* Notify the user layer about a connection error. Used to signal select. */
@@ -482,6 +480,12 @@ err_tcp(void *arg, err_t err)
   if (NETCONN_MBOX_VALID(conn, &conn->recvmbox)) {
     /* use trypost to prevent deadlock */
     sys_mbox_trypost(&conn->recvmbox, mbox_msg);
+#if USE_LIBOS
+    if ((old_state == NETCONN_WRITE) || (old_state == NETCONN_CLOSE) ||
+        (old_state == NETCONN_CONNECT)) {
+      add_recv_list(conn->socket);
+    }
+#endif
   }
   /* pass error message to acceptmbox to wake up pending accept */
   if (NETCONN_MBOX_VALID(conn, &conn->acceptmbox)) {
@@ -1345,11 +1349,7 @@ lwip_netconn_do_connected(void *arg, struct tcp_pcb *pcb, err_t err)
     int s = conn->socket;
     struct lwip_sock *sock = get_socket_without_errno(s);
   
-    if (!!sock && !!sock->epoll_data) {
-      struct epoll_event ee = {0};
-      ee.data.fd = s;
-      ee.events |= EPOLLIN | EPOLLOUT | EPOLLERR;
-      posix_api->epoll_ctl_fn(sock->epoll_data->fd, EPOLL_CTL_DEL, s, &ee);
+    if (!!sock) {
       posix_api->shutdown_fn(s, SHUT_RDWR);
       LWIP_DEBUGF(API_MSG_DEBUG,
           ("linux outgoing connection abort fd=%d\n", s));
diff --git a/src/api/posix_api.c b/src/api/posix_api.c
index a917cea..eff9f46 100644
--- a/src/api/posix_api.c
+++ b/src/api/posix_api.c
@@ -143,11 +143,10 @@ int posix_api_init(void)
 
     /* lstack helper api */
     posix_api->get_socket = get_socket;
-    posix_api->is_epfd = lwip_is_epfd;
-    posix_api->epoll_close_fn = lwip_epoll_close;
+    posix_api->epoll_close_fn = lstack_epoll_close;
 
     /* support fork */
-    posix_api->is_chld = 0;
+    posix_api->is_chld = 1;
     return ERR_OK;
 
 err_out:
diff --git a/src/api/sockets.c b/src/api/sockets.c
index 31dd51c..fe19d7d 100644
--- a/src/api/sockets.c
+++ b/src/api/sockets.c
@@ -69,14 +69,6 @@
 
 #include <string.h>
 
-#if USE_LIBOS
-enum KERNEL_LWIP_PATH {
-  PATH_KERNEL = 0,
-  PATH_LWIP,
-  PATH_ERR,
-};
-#endif
-
 #ifdef LWIP_HOOK_FILENAME
 #include LWIP_HOOK_FILENAME
 #endif
@@ -644,8 +636,6 @@ alloc_socket(struct netconn *newconn, int accepted)
      * (unless it has been created by accept()). */
     sockets[i].sendevent  = (NETCONNTYPE_GROUP(newconn->type) == NETCONN_TCP ? (accepted != 0) : 1);
     sockets[i].errevent   = 0;
-    sockets[i].epoll_data = NULL;
-    init_list_node_null(&sockets[i].list);
     return i + LWIP_SOCKET_OFFSET;
   }
 
@@ -754,13 +744,6 @@ free_socket(struct lwip_sock *sock, int is_tcp)
   /* Protect socket array */
   SYS_ARCH_PROTECT(lev);
 
-#if USE_LIBOS
-  sock->epoll = LIBOS_EPOLLNONE;
-  sock->events = 0;
-  sock->epoll_data = NULL;
-  list_del_node_null(&sock->list);
-#endif
-
   freed = free_socket_locked(sock, is_tcp, &conn, &lastdata);
   SYS_ARCH_UNPROTECT(lev);
   /* don't use 'sock' after this line, as another task might have allocated it */
@@ -789,34 +772,10 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
   SYS_ARCH_DECL_PROTECT(lev);
 
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_accept(%d)...\n", s));
-#if USE_LIBOS
-  int sys_errno = 0;
-  
-  sock = posix_api->get_socket(s);
-  /*AF_UNIX case*/
-  if (!sock) {
-    return posix_api->accept_fn(s, addr, addrlen);
-  }
-  
-  /*for AF_INET, we may try both linux and lwip*/
-  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
-    set_errno(EINVAL);
-    return -1;
-  }
-  
-  /* raise accept syscall in palce */
-  newsock = posix_api->accept_fn(s, addr, addrlen);
-  if (newsock >= 0) {
-    return newsock;
-  }
-  sys_errno = errno;
-#else
   sock = get_socket(s);
   if (!sock) {
     return -1;
   }
-#endif
 
   /* wait for a new connection */
   err = netconn_accept(sock->conn, &newconn);
@@ -830,9 +789,6 @@ lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
       set_errno(err_to_errno(err));
     }
     done_socket(sock);
-#if USE_LIBOS
-    set_errno(sys_errno);
-#endif /* USE_LIBOS */
     return -1;
   }
   LWIP_ASSERT("newconn != NULL", newconn != NULL);
@@ -915,24 +871,10 @@ lwip_bind(int s, const struct sockaddr *name, socklen_t namelen)
   ip_addr_t local_addr;
   u16_t local_port;
   err_t err;
-#if USE_LIBOS
-  sock = posix_api->get_socket(s);
-  /*AF_UNIX case*/
-  if (!sock) {
-    return posix_api->bind_fn(s, name, namelen);
-  }
-  /*for AF_INET, we may try both linux and lwip*/
-  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
-    set_errno(EINVAL);
-    return -1;
-  }
-#else
   sock = get_socket(s);
   if (!sock) {
     return -1;
   }
-#endif
 
   if (!SOCK_ADDR_TYPE_MATCH(name, sock)) {
     /* sockaddr does not match socket type (IPv4/IPv6) */
@@ -952,18 +894,6 @@ lwip_bind(int s, const struct sockaddr *name, socklen_t namelen)
   ip_addr_debug_print_val(SOCKETS_DEBUG, local_addr);
   LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F")\n", local_port));
 
-#if USE_LIBOS
-  /* Supports kernel NIC IP address. */
-  int ret = posix_api->bind_fn(s, name, namelen);
-  if (ret < 0) {
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("bind syscall failed\n"));
-    /* bind must succeed on both linux and libos */
-    if (!is_host_ipv4(local_addr.addr)) {
-      return ret;
-    }
-  }
-#endif /* USE_LIBOS */
-
 #if LWIP_IPV4 && LWIP_IPV6
   /* Dual-stack: Unmap IPv4 mapped IPv6 addresses */
   if (IP_IS_V6_VAL(local_addr) && ip6_addr_isipv4mappedipv6(ip_2_ip6(&local_addr))) {
@@ -993,32 +923,12 @@ lwip_close(int s)
   struct lwip_sock *sock;
   int is_tcp = 0;
   err_t err;
-  int ret = 0;
 
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_close(%d)\n", s));
-
-#if USE_LIBOS
-  if (posix_api->is_epfd(s)) {
-    return posix_api->epoll_close_fn(s);
-  }
-
-  /* No matter what the result of close, lwip_sock resources should release
-   * to prevent the potential double freee problem caused by reporting events after the close */
-  ret = posix_api->close_fn(s);
-  if ((ret < 0) && (errno == EINTR))
-    ret = posix_api->close_fn(s);
-
-  sock = posix_api->get_socket(s);
-  /*AF_UNIX case*/
-  if (!sock) {
-    return ret;
-  }
-#else
   sock = get_socket(s);
   if (!sock) {
     return -1;
   }
-#endif /* USE_LIBOS */
 
   if (sock->conn != NULL) {
     is_tcp = NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP;
@@ -1044,7 +954,7 @@ lwip_close(int s)
 
   free_socket(sock, is_tcp);
   set_errno(0);
-  return ret;
+  return 0;
 }
 
 int
@@ -1053,28 +963,10 @@ lwip_connect(int s, const struct sockaddr *name, socklen_t namelen)
   struct lwip_sock *sock;
   err_t err;
 
-#if USE_LIBOS
-  int ret;
-  
-  sock = posix_api->get_socket(s);
-  if (!sock) {
-    return posix_api->connect_fn(s, name, namelen);
-  }
-  
-  /* raise connect syscall in place */
-  ADD_CONN_TYPE_INPRG(sock->conn);
-  ret = posix_api->connect_fn(s, name, namelen);
-  if (!ret) {
-    SET_CONN_TYPE_HOST(sock->conn);
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("linux connect succeed fd=%d\n", s));
-    return ret;
-  }
-#else
   sock = get_socket(s);
   if (!sock) {
     return -1;
   }
-#endif
 
   if (!SOCK_ADDR_TYPE_MATCH_OR_UNSPEC(name, sock)) {
     /* sockaddr does not match socket type (IPv4/IPv6) */
@@ -1146,29 +1038,10 @@ lwip_listen(int s, int backlog)
 
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_listen(%d, backlog=%d)\n", s, backlog));
 
-#if USE_LIBOS
-  int ret;
-  
-  sock = posix_api->get_socket(s);
-  /*AF_UNIX case*/
-  if (!sock) {
-    return posix_api->listen_fn(s, backlog);
-  }
-  /*for AF_INET, we may try both linux and lwip*/
-  if (!CONN_TYPE_HAS_LIBOS_AND_HOST(sock->conn)) {
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type has libos and host bits"));
-    set_errno(EADDRINUSE);
-    return -1;
-  }
-  
-  if ((ret = posix_api->listen_fn(s, backlog)) == -1)
-    return ret;
-#else
   sock = get_socket(s);
   if (!sock) {
     return -1;
   }
-#endif
 
   /* limit the "backlog" parameter to fit in an u8_t */
   backlog = LWIP_MIN(LWIP_MAX(backlog, 0), 0xff);
@@ -1200,11 +1073,12 @@ static ssize_t
 lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
 {
   u8_t apiflags = NETCONN_NOAUTORCVD;
+  ssize_t recvd = 0;
 #if USE_LIBOS
   apiflags = 0;
-#endif
-  ssize_t recvd = 0;
+#else
   ssize_t recv_left = (len <= SSIZE_MAX) ? (ssize_t)len : SSIZE_MAX;
+#endif
 
   LWIP_ASSERT("no socket given", sock != NULL);
   LWIP_ASSERT("this should be checked internally", NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP);
@@ -1213,6 +1087,7 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
     apiflags |= NETCONN_DONTBLOCK;
   }
 
+#if !USE_LIBOS
   do {
     struct pbuf *p;
     err_t err;
@@ -1222,13 +1097,6 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
     /* Check if there is data left from the last recv operation. */
     if (sock->lastdata.pbuf) {
       p = sock->lastdata.pbuf;
-#if USE_LIBOS
-      if (((flags & MSG_PEEK) == 0) && ((sock->epoll & EPOLLET) == 0)) {
-        if ((NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP)) {
-          del_epoll_event(sock->conn, EPOLLIN);
-        }
-      }
-#endif
     } else {
       /* No data was left from the previous operation, so we try to get
          some from the network. */
@@ -1299,24 +1167,18 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
     /* @todo: do we need to support peeking more than one pbuf? */
   } while ((recv_left > 0) && !(flags & MSG_PEEK));
 lwip_recv_tcp_done:
-#if USE_LIBOS
-  if (apiflags & NETCONN_NOAUTORCVD)
-#endif
-  {
+#else /* USE_LIBOS */
+  recvd = read_lwip_data(sock, flags, apiflags);
+  if (recvd <= 0) {
+    return recvd;
+  }
+#endif /* USE_LIBOS */
+  if (apiflags & NETCONN_NOAUTORCVD) {
     if ((recvd > 0) && !(flags & MSG_PEEK)) {
       /* ensure window update after copying all data */
       netconn_tcp_recvd(sock->conn, (size_t)recvd);
     }
   }
-#if USE_LIBOS
-  if ((flags & MSG_PEEK) == 0) {
-    if (((NETCONNTYPE_GROUP(netconn_type(sock->conn)) == NETCONN_TCP)) && sock->lastdata.pbuf) {
-      add_epoll_event(sock->conn, EPOLLIN);
-    }
-  }
-#else
-  *fromlen = LWIP_MIN(*fromlen, sizeof(saddr));
-#endif
   set_errno(0);
   return recvd;
 }
@@ -1501,37 +1363,6 @@ lwip_recvfrom_udp_raw(struct lwip_sock *sock, int flags, struct msghdr *msg, u16
   return ERR_OK;
 }
 
-#if USE_LIBOS
-static inline enum KERNEL_LWIP_PATH select_path(int s)
-{
-  struct lwip_sock *sock;
-  
-  sock = posix_api->get_socket(s);
-  /*AF_UNIX case*/
-  if (!sock) {
-    return PATH_KERNEL;
-  }
-
-  if (CONN_TYPE_HAS_INPRG(sock->conn)) {
-    set_errno(EWOULDBLOCK);
-    return PATH_ERR;
-  }
-
-  /*for AF_INET, we can try erther linux or lwip*/
-  if (CONN_TYPE_IS_HOST(sock->conn)) {
-    return PATH_KERNEL;
-  }
-
-  if (!CONN_TYPE_IS_LIBOS(sock->conn)) {
-    LWIP_DEBUGF(SOCKETS_DEBUG, ("conn->type is not libos bit type=%x", netconn_type(sock->conn)));
-    set_errno(EINVAL);
-    return PATH_ERR;
-  }
-
-  return PATH_LWIP;
-}
-#endif
-
 ssize_t
 lwip_recvfrom(int s, void *mem, size_t len, int flags,
               struct sockaddr *from, socklen_t *fromlen)
@@ -1539,15 +1370,6 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
   struct lwip_sock *sock;
   ssize_t ret;
 
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->recv_from(s, mem, len, flags, from, fromlen);
-  }
-#endif
-
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, %p, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags));
   sock = get_socket(s);
   if (!sock) {
@@ -1597,14 +1419,6 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
 ssize_t
 lwip_read(int s, void *mem, size_t len)
 {
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->read_fn(s, mem, len);
-  }
-#endif
   return lwip_recvfrom(s, mem, len, 0, NULL, NULL);
 }
 
@@ -1638,15 +1452,6 @@ lwip_recvmsg(int s, struct msghdr *message, int flags)
   msg_iovlen_t i;
   ssize_t buflen;
 
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->recv_msg(s, message, flags);
-  }
-#endif
-
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=%p, flags=0x%x)\n", s, (void *)message, flags));
   LWIP_ERROR("lwip_recvmsg: invalid message pointer", message != NULL, return ERR_ARG;);
   LWIP_ERROR("lwip_recvmsg: unsupported flags", (flags & ~(MSG_PEEK|MSG_DONTWAIT)) == 0,
@@ -1791,15 +1596,6 @@ lwip_sendmsg(int s, const struct msghdr *msg, int flags)
 #endif
   err_t err = ERR_OK;
 
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->send_msg(s, msg, flags);
-  }
-#endif
-
   sock = get_socket(s);
   if (!sock) {
     return -1;
@@ -1963,15 +1759,6 @@ lwip_sendto(int s, const void *data, size_t size, int flags,
   u16_t remote_port;
   struct netbuf buf;
 
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->send_to(s, data, size, flags, to, tolen);
-  }
-#endif
-
   sock = get_socket(s);
   if (!sock) {
     return -1;
@@ -2070,11 +1857,6 @@ lwip_socket(int domain, int type, int protocol)
 
   LWIP_UNUSED_ARG(domain); /* @todo: check this */
 
-#if USE_LIBOS
-  if ((domain != AF_INET && domain != AF_UNSPEC) || posix_api->is_chld)
-    return posix_api->socket_fn(domain, type, protocol);
-#endif
-
   /* create a netconn */
   switch (type) {
     case SOCK_RAW:
@@ -2131,14 +1913,6 @@ lwip_socket(int domain, int type, int protocol)
 ssize_t
 lwip_write(int s, const void *data, size_t size)
 {
-#if USE_LIBOS
-  enum KERNEL_LWIP_PATH path = select_path(s);
-  if (path == PATH_ERR) {
-    return -1;
-  } else if (path == PATH_KERNEL) {
-    return posix_api->write_fn(s, data, size);
-  }
-#endif
   return lwip_send(s, data, size, 0);
 }
 
@@ -2924,20 +2698,16 @@ event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len)
         check_waiters = 0;
       }
 #if USE_LIBOS
-      if (sock->epoll & EPOLLET) {
-        list_del_node_null(&sock->list);
+      if (conn->state == NETCONN_LISTEN) {
+        add_epoll_event(conn, EPOLLIN);
+      } else {
+        add_recv_list(conn->socket);
       }
-      add_epoll_event(conn, EPOLLIN);
 #endif
       break;
     case NETCONN_EVT_RCVMINUS:
       sock->rcvevent--;
       check_waiters = 0;
-#if USE_LIBOS
-      if ((sock->epoll & EPOLLET) == 0) {
-        del_epoll_event(conn, EPOLLIN);
-      }
-#endif
       break;
     case NETCONN_EVT_SENDPLUS:
       if (sock->sendevent) {
@@ -2945,27 +2715,16 @@ event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len)
       }
       sock->sendevent = 1;
 #if USE_LIBOS
-      if (sock->epoll & EPOLLET) {
-        list_del_node_null(&sock->list);
-      }
       add_epoll_event(conn, EPOLLOUT);
 #endif
       break;
     case NETCONN_EVT_SENDMINUS:
       sock->sendevent = 0;
       check_waiters = 0;
-#if USE_LIBOS
-      if ((sock->epoll & EPOLLET) == 0) {
-        del_epoll_event(conn, EPOLLOUT);
-      }
-#endif
       break;
     case NETCONN_EVT_ERROR:
       sock->errevent = 1;
 #if USE_LIBOS
-      if (sock->epoll & EPOLLET) {
-        list_del_node_null(&sock->list);
-      }
       add_epoll_event(conn, EPOLLERR);
 #endif
       break;
@@ -3179,41 +2938,12 @@ lwip_getaddrname(int s, struct sockaddr *name, socklen_t *namelen, u8_t local)
 int
 lwip_getpeername(int s, struct sockaddr *name, socklen_t *namelen)
 {
-#if USE_LIBOS
-  struct lwip_sock *sock;
-  
-  sock = posix_api->get_socket(s);
-  if (!sock) {
-    return posix_api->getpeername_fn(s, name, namelen);
-  }
-  /*for AF_INET, if has only host type bit, just call linux api,
-   *if has libos and host type bits, it's a not connected fd, call
-   *linux api and return -1(errno == ENOTCONN) is also ok*/
-  if (CONN_TYPE_HAS_HOST(sock->conn)) {
-    return posix_api->getpeername_fn(s, name, namelen);
-  }
-#endif
-
   return lwip_getaddrname(s, name, namelen, 0);
 }
 
 int
 lwip_getsockname(int s, struct sockaddr *name, socklen_t *namelen)
 {
-#if USE_LIBOS
-  struct lwip_sock *sock;
-  
-  sock = posix_api->get_socket(s);
-  if (!sock) {
-    return posix_api->getsockname_fn(s, name, namelen);
-  }
-  /*for AF_INET, if has only host type bit, just call linux api,
-   *if has libos and host type bits, also call linux api*/
-  if (CONN_TYPE_HAS_HOST(sock->conn)) {
-    return posix_api->getsockname_fn(s, name, namelen);
-  }
-#endif
-
   return lwip_getaddrname(s, name, namelen, 1);
 }
 
@@ -3226,23 +2956,11 @@ lwip_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen)
   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
 #endif /* !LWIP_TCPIP_CORE_LOCKING */
 
-#if USE_LIBOS
-  struct lwip_sock *sock = posix_api->get_socket(s);
-  
-  if (!sock) {
-    return posix_api->getsockopt_fn(s, level, optname, optval, optlen);
-  }
-  /*for AF_INET, we return linux result? */
-  if (CONN_TYPE_HAS_HOST(sock->conn)) {
-    return posix_api->getsockopt_fn(s, level, optname, optval, optlen);
-  }
-#else
   struct lwip_sock *sock = get_socket(s);
 
   if (!sock) {
     return -1;
   }
-#endif /* USE_LIBOS */
 
   if ((NULL == optval) || (NULL == optlen)) {
     set_errno(EFAULT);
@@ -3685,25 +3403,11 @@ lwip_setsockopt(int s, int level, int optname, const void *optval, socklen_t opt
   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
 #endif /* !LWIP_TCPIP_CORE_LOCKING */
 
-#if USE_LIBOS
-  struct lwip_sock *sock = posix_api->get_socket(s);
-
-  if (!sock) {
-    return posix_api->setsockopt_fn(s, level, optname, optval, optlen);
-  }
-  /*for AF_INET, we may try both linux and lwip*/
-  if (CONN_TYPE_HAS_HOST(sock->conn)) {
-    if (posix_api->setsockopt_fn(s, level, optname, optval, optlen) < 0) {
-      return -1;
-    }
-  }
-#else
   struct lwip_sock *sock = get_socket(s);
 
   if (!sock) {
     return -1;
   }
-#endif /* USE_LIBOS */
 
   if (NULL == optval) {
     set_errno(EFAULT);
@@ -4348,26 +4052,6 @@ lwip_ioctl(int s, long cmd, void *argp)
  * the flag O_NONBLOCK is implemented for F_SETFL.
  */
 int
-#if USE_LIBOS
-lwip_fcntl(int s, int cmd, ...)
-{
-  struct lwip_sock *sock = posix_api->get_socket(s);
-  int val, ret = -1;
-  int op_mode = 0;
-  va_list ap;
-
-  va_start(ap, cmd);
-  val = va_arg(ap, int);
-  va_end(ap);
-
-  if (!sock) {
-    return posix_api->fcntl_fn(s, cmd, val);
-  }
-  if (CONN_TYPE_HAS_HOST(sock->conn)) {
-    if ((ret = posix_api->fcntl_fn(s, cmd, val)) == -1)
-      return ret;
-  }
-#else /* USE_LIBOS */
 lwip_fcntl(int s, int cmd, int val)
 {
   struct lwip_sock *sock = get_socket(s);
@@ -4377,7 +4061,6 @@ lwip_fcntl(int s, int cmd, int val)
   if (!sock) {
     return -1;
   }
-#endif /* USE_LIBOS */
 
   switch (cmd) {
     case F_GETFL:
diff --git a/src/api/sys_arch.c b/src/api/sys_arch.c
index 55561b1..9a92143 100644
--- a/src/api/sys_arch.c
+++ b/src/api/sys_arch.c
@@ -76,8 +76,8 @@ struct sys_mem_stats {
 
 static PER_THREAD struct sys_mem_stats hugepage_stats;
 
-static PER_THREAD uint64_t cycles_per_ms __attribute__((aligned(64)));
-static PER_THREAD uint64_t sys_start_ms __attribute__((aligned(64)));
+static uint64_t cycles_per_ms __attribute__((aligned(64)));
+static uint64_t sys_start_ms __attribute__((aligned(64)));
 
 /*
  * Mailbox
@@ -337,8 +337,12 @@ void sys_calibrate_tsc(void)
 #define MS_PER_SEC  1E3
     uint64_t freq = rte_get_tsc_hz();
 
-    cycles_per_ms = (freq + MS_PER_SEC - 1) / MS_PER_SEC;
-    sys_start_ms = rte_rdtsc() / cycles_per_ms;
+    if (cycles_per_ms == 0) {
+        cycles_per_ms = (freq + MS_PER_SEC - 1) / MS_PER_SEC;
+    }
+    if (sys_start_ms == 0) {
+        sys_start_ms = rte_rdtsc() / cycles_per_ms;
+    }
 }
 
 uint32_t sys_now(void)
diff --git a/src/core/tcp_out.c b/src/core/tcp_out.c
index 74b22b0..1164c56 100644
--- a/src/core/tcp_out.c
+++ b/src/core/tcp_out.c
@@ -475,6 +475,7 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags)
    * pos records progress as data is segmented.
    */
 
+#if !USE_LIBOS
   /* Find the tail of the unsent queue. */
   if (pcb->unsent != NULL) {
     u16_t space;
@@ -590,6 +591,13 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags)
                 pcb->unsent_oversize == 0);
 #endif /* TCP_OVERSIZE */
   }
+#else /* USE_LIBOS */
+  if (pcb->unsent != NULL) {
+    /* @todo: this could be sped up by keeping last_unsent in the pcb */
+    for (last_unsent = pcb->unsent; last_unsent->next != NULL;
+         last_unsent = last_unsent->next);
+  }
+#endif /* USE_LIBOS */
 
   /*
    * Phase 3: Create new segments.
@@ -607,6 +615,7 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags)
     u8_t chksum_swapped = 0;
 #endif /* TCP_CHECKSUM_ON_COPY */
 
+#if !USE_LIBOS
     if (apiflags & TCP_WRITE_FLAG_COPY) {
       /* If copy is set, memory should be allocated and data copied
        * into pbuf */
@@ -653,6 +662,10 @@ tcp_write(struct tcp_pcb *pcb, const void *arg, u16_t len, u8_t apiflags)
       /* Concatenate the headers and data pbufs together. */
       pbuf_cat(p/*header*/, p2/*data*/);
     }
+#else  /* USE_LIBOS */
+    p = (struct pbuf *)arg;
+    seglen = p->len;
+#endif  /* USE_LIBOS */
 
     queuelen += pbuf_clen(p);
 
diff --git a/src/include/eventpoll.h b/src/include/eventpoll.h
index f525bc2..aacc1d2 100644
--- a/src/include/eventpoll.h
+++ b/src/include/eventpoll.h
@@ -63,9 +63,7 @@ struct libos_epoll {
     int efd; /* eventfd */
 };
 
-extern int add_epoll_event(struct netconn*, uint32_t);
-extern int del_epoll_event(struct netconn*, uint32_t);
-extern int lwip_epoll_close(int);
-extern int lwip_is_epfd(int);
+extern void add_epoll_event(struct netconn*, uint32_t);
+extern int32_t lstack_epoll_close(int32_t);
 
 #endif /* __EVENTPOLL_H__ */
diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h
index f666f8f..783acf5 100644
--- a/src/include/lwip/priv/tcp_priv.h
+++ b/src/include/lwip/priv/tcp_priv.h
@@ -346,7 +346,7 @@ static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pc
 {
   LWIP_ASSERT("Invalid parameter", pcb != NULL);
 
-  struct libnet_quintuple qtuple;
+  struct gazelle_quintuple qtuple;
   qtuple.protocol   = 0;
   qtuple.src_ip     = pcb->local_ip.addr;
   qtuple.src_port   = lwip_htons(pcb->local_port);
diff --git a/src/include/lwip/sockets.h b/src/include/lwip/sockets.h
index c8f0fab..1d63e8f 100644
--- a/src/include/lwip/sockets.h
+++ b/src/include/lwip/sockets.h
@@ -666,7 +666,7 @@ int lwip_poll(struct pollfd *fds, nfds_t nfds, int timeout);
 
 #if USE_LIBOS
 int lwip_ioctl(int s, long cmd, ...);
-int lwip_fcntl(int s, int cmd, ...);
+int lwip_fcntl(int s, int cmd, int val);
 #else
 int lwip_ioctl(int s, long cmd, void *argp);
 int lwip_fcntl(int s, int cmd, int val);
diff --git a/src/include/lwipsock.h b/src/include/lwipsock.h
index e9ffbb1..069cdcb 100644
--- a/src/include/lwipsock.h
+++ b/src/include/lwipsock.h
@@ -60,6 +60,10 @@ union lwip_sock_lastdata {
   struct pbuf *pbuf;
 };
 
+#if USE_LIBOS
+struct protocol_stack;
+struct weakup_poll;
+#endif
 /** Contains all internal pointers and states used for a socket */
 struct lwip_sock {
   /** sockets currently are built on netconns, each socket has one netconn */
@@ -88,14 +92,19 @@ struct lwip_sock {
 #endif
 
 #if USE_LIBOS
-  struct list_node list;
-  /* registered events */
-  uint32_t epoll;
-  /* available events */
-  uint32_t events;
+  uint32_t epoll_events; /* registered events */
+  uint32_t events; /* available events */
+  int32_t in_event; /* avoid recurring events */
   epoll_data_t ep_data;
-  /* libos_epoll pointer in use */
-  struct libos_epoll *epoll_data;
+  struct weakup_poll *weakup;
+  struct protocol_stack *stack;
+  void *recv_ring;
+  struct pbuf *recv_lastdata; /* unread data in one pbuf */
+  struct pbuf *send_lastdata; /* unread data in one pbuf */
+  void *send_ring;
+  int32_t recv_flags;
+  int32_t nextfd; /* listenfd list */
+  struct list_node recv_list;
 #endif
 };
 
@@ -138,6 +147,10 @@ get_socket_without_errno(int s)
 
   return sock;
 }
+
+extern void add_recv_list(int32_t fd);
+extern ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags);
+extern void gazelle_clean_sock(int32_t fd);
 #endif /* USE_LIBOS */
 
 struct lwip_sock *get_socket(int s);
@@ -145,6 +158,4 @@ struct lwip_sock *get_socket_by_fd(int s);
 void lwip_sock_init(void);
 void lwip_exit(void);
 
-extern int is_host_ipv4(uint32_t ipv4);
-
 #endif /* __LWIPSOCK_H__ */
diff --git a/src/include/posix_api.h b/src/include/posix_api.h
index 0dca8eb..2afd266 100644
--- a/src/include/posix_api.h
+++ b/src/include/posix_api.h
@@ -34,7 +34,7 @@
 #define __POSIX_API_H__
 
 #include <signal.h>
-#include <poll.h>
+#include <sys/poll.h>
 #include <sys/epoll.h>
 #include <sys/eventfd.h>
 
diff --git a/src/include/reg_sock.h b/src/include/reg_sock.h
index 76d4c48..76673da 100644
--- a/src/include/reg_sock.h
+++ b/src/include/reg_sock.h
@@ -41,7 +41,7 @@ enum reg_ring_type {
     RING_REG_MAX,
 };
 
-struct libnet_quintuple {
+struct gazelle_quintuple {
 	uint32_t protocol;
     /* net byte order */
     uint16_t src_port;
@@ -54,9 +54,9 @@ struct reg_ring_msg {
     enum reg_ring_type type;
 
     uint32_t tid;
-    struct libnet_quintuple qtuple;
+    struct gazelle_quintuple qtuple;
 };
 
-extern int vdev_reg_xmit(enum reg_ring_type type, struct libnet_quintuple *qtuple);
+extern int vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple);
 
-#endif /* __REG_SOCK_H__ */
\ No newline at end of file
+#endif /* __REG_SOCK_H__ */
-- 
2.30.0