From 476b225bc50737678223fa29757b0ac08f1a500d Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Sun, 8 Oct 2023 20:59:57 +0800
Subject: [PATCH] wrap: add run-to-completion/wakeup mode api

---
 src/lstack/api/dir.mk                         |   2 +-
 src/lstack/api/lstack_rtc_api.c               |  72 ++++++
 src/lstack/api/lstack_rtw_api.c               | 239 ++++++++++++++++++
 src/lstack/api/lstack_wrap.c                  | 206 +++++++--------
 src/lstack/core/lstack_init.c                 |   3 +
 src/lstack/core/lstack_lwip.c                 |   2 +-
 src/lstack/include/lstack_lwip.h              |   2 +-
 .../lstack_socket.h => lstack_rtc_api.h}      |  59 +++--
 src/lstack/include/lstack_rtw_api.h           |  47 ++++
 src/lstack/include/lstack_wrap.h              |  19 ++
 10 files changed, 514 insertions(+), 137 deletions(-)
 create mode 100644 src/lstack/api/lstack_rtc_api.c
 create mode 100644 src/lstack/api/lstack_rtw_api.c
 rename src/lstack/include/{posix/lstack_socket.h => lstack_rtc_api.h} (53%)
 create mode 100644 src/lstack/include/lstack_rtw_api.h
 create mode 100644 src/lstack/include/lstack_wrap.h

diff --git a/src/lstack/api/dir.mk b/src/lstack/api/dir.mk
index f5370a2..ffbb137 100644
--- a/src/lstack/api/dir.mk
+++ b/src/lstack/api/dir.mk
@@ -8,7 +8,7 @@
 # PURPOSE.
 # See the Mulan PSL v2 for more details.
 
-SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c
+SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c
 
 $(eval $(call register_dir, api, $(SRC)))
 
diff --git a/src/lstack/api/lstack_rtc_api.c b/src/lstack/api/lstack_rtc_api.c
new file mode 100644
index 0000000..b7c6380
--- /dev/null
+++ b/src/lstack/api/lstack_rtc_api.c
@@ -0,0 +1,72 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#include <fcntl.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#include <lwip/posix_api.h>
+#include <lwip/lwipsock.h>
+#include "posix/lstack_epoll.h"
+#include "lstack_log.h"
+#include "lstack_cfg.h"
+#include "lstack_protocol_stack.h"
+#include "lstack_rtc_api.h"
+
+int rtc_poll(struct pollfd *fds, nfds_t nfds, int timeout)
+{
+    return -1;
+}
+
+int rtc_epoll_wait(int epfd, struct epoll_event* events, int maxevents, int timeout)
+{
+    return -1;
+}
+
+int rtc_socket(int domain, int type, int protocol)
+{
+    int ret;
+    
+    /* need call stack thread init function */
+    ret = lwip_socket(domain, type, protocol);
+    struct lwip_sock *sock = get_socket(ret);
+    if (sock != NULL) {
+        sock->stack = get_protocol_stack();
+        sock->epoll_events = 0;
+        sock->events = 0;
+        sock->wakeup = NULL;
+        init_list_node_null(&sock->event_list);
+    }
+    return ret;
+}
+
+int rtc_close(int s)
+{
+    return lwip_close(s);
+}
+
+int rtc_epoll_create(int flags)
+{
+    /* need call stack thread init function */
+    return lstack_epoll_create(flags);
+}
+
+int rtc_epoll_create1(int flags)
+{
+    /* need call stack thread init function */
+    return lstack_epoll_create1(flags);
+}
+
+int rtc_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
+{
+    return lstack_epoll_ctl(epfd, op, fd, event);
+}
diff --git a/src/lstack/api/lstack_rtw_api.c b/src/lstack/api/lstack_rtw_api.c
new file mode 100644
index 0000000..9b8678a
--- /dev/null
+++ b/src/lstack/api/lstack_rtw_api.c
@@ -0,0 +1,239 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#include <fcntl.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <lwip/lwipsock.h>
+
+#include "lstack_thread_rpc.h"
+#include "posix/lstack_epoll.h"
+#include "lstack_protocol_stack.h"
+#include "lstack_cfg.h"
+#include "lstack_lwip.h"
+#include "lstack_rtw_api.h"
+
+int rtw_socket(int domain, int type, int protocol)
+{
+    return rpc_call_socket(domain, type, protocol);
+}
+
+int rtw_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
+{
+    return stack_broadcast_accept(s, addr, addrlen);
+}
+
+int rtw_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags)
+{
+    return stack_broadcast_accept4(s, addr, addrlen, flags);
+}
+
+int rtw_bind(int s, const struct sockaddr *name, socklen_t namelen)
+{
+    struct lwip_sock *sock = get_socket_by_fd(s);
+
+    if (NETCONN_IS_UDP(sock) && get_global_cfg_params()->listen_shadow) {
+        return stack_broadcast_bind(s, name, namelen);
+    } else {
+        return stack_single_bind(s, name, namelen);
+    }
+}
+
+int rtw_listen(int s, int backlog)
+{
+    if (!get_global_cfg_params()->tuple_filter &&
+        !get_global_cfg_params()->listen_shadow) {
+        return stack_single_listen(s, backlog);
+    } else {
+        return stack_broadcast_listen(s, backlog);
+    }
+}
+
+int rtw_connect(int s, const struct sockaddr *name, socklen_t namelen)
+{
+    return rpc_call_connect(s, name, namelen);
+}
+
+int rtw_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen)
+{
+    return rpc_call_setsockopt(s, level, optname, optval, optlen);
+}
+
+int rtw_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen)
+{
+    return rpc_call_getsockopt(s, level, optname, optval, optlen);
+}
+
+int rtw_getpeername(int s, struct sockaddr *name, socklen_t *namelen)
+{
+    return rpc_call_getpeername(s, name, namelen);
+}
+
+int rtw_getsockname(int s, struct sockaddr *name, socklen_t *namelen)
+{
+    return rpc_call_getsockname(s, name, namelen);
+}
+
+ssize_t rtw_read(int s, void *mem, size_t len)
+{
+    return do_lwip_read_from_stack(s, mem, len, 0, NULL, NULL);
+}
+
+ssize_t rtw_readv(int s, const struct iovec *iov, int iovcnt)
+{
+    struct msghdr msg;
+
+    msg.msg_name = NULL;
+    msg.msg_namelen = 0;
+    msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov);
+    msg.msg_iovlen = iovcnt;
+    msg.msg_control = NULL;
+    msg.msg_controllen = 0;
+    msg.msg_flags = 0;
+    ssize_t result = do_lwip_recvmsg_from_stack(s, &msg, 0);
+    if (result == -1 && errno == EAGAIN) {
+        errno = 0;
+        return 0;
+    }
+    return result;
+}
+
+ssize_t rtw_write(int s, const void *mem, size_t size)
+{
+    return do_lwip_send_to_stack(s, mem, size, 0, NULL, 0);
+}
+
+ssize_t rtw_writev(int s, const struct iovec *iov, int iovcnt)
+{
+    struct lwip_sock *sock = get_socket_by_fd(s);
+    struct msghdr msg;
+
+    msg.msg_name = NULL;
+    msg.msg_namelen = 0;
+    msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov);
+    msg.msg_iovlen = iovcnt;
+    msg.msg_control = NULL;
+    msg.msg_controllen = 0;
+    msg.msg_flags = 0;
+    return do_lwip_sendmsg_to_stack(sock, s, &msg, 0);
+}
+
+ssize_t rtw_recv(int sockfd, void *buf, size_t len, int flags)
+{
+    return do_lwip_read_from_stack(sockfd, buf, len, flags, NULL, NULL);
+}
+
+ssize_t rtw_send(int sockfd, const void *buf, size_t len, int flags)
+{
+    return do_lwip_send_to_stack(sockfd, buf, len, flags, NULL, 0);
+}
+
+ssize_t rtw_recvmsg(int s, const struct msghdr *message, int flags)
+{
+    return do_lwip_recvmsg_from_stack(s, message, flags);
+}
+
+ssize_t rtw_sendmsg(int s, const struct msghdr *message, int flags)
+{
+    struct lwip_sock *sock = get_socket_by_fd(s);
+    return do_lwip_sendmsg_to_stack(sock, s, message, flags);
+}
+
+static ssize_t rtw_udp_recvfrom(int sockfd, void *buf, size_t len, int flags,
+                                struct sockaddr *addr, socklen_t *addrlen)
+{
+    struct lwip_sock *sock = get_socket_by_fd(sockfd);
+    int ret;
+
+    while (1) {
+        ret = do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
+        if (ret > 0) {
+            return ret;
+        }
+        if (ret <= 0 && errno != EAGAIN) {
+            return -1;
+        }
+        sock = sock->listen_next;
+        if (sock != NULL && sock->conn != NULL) {
+            sockfd = sock->conn->socket;
+        } else {
+            if (sock == NULL) {
+                errno = EAGAIN;
+                return -1;
+            } else {
+                errno = ENOTCONN;
+                return -1;
+            }
+        }
+    }
+}
+
+static inline ssize_t rtw_tcp_recvfrom(int sockfd, void *buf, size_t len, int flags,
+                                       struct sockaddr *addr, socklen_t *addrlen)
+{
+    return do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
+}
+
+
+ssize_t rtw_recvfrom(int sockfd, void *buf, size_t len, int flags,
+                     struct sockaddr *addr, socklen_t *addrlen)
+{
+    struct lwip_sock *sock = get_socket_by_fd(sockfd);
+    if (NETCONN_IS_UDP(sock)) {
+        return rtw_udp_recvfrom(sockfd, buf, len, flags, addr, addrlen);
+    } else {
+        return rtw_tcp_recvfrom(sockfd, buf, len, flags, addr, addrlen);
+    }
+}
+
+ssize_t rtw_sendto(int sockfd, const void *buf, size_t len, int flags,
+                   const struct sockaddr *addr, socklen_t addrlen)
+{
+    return do_lwip_send_to_stack(sockfd, buf, len, flags, addr, addrlen);
+}
+
+
+int rtw_epoll_wait(int epfd, struct epoll_event* events, int maxevents, int timeout)
+{
+    return lstack_epoll_wait(epfd, events, maxevents, timeout);
+}
+
+int rtw_poll(struct pollfd *fds, nfds_t nfds, int timeout)
+{
+    return lstack_poll(fds, nfds, timeout);
+}
+
+int rtw_close(int s)
+{
+    struct lwip_sock *sock = get_socket(s);
+    if (sock && sock->wakeup && sock->wakeup->epollfd == s) {
+        return lstack_epoll_close(s);
+    }
+    return stack_broadcast_close(s);
+}
+
+int rtw_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)
+{
+    return lstack_epoll_ctl(epfd, op, fd, event);
+}
+
+int rtw_epoll_create1(int flags)
+{
+    return lstack_epoll_create1(flags);
+}
+
+int rtw_epoll_create(int flags)
+{
+    return lstack_epoll_create(flags);
+}
diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index ab39a68..a808ee8 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -12,17 +12,12 @@
 
 #define _GNU_SOURCE
 #include <dlfcn.h>
-#include <string.h>
 
 #include <signal.h>
-#include <sys/socket.h>
 #include <fcntl.h>
-#include <sys/socket.h>
-#include <stdarg.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <sys/epoll.h>
-#include <unistd.h>
 #include <net/if.h>
 #include <securec.h>
 
@@ -30,29 +25,94 @@
 #include <lwip/lwipsock.h>
 #include <lwip/tcp.h>
 
-#include "posix/lstack_epoll.h"
 #include "posix/lstack_unistd.h"
-#include "posix/lstack_socket.h"
 #include "lstack_log.h"
 #include "lstack_cfg.h"
 #include "lstack_lwip.h"
-#include "lstack_protocol_stack.h"
 #include "gazelle_base_func.h"
-#include "lstack_thread_rpc.h"
-
 #include "lstack_preload.h"
 
+#include "lstack_rtc_api.h"
+#include "lstack_rtw_api.h"
+
 #ifndef SOCK_TYPE_MASK
 #define SOCK_TYPE_MASK 0xf
 #endif
 
+posix_api_t g_wrap_api_value;
+posix_api_t *g_wrap_api;
+
+void wrap_api_init(void)
+{
+    if (g_wrap_api != NULL) {
+        return;
+    }
+
+    g_wrap_api = &g_wrap_api_value;
+    if (get_global_cfg_params()->stack_mode_rtc) {
+        g_wrap_api->socket_fn        = rtc_socket;
+        g_wrap_api->accept_fn        = lwip_accept;
+        g_wrap_api->accept4_fn       = lwip_accept4;
+        g_wrap_api->bind_fn          = lwip_bind;
+        g_wrap_api->listen_fn        = lwip_listen;
+        g_wrap_api->connect_fn       = lwip_connect;
+        g_wrap_api->setsockopt_fn    = lwip_setsockopt;
+        g_wrap_api->getsockopt_fn    = lwip_getsockopt;
+        g_wrap_api->getpeername_fn   = lwip_getpeername;
+        g_wrap_api->getsockname_fn   = lwip_getsockname;
+        g_wrap_api->read_fn          = lwip_read;
+        g_wrap_api->readv_fn         = lwip_readv;
+        g_wrap_api->write_fn         = lwip_write;
+        g_wrap_api->writev_fn        = lwip_writev;
+        g_wrap_api->recv_fn          = lwip_recv;
+        g_wrap_api->send_fn          = lwip_send;
+        g_wrap_api->recv_msg         = lwip_recvmsg;
+        g_wrap_api->send_msg         = lwip_sendmsg;
+        g_wrap_api->recv_from        = lwip_recvfrom;
+        g_wrap_api->send_to          = lwip_sendto;
+        g_wrap_api->epoll_wait_fn    = rtc_epoll_wait;
+        g_wrap_api->poll_fn          = rtc_poll;
+        g_wrap_api->close_fn         = rtc_close;
+        g_wrap_api->epoll_ctl_fn     = rtc_epoll_ctl;
+        g_wrap_api->epoll_create1_fn = rtc_epoll_create1;
+        g_wrap_api->epoll_create_fn  = rtc_epoll_create;
+    } else {
+        g_wrap_api->socket_fn        = rtw_socket;
+        g_wrap_api->accept_fn        = rtw_accept;
+        g_wrap_api->accept4_fn       = rtw_accept4;
+        g_wrap_api->bind_fn          = rtw_bind;
+        g_wrap_api->listen_fn        = rtw_listen;
+        g_wrap_api->connect_fn       = rtw_connect;
+        g_wrap_api->setsockopt_fn    = rtw_setsockopt;
+        g_wrap_api->getsockopt_fn    = rtw_getsockopt;
+        g_wrap_api->getpeername_fn   = rtw_getpeername;
+        g_wrap_api->getsockname_fn   = rtw_getsockname;
+        g_wrap_api->read_fn          = rtw_read;
+        g_wrap_api->readv_fn         = rtw_readv;
+        g_wrap_api->write_fn         = rtw_write;
+        g_wrap_api->writev_fn        = rtw_writev;
+        g_wrap_api->recv_fn          = rtw_recv;
+        g_wrap_api->send_fn          = rtw_send;
+        g_wrap_api->recv_msg         = rtw_recvmsg;
+        g_wrap_api->send_msg         = rtw_sendmsg;
+        g_wrap_api->recv_from        = rtw_recvfrom;
+        g_wrap_api->send_to          = rtw_sendto;
+        g_wrap_api->epoll_wait_fn    = rtw_epoll_wait;
+        g_wrap_api->poll_fn          = rtw_poll;
+        g_wrap_api->close_fn         = rtw_close;
+        g_wrap_api->epoll_ctl_fn     = rtw_epoll_ctl;
+        g_wrap_api->epoll_create1_fn = rtw_epoll_create1;
+        g_wrap_api->epoll_create_fn  = rtw_epoll_create;
+    }
+}
+
 static inline int32_t do_epoll_create1(int32_t flags)
 {
     if (select_posix_path() == PATH_KERNEL) {
         return posix_api->epoll_create1_fn(flags);
     }
 
-    return lstack_epoll_create1(flags);
+    return g_wrap_api->epoll_create1_fn(flags);
 }
 
 static inline int32_t do_epoll_create(int32_t size)
@@ -61,7 +121,7 @@ static inline int32_t do_epoll_create(int32_t size)
         return posix_api->epoll_create_fn(size);
     }
 
-    return lstack_epoll_create(size);
+    return g_wrap_api->epoll_create_fn(size);
 }
 
 static inline int32_t do_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct epoll_event* event)
@@ -70,7 +130,7 @@ static inline int32_t do_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct
         return posix_api->epoll_ctl_fn(epfd, op, fd, event);
     }
 
-    return lstack_epoll_ctl(epfd, op, fd, event);
+    return g_wrap_api->epoll_ctl_fn(epfd, op, fd, event);
 }
 
 static inline int32_t do_epoll_wait(int32_t epfd, struct epoll_event* events, int32_t maxevents, int32_t timeout)
@@ -87,7 +147,7 @@ static inline int32_t do_epoll_wait(int32_t epfd, struct epoll_event* events, in
         GAZELLE_RETURN(EINVAL);
     }
 
-    return lstack_epoll_wait(epfd, events, maxevents, timeout);
+    return g_wrap_api->epoll_wait_fn(epfd, events, maxevents, timeout);
 }
 
 static inline int32_t do_accept(int32_t s, struct sockaddr *addr, socklen_t *addrlen)
@@ -96,7 +156,7 @@ static inline int32_t do_accept(int32_t s, struct sockaddr *addr, socklen_t *add
         return posix_api->accept_fn(s, addr, addrlen);
     }
 
-    int32_t fd = stack_broadcast_accept(s, addr, addrlen);
+    int32_t fd = g_wrap_api->accept_fn(s, addr, addrlen);
     if (fd >= 0) {
         return fd;
     }
@@ -114,7 +174,7 @@ static int32_t do_accept4(int32_t s, struct sockaddr *addr, socklen_t *addrlen,
         return posix_api->accept4_fn(s, addr, addrlen, flags);
     }
 
-    int32_t fd = stack_broadcast_accept4(s, addr, addrlen, flags);
+    int32_t fd = g_wrap_api->accept4_fn(s, addr, addrlen, flags);
     if (fd >= 0) {
         return fd;
     }
@@ -158,11 +218,7 @@ static int32_t do_bind(int32_t s, const struct sockaddr *name, socklen_t namelen
     if (match_host_addr(((struct sockaddr_in *)name)->sin_addr.s_addr)) {
         /* maybe kni addr */
         posix_api->bind_fn(s, name, namelen);
-        if (NETCONN_IS_UDP(sock) && get_global_cfg_params()->listen_shadow) {
-            return stack_broadcast_bind(s, name, namelen);
-        } else {
-            return stack_single_bind(s, name, namelen);
-        }
+        return g_wrap_api->bind_fn(s, name, namelen);
     } else {
         SET_CONN_TYPE_HOST(sock->conn);
         return posix_api->bind_fn(s, name, namelen);
@@ -253,7 +309,7 @@ static int32_t do_connect(int32_t s, const struct sockaddr *name, socklen_t name
         ret = posix_api->connect_fn(s, name, namelen);
         SET_CONN_TYPE_HOST(sock->conn);
     } else {
-        ret = rpc_call_connect(s, name, namelen);
+        ret = g_wrap_api->connect_fn(s, name, namelen);
         SET_CONN_TYPE_LIBOS(sock->conn);
     }
 
@@ -266,13 +322,7 @@ static inline int32_t do_listen(int32_t s, int32_t backlog)
         return posix_api->listen_fn(s, backlog);
     }
 
-    int32_t ret;
-    if (!get_global_cfg_params()->tuple_filter &&
-        !get_global_cfg_params()->listen_shadow) {
-        ret = stack_single_listen(s, backlog);
-    } else {
-        ret = stack_broadcast_listen(s, backlog);
-    }
+    int32_t ret = g_wrap_api->listen_fn(s, backlog);
     if (ret != 0) {
         return ret;
     }
@@ -287,7 +337,7 @@ static inline int32_t do_getpeername(int32_t s, struct sockaddr *name, socklen_t
     }
 
     if (select_fd_posix_path(s, NULL) == PATH_LWIP) {
-        return rpc_call_getpeername(s, name, namelen);
+        return g_wrap_api->getpeername_fn(s, name, namelen);
     }
 
     return posix_api->getpeername_fn(s, name, namelen);
@@ -300,7 +350,7 @@ static inline int32_t do_getsockname(int32_t s, struct sockaddr *name, socklen_t
     }
 
     if (select_fd_posix_path(s, NULL) == PATH_LWIP) {
-        return rpc_call_getsockname(s, name, namelen);
+        return g_wrap_api->getsockname_fn(s, name, namelen);
     }
 
     return posix_api->getsockname_fn(s, name, namelen);
@@ -321,7 +371,7 @@ static bool unsupport_optname(int32_t optname)
 static inline int32_t do_getsockopt(int32_t s, int32_t level, int32_t optname, void *optval, socklen_t *optlen)
 {
     if (select_fd_posix_path(s, NULL) == PATH_LWIP && !unsupport_optname(optname)) {
-        return rpc_call_getsockopt(s, level, optname, optval, optlen);
+        return g_wrap_api->getsockopt_fn(s, level, optname, optval, optlen);
     }
 
     return posix_api->getsockopt_fn(s, level, optname, optval, optlen);
@@ -339,7 +389,7 @@ static inline int32_t do_setsockopt(int32_t s, int32_t level, int32_t optname, c
         return ret;
     }
 
-    return rpc_call_setsockopt(s, level, optname, optval, optlen);
+    return g_wrap_api->setsockopt_fn(s, level, optname, optval, optlen);
 }
 
 static inline int32_t do_socket(int32_t domain, int32_t type, int32_t protocol)
@@ -355,7 +405,7 @@ static inline int32_t do_socket(int32_t domain, int32_t type, int32_t protocol)
         return posix_api->socket_fn(domain, type, protocol);
     }
 
-    ret = rpc_call_socket(domain, type, protocol);
+    ret = g_wrap_api->socket_fn(domain, type, protocol);
     /* if udp_enable = 1 in lstack.conf, udp protocol must be in user path currently */
     if ((ret >= 0) && (type & SOCK_DGRAM)) {
         struct lwip_sock *sock = get_socket(ret);
@@ -382,7 +432,7 @@ static inline ssize_t do_recv(int32_t sockfd, void *buf, size_t len, int32_t fla
         return posix_api->recv_fn(sockfd, buf, len, flags);
     }
 
-    return do_lwip_read_from_stack(sockfd, buf, len, flags, NULL, NULL);
+    return g_wrap_api->recv_fn(sockfd, buf, len, flags);
 }
 
 static inline ssize_t do_read(int32_t s, void *mem, size_t len)
@@ -400,7 +450,7 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len)
         return posix_api->read_fn(s, mem, len);
     }
 
-    return do_lwip_read_from_stack(s, mem, len, 0, NULL, NULL);
+    return g_wrap_api->read_fn(s, mem, len);
 }
 
 static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt)
@@ -410,21 +460,7 @@ static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt)
         return posix_api->readv_fn(s, iov, iovcnt);
     }
 
-    struct msghdr msg;
-
-    msg.msg_name = NULL;
-    msg.msg_namelen = 0;
-    msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov);
-    msg.msg_iovlen = iovcnt;
-    msg.msg_control = NULL;
-    msg.msg_controllen = 0;
-    msg.msg_flags = 0;
-    ssize_t result = do_lwip_recvmsg_from_stack(s, &msg, 0);
-    if (result == -1 && errno == EAGAIN) {
-        errno = 0;
-        return 0;
-    }
-    return result;
+    return g_wrap_api->readv_fn(s, iov, iovcnt);
 }
 
 static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32_t flags)
@@ -434,7 +470,7 @@ static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32
         return posix_api->send_fn(sockfd, buf, len, flags);
     }
 
-    return do_lwip_send_to_stack(sockfd, buf, len, flags, NULL, 0);
+    return g_wrap_api->send_fn(sockfd, buf, len, flags);
 }
 
 static inline ssize_t do_write(int32_t s, const void *mem, size_t size)
@@ -444,7 +480,7 @@ static inline ssize_t do_write(int32_t s, const void *mem, size_t size)
         return posix_api->write_fn(s, mem, size);
     }
 
-    return do_lwip_send_to_stack(s, mem, size, 0, NULL, 0);
+    return g_wrap_api->write_fn(s, mem, size);
 }
 
 static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt)
@@ -455,16 +491,7 @@ static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt)
         return posix_api->writev_fn(s, iov, iovcnt);
     }
 
-    struct msghdr msg;
-
-    msg.msg_name = NULL;
-    msg.msg_namelen = 0;
-    msg.msg_iov = LWIP_CONST_CAST(struct iovec *, iov);
-    msg.msg_iovlen = iovcnt;
-    msg.msg_control = NULL;
-    msg.msg_controllen = 0;
-    msg.msg_flags = 0;
-    return do_lwip_sendmsg_to_stack(sock, s, &msg, 0);
+    return g_wrap_api->writev_fn(s, iov, iovcnt);
 }
 
 static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flags)
@@ -478,7 +505,7 @@ static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flag
         return posix_api->recv_msg(s, message, flags);
     }
 
-    return do_lwip_recvmsg_from_stack(s, message, flags);
+    return g_wrap_api->recv_msg(s, message, flags);
 }
 
 static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_t flags)
@@ -493,39 +520,7 @@ static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_
         return posix_api->send_msg(s, message, flags);
     }
 
-    return do_lwip_sendmsg_to_stack(sock, s, message, flags);
-}
-
-static inline ssize_t udp_recvfrom(struct lwip_sock *sock, int32_t sockfd, void *buf, size_t len, int32_t flags,
-                                   struct sockaddr *addr, socklen_t *addrlen)
-{
-    int32_t ret;
-
-    while (1) {
-        ret = do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
-        if (ret > 0) {
-            return ret;
-        }
-        if (ret <= 0 && errno != EAGAIN) {
-            return -1;
-        }
-        sock = sock->listen_next;
-        if (sock != NULL && sock->conn != NULL) {
-            sockfd = sock->conn->socket;
-        } else {
-            if (sock == NULL) {
-                GAZELLE_RETURN(EAGAIN);
-            } else {
-                GAZELLE_RETURN(ENOTCONN);
-            }
-        }
-    }
-}
-
-static inline ssize_t tcp_recvfrom(struct lwip_sock *sock, int32_t sockfd, void *buf, size_t len, int32_t flags,
-                                   struct sockaddr *addr, socklen_t *addrlen)
-{
-    return do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
+    return g_wrap_api->send_msg(s, message, flags);
 }
 
 static inline ssize_t do_recvfrom(int32_t sockfd, void *buf, size_t len, int32_t flags,
@@ -541,11 +536,7 @@ static inline ssize_t do_recvfrom(int32_t sockfd, void *buf, size_t len, int32_t
 
     struct lwip_sock *sock = NULL;
     if (select_fd_posix_path(sockfd, &sock) == PATH_LWIP) {
-        if (NETCONN_IS_UDP(sock)) {
-            return udp_recvfrom(sock, sockfd, buf, len, flags, addr, addrlen);
-        } else {
-            return tcp_recvfrom(sock, sockfd, buf, len, flags, addr, addrlen);
-        }
+        return g_wrap_api->recv_from(sockfd, buf, len, flags, addr, addrlen);
     }
 
     return posix_api->recv_from(sockfd, buf, len, flags, addr, addrlen);
@@ -559,7 +550,7 @@ static inline ssize_t do_sendto(int32_t sockfd, const void *buf, size_t len, int
         return posix_api->send_to(sockfd, buf, len, flags, addr, addrlen);
     }
 
-    return do_lwip_send_to_stack(sockfd, buf, len, flags, addr, addrlen);
+    return g_wrap_api->send_to(sockfd, buf, len, flags, addr, addrlen);
 }
 
 static inline int32_t do_close(int32_t s)
@@ -570,16 +561,13 @@ static inline int32_t do_close(int32_t s)
         /* we called lwip_socket, even if kernel fd */
         if (posix_api != NULL && !posix_api->ues_posix &&
             /* contain posix_api->close_fn if success */
-            stack_broadcast_close(s) == 0) {
+            g_wrap_api->close_fn(s) == 0) {
             return 0;
         } else {
             return posix_api->close_fn(s);
         }
     }
-    if (sock && sock->wakeup && sock->wakeup->epollfd == s) {
-        return lstack_epoll_close(s);
-    }
-    return stack_broadcast_close(s);
+    return g_wrap_api->close_fn(s);
 }
 
 static int32_t do_poll(struct pollfd *fds, nfds_t nfds, int32_t timeout)
@@ -588,7 +576,7 @@ static int32_t do_poll(struct pollfd *fds, nfds_t nfds, int32_t timeout)
         return posix_api->poll_fn(fds, nfds, timeout);
     }
 
-    return lstack_poll(fds, nfds, timeout);
+    return g_wrap_api->poll_fn(fds, nfds, timeout);
 }
 
 static int32_t do_ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *tmo_p, const sigset_t *sigmask)
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index b6e9bf1..950fa8d 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -47,6 +47,7 @@
 #include "gazelle_base_func.h"
 #include "lstack_protocol_stack.h"
 #include "lstack_preload.h"
+#include "lstack_wrap.h"
 
 static void check_process_start(void)
 {
@@ -235,6 +236,8 @@ __attribute__((constructor)) void gazelle_network_init(void)
     }
     LSTACK_PRE_LOG(LSTACK_INFO, "cfg_init success\n");
 
+    wrap_api_init();
+
     /* check primary process start */
     check_process_start();
 
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index a98b1b8..cdb0089 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -744,7 +744,7 @@ static int32_t check_msg_vaild(const struct msghdr *message)
     return 0;
 }
 
-ssize_t do_lwip_recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags)
+ssize_t do_lwip_recvmsg_from_stack(int32_t s, const struct msghdr *message, int32_t flags)
 {
     ssize_t buflen = 0;
 
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index 55a483e..4a13204 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -41,7 +41,7 @@ ssize_t do_lwip_read_from_lwip(struct lwip_sock *sock, int32_t flags, uint8_t ap
 /* app write/read ring */
 ssize_t do_lwip_sendmsg_to_stack(struct lwip_sock *sock, int32_t s,
                                  const struct msghdr *message, int32_t flags);
-ssize_t do_lwip_recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags);
+ssize_t do_lwip_recvmsg_from_stack(int32_t s, const struct msghdr *message, int32_t flags);
 
 ssize_t do_lwip_send_to_stack(int32_t fd, const void *buf, size_t len, int32_t flags,
                               const struct sockaddr *addr, socklen_t addrlen);
diff --git a/src/lstack/include/posix/lstack_socket.h b/src/lstack/include/lstack_rtc_api.h
similarity index 53%
rename from src/lstack/include/posix/lstack_socket.h
rename to src/lstack/include/lstack_rtc_api.h
index a3ce1eb..563cbd8 100644
--- a/src/lstack/include/posix/lstack_socket.h
+++ b/src/lstack/include/lstack_rtc_api.h
@@ -10,37 +10,46 @@
 * See the Mulan PSL v2 for more details.
 */
 
-#ifndef _GAZELLE_SOCKET_H_
-#define _GAZELLE_SOCKET_H_
+#ifndef _LSTACK_RTC_API_H_
+#define _LSTACK_RTC_API_H_
+#include <sys/epoll.h>
+#include <sys/socket.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int lwip_socket(int domain, int type, int protocol);
-int lwip_bind(int s, const struct sockaddr *name, socklen_t namelen);
-int lwip_connect(int s, const struct sockaddr *name, socklen_t namelen);
-int lwip_listen(int s, int backlog);
+/* don't include lwip/sockets.h, conflict with sys/socket.h */
+/* extern lwip_api here */
+int lwip_fcntl(int s, int cmd, int val);
+int lwip_ioctl(int s, long cmd, ...);
 int lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
-int lwip_close(int s);
+int lwip_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags);
+int lwip_bind(int s, const struct sockaddr *name, socklen_t namelen);
+int lwip_shutdown(int s, int how);
 int lwip_getpeername(int s, struct sockaddr *name, socklen_t *namelen);
 int lwip_getsockname(int s, struct sockaddr *name, socklen_t *namelen);
 int lwip_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen);
 int lwip_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen);
-
-ssize_t lwip_write(int s, const void *dataptr, size_t size);
-ssize_t lwip_send(int s, const void *data, size_t size, int flags);
-ssize_t lwip_recvmsg(int s, struct msghdr *message, int flags);
-ssize_t lwip_sendmsg(int s, const struct msghdr *message, int flags);
-ssize_t lwip_read(int s, void *mem, size_t len);
-ssize_t lwip_recvfrom(int s, void *mem, size_t len, int flags, void *from, void *fromlen);
+int lwip_close(int s);
+int lwip_connect(int s, const struct sockaddr *name, socklen_t namelen);
+int lwip_listen(int s, int backlog);
 ssize_t lwip_recv(int s, void *mem, size_t len, int flags);
+ssize_t lwip_read(int s, void *mem, size_t len);
+ssize_t lwip_readv(int s, const struct iovec *iov, int iovcnt);
+ssize_t lwip_recvfrom(int s, void *mem, size_t len, int flags,
+                      struct sockaddr *from, socklen_t *fromlen);
+ssize_t lwip_recvmsg(int s, const struct msghdr *message, int flags);
+ssize_t lwip_send(int s, const void *dataptr, size_t size, int flags);
+ssize_t lwip_sendmsg(int s, const struct msghdr *message, int flags);
+ssize_t lwip_sendto(int s, const void *dataptr, size_t size, int flags,
+                    const struct sockaddr *to, socklen_t tolen);
+int lwip_socket(int domain, int type, int protocol);
+ssize_t lwip_write(int s, const void *dataptr, size_t size);
+ssize_t lwip_writev(int s, const struct iovec *iov, int iovcnt);
 
-int lwip_fcntl(int s, int cmd, int val);
-int lwip_ioctl(int s, int cmd, ...);
-
-#ifdef __cplusplus
-}
-#endif
+int rtc_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+int rtc_epoll_wait(int epfd, struct epoll_event* events, int maxevents, int timeout);
+int rtc_socket(int domain, int type, int protocol);
+int rtc_close(int s);
+int rtc_epoll_create(int flags);
+int rtc_epoll_create1(int flags);
+int rtc_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
 
-#endif /* _GAZELLE_SOCKET_H_ */
+#endif /* __LSTACK_RTC_API_H_  */
diff --git a/src/lstack/include/lstack_rtw_api.h b/src/lstack/include/lstack_rtw_api.h
new file mode 100644
index 0000000..facf4c0
--- /dev/null
+++ b/src/lstack/include/lstack_rtw_api.h
@@ -0,0 +1,47 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef _LSTACK_RTW_API_H_
+#define _LSTACK_RTW_API_H_
+
+#include <sys/epoll.h>
+#include <sys/socket.h>
+
+int rtw_socket(int domain, int type, int protocol);
+int rtw_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
+int rtw_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags);
+int rtw_bind(int s, const struct sockaddr *name, socklen_t namelen);
+int rtw_listen(int s, int backlog);
+int rtw_connect(int s, const struct sockaddr *name, socklen_t namelen);
+int rtw_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen);
+int rtw_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen);
+int rtw_getpeername(int s, struct sockaddr *name, socklen_t *namelen);
+int rtw_getsockname(int s, struct sockaddr *name, socklen_t *namelen);
+ssize_t rtw_read(int s, void *mem, size_t len);
+ssize_t rtw_readv(int s, const struct iovec *iov, int iovcnt);
+ssize_t rtw_write(int s, const void *mem, size_t size);
+ssize_t rtw_writev(int s, const struct iovec *iov, int iovcnt);
+ssize_t rtw_recv(int sockfd, void *buf, size_t len, int flags);
+ssize_t rtw_send(int sockfd, const void *buf, size_t len, int flags);
+ssize_t rtw_recvmsg(int s, const struct msghdr *message, int flags);
+ssize_t rtw_sendmsg(int s, const struct msghdr *message, int flags);
+ssize_t rtw_recvfrom(int sockfd, void *buf, size_t len, int flags,
+                     struct sockaddr *addr, socklen_t *addrlen);
+ssize_t rtw_sendto(int sockfd, const void *buf, size_t len, int flags,
+                   const struct sockaddr *addr, socklen_t addrlen);
+int rtw_epoll_wait(int epfd, struct epoll_event* events, int maxevents, int timeout);
+int rtw_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+int rtw_close(int s);
+int rtw_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
+int rtw_epoll_create1(int flags);
+int rtw_epoll_create(int flags);
+#endif /* _LSTACK_RTW_API_H_ */
diff --git a/src/lstack/include/lstack_wrap.h b/src/lstack/include/lstack_wrap.h
new file mode 100644
index 0000000..80e5f3c
--- /dev/null
+++ b/src/lstack/include/lstack_wrap.h
@@ -0,0 +1,19 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef _LSTACK_WRAP_H_
+#define _LSTACK_WRAP_H_
+
+void wrap_api_init(void);
+
+#endif
+
-- 
2.27.0