From 9ba0b1d9df98b648eecdf5c5ed31611574d25272 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Wed, 20 Sep 2023 19:48:22 +0800
Subject: [PATCH] lstack_lwip: external api start with do_lwip_ prefix

---
 src/lstack/api/lstack_wrap.c               |  22 +--
 src/lstack/core/lstack_dpdk.c              |  12 +-
 src/lstack/core/lstack_lwip.c              | 198 ++++-----------------
 src/lstack/core/lstack_protocol_stack.c    | 138 +++++++++++++-
 src/lstack/core/lstack_thread_rpc.c        |   8 +-
 src/lstack/include/lstack_dpdk.h           |   1 +
 src/lstack/include/lstack_lwip.h           |  61 ++++---
 src/lstack/include/lstack_protocol_stack.h |   7 +
 src/lstack/netif/lstack_ethdev.c           |   8 +-
 src/lstack/netif/lstack_vdev.c             |   2 +-
 10 files changed, 241 insertions(+), 216 deletions(-)

diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index 956d661..236c689 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -438,7 +438,7 @@ static inline ssize_t do_recv(int32_t sockfd, void *buf, size_t len, int32_t fla
 
     struct lwip_sock *sock = NULL;
     if (select_path(sockfd, &sock) == PATH_LWIP) {
-        return read_stack_data(sockfd, buf, len, flags, NULL, NULL);
+        return do_lwip_read_from_stack(sockfd, buf, len, flags, NULL, NULL);
     }
 
     return posix_api->recv_fn(sockfd, buf, len, flags);
@@ -456,7 +456,7 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len)
 
     struct lwip_sock *sock = NULL;
     if (select_path(s, &sock) == PATH_LWIP) {
-        return read_stack_data(s, mem, len, 0, NULL, NULL);
+        return do_lwip_read_from_stack(s, mem, len, 0, NULL, NULL);
     }
     return posix_api->read_fn(s, mem, len);
 }
@@ -477,7 +477,7 @@ static inline ssize_t do_readv(int32_t s, const struct iovec *iov, int iovcnt)
     msg.msg_control = NULL;
     msg.msg_controllen = 0;
     msg.msg_flags = 0;
-    ssize_t result = recvmsg_from_stack(s, &msg, 0);
+    ssize_t result = do_lwip_recvmsg_from_stack(s, &msg, 0);
     if (result == -1 && errno == EAGAIN) {
         errno = 0;
         return 0;
@@ -492,7 +492,7 @@ static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32
         return posix_api->send_fn(sockfd, buf, len, flags);
     }
 
-    return gazelle_send(sockfd, buf, len, flags, NULL, 0);
+    return do_lwip_send_to_stack(sockfd, buf, len, flags, NULL, 0);
 }
 
 static inline ssize_t do_write(int32_t s, const void *mem, size_t size)
@@ -502,7 +502,7 @@ static inline ssize_t do_write(int32_t s, const void *mem, size_t size)
         return posix_api->write_fn(s, mem, size);
     }
 
-    return gazelle_send(s, mem, size, 0, NULL, 0);
+    return do_lwip_send_to_stack(s, mem, size, 0, NULL, 0);
 }
 
 static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt)
@@ -521,7 +521,7 @@ static inline ssize_t do_writev(int32_t s, const struct iovec *iov, int iovcnt)
     msg.msg_control = NULL;
     msg.msg_controllen = 0;
     msg.msg_flags = 0;
-    return sendmsg_to_stack(sock, s, &msg, 0);
+    return do_lwip_sendmsg_to_stack(sock, s, &msg, 0);
 }
 
 static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flags)
@@ -532,7 +532,7 @@ static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flag
 
     struct lwip_sock *sock = NULL;
     if (select_path(s, &sock) == PATH_LWIP) {
-        return recvmsg_from_stack(s, message, flags);
+        return do_lwip_recvmsg_from_stack(s, message, flags);
     }
 
     return posix_api->recv_msg(s, message, flags);
@@ -546,7 +546,7 @@ static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_
 
     struct lwip_sock *sock = NULL;
     if (select_path(s, &sock) == PATH_LWIP) {
-        return sendmsg_to_stack(sock, s, message, flags);
+        return do_lwip_sendmsg_to_stack(sock, s, message, flags);
     }
 
     return posix_api->send_msg(s, message, flags);
@@ -558,7 +558,7 @@ static inline ssize_t udp_recvfrom(struct lwip_sock *sock, int32_t sockfd, void
     int32_t ret;
 
     while (1) {
-        ret = read_stack_data(sockfd, buf, len, flags, addr, addrlen);
+        ret = do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
         if (ret > 0) {
             return ret;
         }
@@ -581,7 +581,7 @@ static inline ssize_t udp_recvfrom(struct lwip_sock *sock, int32_t sockfd, void
 static inline ssize_t tcp_recvfrom(struct lwip_sock *sock, int32_t sockfd, void *buf, size_t len, int32_t flags,
                                    struct sockaddr *addr, socklen_t *addrlen)
 {
-    return read_stack_data(sockfd, buf, len, flags, addr, addrlen);
+    return do_lwip_read_from_stack(sockfd, buf, len, flags, addr, addrlen);
 }
 
 static inline ssize_t do_recvfrom(int32_t sockfd, void *buf, size_t len, int32_t flags,
@@ -615,7 +615,7 @@ static inline ssize_t do_sendto(int32_t sockfd, const void *buf, size_t len, int
         return posix_api->send_to(sockfd, buf, len, flags, addr, addrlen);
     }
 
-    return gazelle_send(sockfd, buf, len, flags, addr, addrlen);
+    return do_lwip_send_to_stack(sockfd, buf, len, flags, addr, addrlen);
 }
 
 static inline int32_t do_close(int32_t s)
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index cd42047..b4e29cb 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -281,6 +281,16 @@ int32_t create_shared_ring(struct protocol_stack *stack)
     return 0;
 }
 
+int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num)
+{
+    int32_t ret = rte_pktmbuf_alloc_bulk(pool, mbufs, num);
+    if (ret != 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
 int32_t fill_mbuf_to_ring(struct rte_mempool *mempool, struct rte_ring *ring, uint32_t mbuf_num)
 {
     int32_t ret;
@@ -291,7 +301,7 @@ int32_t fill_mbuf_to_ring(struct rte_mempool *mempool, struct rte_ring *ring, ui
     while (remain > 0) {
         batch = LWIP_MIN(remain, RING_SIZE(FREE_RX_QUEUE_SZ));
 
-        ret = gazelle_alloc_pktmbuf(mempool, free_buf, batch);
+        ret = dpdk_alloc_pktmbuf(mempool, free_buf, batch);
         if (ret != 0) {
             LSTACK_LOG(ERR, LSTACK, "cannot alloc mbuf for ring, count: %u ret=%d\n", batch, ret);
             return -1;
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index bcdbac0..9ab8446 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -167,7 +167,7 @@ static bool replenish_send_idlembuf(struct protocol_stack *stack, struct lwip_so
     return false;
 }
 
-void gazelle_init_sock(int32_t fd)
+void do_lwip_init_sock(int32_t fd)
 {
     static _Atomic uint32_t name_tick = 0;
     struct protocol_stack *stack = get_protocol_stack();
@@ -200,7 +200,7 @@ void gazelle_init_sock(int32_t fd)
     init_list_node_null(&sock->event_list);
 }
 
-void gazelle_clean_sock(int32_t fd)
+void do_lwip_clean_sock(int32_t fd)
 {
     struct lwip_sock *sock = get_socket_by_fd(fd);
     if (sock == NULL || sock->stack == NULL) {
@@ -220,7 +220,7 @@ void gazelle_clean_sock(int32_t fd)
     list_del_node_null(&sock->recv_list);
 }
 
-void gazelle_free_pbuf(struct pbuf *pbuf)
+void do_lwip_free_pbuf(struct pbuf *pbuf)
 {
     if (pbuf == NULL) {
         return;
@@ -231,17 +231,7 @@ void gazelle_free_pbuf(struct pbuf *pbuf)
     rte_pktmbuf_free_seg(mbuf);
 }
 
-int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num)
-{
-    int32_t ret = rte_pktmbuf_alloc_bulk(pool, mbufs, num);
-    if (ret != 0) {
-        return ret;
-    }
-
-    return 0;
-}
-
-struct pbuf *lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type)
+struct pbuf *do_lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type)
 {
     struct rte_mbuf *mbuf;
     struct protocol_stack *stack = get_protocol_stack();
@@ -254,7 +244,7 @@ struct pbuf *lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type)
     return init_mbuf_to_pbuf(mbuf, layer, length, type);
 }
 
-struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags)
+struct pbuf *do_lwip_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags)
 {
     struct pbuf *pbuf = NULL;
 
@@ -321,7 +311,7 @@ struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8
     return pbuf;
 }
 
-void write_lwip_over(struct lwip_sock *sock)
+void do_lwip_get_from_sendring_over(struct lwip_sock *sock)
 {
     sock->send_pre_del = NULL;
     sock->stack->stats.write_lwip_cnt++;
@@ -549,8 +539,8 @@ int sem_timedwait_nsecs(sem_t *sem)
     return sem_timedwait(sem, &ts);
 }
 
-ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len,
-                         const struct sockaddr *addr, socklen_t addrlen)
+static ssize_t do_lwip_fill_sendring(struct lwip_sock *sock, const void *buf, size_t len,
+                                     const struct sockaddr *addr, socklen_t addrlen)
 {
     if (sock->errevent > 0) {
         GAZELLE_RETURN(ENOTCONN);
@@ -639,7 +629,7 @@ END:
     return send_len;
 }
 
-static inline bool replenish_send_ring(struct protocol_stack *stack, struct lwip_sock *sock)
+bool do_lwip_replenish_sendring(struct protocol_stack *stack, struct lwip_sock *sock)
 {
     bool replenish_again = false;
 
@@ -652,16 +642,8 @@ static inline bool replenish_send_ring(struct protocol_stack *stack, struct lwip
     return replenish_again;
 }
 
-void rpc_replenish(struct rpc_msg *msg)
-{
-    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_0].p;
-    struct lwip_sock *sock = (struct lwip_sock *)msg->args[MSG_ARG_1].p;
-
-    msg->result = replenish_send_ring(stack, sock);
-}
-
-static inline bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
-                                size_t len, int32_t flags)
+bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
+                  size_t len, int32_t flags)
 {
     /* send all send_ring, so len set lwip send max. */
     if (NETCONN_IS_UDP(sock)) {
@@ -670,38 +652,7 @@ static inline bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct
         (void)lwip_send(fd, sock, UINT16_MAX, flags);
     }
 
-    return replenish_send_ring(stack, sock);
-}
-
-void stack_send(struct rpc_msg *msg)
-{
-    int32_t fd = msg->args[MSG_ARG_0].i;
-    size_t len = msg->args[MSG_ARG_1].size;
-    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p;
-    bool replenish_again;
-
-    struct lwip_sock *sock = get_socket(fd);
-    if (sock == NULL) {
-        msg->result = -1;
-        LSTACK_LOG(ERR, LSTACK, "stack_send: sock error!\n");
-        rpc_msg_free(msg);
-        return;
-    }
-
-    replenish_again = do_lwip_send(stack, sock->conn->socket, sock, len, 0);
-    __sync_fetch_and_sub(&sock->call_num, 1);
-    if (!NETCONN_IS_DATAOUT(sock) && !replenish_again) {
-        rpc_msg_free(msg);
-        return;
-    } else {
-        if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0) {
-            rpc_call(&stack->rpc_queue, msg);
-             __sync_fetch_and_add(&sock->call_num, 1);
-        } else {
-            rpc_msg_free(msg);
-            return;
-        }
-    }
+    return do_lwip_replenish_sendring(stack, sock);
 }
 
 static inline void free_recv_ring_readover(struct rte_ring *ring)
@@ -721,7 +672,7 @@ static inline struct pbuf *pbuf_last(struct pbuf *pbuf)
     return pbuf;
 }
 
-ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
+ssize_t do_lwip_read_from_lwip(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
 {
     if (sock->conn->recvmbox == NULL) {
         return 0;
@@ -808,7 +759,7 @@ static int32_t check_msg_vaild(const struct msghdr *message)
     return 0;
 }
 
-ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags)
+ssize_t do_lwip_recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags)
 {
     ssize_t buflen = 0;
 
@@ -821,8 +772,8 @@ ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags)
             continue;
         }
 
-        ssize_t recvd_local = read_stack_data(s, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len,
-                                              flags, NULL, NULL);
+        ssize_t recvd_local = do_lwip_read_from_stack(s, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len,
+                                                      flags, NULL, NULL);
         if (recvd_local > 0) {
             buflen += recvd_local;
         }
@@ -956,8 +907,8 @@ static inline void thread_bind_stack(struct lwip_sock *sock)
     }
 }
 
-ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags,
-                     const struct sockaddr *addr, socklen_t addrlen)
+ssize_t do_lwip_send_to_stack(int32_t fd, const void *buf, size_t len, int32_t flags,
+                              const struct sockaddr *addr, socklen_t addrlen)
 {
     if (buf == NULL) {
         GAZELLE_RETURN(EINVAL);
@@ -974,7 +925,7 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags,
     if (sock->same_node_tx_ring != NULL) {
         return gazelle_same_node_ring_send(sock, buf, len, flags);
     }
-    ssize_t send = write_stack_data(sock, buf, len, addr, addrlen);
+    ssize_t send = do_lwip_fill_sendring(sock, buf, len, addr, addrlen);
     if (send <= 0) {
         return send;
     }
@@ -983,7 +934,7 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags,
     return send;
 }
 
-ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr *message, int32_t flags)
+ssize_t do_lwip_sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr *message, int32_t flags)
 {
     int32_t ret;
     int32_t i;
@@ -998,7 +949,7 @@ ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr
             continue;
         }
 
-        ret = write_stack_data(sock, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len, NULL, 0);
+        ret = do_lwip_fill_sendring(sock, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len, NULL, 0);
         if (ret <= 0) {
             buflen = (buflen == 0) ? ret : buflen;
             break;
@@ -1037,7 +988,8 @@ static struct pbuf *pbuf_free_partial(struct pbuf *pbuf, uint16_t free_len)
     return pbuf;
 }
 
-ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags, struct sockaddr *addr, socklen_t *addrlen)
+ssize_t do_lwip_read_from_stack(int32_t fd, void *buf, size_t len, int32_t flags,
+                                struct sockaddr *addr, socklen_t *addrlen)
 {
     size_t recv_left = len;
     struct pbuf *pbuf = NULL;
@@ -1107,7 +1059,7 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags, struct
     return recvd;
 }
 
-void add_recv_list(int32_t fd)
+void do_lwip_add_recvlist(int32_t fd)
 {
     struct lwip_sock *sock = get_socket_by_fd(fd);
 
@@ -1131,7 +1083,7 @@ void read_same_node_recv_list(struct protocol_stack *stack)
     }
 }
 
-void read_recv_list(struct protocol_stack *stack, uint32_t max_num)
+void do_lwip_read_recvlist(struct protocol_stack *stack, uint32_t max_num)
 {
     struct list_node *list = &(stack->recv_list);
     struct list_node *node, *temp;
@@ -1168,7 +1120,7 @@ void read_recv_list(struct protocol_stack *stack, uint32_t max_num)
     }
 }
 
-void gazelle_connected_callback(struct netconn *conn)
+void do_lwip_connected_callback(struct netconn *conn)
 {
     if (conn == NULL) {
         return;
@@ -1225,7 +1177,7 @@ static void copy_pcb_to_conn(struct gazelle_stat_lstack_conn_info *conn, const s
     }
 }
 
-static inline void clone_lwip_socket_opt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock)
+void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock)
 {
     dst_sock->conn->pcb.ip->so_options = src_sock->conn->pcb.ip->so_options;
     dst_sock->conn->pcb.ip->ttl = src_sock->conn->pcb.ip->ttl;
@@ -1244,19 +1196,19 @@ static inline void clone_lwip_socket_opt(struct lwip_sock *dst_sock, struct lwip
     }
 }
 
-int32_t gazelle_socket(int domain, int type, int protocol)
+int32_t do_lwip_socket(int domain, int type, int protocol)
 {
     int32_t fd = lwip_socket(AF_INET, type, 0);
     if (fd < 0) {
         return fd;
     }
 
-    gazelle_init_sock(fd);
+    do_lwip_init_sock(fd);
 
     struct lwip_sock *sock = get_socket(fd);
     if (sock == NULL || sock->stack == NULL) {
         lwip_close(fd);
-        gazelle_clean_sock(fd);
+        do_lwip_clean_sock(fd);
         posix_api->close_fn(fd);
         return -1;
     }
@@ -1264,66 +1216,14 @@ int32_t gazelle_socket(int domain, int type, int protocol)
     return fd;
 }
 
-void create_shadow_fd(struct rpc_msg *msg)
-{
-    int32_t fd = msg->args[MSG_ARG_0].i;
-    struct sockaddr *addr = msg->args[MSG_ARG_1].p;
-    socklen_t addr_len = msg->args[MSG_ARG_2].socklen;
-
-    int32_t clone_fd = 0;
-    struct lwip_sock *sock = get_socket_by_fd(fd);
-    if (sock == NULL) {
-        LSTACK_LOG(ERR, LSTACK, "get sock null fd=%d\n", fd);
-        msg->result = -1;
-        return;
-    }
-
-    if (NETCONN_IS_UDP(sock)) {
-        clone_fd = gazelle_socket(AF_INET, SOCK_DGRAM, 0);
-    } else {
-        clone_fd = gazelle_socket(AF_INET, SOCK_STREAM, 0);
-    }
-
-    if (clone_fd < 0) {
-        LSTACK_LOG(ERR, LSTACK, "clone socket failed clone_fd=%d errno=%d\n", clone_fd, errno);
-        msg->result = clone_fd;
-        return;
-    }
-
-    struct lwip_sock *clone_sock = get_socket_by_fd(clone_fd);
-    if (clone_sock == NULL) {
-        LSTACK_LOG(ERR, LSTACK, "get sock null fd=%d clone_fd=%d\n", fd, clone_fd);
-        msg->result = -1;
-        return;
-    }
-
-    clone_lwip_socket_opt(clone_sock, sock);
-
-    while (sock->listen_next) {
-        sock = sock->listen_next;
-    }
-    sock->listen_next = clone_sock;
-
-    int32_t ret = lwip_bind(clone_fd, addr, addr_len);
-    if (ret < 0) {
-        LSTACK_LOG(ERR, LSTACK, "clone bind failed clone_fd=%d errno=%d\n", clone_fd, errno);
-        msg->result = ret;
-        return;
-    }
-
-    msg->result = clone_fd;
-}
-
-void get_lwip_conntable(struct rpc_msg *msg)
+uint32_t do_lwip_get_conntable(struct gazelle_stat_lstack_conn_info *conn,
+                               uint32_t max_num)
 {
     struct tcp_pcb *pcb = NULL;
     uint32_t conn_num = 0;
-    struct gazelle_stat_lstack_conn_info *conn = (struct gazelle_stat_lstack_conn_info *)msg->args[MSG_ARG_0].p;
-    uint32_t max_num = msg->args[MSG_ARG_1].u;
 
     if (conn == NULL) {
-        msg->result = -1;
-        return;
+        return -1;
     }
 
     for (pcb = tcp_active_pcbs; pcb != NULL && conn_num < max_num; pcb = pcb->next) {
@@ -1352,10 +1252,10 @@ void get_lwip_conntable(struct rpc_msg *msg)
         conn_num++;
     }
 
-    msg->result = conn_num;
+    return conn_num;
 }
 
-void get_lwip_connnum(struct rpc_msg *msg)
+uint32_t do_lwip_get_connnum(void)
 {
     struct tcp_pcb *pcb = NULL;
     struct tcp_pcb_listen *pcbl = NULL;
@@ -1373,33 +1273,7 @@ void get_lwip_connnum(struct rpc_msg *msg)
         conn_num++;
     }
 
-    msg->result = conn_num;
-}
-
-static uint32_t get_list_count(struct list_node *list)
-{
-    struct list_node *node, *temp;
-    uint32_t count = 0;
-
-    list_for_each_safe(node, temp, list) {
-        count++;
-    }
-
-    return count;
-}
-
-void stack_mempool_size(struct rpc_msg *msg)
-{
-    struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p;
-
-    msg->result = rte_mempool_avail_count(stack->rxtx_pktmbuf_pool);
-}
-
-void stack_recvlist_count(struct rpc_msg *msg)
-{
-    struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p;
-
-    msg->result = get_list_count(&stack->recv_list);
+    return conn_num;
 }
 
 void netif_poll(struct netif *netif)
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index ca0b046..5695438 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -484,7 +484,7 @@ static void* gazelle_stack_thread(void *arg)
                 read_same_node_recv_list(stack);
             }
         }
-        read_recv_list(stack, read_connect_number);
+        do_lwip_read_recvlist(stack, read_connect_number);
 
         if ((wakeup_tick & 0xf) == 0) {
             wakeup_stack_epoll(stack);
@@ -643,9 +643,9 @@ void stack_arp(struct rpc_msg *msg)
 
 void stack_socket(struct rpc_msg *msg)
 {
-    msg->result = gazelle_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
+    msg->result = do_lwip_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
     if (msg->result < 0) {
-        msg->result = gazelle_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
+        msg->result = do_lwip_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
         if (msg->result < 0) {
             LSTACK_LOG(ERR, LSTACK, "tid %ld, %ld socket failed\n", get_stack_tid(), msg->result);
         }
@@ -661,7 +661,7 @@ void stack_close(struct rpc_msg *msg)
         LSTACK_LOG(ERR, LSTACK, "tid %ld, fd %d failed %ld\n", get_stack_tid(), msg->args[MSG_ARG_0].i, msg->result);
     }
 
-    gazelle_clean_sock(fd);
+    do_lwip_clean_sock(fd);
 
     posix_api->close_fn(fd);
 }
@@ -706,7 +706,7 @@ void stack_accept(struct rpc_msg *msg)
     struct lwip_sock *sock = get_socket(accept_fd);
     if (sock == NULL || sock->stack == NULL) {
         lwip_close(accept_fd);
-        gazelle_clean_sock(accept_fd);
+        do_lwip_clean_sock(accept_fd);
         posix_api->close_fn(accept_fd);
         LSTACK_LOG(ERR, LSTACK, "fd %d ret %d\n", fd, accept_fd);
         return;
@@ -715,7 +715,7 @@ void stack_accept(struct rpc_msg *msg)
     msg->result = accept_fd;
     sock->stack->conn_num++;
     if (rte_ring_count(sock->conn->recvmbox->ring)) {
-        add_recv_list(accept_fd);
+        do_lwip_add_recvlist(accept_fd);
     }
 }
 
@@ -783,6 +783,37 @@ void stack_recv(struct rpc_msg *msg)
         msg->args[MSG_ARG_3].i);
 }
 
+void stack_send(struct rpc_msg *msg)
+{
+    int32_t fd = msg->args[MSG_ARG_0].i;
+    size_t len = msg->args[MSG_ARG_1].size;
+    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p;
+    bool replenish_again;
+
+    struct lwip_sock *sock = get_socket(fd);
+    if (sock == NULL) {
+        msg->result = -1;
+        LSTACK_LOG(ERR, LSTACK, "stack_send: sock error!\n");
+        rpc_msg_free(msg);
+        return;
+    }
+
+    replenish_again = do_lwip_send(stack, sock->conn->socket, sock, len, 0);
+    __sync_fetch_and_sub(&sock->call_num, 1);
+    if (!NETCONN_IS_DATAOUT(sock) && !replenish_again) {
+        rpc_msg_free(msg);
+        return;
+    } else {
+        if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0) {
+            rpc_call(&stack->rpc_queue, msg);
+             __sync_fetch_and_add(&sock->call_num, 1);
+        } else {
+            rpc_msg_free(msg);
+            return;
+        }
+    }
+}
+
 /* any protocol stack thread receives arp packet and sync it to other threads so that it can have the arp table */
 void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack)
 {
@@ -797,7 +828,7 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack
             continue;
         }
 
-        ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
         if (ret != 0) {
             stack->stats.rx_allocmbuf_fail++;
             return;
@@ -830,6 +861,99 @@ void stack_clean_epoll(struct rpc_msg *msg)
     list_del_node_null(&wakeup->wakeup_list[stack->stack_idx]);
 }
 
+void stack_mempool_size(struct rpc_msg *msg)
+{
+    struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p;
+
+    msg->result = rte_mempool_avail_count(stack->rxtx_pktmbuf_pool);
+}
+
+void stack_create_shadow_fd(struct rpc_msg *msg)
+{
+    int32_t fd = msg->args[MSG_ARG_0].i;
+    struct sockaddr *addr = msg->args[MSG_ARG_1].p;
+    socklen_t addr_len = msg->args[MSG_ARG_2].socklen;
+
+    int32_t clone_fd = 0;
+    struct lwip_sock *sock = get_socket_by_fd(fd);
+    if (sock == NULL) {
+        LSTACK_LOG(ERR, LSTACK, "get sock null fd=%d\n", fd);
+        msg->result = -1;
+        return;
+    }
+
+    if (NETCONN_IS_UDP(sock)) {
+        clone_fd = do_lwip_socket(AF_INET, SOCK_DGRAM, 0);
+    } else {
+        clone_fd = do_lwip_socket(AF_INET, SOCK_STREAM, 0);
+    }
+
+    if (clone_fd < 0) {
+        LSTACK_LOG(ERR, LSTACK, "clone socket failed clone_fd=%d errno=%d\n", clone_fd, errno);
+        msg->result = clone_fd;
+        return;
+    }
+
+    struct lwip_sock *clone_sock = get_socket_by_fd(clone_fd);
+    if (clone_sock == NULL) {
+        LSTACK_LOG(ERR, LSTACK, "get sock null fd=%d clone_fd=%d\n", fd, clone_fd);
+        msg->result = -1;
+        return;
+    }
+
+    do_lwip_clone_sockopt(clone_sock, sock);
+
+    while (sock->listen_next) {
+        sock = sock->listen_next;
+    }
+    sock->listen_next = clone_sock;
+
+    int32_t ret = lwip_bind(clone_fd, addr, addr_len);
+    if (ret < 0) {
+        LSTACK_LOG(ERR, LSTACK, "clone bind failed clone_fd=%d errno=%d\n", clone_fd, errno);
+        msg->result = ret;
+        return;
+    }
+
+    msg->result = clone_fd;
+}
+
+void stack_replenish_sendring(struct rpc_msg *msg)
+{
+    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_0].p;
+    struct lwip_sock *sock = (struct lwip_sock *)msg->args[MSG_ARG_1].p;
+
+    msg->result = do_lwip_replenish_sendring(stack, sock);
+}
+
+void stack_get_conntable(struct rpc_msg *msg)
+{
+    struct gazelle_stat_lstack_conn_info *conn = (struct gazelle_stat_lstack_conn_info *)msg->args[MSG_ARG_0].p;
+    uint32_t max_num = msg->args[MSG_ARG_1].u;
+
+    msg->result = do_lwip_get_conntable(conn, max_num);
+}
+
+void stack_get_connnum(struct rpc_msg *msg)
+{
+    msg->result = do_lwip_get_connnum();
+}
+
+void stack_recvlist_count(struct rpc_msg *msg)
+{
+    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_0].p;
+    struct list_node *list = &stack->recv_list;
+    uint32_t count = 0;
+    struct list_node *node;
+    struct list_node *temp;
+
+    list_for_each_safe(node, temp, list) {
+        count++;
+    }
+    
+    msg->result = count;
+}
+
 /* when fd is listenfd, listenfd of all protocol stack thread will be closed */
 int32_t stack_broadcast_close(int32_t fd)
 {
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 1234bc6..92c58df 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -122,7 +122,7 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
 
 int32_t rpc_call_conntable(struct protocol_stack *stack, void *conn_table, uint32_t max_conn)
 {
-    struct rpc_msg *msg = rpc_msg_alloc(stack, get_lwip_conntable);
+    struct rpc_msg *msg = rpc_msg_alloc(stack, stack_get_conntable);
     if (msg == NULL) {
         return -1;
     }
@@ -135,7 +135,7 @@ int32_t rpc_call_conntable(struct protocol_stack *stack, void *conn_table, uint3
 
 int32_t rpc_call_connnum(struct protocol_stack *stack)
 {
-    struct rpc_msg *msg = rpc_msg_alloc(stack, get_lwip_connnum);
+    struct rpc_msg *msg = rpc_msg_alloc(stack, stack_get_connnum);
     if (msg == NULL) {
         return -1;
     }
@@ -145,7 +145,7 @@ int32_t rpc_call_connnum(struct protocol_stack *stack)
 
 int32_t rpc_call_shadow_fd(struct protocol_stack *stack, int32_t fd, const struct sockaddr *addr, socklen_t addrlen)
 {
-    struct rpc_msg *msg = rpc_msg_alloc(stack, create_shadow_fd);
+    struct rpc_msg *msg = rpc_msg_alloc(stack, stack_create_shadow_fd);
     if (msg == NULL) {
         return -1;
     }
@@ -434,7 +434,7 @@ int32_t rpc_call_ioctl(int fd, long cmd, void *argp)
 
 int32_t rpc_call_replenish(struct protocol_stack *stack, struct lwip_sock *sock)
 {
-    struct rpc_msg *msg = rpc_msg_alloc(stack, rpc_replenish);
+    struct rpc_msg *msg = rpc_msg_alloc(stack, stack_replenish_sendring);
     if (msg == NULL) {
         return -1;
     }
diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h
index c233120..6ca4f3b 100644
--- a/src/lstack/include/lstack_dpdk.h
+++ b/src/lstack/include/lstack_dpdk.h
@@ -59,4 +59,5 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
                                            uint32_t mbuf_cache_size, uint16_t queue_id);
 
 void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id);
+int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num);
 #endif /* GAZELLE_DPDK_H */
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index 223ff93..55a483e 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -12,6 +12,9 @@
 
 #ifndef __GAZELLE_LWIP_H__
 #define __GAZELLE_LWIP_H__
+#include <stdbool.h>
+
+#include "gazelle_dfx_msg.h"
 
 #define NETCONN_IS_ACCEPTIN(sock)   (((sock)->conn->acceptmbox != NULL) && !sys_mbox_empty((sock)->conn->acceptmbox))
 #define NETCONN_IS_DATAIN(sock)     ((gazelle_ring_readable_count((sock)->recv_ring) || (sock)->recv_lastdata) || (sock->same_node_rx_ring != NULL && same_node_ring_count(sock)))
@@ -24,32 +27,38 @@ struct rte_mempool;
 struct rpc_msg;
 struct rte_mbuf;
 struct protocol_stack;
-void create_shadow_fd(struct rpc_msg *msg);
-void gazelle_init_sock(int32_t fd);
-int32_t gazelle_socket(int domain, int type, int protocol);
-void gazelle_clean_sock(int32_t fd);
-struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags);
-void write_lwip_over(struct lwip_sock *sock);
-ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len,
-                         const struct sockaddr *addr, socklen_t addrlen);
-ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags, struct sockaddr *addr, socklen_t *addrlen);
-ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, uint8_t apiflags);
-void read_recv_list(struct protocol_stack *stack, uint32_t max_num);
+
+int32_t do_lwip_socket(int domain, int type, int protocol);
+void do_lwip_init_sock(int32_t fd);
+void do_lwip_clean_sock(int32_t fd);
+void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock);
+
+struct pbuf *do_lwip_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags);
+void do_lwip_get_from_sendring_over(struct lwip_sock *sock);
+bool do_lwip_replenish_sendring(struct protocol_stack *stack, struct lwip_sock *sock);
+ssize_t do_lwip_read_from_lwip(struct lwip_sock *sock, int32_t flags, uint8_t apiflags);
+
+/* app write/read ring */
+ssize_t do_lwip_sendmsg_to_stack(struct lwip_sock *sock, int32_t s,
+                                 const struct msghdr *message, int32_t flags);
+ssize_t do_lwip_recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags);
+
+ssize_t do_lwip_send_to_stack(int32_t fd, const void *buf, size_t len, int32_t flags,
+                              const struct sockaddr *addr, socklen_t addrlen);
+ssize_t do_lwip_read_from_stack(int32_t fd, void *buf, size_t len, int32_t flags,
+                                struct sockaddr *addr, socklen_t *addrlen);
+
+void do_lwip_read_recvlist(struct protocol_stack *stack, uint32_t max_num);
+void do_lwip_add_recvlist(int32_t fd);
+bool do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock,
+                  size_t len, int32_t flags);
+
+uint32_t do_lwip_get_conntable(struct gazelle_stat_lstack_conn_info *conn, uint32_t max_num);
+uint32_t do_lwip_get_connnum(void);
+
+void do_lwip_free_pbuf(struct pbuf *pbuf);
+struct pbuf *do_lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type);
+
 void read_same_node_recv_list(struct protocol_stack *stack);
-void send_stack_list(struct protocol_stack *stack, uint32_t send_max);
-void add_recv_list(int32_t fd);
-void get_lwip_conntable(struct rpc_msg *msg);
-void get_lwip_connnum(struct rpc_msg *msg);
-void stack_recvlist_count(struct rpc_msg *msg);
-void stack_send(struct rpc_msg *msg);
-void app_rpc_write(struct rpc_msg *msg);
-int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num);
-void gazelle_free_pbuf(struct pbuf *pbuf);
-ssize_t sendmsg_to_stack(struct lwip_sock *sock, int32_t s, const struct msghdr *message, int32_t flags);
-ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags);
-ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags,
-                     const struct sockaddr *addr, socklen_t addrlen);
-void rpc_replenish(struct rpc_msg *msg);
-void stack_mempool_size(struct rpc_msg *msg);
 
 #endif
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index a23ddff..f8455bd 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -161,5 +161,12 @@ void stack_getsockopt(struct rpc_msg *msg);
 void stack_setsockopt(struct rpc_msg *msg);
 void stack_fcntl(struct rpc_msg *msg);
 void stack_ioctl(struct rpc_msg *msg);
+void stack_send(struct rpc_msg *msg);
+void stack_mempool_size(struct rpc_msg *msg);
+void stack_create_shadow_fd(struct rpc_msg *msg);
+void stack_replenish_sendring(struct rpc_msg *msg);
+void stack_get_conntable(struct rpc_msg *msg);
+void stack_get_connnum(struct rpc_msg *msg);
+void stack_recvlist_count(struct rpc_msg *msg);
 void kni_handle_tx(struct rte_mbuf *mbuf);
 #endif
diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c
index e3e823b..30c3c9a 100644
--- a/src/lstack/netif/lstack_ethdev.c
+++ b/src/lstack/netif/lstack_ethdev.c
@@ -540,9 +540,9 @@ void parse_arp_and_transefer(char* buf)
     int32_t ret;
     for (int32_t i = 0; i < stack_group->stack_num; i++) {
         stack = stack_group->stacks[i];
-        ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
         while (ret != 0) {
-            ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
+            ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
             stack->stats.rx_allocmbuf_fail++;
         }
         copy_mbuf(mbuf_copy, mbuf);
@@ -569,9 +569,9 @@ void parse_tcp_and_transefer(char* buf)
     struct rte_mbuf *mbuf_copy = NULL;
     struct protocol_stack *stack = stack_group->stacks[stk_index];
 
-    int32_t ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
+    int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
     while (ret != 0) {
-        ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
         stack->stats.rx_allocmbuf_fail++;
     }
 
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index 4307f24..81b48dc 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -56,7 +56,7 @@ static uint32_t ltran_rx_poll(struct protocol_stack *stack, struct rte_mbuf **pk
     stack->rx_ring_used += rcvd_pkts;
     if (unlikely(stack->rx_ring_used >= USED_RX_PKTS_WATERMARK)) {
         uint32_t free_cnt = LWIP_MIN(stack->rx_ring_used, RING_SIZE(DPDK_PKT_BURST_SIZE));
-        int32_t ret = gazelle_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)free_buf, free_cnt);
+        int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)free_buf, free_cnt);
         if (likely(ret == 0)) {
             nr_pkts = gazelle_ring_sp_enqueue(stack->rx_ring, (void **)free_buf, free_cnt);
             stack->rx_ring_used -= nr_pkts;
-- 
2.27.0