From 95c0a884ff26b42a75ee35639d789b40af131fd3 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Tue, 12 Dec 2023 19:24:14 +0800
Subject: [PATCH] optimize gazelle exit process 1. close all fds 2. lstack
 thread exits, then gazelle process exits

---
 src/common/gazelle_base_func.h             |  2 +
 src/common/gazelle_dfx_msg.c               |  8 +-
 src/lstack/api/dir.mk                      |  2 +-
 src/lstack/api/lstack_dummy_api.c          | 55 +++++++++++++
 src/lstack/api/lstack_rtc_api.c            |  1 +
 src/lstack/api/lstack_signal.c             |  5 +-
 src/lstack/api/lstack_wrap.c               | 12 +++
 src/lstack/core/lstack_dpdk.c              |  2 +-
 src/lstack/core/lstack_init.c              | 18 ++++-
 src/lstack/core/lstack_lwip.c              | 16 ++--
 src/lstack/core/lstack_protocol_stack.c    | 93 ++++++++++++++++++----
 src/lstack/core/lstack_thread_rpc.c        | 22 ++++-
 src/lstack/include/lstack_dummy_api.h      | 23 ++++++
 src/lstack/include/lstack_lwip.h           |  4 +-
 src/lstack/include/lstack_protocol_stack.h |  6 +-
 src/lstack/include/lstack_thread_rpc.h     |  3 +-
 src/lstack/include/lstack_wrap.h           |  1 +
 17 files changed, 237 insertions(+), 36 deletions(-)
 create mode 100644 src/lstack/api/lstack_dummy_api.c
 create mode 100644 src/lstack/include/lstack_dummy_api.h

diff --git a/src/common/gazelle_base_func.h b/src/common/gazelle_base_func.h
index d21ef5f..2d629c1 100644
--- a/src/common/gazelle_base_func.h
+++ b/src/common/gazelle_base_func.h
@@ -34,6 +34,8 @@ int32_t check_and_set_run_dir(void);
 
 int32_t filename_check(const char* args);
 
+void gazelle_exit(void);
+
 #undef container_of
 #define container_of(ptr, type, field) ((type *)(void*)(((char *)(ptr)) - offsetof(type, field)))
 
diff --git a/src/common/gazelle_dfx_msg.c b/src/common/gazelle_dfx_msg.c
index 5fe4e06..ec23401 100644
--- a/src/common/gazelle_dfx_msg.c
+++ b/src/common/gazelle_dfx_msg.c
@@ -29,19 +29,19 @@ int read_specied_len(int fd, char *buf, size_t target_size)
     while (total_read < target_size) {
         int ret = poll(fds, 1, GAZELLECTL_TIMEOUT);
         if (ret < 0) {
-            printf("read_specied_len:: poll ret=%d \n", ret);
+            printf("read_specied_len: poll ret=%d \n", ret);
             return -1;
         } else if (ret == 0) {
-            printf("read_specied_len:: time out \n");
+            printf("read_specied_len: time out \n");
             return -1;
         }
         if (fds[0].revents & POLLIN) {
             int n = read(fd, buf + total_read, target_size - total_read);
             if (n < 0) {
-                printf("read_specied_len:: read ret=%d \n", ret);
+                printf("read_specied_len: read ret=%d \n", ret);
                 return -1;
             } else if (n == 0) {
-                printf("read_specied_len:: Connection closed \n");
+                printf("read_specied_len: Connection closed \n");
                 return -1;
             }
             total_read += n;
diff --git a/src/lstack/api/dir.mk b/src/lstack/api/dir.mk
index ffbb137..729690d 100644
--- a/src/lstack/api/dir.mk
+++ b/src/lstack/api/dir.mk
@@ -8,7 +8,7 @@
 # PURPOSE.
 # See the Mulan PSL v2 for more details.
 
-SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c
+SRC = lstack_epoll.c lstack_signal.c lstack_fork.c lstack_wrap.c lstack_rtw_api.c lstack_rtc_api.c lstack_dummy_api.c
 
 $(eval $(call register_dir, api, $(SRC)))
 
diff --git a/src/lstack/api/lstack_dummy_api.c b/src/lstack/api/lstack_dummy_api.c
new file mode 100644
index 0000000..f327916
--- /dev/null
+++ b/src/lstack/api/lstack_dummy_api.c
@@ -0,0 +1,55 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+#include <sys/socket.h>
+#include <unistd.h>
+#include <errno.h>
+
+#define DUMMY_SLEEP_S 5
+
+static inline ssize_t dummy_exit(void)
+{
+    sleep(DUMMY_SLEEP_S);
+    errno = ENOTCONN;
+    return -1;
+}
+
+int dummy_socket(int domain, int type, int protocol)
+{
+    sleep(DUMMY_SLEEP_S);
+    return -1;
+}
+
+ssize_t dummy_write(int s, const void *mem, size_t size)
+{
+    return dummy_exit();
+}
+
+ssize_t dummy_writev(int s, const struct iovec *iov, int iovcnt)
+{
+    return dummy_exit();
+}
+
+ssize_t dummy_send(int sockfd, const void *buf, size_t len, int flags)
+{
+    return dummy_exit();
+}
+
+ssize_t dummy_sendmsg(int s, const struct msghdr *message, int flags)
+{
+    return dummy_exit();
+}
+
+ssize_t dummy_sendto(int sockfd, const void *buf, size_t len, int flags,
+                     const struct sockaddr *addr, socklen_t addrlen)
+{
+    return dummy_exit();
+}
diff --git a/src/lstack/api/lstack_rtc_api.c b/src/lstack/api/lstack_rtc_api.c
index 50d72bc..d29e51e 100644
--- a/src/lstack/api/lstack_rtc_api.c
+++ b/src/lstack/api/lstack_rtc_api.c
@@ -20,6 +20,7 @@
 #include "lstack_log.h"
 #include "lstack_cfg.h"
 #include "lstack_protocol_stack.h"
+#include "lstack_thread_rpc.h"
 #include "lstack_rtc_api.h"
 
 int rtc_poll(struct pollfd *fds, nfds_t nfds, int timeout)
diff --git a/src/lstack/api/lstack_signal.c b/src/lstack/api/lstack_signal.c
index 285aaf3..6da6eb4 100644
--- a/src/lstack/api/lstack_signal.c
+++ b/src/lstack/api/lstack_signal.c
@@ -18,6 +18,7 @@
 #include <lwip/lwipsock.h>
 #include <lwip/posix_api.h>
 
+#include "gazelle_base_func.h"
 #include "lstack_cfg.h"
 #include "dpdk_common.h"
 #include "lstack_log.h"
@@ -60,15 +61,13 @@ static void lstack_sig_default_handler(int sig)
     if (get_global_cfg_params() && get_global_cfg_params()->is_primary) {
         delete_primary_path();
     }
-    if (!use_ltran()) {
-        dpdk_kni_release();
-    }
     control_fd_close();
     /* When operations such as pressing Ctrl+C or Kill, the call stack exit is not displayed. */
     if (sig != SIGINT && sig != SIGTERM && sig != SIGKILL) {
         dump_stack();
     }
     lwip_exit();
+    gazelle_exit();
     (void)kill(getpid(), sig);
 }
 
diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index 65a0a5a..89f54f8 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -36,6 +36,7 @@
 
 #include "lstack_rtc_api.h"
 #include "lstack_rtw_api.h"
+#include "lstack_dummy_api.h"
 
 #ifndef SOCK_TYPE_MASK
 #define SOCK_TYPE_MASK 0xf
@@ -112,6 +113,17 @@ void wrap_api_init(void)
     }
 }
 
+void wrap_api_set_dummy(void)
+{
+    g_wrap_api->socket_fn  = dummy_socket;
+    g_wrap_api->send_fn    = dummy_send;
+    g_wrap_api->write_fn   = dummy_write;
+    g_wrap_api->writev_fn  = dummy_writev;
+    g_wrap_api->send_msg   = dummy_sendmsg;
+    g_wrap_api->send_to    = dummy_sendto;
+    rte_wmb();
+}
+
 static inline int32_t do_epoll_create1(int32_t flags)
 {
     if (select_posix_path() == PATH_KERNEL) {
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 8950591..e20dea8 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -897,4 +897,4 @@ void dpdk_nic_features_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id)
     dfx->data.nic_features.tx_offload = dev_conf.txmode.offloads;
     dfx->data.nic_features.rx_offload = dev_conf.rxmode.offloads;
     return;
-}
\ No newline at end of file
+}
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index a3ca4ff..fef2942 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -107,10 +107,24 @@ static int32_t check_process_conflict(void)
     return 0;
 }
 
+void gazelle_exit(void)
+{
+    if (!get_global_cfg_params()->stack_mode_rtc) {
+        wrap_api_set_dummy();
+        /* 1: wait until app thread call send functio complete */
+        sleep(1);
+        stack_group_exit();
+    }
+    if (!use_ltran()) {
+        dpdk_kni_release();
+    }
+}
+
 __attribute__((destructor)) void gazelle_network_exit(void)
 {
     if (posix_api != NULL && !posix_api->ues_posix) {
         lwip_exit();
+        gazelle_exit();
     }
 
     if (!use_ltran()) {
@@ -118,8 +132,6 @@ __attribute__((destructor)) void gazelle_network_exit(void)
         if (ret < 0) {
             LSTACK_LOG(ERR, LSTACK, "rte_pdump_uninit failed\n");
         }
-
-        dpdk_kni_release();
     }
 }
 
@@ -289,6 +301,7 @@ __attribute__((constructor)) void gazelle_network_init(void)
 
     if (!get_global_cfg_params()->stack_mode_rtc) {
         if (stack_setup_thread() != 0) {
+            gazelle_exit();
             LSTACK_EXIT(1, "stack_setup_thread failed\n");
         }
     }
@@ -301,6 +314,7 @@ __attribute__((constructor)) void gazelle_network_init(void)
     }
 
     if (set_process_start_flag() != 0) {
+        gazelle_exit();
         LSTACK_EXIT(1, "set_process_start_flag failed\n");
     }
 
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index fb286d6..15f99f9 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -201,7 +201,7 @@ void do_lwip_init_sock(int32_t fd)
     init_list_node_null(&sock->event_list);
 }
 
-void do_lwip_clean_sock(int32_t fd)
+void do_lwip_clean_sock(int fd)
 {
     struct lwip_sock *sock = get_socket_by_fd(fd);
     if (sock == NULL || sock->stack == NULL) {
@@ -1193,7 +1193,15 @@ void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_soc
     }
 }
 
-int32_t do_lwip_socket(int domain, int type, int protocol)
+int do_lwip_close(int fd)
+{
+    int ret = lwip_close(fd);
+    do_lwip_clean_sock(fd);
+    posix_api->close_fn(fd);
+    return ret;
+}
+
+int do_lwip_socket(int domain, int type, int protocol)
 {
     int32_t fd = lwip_socket(domain, type, 0);
     if (fd < 0) {
@@ -1204,9 +1212,7 @@ int32_t do_lwip_socket(int domain, int type, int protocol)
 
     struct lwip_sock *sock = get_socket(fd);
     if (sock == NULL || sock->stack == NULL) {
-        lwip_close(fd);
-        do_lwip_clean_sock(fd);
-        posix_api->close_fn(fd);
+        do_lwip_close(fd);
         return -1;
     }
 
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 8dbd9ad..3123ca3 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -45,6 +45,22 @@ static struct protocol_stack_group g_stack_group = {0};
 
 typedef void *(*stack_thread_func)(void *arg);
 
+static void stack_set_state(struct protocol_stack *stack, enum rte_lcore_state_t state)
+{
+    __atomic_store_n(&stack->state, state, __ATOMIC_RELEASE);
+}
+
+static enum rte_lcore_state_t stack_get_state(struct protocol_stack *stack)
+{
+    return __atomic_load_n(&stack->state, __ATOMIC_ACQUIRE);
+}
+
+static void stack_wait_quit(struct protocol_stack *stack)
+{
+    while (__atomic_load_n(&stack->state, __ATOMIC_ACQUIRE) != WAIT) {
+        rte_pause();
+    }
+}
 
 void bind_to_stack_numa(struct protocol_stack *stack)
 {
@@ -436,8 +452,9 @@ END:
     return NULL;
 }
 
-void stack_polling(uint32_t wakeup_tick)
+int stack_polling(uint32_t wakeup_tick)
 {
+    int force_quit;
     struct cfg_params *cfg = get_global_cfg_params();
     uint8_t use_ltran_flag = cfg->use_ltran;
     bool kni_switch = cfg->kni_switch;
@@ -448,7 +465,7 @@ void stack_polling(uint32_t wakeup_tick)
     uint32_t read_connect_number = cfg->read_connect_number;
     struct protocol_stack *stack = get_protocol_stack();
 
-    poll_rpc_msg(stack, rpc_number);
+    force_quit = poll_rpc_msg(stack, rpc_number);
     gazelle_eth_dev_poll(stack, use_ltran_flag, nic_read_number);
     sys_timer_run();
     if (cfg->low_power_mod != 0) {
@@ -456,7 +473,7 @@ void stack_polling(uint32_t wakeup_tick)
     }
 
     if (stack_mode_rtc) {
-        return;
+        return force_quit;
     }
 
     do_lwip_read_recvlist(stack, read_connect_number);
@@ -482,7 +499,7 @@ void stack_polling(uint32_t wakeup_tick)
             kni_handle_rx(stack->port_id);
         }
     }
-    return;
+    return force_quit;
 }
 
 static void* gazelle_stack_thread(void *arg)
@@ -512,11 +529,14 @@ static void* gazelle_stack_thread(void *arg)
         return NULL;
     }
 
-    for (;;) {
-        stack_polling(wakeup_tick);
+    stack_set_state(stack, RUNNING);
+
+    while (stack_polling(wakeup_tick) == 0) {
         wakeup_tick++;
     }
 
+    stack_set_state(stack, WAIT);
+
     return NULL;
 }
 
@@ -576,6 +596,7 @@ int32_t stack_group_init(void)
         LSTACK_LOG(ERR, LSTACK, "sem_init failed errno=%d\n", errno);
         return -1;
     }
+
     stack_group->stack_setup_fail = 0;
 
     if (get_global_cfg_params()->is_primary) {
@@ -705,14 +726,10 @@ void stack_close(struct rpc_msg *msg)
         return;
     }
     
-    msg->result = lwip_close(fd);
+    msg->result = do_lwip_close(fd);
     if (msg->result != 0) {
         LSTACK_LOG(ERR, LSTACK, "tid %ld, fd %d failed %ld\n", get_stack_tid(), msg->args[MSG_ARG_0].i, msg->result);
     }
-
-    do_lwip_clean_sock(fd);
-
-    posix_api->close_fn(fd);
 }
 
 void stack_shutdown(struct rpc_msg *msg)
@@ -775,9 +792,7 @@ void stack_accept(struct rpc_msg *msg)
 
     struct lwip_sock *sock = get_socket(accept_fd);
     if (sock == NULL || sock->stack == NULL) {
-        lwip_close(accept_fd);
-        do_lwip_clean_sock(accept_fd);
-        posix_api->close_fn(accept_fd);
+        do_lwip_close(accept_fd);
         LSTACK_LOG(ERR, LSTACK, "fd %d ret %d\n", fd, accept_fd);
         return;
     }
@@ -866,7 +881,6 @@ void stack_send(struct rpc_msg *msg)
     if (sock == NULL) {
         msg->result = -1;
         LSTACK_LOG(ERR, LSTACK, "get sock error! fd=%d, len=%ld\n", fd, len);
-        __sync_fetch_and_sub(&sock->call_num, 1);
         return;
     }
 
@@ -1260,3 +1274,52 @@ int32_t stack_broadcast_accept(int32_t fd, struct sockaddr *addr, socklen_t *add
     return stack_broadcast_accept4(fd, addr, addrlen, 0);
 }
 
+static void stack_all_fds_close(void)
+{
+    for (int i = 3; i < GAZELLE_MAX_CLIENTS + GAZELLE_RESERVED_CLIENTS; i++) {
+        struct lwip_sock *sock = get_socket(i);
+        if (sock && sock->stack == get_protocol_stack()) {
+            do_lwip_close(i);
+        }
+    }
+}
+
+static void stack_exit(void)
+{
+    stack_all_fds_close();
+}
+
+void stack_exit_by_rpc(struct rpc_msg *msg)
+{
+    stack_exit();
+}
+
+void stack_group_exit(void)
+{
+    int i;
+    struct protocol_stack_group *stack_group = get_protocol_stack_group();
+    struct protocol_stack *stack = get_protocol_stack();
+
+    for (i = 0; i < stack_group->stack_num; i++) {
+        if ((stack_group->stacks[i] == NULL) ||
+            stack_get_state(stack_group->stacks[i]) != RUNNING) {
+            continue;
+        }
+
+        if (stack != stack_group->stacks[i]) {
+            rpc_call_stack_exit(stack_group->stacks[i]);
+        }
+    }
+
+    if (stack != NULL) {
+        stack_exit();
+    }
+
+    for (i = 0; i < stack_group->stack_num; i++) {
+        if (stack_group->stacks[i] == NULL || stack == stack_group->stacks[i]) {
+            continue;
+        }
+        /* wait stack thread quit */
+        stack_wait_quit(stack_group->stacks[i]);
+    }
+}
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 0b2a62a..2af30d7 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -90,8 +90,9 @@ static inline __attribute__((always_inline)) int32_t rpc_sync_call(lockless_queu
     return ret;
 }
 
-void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
+int poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
 {
+    int force_quit = 0;
     struct rpc_msg *msg = NULL;
 
     while (max_num--) {
@@ -108,6 +109,10 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
             stack->stats.call_null++;
         }
 
+        if (msg->func == stack_exit_by_rpc) {
+            force_quit = 1;
+        }
+
         if (!msg->recall_flag) {
             if (msg->sync_flag) {
                 pthread_spin_unlock(&msg->lock);
@@ -118,6 +123,8 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
             msg->recall_flag = 0;
         }
     }
+
+    return force_quit;
 }
 
 int32_t rpc_call_conntable(struct protocol_stack *stack, void *conn_table, uint32_t max_conn)
@@ -246,6 +253,7 @@ int32_t rpc_call_arp(struct protocol_stack *stack, struct rte_mbuf *mbuf)
 int32_t rpc_call_socket(int32_t domain, int32_t type, int32_t protocol)
 {
     struct protocol_stack *stack = get_bind_protocol_stack();
+
     struct rpc_msg *msg = rpc_msg_alloc(stack, stack_socket);
     if (msg == NULL) {
         return -1;
@@ -271,6 +279,18 @@ int32_t rpc_call_close(int fd)
     return rpc_sync_call(&stack->rpc_queue, msg);
 }
 
+int32_t rpc_call_stack_exit(struct protocol_stack *stack)
+{
+    struct rpc_msg *msg = rpc_msg_alloc(stack, stack_exit_by_rpc);
+    if (msg == NULL) {
+        LSTACK_LOG(INFO, LSTACK, "rpc msg alloc failed\n");
+        return -1;
+    }
+
+    rpc_call(&stack->rpc_queue, msg);
+    return 0;
+}
+
 int32_t rpc_call_shutdown(int fd, int how)
 {
     struct protocol_stack *stack = get_protocol_stack_by_fd(fd);
diff --git a/src/lstack/include/lstack_dummy_api.h b/src/lstack/include/lstack_dummy_api.h
new file mode 100644
index 0000000..48bce31
--- /dev/null
+++ b/src/lstack/include/lstack_dummy_api.h
@@ -0,0 +1,23 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef _LSTACK_DUMMY_API_H_
+#define _LSTACK_DUMMY_API_H_
+
+int dummy_socket(int domain, int type, int protocol);
+ssize_t dummy_write(int s, const void *mem, size_t size);
+ssize_t dummy_writev(int s, const struct iovec *iov, int iovcnt);
+ssize_t dummy_sendmsg(int s, const struct msghdr *message, int flags);
+ssize_t dummy_send(int sockfd, const void *buf, size_t len, int flags);
+ssize_t dummy_sendto(int sockfd, const void *buf, size_t len, int flags,
+                     const struct sockaddr *addr, socklen_t addrlen);
+#endif /* __LSTACK_DUMMY_API_H_  */
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index a11489c..0a82781 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -28,9 +28,9 @@ struct rpc_msg;
 struct rte_mbuf;
 struct protocol_stack;
 
-int32_t do_lwip_socket(int domain, int type, int protocol);
+int do_lwip_socket(int domain, int type, int protocol);
+int do_lwip_close(int32_t fd);
 void do_lwip_init_sock(int32_t fd);
-void do_lwip_clean_sock(int32_t fd);
 void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock);
 
 struct pbuf *do_lwip_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 6638984..8e2e807 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -46,6 +46,7 @@ struct protocol_stack {
     uint32_t stack_idx;
     cpu_set_t idle_cpuset; /* idle cpu in numa of stack, app thread bind to it */
     int32_t epollfd; /* kernel event thread epoll fd */
+    volatile enum rte_lcore_state_t state;
 
     struct rte_mempool *rxtx_mbuf_pool;
     struct rte_ring  *rx_ring;
@@ -114,6 +115,7 @@ struct protocol_stack *get_bind_protocol_stack(void);
 struct protocol_stack_group *get_protocol_stack_group(void);
 
 int32_t stack_group_init(void);
+void stack_group_exit(void);
 int32_t stack_setup_thread(void);
 int32_t stack_setup_app_thread(void);
 
@@ -176,6 +178,8 @@ void stack_replenish_sendring(struct rpc_msg *msg);
 void stack_get_conntable(struct rpc_msg *msg);
 void stack_get_connnum(struct rpc_msg *msg);
 void stack_recvlist_count(struct rpc_msg *msg);
-void stack_polling(uint32_t wakeup_tick);
+void stack_exit_by_rpc(struct rpc_msg *msg);
+
+int stack_polling(uint32_t wakeup_tick);
 void kni_handle_tx(struct rte_mbuf *mbuf);
 #endif
diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h
index ca8a510..633ef93 100644
--- a/src/lstack/include/lstack_thread_rpc.h
+++ b/src/lstack/include/lstack_thread_rpc.h
@@ -62,7 +62,7 @@ struct protocol_stack;
 struct rte_mbuf;
 struct wakeup_poll;
 struct lwip_sock;
-void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num);
+int poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num);
 void rpc_call_clean_epoll(struct protocol_stack *stack, struct wakeup_poll *wakeup);
 int32_t rpc_call_msgcnt(struct protocol_stack *stack);
 int32_t rpc_call_shadow_fd(struct protocol_stack *stack, int32_t fd, const struct sockaddr *addr, socklen_t addrlen);
@@ -89,6 +89,7 @@ int32_t rpc_call_ioctl(int fd, long cmd, void *argp);
 int32_t rpc_call_replenish(struct protocol_stack *stack, struct lwip_sock *sock);
 int32_t rpc_call_mbufpoolsize(struct protocol_stack *stack);
 int32_t rpc_call_rpcpool_size(struct protocol_stack *stack);
+int32_t rpc_call_stack_exit(struct protocol_stack *stack);
 
 static inline __attribute__((always_inline)) void rpc_call(lockless_queue *queue, struct rpc_msg *msg)
 {
diff --git a/src/lstack/include/lstack_wrap.h b/src/lstack/include/lstack_wrap.h
index 80e5f3c..dab5222 100644
--- a/src/lstack/include/lstack_wrap.h
+++ b/src/lstack/include/lstack_wrap.h
@@ -14,6 +14,7 @@
 #define _LSTACK_WRAP_H_
 
 void wrap_api_init(void);
+void wrap_api_set_dummy(void);
 
 #endif
 
-- 
2.27.0