From a0371ba4a351362d7a804d10ed1ee80884da0d85 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Sat, 7 Oct 2023 21:35:52 +0800
Subject: [PATCH] init: remove sync sem between lstack thread and main thread

---
 src/lstack/core/lstack_dpdk.c              |  14 ---
 src/lstack/core/lstack_init.c              |  26 +---
 src/lstack/core/lstack_protocol_stack.c    | 133 +++++++--------------
 src/lstack/include/lstack_protocol_stack.h |   7 +-
 4 files changed, 51 insertions(+), 129 deletions(-)

diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index cd42047..d848275 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -568,18 +568,6 @@ int32_t dpdk_ethdev_init(int port_id, bool bond_port)
     stack_group->nb_queues = nb_queues;
 
     if (get_global_cfg_params()->is_primary) {
-        for (uint32_t i = 0; i < stack_group->stack_num; i++) {
-            struct protocol_stack *stack = stack_group->stacks[i];
-            if (likely(stack)) {
-                    stack->port_id = stack_group->port_id;
-            } else {
-                LSTACK_LOG(ERR, LSTACK, "empty stack at stack_num %d\n", i);
-                stack_group->eth_params = NULL;
-                free(eth_params);
-                return -EINVAL;
-            }
-        }
-
         ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &eth_params->conf);
         if (ret < 0) {
             LSTACK_LOG(ERR, LSTACK, "cannot config eth dev at port %d: %s\n", port_id, rte_strerror(-ret));
@@ -760,8 +748,6 @@ int32_t init_dpdk_ethdev(void)
         }
     }
 
-    struct protocol_stack_group *stack_group = get_protocol_stack_group();
-    sem_post(&stack_group->ethdev_init);
     return 0;
 }
 
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index db203d2..2183e3a 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -48,18 +48,6 @@
 #include "lstack_protocol_stack.h"
 #include "lstack_preload.h"
 
-static volatile bool g_init_fail = false;
-
-void set_init_fail(void)
-{
-    g_init_fail = true;
-}
-
-bool get_init_fail(void)
-{
-    return g_init_fail;
-}
-
 static void check_process_start(void)
 {
     if (get_global_cfg_params()->is_primary) {
@@ -286,8 +274,8 @@ __attribute__((constructor)) void gazelle_network_init(void)
     lstack_log_level_init();
     lstack_prelog_uninit();
 
-    if (init_protocol_stack() != 0) {
-        LSTACK_EXIT(1, "init_protocol_stack failed\n");
+    if (stack_group_init() != 0) {
+        LSTACK_EXIT(1, "stack_group_init failed\n");
     }
 
     if (!use_ltran()) {
@@ -296,15 +284,13 @@ __attribute__((constructor)) void gazelle_network_init(void)
         }
     }
 
+    if (stack_thread_setup() != 0) {
+        LSTACK_EXIT(1, "stack_init_in_setup failed\n");
+    }
+
     /* lwip initialization */
     lwip_sock_init();
 
-    /* wait stack thread and kernel_event thread init finish */
-    wait_sem_value(&get_protocol_stack_group()->all_init, get_protocol_stack_group()->stack_num);
-    if (g_init_fail) {
-        LSTACK_EXIT(1, "stack thread or kernel_event thread failed\n");
-    }
-
     if (get_global_cfg_params()->kni_switch) {
         set_kni_ip_mac();
     }
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index ca0b046..8071bda 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -43,8 +43,6 @@
 static PER_THREAD struct protocol_stack *g_stack_p = NULL;
 static struct protocol_stack_group g_stack_group = {0};
 
-void set_init_fail(void);
-bool get_init_fail(void);
 typedef void *(*stack_thread_func)(void *arg);
 
 
@@ -286,12 +284,11 @@ static void* gazelle_kernelevent_thread(void *arg)
     struct thread_params *t_params = (struct thread_params*) arg;
     uint16_t idx = t_params->idx;
     struct protocol_stack *stack = get_protocol_stack_group()->stacks[idx];
-    struct protocol_stack_group *stack_group = get_protocol_stack_group();
 
-    sem_post(&stack_group->thread_phase1);
     bind_to_stack_numa(stack);
 
     LSTACK_LOG(INFO, LSTACK, "kernelevent_%02hu start\n", idx);
+    free(arg);
 
     for (;;) {
         stack->kernel_event_num = posix_api->epoll_wait_fn(stack->epollfd, stack->kernel_events, KERNEL_EPOLL_MAX, -1);
@@ -311,6 +308,7 @@ static int32_t init_stack_value(struct protocol_stack *stack, void *arg)
 
     stack->tid = rte_gettid();
     stack->queue_id = t_params->queue_id;
+    stack->port_id = stack_group->port_id;
     stack->stack_idx = t_params->idx;
     stack->lwip_stats = &lwip_stats;
 
@@ -370,7 +368,12 @@ void wait_sem_value(sem_t *sem, int32_t wait_value)
 
 static int32_t create_affiliate_thread(void *arg)
 {
-    if (create_thread(arg, "gazellekernel", gazelle_kernelevent_thread) != 0) {
+    struct thread_params *params = malloc(sizeof(struct thread_params));
+    if (params == NULL) {
+        return -1;
+    }
+    memcpy_s(params, sizeof(*params), arg, sizeof(struct thread_params));
+    if (create_thread((void *)params, "gazellekernel", gazelle_kernelevent_thread) != 0) {
         LSTACK_LOG(ERR, LSTACK, "gazellekernel errno=%d\n", errno);
         return -1;
     }
@@ -380,60 +383,49 @@ static int32_t create_affiliate_thread(void *arg)
 
 static struct protocol_stack *stack_thread_init(void *arg)
 {
-    struct protocol_stack_group *stack_group = get_protocol_stack_group();
     struct protocol_stack *stack = calloc(1, sizeof(*stack));
     if (stack == NULL) {
         LSTACK_LOG(ERR, LSTACK, "malloc stack failed\n");
-        goto END2;
+        goto END;
     }
 
     if (init_stack_value(stack, arg) != 0) {
-        goto END2;
+        goto END;
     }
 
     if (init_stack_numa_cpuset(stack) < 0) {
-        goto END2;
+        goto END;
     }
     if (create_affiliate_thread(arg) < 0) {
-        goto END2;
+        goto END;
     }
 
     if (thread_affinity_init(stack->cpu_id) != 0) {
-        goto END1;
+        goto END;
     }
     RTE_PER_LCORE(_lcore_id) = stack->cpu_id;
 
     if (hugepage_init() != 0) {
         LSTACK_LOG(ERR, LSTACK, "hugepage init failed\n");
-        goto END1;
+        goto END;
     }
 
     tcpip_init(NULL, NULL);
 
     if (use_ltran()) {
         if (client_reg_thrd_ring() != 0) {
-            goto END1;
+            goto END;
         }
     }
 
-    sem_post(&stack_group->thread_phase1);
-
-    if (!use_ltran()) {
-        wait_sem_value(&stack_group->ethdev_init, 1);
-    }
-
     usleep(SLEEP_US_BEFORE_LINK_UP);
 
     if (ethdev_init(stack) != 0) {
-        goto END1;
+        goto END;
     }
 
     return stack;
-/* kernel event thread dont create, stack thread post sem twice */
-END2:
-    sem_post(&stack_group->thread_phase1);
-END1:
-    sem_post(&stack_group->thread_phase1);
+END:
     if (stack != NULL) {
         free(stack);
     }
@@ -453,23 +445,19 @@ static void* gazelle_stack_thread(void *arg)
     uint32_t rpc_number = cfg->rpc_number;
     uint32_t nic_read_number = cfg->nic_read_number;
     uint32_t wakeup_tick = 0;
-    struct protocol_stack_group *stack_group = get_protocol_stack_group();
 
     struct protocol_stack *stack = stack_thread_init(arg);
 
+    free(arg);
     if (stack == NULL) {
-        /* exit in main thread, avoid create mempool and exit at the same time */
-        set_init_fail();
-        sem_post(&stack_group->all_init);
         LSTACK_LOG(ERR, LSTACK, "stack_thread_init failed queue_id=%hu\n", queue_id);
-        return NULL;
+        /* exit in signal thread */
+        raise(SIGTERM);
     }
     if (!use_ltran() && queue_id == 0) {
         init_listen_and_user_ports();
     }
 
-    sem_post(&stack_group->all_init);
-
     LSTACK_LOG(INFO, LSTACK, "stack_%02hu init success\n", queue_id);
 
     for (;;) {
@@ -517,40 +505,9 @@ static void libnet_listen_thread(void *arg)
     recv_pkts_from_other_process(cfg_param->process_idx, arg);
 }
 
-static int32_t init_protocol_sem(void)
+int32_t stack_group_init(void)
 {
-    int32_t ret;
     struct protocol_stack_group *stack_group = get_protocol_stack_group();
-
-    if (!use_ltran()) {
-        ret = sem_init(&stack_group->ethdev_init, 0, 0);
-        if (ret < 0) {
-            LSTACK_LOG(ERR, PORT, "sem_init failed ret=%d errno=%d\n", ret, errno);
-            return -1;
-        }
-    }
-
-    ret = sem_init(&stack_group->thread_phase1, 0, 0);
-    if (ret < 0) {
-        LSTACK_LOG(ERR, PORT, "sem_init failed ret=%d errno=%d\n", ret, errno);
-        return -1;
-    }
-
-    ret = sem_init(&stack_group->all_init, 0, 0);
-    if (ret < 0) {
-        LSTACK_LOG(ERR, PORT, "sem_init failed ret=%d errno=%d\n", ret, errno);
-        return -1;
-    }
-
-    return 0;
-}
-
-int32_t init_protocol_stack(void)
-{
-    struct protocol_stack_group *stack_group = get_protocol_stack_group();
-    int32_t ret;
-    char name[PATH_MAX];
-
     if (!get_global_cfg_params()->seperate_send_recv) {
         stack_group->stack_num = get_global_cfg_params()->num_cpu;
     } else {
@@ -560,13 +517,6 @@ int32_t init_protocol_stack(void)
     init_list_node(&stack_group->poll_list);
     pthread_spin_init(&stack_group->poll_list_lock, PTHREAD_PROCESS_PRIVATE);
     pthread_spin_init(&stack_group->socket_lock, PTHREAD_PROCESS_PRIVATE);
-    
-    if (init_protocol_sem() != 0) {
-        return -1;
-    }
-    int queue_num = get_global_cfg_params()->num_queue;
-    struct thread_params *t_params[queue_num];
-    int process_index = get_global_cfg_params()->process_idx;
 
     if (get_global_cfg_params()->is_primary) {
         uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count;
@@ -580,6 +530,27 @@ int32_t init_protocol_stack(void)
         }
     }
 
+    if (!use_ltran()) {
+        char name[PATH_MAX];
+        sem_init(&stack_group->sem_listen_thread, 0, 0);
+        sprintf_s(name, sizeof(name), "%s", "listen_thread");
+        struct sys_thread *thread = sys_thread_new(name, libnet_listen_thread,
+            (void*)(&stack_group->sem_listen_thread), 0, 0);
+        free(thread);
+        sem_wait(&stack_group->sem_listen_thread);
+    }
+
+    return 0;
+}
+
+int32_t stack_thread_setup(void)
+{
+    int32_t ret;
+    char name[PATH_MAX];
+    int queue_num = get_global_cfg_params()->num_queue;
+    struct thread_params *t_params[queue_num];
+    int process_index = get_global_cfg_params()->process_idx;
+
     for (uint32_t i = 0; i < queue_num; i++) {
         if (get_global_cfg_params()->seperate_send_recv) {
             if (i % 2 == 0) {
@@ -610,26 +581,6 @@ int32_t init_protocol_stack(void)
         }
     }
 
-    /* stack_num * 2: stack thread and kernel event thread will post sem */
-    wait_sem_value(&stack_group->thread_phase1, stack_group->stack_num * 2);
-
-    for (int idx = 0; idx < queue_num; idx++){
-        free(t_params[idx]);
-    }
-
-    if (!use_ltran()) {
-        ret = sem_init(&stack_group->sem_listen_thread, 0, 0);
-        ret = sprintf_s(name, sizeof(name), "%s", "listen_thread");
-        struct sys_thread *thread = sys_thread_new(name, libnet_listen_thread,
-            (void*)(&stack_group->sem_listen_thread), 0, 0);
-        free(thread);
-        sem_wait(&stack_group->sem_listen_thread);
-    }
-
-    if (get_init_fail()) {
-        return -1;
-    }
-
     return 0;
 }
 
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index a23ddff..2d723d4 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -85,9 +85,6 @@ struct eth_params;
 struct protocol_stack_group {
     uint16_t stack_num;
     uint16_t port_id;
-    sem_t thread_phase1;
-    sem_t ethdev_init;
-    sem_t all_init;
     uint64_t rx_offload;
     uint64_t tx_offload;
     uint32_t reta_mask;
@@ -112,7 +109,9 @@ struct protocol_stack *get_protocol_stack_by_fd(int32_t fd);
 struct protocol_stack *get_bind_protocol_stack(void);
 struct protocol_stack_group *get_protocol_stack_group(void);
 
-int32_t init_protocol_stack(void);
+int32_t stack_group_init(void);
+int32_t stack_thread_setup(void);
+
 void bind_to_stack_numa(struct protocol_stack *stack);
 int32_t init_dpdk_ethdev(void);
 
-- 
2.27.0