From 76dc40bf3d8730e4155c142002bc4a27325672a8 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Mon, 9 Oct 2023 10:18:20 +0800
Subject: [PATCH] init: stack setup in app thread when app call
 socket/epoll_create first in rtc mode

---
 src/lstack/api/lstack_rtc_api.c            | 14 ++++++-
 src/lstack/core/lstack_init.c              |  6 ++-
 src/lstack/core/lstack_protocol_stack.c    | 43 ++++++++++++++++++++--
 src/lstack/include/lstack_protocol_stack.h |  3 +-
 4 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/src/lstack/api/lstack_rtc_api.c b/src/lstack/api/lstack_rtc_api.c
index b7c6380..276bec0 100644
--- a/src/lstack/api/lstack_rtc_api.c
+++ b/src/lstack/api/lstack_rtc_api.c
@@ -35,6 +35,10 @@ int rtc_epoll_wait(int epfd, struct epoll_event* events, int maxevents, int time
 int rtc_socket(int domain, int type, int protocol)
 {
     int ret;
+
+    if (stack_setup_app_thread() < 0) {
+        LSTACK_EXIT(1, "stack_setup_app_thread failed!\n");
+    }
     
     /* need call stack thread init function */
     ret = lwip_socket(domain, type, protocol);
@@ -56,13 +60,19 @@ int rtc_close(int s)
 
 int rtc_epoll_create(int flags)
 {
-    /* need call stack thread init function */
+    if (stack_setup_app_thread() < 0) {
+        LSTACK_EXIT(1, "stack_setup_app_thread failed!\n");
+    }
+
     return lstack_epoll_create(flags);
 }
 
 int rtc_epoll_create1(int flags)
 {
-    /* need call stack thread init function */
+    if (stack_setup_app_thread() < 0) {
+        LSTACK_EXIT(1, "stack_setup_app_thread failed!\n");
+    }
+
     return lstack_epoll_create1(flags);
 }
 
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index 950fa8d..a3ca4ff 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -287,8 +287,10 @@ __attribute__((constructor)) void gazelle_network_init(void)
         }
     }
 
-    if (stack_thread_setup() != 0) {
-        LSTACK_EXIT(1, "stack_thread_setup failed\n");
+    if (!get_global_cfg_params()->stack_mode_rtc) {
+        if (stack_setup_thread() != 0) {
+            LSTACK_EXIT(1, "stack_setup_thread failed\n");
+        }
     }
 
     /* lwip initialization */
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index ea85bc1..69897c7 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -501,6 +501,9 @@ static void* gazelle_stack_thread(void *arg)
     }
 
     LSTACK_LOG(INFO, LSTACK, "stack_%02hu init success\n", queue_id);
+    if (get_global_cfg_params()->stack_mode_rtc) {
+        return NULL;
+    }
 
     for (;;) {
         stack_polling(wakeup_tick);
@@ -510,7 +513,7 @@ static void* gazelle_stack_thread(void *arg)
     return NULL;
 }
 
-static void libnet_listen_thread(void *arg)
+static void gazelle_listen_thread(void *arg)
 {
     struct cfg_params *cfg_param = get_global_cfg_params();
     recv_pkts_from_other_process(cfg_param->process_idx, arg);
@@ -541,11 +544,12 @@ int32_t stack_group_init(void)
         }
     }
 
-    if (!use_ltran()) {
+    /* run to completion mode does not currently support multiple process */
+    if (!use_ltran() && !get_global_cfg_params()->stack_mode_rtc) {
         char name[PATH_MAX];
         sem_init(&stack_group->sem_listen_thread, 0, 0);
         sprintf_s(name, sizeof(name), "%s", "listen_thread");
-        struct sys_thread *thread = sys_thread_new(name, libnet_listen_thread,
+        struct sys_thread *thread = sys_thread_new(name, gazelle_listen_thread,
             (void*)(&stack_group->sem_listen_thread), 0, 0);
         free(thread);
         sem_wait(&stack_group->sem_listen_thread);
@@ -554,7 +558,33 @@ int32_t stack_group_init(void)
     return 0;
 }
 
-int32_t stack_thread_setup(void)
+int32_t stack_setup_app_thread(void)
+{
+    static PER_THREAD int first_flags = 1;
+    static _Atomic uint32_t queue_id = 0;
+
+    if (likely(first_flags == 0)) {
+        return 0;
+    }
+    first_flags=0;
+
+    uint32_t cur_queue_id = atomic_fetch_add(&queue_id, 1);
+    struct thread_params *t_params = malloc(sizeof(struct thread_params));
+    if (t_params == NULL) {
+        return -1;
+    }
+    t_params->idx = cur_queue_id;
+    t_params->queue_id = cur_queue_id;
+
+    if (stack_thread_init(t_params) == NULL) {
+        LSTACK_LOG(INFO, LSTACK, "stack setup failed in app thread\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int32_t stack_setup_thread(void)
 {
     int32_t ret;
     char name[PATH_MAX];
@@ -790,6 +820,11 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack
             continue;
         }
 
+        /* stack maybe not init in app thread yet */
+        if (stack == NULL || !(netif_is_up(&stack->netif))) {
+            continue;
+        }
+
         ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1);
         if (ret != 0) {
             stack->stats.rx_allocmbuf_fail++;
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 7cb37bf..4f1b127 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -110,7 +110,8 @@ struct protocol_stack *get_bind_protocol_stack(void);
 struct protocol_stack_group *get_protocol_stack_group(void);
 
 int32_t stack_group_init(void);
-int32_t stack_thread_setup(void);
+int32_t stack_setup_thread(void);
+int32_t stack_setup_app_thread(void);
 
 void bind_to_stack_numa(struct protocol_stack *stack);
 int32_t init_dpdk_ethdev(void);
-- 
2.27.0