From 1a6c2c86c51997d541c6243a80580c9f54f99bb2 Mon Sep 17 00:00:00 2001
From: yinbin <yinbin8@huawei.com>
Date: Tue, 28 Nov 2023 11:30:49 +0800
Subject: [PATCH] fix coredump because sock closed before send data fully

---
 src/lstack/core/lstack_protocol_stack.c | 15 ++++++++++-----
 src/lstack/core/lstack_thread_rpc.c     | 20 +++++++++++---------
 src/lstack/include/lstack_thread_rpc.h  |  3 ++-
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 54cf9e8..71e7bcc 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -696,7 +696,15 @@ void stack_socket(struct rpc_msg *msg)
 void stack_close(struct rpc_msg *msg)
 {
     int32_t fd = msg->args[MSG_ARG_0].i;
-
+    struct protocol_stack *stack = get_protocol_stack_by_fd(fd);
+    struct lwip_sock *sock = get_socket(fd);
+    
+    if (NETCONN_IS_DATAOUT(sock)) {
+        msg->recall_flag = 1;
+        rpc_call(&stack->rpc_queue, msg); /* until stack_send recall finish */
+	return;
+    }
+    
     msg->result = lwip_close(fd);
     if (msg->result != 0) {
         LSTACK_LOG(ERR, LSTACK, "tid %ld, fd %d failed %ld\n", get_stack_tid(), msg->args[MSG_ARG_0].i, msg->result);
@@ -842,15 +850,12 @@ void stack_send(struct rpc_msg *msg)
     replenish_again = do_lwip_send(stack, sock->conn->socket, sock, len, 0);
     __sync_fetch_and_sub(&sock->call_num, 1);
     if (!NETCONN_IS_DATAOUT(sock) && !replenish_again) {
-        rpc_msg_free(msg);
         return;
     } else {
         if (__atomic_load_n(&sock->call_num, __ATOMIC_ACQUIRE) == 0) {
+	    msg->recall_flag = 1;
             rpc_call(&stack->rpc_queue, msg);
              __sync_fetch_and_add(&sock->call_num, 1);
-        } else {
-            rpc_msg_free(msg);
-            return;
         }
     }
 }
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index 92c58df..03e014d 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -71,8 +71,8 @@ static struct rpc_msg *rpc_msg_alloc(struct protocol_stack *stack, rpc_msg_func
 
     pthread_spin_init(&msg->lock, PTHREAD_PROCESS_PRIVATE);
     msg->func = func;
-    msg->self_release = 1;
-
+    msg->sync_flag = 1;
+    msg->recall_flag = 0;
     return msg;
 }
 
@@ -94,6 +94,7 @@ static inline __attribute__((always_inline)) int32_t rpc_sync_call(lockless_queu
 void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
 {
     struct rpc_msg *msg = NULL;
+    struct lwip_sock *sock = NULL;
 
     while (max_num--) {
         lockless_queue_node *node = lockless_queue_mpsc_pop(&stack->rpc_queue);
@@ -109,14 +110,15 @@ void poll_rpc_msg(struct protocol_stack *stack, uint32_t max_num)
             stack->stats.call_null++;
         }
 
-        /* stack_send free msg in stack_send */
-        if (msg->func != stack_send) {
-            if (msg->self_release) {
+        if (!msg->recall_flag) {
+	    if (msg->sync_flag) {
                 pthread_spin_unlock(&msg->lock);
             } else {
                 rpc_msg_free(msg);
             }
-        }
+        } else {
+	  msg->recall_flag = 0;
+	}
     }
 }
 
@@ -224,7 +226,7 @@ int32_t rpc_call_arp(struct protocol_stack *stack, struct rte_mbuf *mbuf)
         return -1;
     }
 
-    msg->self_release = 0;
+    msg->sync_flag = 0;
     msg->args[MSG_ARG_0].p = mbuf;
     msg->args[MSG_ARG_1].p = stack;
 
@@ -458,8 +460,8 @@ int32_t rpc_call_send(int fd, const void *buf, size_t len, int flags)
     msg->args[MSG_ARG_1].size = len;
     msg->args[MSG_ARG_2].i = flags;
     msg->args[MSG_ARG_3].p = stack;
-    msg->self_release = 0;
-
+    msg->sync_flag = 0;
+    
     rpc_call(&stack->rpc_queue, msg);
 
     return 0;
diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h
index bcb40dd..80e254f 100644
--- a/src/lstack/include/lstack_thread_rpc.h
+++ b/src/lstack/include/lstack_thread_rpc.h
@@ -44,7 +44,8 @@ union rpc_msg_arg {
 struct rpc_msg_pool;
 struct rpc_msg {
     pthread_spinlock_t lock; /* msg handler unlock notice sender msg process done */
-    int32_t self_release; /* 0:msg handler release msg  1:msg sender release msg */
+    int8_t sync_flag : 1;
+    int8_t recall_flag : 1;
     int64_t result; /* func return val */
     lockless_queue_node queue_node;
     struct rpc_msg_pool *pool;
-- 
2.27.0