From 506a252ff94be857ab89b1c1d4aea0ef66ed9582 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Mon, 9 Oct 2023 10:36:50 +0800
Subject: [PATCH] epoll: distinguish add/del_sock_event and
 add/del_sock_event_nolock

---
 src/lstack/api/lstack_epoll.c           | 59 +++++++++++++++++--------
 src/lstack/core/lstack_lwip.c           | 47 ++++----------------
 src/lstack/include/posix/lstack_epoll.h |  6 +++
 3 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c
index 55ca4fe..7c40792 100644
--- a/src/lstack/api/lstack_epoll.c
+++ b/src/lstack/api/lstack_epoll.c
@@ -46,37 +46,60 @@ static void update_epoll_max_stack(struct wakeup_poll *wakeup);
 static void change_epollfd_kernel_thread(struct wakeup_poll *wakeup, struct protocol_stack *old_stack,
     struct protocol_stack *new_stack);
 
+static inline void add_wakeup_to_stack_wakeuplist(struct wakeup_poll *wakeup, struct protocol_stack *stack)
+{
+    if (list_is_null(&wakeup->wakeup_list[stack->stack_idx])) {
+        list_add_node(&stack->wakeup_list, &wakeup->wakeup_list[stack->stack_idx]);
+    }
+}
+
+void add_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
+{
+    struct wakeup_poll *wakeup = sock->wakeup;
+
+    if (wakeup == NULL || wakeup->type == WAKEUP_CLOSE || (event & sock->epoll_events) == 0) {
+        return;
+    }
+    sock->events |= (event == EPOLLERR) ? (EPOLLIN | EPOLLERR) : (event & sock->epoll_events);
+    if (list_is_null(&sock->event_list)) {
+        list_add_node(&wakeup->event_list, &sock->event_list);
+    }
+    return;
+}
+
 void add_sock_event(struct lwip_sock *sock, uint32_t event)
 {
     struct wakeup_poll *wakeup = sock->wakeup;
+    struct protocol_stack *stack = sock->stack;
     if (wakeup == NULL || wakeup->type == WAKEUP_CLOSE || (event & sock->epoll_events) == 0) {
         return;
     }
 
     if (wakeup->type == WAKEUP_EPOLL) {
         pthread_spin_lock(&wakeup->event_list_lock);
-
-        /* app thread have read/write, event is outdated */
-        if (event == EPOLLIN && sock->conn->state != NETCONN_LISTEN && !NETCONN_IS_DATAIN(sock)) {
-            pthread_spin_unlock(&wakeup->event_list_lock);
-            return;
-        }
-        if (event == EPOLLOUT && !NETCONN_IS_OUTIDLE(sock)) {
-            pthread_spin_unlock(&wakeup->event_list_lock);
-            return;
-        }
-
-        sock->events |= (event == EPOLLERR) ? (EPOLLIN | EPOLLERR) : (event & sock->epoll_events);
-        if (list_is_null(&sock->event_list)) {
-            list_add_node(&wakeup->event_list, &sock->event_list);
-        }
+        add_sock_event_nolock(sock, event);
         pthread_spin_unlock(&wakeup->event_list_lock);
     }
 
-    struct protocol_stack *stack = sock->stack;
-    if (list_is_null(&wakeup->wakeup_list[stack->stack_idx])) {
-        list_add_node(&stack->wakeup_list, &wakeup->wakeup_list[stack->stack_idx]);
+    add_wakeup_to_stack_wakeuplist(wakeup, stack);
+    return;
+}
+
+void del_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
+{
+    sock->events &= ~event;
+
+    if (sock->events == 0) {
+        list_del_node_null(&sock->event_list);
     }
+    return;
+}
+
+void del_sock_event(struct lwip_sock *sock, uint32_t event)
+{
+    pthread_spin_lock(&sock->wakeup->event_list_lock);
+    del_sock_event_nolock(sock, event);
+    pthread_spin_unlock(&sock->wakeup->event_list_lock);
 }
 
 void wakeup_stack_epoll(struct protocol_stack *stack)
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 9ab8446..a98b1b8 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -317,22 +317,6 @@ void do_lwip_get_from_sendring_over(struct lwip_sock *sock)
     sock->stack->stats.write_lwip_cnt++;
 }
 
-static inline void del_data_out_event(struct lwip_sock *sock)
-{
-    pthread_spin_lock(&sock->wakeup->event_list_lock);
-
-    /* check again avoid cover event add in stack thread */
-    if (!NETCONN_IS_OUTIDLE(sock)) {
-        sock->events &= ~EPOLLOUT;
-
-        if (sock->events == 0) {
-            list_del_node_null(&sock->event_list);
-        }
-    }
-
-    pthread_spin_unlock(&sock->wakeup->event_list_lock);
-}
-
 static ssize_t do_app_write(struct pbuf *pbufs[], void *buf, size_t len, uint32_t write_num)
 {
     ssize_t send_len = 0;
@@ -617,8 +601,9 @@ static ssize_t do_lwip_fill_sendring(struct lwip_sock *sock, const void *buf, si
         wakeup->stat.app_write_cnt += write_num;
     }
 
-    if (wakeup && wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)) {
-        del_data_out_event(sock);
+    if (wakeup && wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)
+        && !NETCONN_IS_OUTIDLE(sock)) {
+        del_sock_event(sock, EPOLLOUT);
     }
 
 END:
@@ -801,22 +786,6 @@ static inline void notice_stack_send(struct lwip_sock *sock, int32_t fd, int32_t
     }
 }
 
-static inline void del_data_in_event(struct lwip_sock *sock)
-{
-    pthread_spin_lock(&sock->wakeup->event_list_lock);
-
-    /* check again avoid cover event add in stack thread */
-    if (!NETCONN_IS_DATAIN(sock)) {
-        sock->events &= ~EPOLLIN;
-
-        if (sock->events == 0) {
-            list_del_node_null(&sock->event_list);
-        }
-    }
-
-    pthread_spin_unlock(&sock->wakeup->event_list_lock);
-}
-
 /* process on same node use ring to recv data */
 ssize_t gazelle_same_node_ring_recv(struct lwip_sock *sock, const void *buf, size_t len, int32_t flags)
 {
@@ -847,8 +816,9 @@ ssize_t gazelle_same_node_ring_recv(struct lwip_sock *sock, const void *buf, siz
 
 END:
     /* rte_ring_count reduce lock */
-    if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)) {
-        del_data_in_event(sock);
+    if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)
+        && (!NETCONN_IS_DATAIN(sock))) {
+        del_sock_event(sock, EPOLLIN);
     }
     return act_len;
 }
@@ -1042,8 +1012,9 @@ ssize_t do_lwip_read_from_stack(int32_t fd, void *buf, size_t len, int32_t flags
     }
 
     /* rte_ring_count reduce lock */
-    if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)) {
-        del_data_in_event(sock);
+    if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)
+        && (!NETCONN_IS_DATAIN(sock))) {
+        del_sock_event(sock, EPOLLIN);
     }
 
     if (pbuf && addr && addrlen) {
diff --git a/src/lstack/include/posix/lstack_epoll.h b/src/lstack/include/posix/lstack_epoll.h
index d6c81a7..699a951 100644
--- a/src/lstack/include/posix/lstack_epoll.h
+++ b/src/lstack/include/posix/lstack_epoll.h
@@ -63,8 +63,14 @@ struct wakeup_poll {
 
 struct netconn;
 struct lwip_sock;
+
 void add_sock_event(struct lwip_sock *sock, uint32_t event);
+void add_sock_event_nolock(struct lwip_sock *sock, uint32_t event);
+void del_sock_event(struct lwip_sock *sock, uint32_t event);
+void del_sock_event_nolock(struct lwip_sock *sock, uint32_t event);
+
 void wakeup_stack_epoll(struct protocol_stack *stack);
+
 int32_t lstack_epoll_create(int32_t size);
 int32_t lstack_epoll_create1(int32_t flags);
 int32_t lstack_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct epoll_event *event);
-- 
2.27.0