From 2f7ca29c8b7a93079a5579062fc6751a6be6fd0c Mon Sep 17 00:00:00 2001
From: Lemmy Huang <huangliming5@huawei.com>
Date: Wed, 10 Jul 2024 11:08:07 +0800
Subject: [PATCH] cleancode: refactor lwipsock.h

Signed-off-by: Lemmy Huang <huangliming5@huawei.com>
---
 src/lstack/api/lstack_rtc_api.c         |  7 +---
 src/lstack/api/lstack_wrap.c            | 11 ++++--
 src/lstack/core/lstack_init.c           |  3 --
 src/lstack/core/lstack_lwip.c           | 45 ++++++++-----------------
 src/lstack/core/lstack_protocol_stack.c | 26 ++++----------
 src/lstack/include/lstack_lwip.h        |  7 ++--
 6 files changed, 33 insertions(+), 66 deletions(-)

diff --git a/src/lstack/api/lstack_rtc_api.c b/src/lstack/api/lstack_rtc_api.c
index e77edec..57ff89f 100644
--- a/src/lstack/api/lstack_rtc_api.c
+++ b/src/lstack/api/lstack_rtc_api.c
@@ -68,12 +68,7 @@ int rtc_close(int s)
         return lstack_epoll_close(s);
     }
 
-    lwip_close(s);
-    if (sock != NULL) {
-        list_del_node_null(&sock->event_list);
-    }
-
-    return posix_api->close_fn(s);
+    return lwip_close(s);
 }
 
 int rtc_shutdown(int fd, int how)
diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index cf0d302..16fc876 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -175,6 +175,8 @@ static inline int32_t do_accept(int32_t s, struct sockaddr *addr, socklen_t *add
 
     int32_t fd = g_wrap_api->accept_fn(s, addr, addrlen);
     if (fd >= 0) {
+        struct lwip_sock *sock = get_socket(fd);
+        SET_CONN_TYPE_LIBOS(sock->conn);
         return fd;
     }
 
@@ -193,6 +195,8 @@ static int32_t do_accept4(int32_t s, struct sockaddr *addr, socklen_t *addrlen,
 
     int32_t fd = g_wrap_api->accept4_fn(s, addr, addrlen, flags);
     if (fd >= 0) {
+        struct lwip_sock *sock = get_socket(fd);
+        SET_CONN_TYPE_LIBOS(sock->conn);
         return fd;
     }
 
@@ -452,10 +456,11 @@ static inline int32_t do_socket(int32_t domain, int32_t type, int32_t protocol)
     }
 
     ret = g_wrap_api->socket_fn(domain, type, protocol);
-    /* if udp_enable = 1 in lstack.conf, udp protocol must be in user path currently */
-    if ((ret >= 0) && (type & SOCK_DGRAM)) {
+    if (ret >= 0) {
         struct lwip_sock *sock = get_socket(ret);
-        if (sock != NULL && sock->conn != NULL) {
+        SET_CONN_TYPE_LIBOS_OR_HOST(sock->conn);
+        /* if udp_enable = 1 in lstack.conf, udp protocol must be in user path currently */
+        if (type & SOCK_DGRAM) {
             SET_CONN_TYPE_LIBOS(sock->conn);
         }
     }
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index a5a4a4e..1b3882e 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -313,9 +313,6 @@ __attribute__((constructor)) void gazelle_network_init(void)
         }
     }
 
-    /* lwip initialization */
-    lwip_sock_init();
-
 #if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
     if (get_global_cfg_params()->kni_switch) {
         set_kni_ip_mac();
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 75ef5f6..6fe4055 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -144,12 +144,12 @@ static bool replenish_send_idlembuf(struct protocol_stack *stack, struct lwip_so
     return false;
 }
 
-void do_lwip_init_sock(int32_t fd)
+int do_lwip_init_sock(int32_t fd)
 {
     struct protocol_stack *stack = get_protocol_stack();
-    struct lwip_sock *sock = get_socket(fd);
+    struct lwip_sock *sock = get_socket_by_fd(fd);
     if (sock == NULL) {
-        return;
+        return -1;
     }
 
     reset_sock_data(sock);
@@ -157,7 +157,7 @@ void do_lwip_init_sock(int32_t fd)
     sock->recv_ring = gazelle_ring_create_fast("sock_recv", SOCK_RECV_RING_SIZE, RING_F_SP_ENQ | RING_F_SC_DEQ);
     if (sock->recv_ring == NULL) {
         LSTACK_LOG(ERR, LSTACK, "sock_recv create failed. errno: %d.\n", rte_errno);
-        return;
+        return -1;
     }
 
     sock->send_ring = gazelle_ring_create_fast("sock_send",
@@ -166,7 +166,7 @@ void do_lwip_init_sock(int32_t fd)
     if (sock->send_ring == NULL) {
         gazelle_ring_free_fast(sock->recv_ring);
         LSTACK_LOG(ERR, LSTACK, "sock_send create failed. errno: %d.\n", rte_errno);
-        return;
+        return -1;
     }
     (void)replenish_send_idlembuf(stack, sock);
 
@@ -174,6 +174,7 @@ void do_lwip_init_sock(int32_t fd)
 
     init_list_node_null(&sock->recv_list);
     init_list_node_null(&sock->event_list);
+    return 0;
 }
 
 void do_lwip_clean_sock(int fd)
@@ -1237,32 +1238,6 @@ void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_soc
     }
 }
 
-int do_lwip_close(int fd)
-{
-    int ret = lwip_close(fd);
-    do_lwip_clean_sock(fd);
-    posix_api->close_fn(fd);
-    return ret;
-}
-
-int do_lwip_socket(int domain, int type, int protocol)
-{
-    int32_t fd = lwip_socket(domain, type, 0);
-    if (fd < 0) {
-        return fd;
-    }
-
-    do_lwip_init_sock(fd);
-
-    struct lwip_sock *sock = get_socket(fd);
-    if (sock == NULL || sock->stack == NULL) {
-        do_lwip_close(fd);
-        return -1;
-    }
-
-    return fd;
-}
-
 uint32_t do_lwip_get_conntable(struct gazelle_stat_lstack_conn_info *conn,
                                uint32_t max_num)
 {
@@ -1583,3 +1558,11 @@ err_t find_same_node_ring(struct tcp_pcb *npcb)
     }
     return 0;
 }
+
+unsigned same_node_ring_count(struct lwip_sock *sock)
+{
+  const unsigned long long cur_begin = __atomic_load_n(&sock->same_node_rx_ring->sndbegin, __ATOMIC_RELAXED);
+  const unsigned long long cur_end = __atomic_load_n(&sock->same_node_rx_ring->sndend, __ATOMIC_RELAXED);
+
+  return cur_end - cur_begin;
+}
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index d8bdd3c..c6075d5 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -424,11 +424,6 @@ static struct protocol_stack *stack_thread_init(void *arg)
     }
     RTE_PER_LCORE(_lcore_id) = stack->cpu_id;
 
-    if (hugepage_init() != 0) {
-        LSTACK_LOG(ERR, LSTACK, "hugepage init failed\n");
-        goto END;
-    }
-
     tcpip_init(NULL, NULL);
 
     if (use_ltran()) {
@@ -714,12 +709,9 @@ void stack_arp(struct rpc_msg *msg)
 
 void stack_socket(struct rpc_msg *msg)
 {
-    msg->result = do_lwip_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
+    msg->result = lwip_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
     if (msg->result < 0) {
-        msg->result = do_lwip_socket(msg->args[MSG_ARG_0].i, msg->args[MSG_ARG_1].i, msg->args[MSG_ARG_2].i);
-        if (msg->result < 0) {
-            LSTACK_LOG(ERR, LSTACK, "tid %ld, %ld socket failed\n", get_stack_tid(), msg->result);
-        }
+        LSTACK_LOG(ERR, LSTACK, "tid %ld, %ld socket failed\n", get_stack_tid(), msg->result);
     }
 }
 
@@ -735,7 +727,7 @@ void stack_close(struct rpc_msg *msg)
         return;
     }
     
-    msg->result = do_lwip_close(fd);
+    msg->result = lwip_close(fd);
     if (msg->result != 0) {
         LSTACK_LOG(ERR, LSTACK, "tid %ld, fd %d failed %ld\n", get_stack_tid(), msg->args[MSG_ARG_0].i, msg->result);
     }
@@ -803,7 +795,7 @@ void stack_accept(struct rpc_msg *msg)
 
     struct lwip_sock *sock = get_socket(accept_fd);
     if (sock == NULL || sock->stack == NULL) {
-        do_lwip_close(accept_fd);
+        lwip_close(accept_fd);
         LSTACK_LOG(ERR, LSTACK, "fd %d ret %d\n", fd, accept_fd);
         return;
     }
@@ -1046,12 +1038,8 @@ void stack_create_shadow_fd(struct rpc_msg *msg)
     }
 
     int domain = addr->sa_family;
-    if (NETCONN_IS_UDP(sock)) {
-        clone_fd = do_lwip_socket(domain, SOCK_DGRAM, 0);
-    } else {
-        clone_fd = do_lwip_socket(domain, SOCK_STREAM, 0);
-    }
-
+    int type = NETCONN_IS_UDP(sock) ? SOCK_DGRAM : SOCK_STREAM;
+    clone_fd = lwip_socket(domain, type, 0);
     if (clone_fd < 0) {
         LSTACK_LOG(ERR, LSTACK, "clone socket failed clone_fd=%d errno=%d\n", clone_fd, errno);
         msg->result = clone_fd;
@@ -1370,7 +1358,7 @@ static void stack_all_fds_close(void)
     for (int i = 3; i < GAZELLE_MAX_CLIENTS + GAZELLE_RESERVED_CLIENTS; i++) {
         struct lwip_sock *sock = get_socket(i);
         if (sock && sock->stack == get_protocol_stack()) {
-            do_lwip_close(i);
+            lwip_close(i);
         }
     }
 }
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index 0b952ec..b972f11 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -16,21 +16,20 @@
 
 #include "common/gazelle_dfx_msg.h"
 
+struct lwip_sock;
+unsigned same_node_ring_count(struct lwip_sock *sock);
+
 #define NETCONN_IS_ACCEPTIN(sock)   (((sock)->conn->acceptmbox != NULL) && !sys_mbox_empty((sock)->conn->acceptmbox))
 #define NETCONN_IS_DATAIN(sock)     ((gazelle_ring_readable_count((sock)->recv_ring) || (sock)->recv_lastdata) || (sock->same_node_rx_ring != NULL && same_node_ring_count(sock)))
 #define NETCONN_IS_DATAOUT(sock)    (gazelle_ring_readover_count((sock)->send_ring) || (sock)->send_pre_del)
 #define NETCONN_IS_OUTIDLE(sock)    gazelle_ring_readable_count((sock)->send_ring)
 #define NETCONN_IS_UDP(sock)        (NETCONNTYPE_GROUP(netconn_type((sock)->conn)) == NETCONN_UDP)
 
-struct lwip_sock;
 struct rte_mempool;
 struct rpc_msg;
 struct rte_mbuf;
 struct protocol_stack;
 
-int do_lwip_socket(int domain, int type, int protocol);
-int do_lwip_close(int32_t fd);
-void do_lwip_init_sock(int32_t fd);
 void do_lwip_clone_sockopt(struct lwip_sock *dst_sock, struct lwip_sock *src_sock);
 
 struct pbuf *do_lwip_tcp_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size);
-- 
2.33.0