From c064bea759414419d7fcae6c31455e517a9556f9 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng12@huawei.com>
Date: Wed, 16 Mar 2022 15:19:53 +0800
Subject: [PATCH 22/34] fix get data error

---
 src/common/gazelle_dfx_msg.h               |  2 +-
 src/lstack/api/lstack_epoll.c              | 10 ++++------
 src/lstack/core/lstack_dpdk.c              |  8 +++-----
 src/lstack/core/lstack_lwip.c              | 15 ++++++++-------
 src/lstack/core/lstack_stack_stat.c        |  1 -
 src/lstack/core/lstack_thread_rpc.c        |  8 --------
 src/lstack/include/lstack_protocol_stack.h |  1 -
 src/lstack/netif/lstack_ethdev.c           |  4 ++--
 src/ltran/ltran_dfx.c                      |  7 +++----
 9 files changed, 21 insertions(+), 35 deletions(-)

diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 41cbefa..3956c33 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -86,10 +86,10 @@ struct gazelle_stat_pkts {
     uint64_t accept_events;
     uint64_t read_null;
     uint64_t recv_empty;
-    uint64_t event_null;
     uint64_t remove_event;
     uint64_t send_self_rpc;
     uint64_t call_null;
+    uint64_t arp_copy_fail;
 };
 
 /* same as define in lwip/stats.h - struct stats_mib2 */
diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c
index b12ce58..885ec45 100644
--- a/src/lstack/api/lstack_epoll.c
+++ b/src/lstack/api/lstack_epoll.c
@@ -78,17 +78,16 @@ void add_epoll_event(struct netconn *conn, uint32_t event)
         sock = sock->shadowed_sock;
     }
 
-    /* sock not in monitoring */
-    if (!sock->weakup) {
+    if ((event & sock->epoll_events) == 0) {
         return;
     }
+    sock->events |= event & sock->epoll_events;
 
-    if ((event & sock->epoll_events) == 0) {
+    /* sock not in monitoring */
+    if (!sock->weakup) {
         return;
     }
 
-    sock->events |= event & sock->epoll_events;
-
     if (report_events(sock, event)) {
         sock->have_event = true;
         weakup_enqueue(sock->stack->weakup_ring, sock);
@@ -286,7 +285,6 @@ static int32_t get_lwip_events(struct weakup_poll *weakup, void *out, uint32_t m
     while (event_num < events_cnt) {
         int32_t ret = rte_ring_sc_dequeue(weakup->event_ring, (void **)&sock);
         if (ret != 0) {
-            get_protocol_stack_group()->event_null++;
             break;
         }
         sock->have_event = false;
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index b8320db..fb0e988 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -167,14 +167,12 @@ int32_t pktmbuf_pool_init(struct protocol_stack *stack, uint16_t stack_num)
         return -1;
     }
 
-    stack->rx_pktmbuf_pool = create_pktmbuf_mempool("rx_mbuf", RX_NB_MBUF / stack_num, RX_MBUF_CACHE_SZ,
-        stack->queue_id);
+    stack->rx_pktmbuf_pool = create_pktmbuf_mempool("rx_mbuf", RX_NB_MBUF / stack_num, 0, stack->queue_id);
     if (stack->rx_pktmbuf_pool == NULL) {
         return -1;
     }
 
-    stack->tx_pktmbuf_pool = create_pktmbuf_mempool("tx_mbuf", TX_NB_MBUF / stack_num, TX_MBUF_CACHE_SZ,
-        stack->queue_id);
+    stack->tx_pktmbuf_pool = create_pktmbuf_mempool("tx_mbuf", TX_NB_MBUF / stack_num, 0, stack->queue_id);
     if (stack->tx_pktmbuf_pool == NULL) {
         return -1;
     }
@@ -557,7 +555,7 @@ int32_t dpdk_init_lstack_kni(void)
 {
     struct protocol_stack_group *stack_group = get_protocol_stack_group();
 
-    stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, KNI_MBUF_CACHE_SZ, 0);
+    stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0);
     if (stack_group->kni_pktmbuf_pool == NULL) {
         return -1;
     }
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 1fc8446..1a9d9b9 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -175,7 +175,6 @@ int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs,
 
     int32_t ret = alloc_mbufs(pool, mbufs, num);
     if (ret != 0) {
-        get_protocol_stack()->stats.tx_allocmbuf_fail++;
         return ret;
     }
 
@@ -289,7 +288,7 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len)
         }
 
         copy_len = (len - send_len > pbuf->len) ? pbuf->len : (len - send_len);
-        pbuf_take(pbuf, buf + send_len, copy_len);
+        pbuf_take(pbuf, (char *)buf + send_len, copy_len);
         pbuf->tot_len = pbuf->len = copy_len;
 
         ret = rte_ring_sp_enqueue(sock->send_ring, pbuf);
@@ -331,13 +330,13 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
 
     uint32_t free_count = rte_ring_free_count(sock->recv_ring);
     uint32_t data_count = rte_ring_count(sock->conn->recvmbox->ring);
-
+    uint32_t read_max = LWIP_MIN(free_count, data_count);
     struct pbuf *pbuf = NULL;
-    uint32_t read_count = LWIP_MIN(free_count, data_count);
+    uint32_t read_count = 0;
     ssize_t recv_len = 0;
     int32_t ret;
 
-    for (uint32_t i = 0; i < read_count; i++) {
+    for (uint32_t i = 0; i < read_max; i++) {
         err_t err = netconn_recv_tcp_pbuf_flags(sock->conn, &pbuf, apiflags);
         if (err != ERR_OK) {
             if (recv_len > 0) {
@@ -356,6 +355,7 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
                 sock->stack->stats.read_lwip_drop++;
                 break;
             }
+            read_count++;
         }
 
         recv_len += pbuf->len;
@@ -364,7 +364,7 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
         apiflags |= NETCONN_DONTBLOCK | NETCONN_NOFIN;
     }
 
-    if (data_count > free_count) {
+    if (data_count > read_count) {
         add_recv_list(sock->conn->socket);
     }
 
@@ -481,6 +481,7 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags)
     while (recv_left > 0) {
         if (sock->recv_lastdata) {
             pbuf = sock->recv_lastdata;
+            sock->recv_lastdata = NULL;
         } else {
             ret = rte_ring_sc_dequeue(sock->recv_ring, (void **)&pbuf);
             if (ret != 0) {
@@ -490,7 +491,7 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags)
         }
 
         copy_len = (recv_left > pbuf->tot_len) ? pbuf->tot_len : (u16_t)recv_left;
-        pbuf_copy_partial(pbuf, buf + recvd, copy_len, 0);
+        pbuf_copy_partial(pbuf, (char *)buf + recvd, copy_len, 0);
 
         recvd += copy_len;
         recv_left -= copy_len;
diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c
index 0bd1c6c..41fe9bf 100644
--- a/src/lstack/core/lstack_stack_stat.c
+++ b/src/lstack/core/lstack_stack_stat.c
@@ -105,7 +105,6 @@ static void get_stack_stats(struct gazelle_stack_dfx_data *dfx, struct protocol_
     lstack_get_low_power_info(&dfx->low_power_info);
     memcpy_s(&dfx->data.pkts, sizeof(dfx->data.pkts), &stack->stats, sizeof(dfx->data.pkts));
     dfx->data.pkts.call_alloc_fail = stack_group->call_alloc_fail;
-    dfx->data.pkts.event_null = stack_group->event_null;
     dfx->data.pkts.weakup_ring_cnt = rte_ring_count(stack->weakup_ring);
     dfx->data.pkts.send_idle_ring_cnt = rte_ring_count(stack->send_idle_ring);
     dfx->data.pkts.call_msg_cnt = rpc_call_msgcnt(stack);
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index b4d57d3..8af1077 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -80,15 +80,11 @@ void poll_rpc_msg(struct protocol_stack *stack)
     struct rpc_msg *msg = NULL;
 
     num = 0;
-    lockless_queue_node *first_node = NULL;
     while (num++ < HANDLE_RPC_MSG_MAX) {
         lockless_queue_node *node = lockless_queue_mpsc_pop(&stack->rpc_queue);
         if (node == NULL) {
             return;
         }
-        if (first_node == NULL) {
-            first_node = node;
-        }
 
         msg = container_of(node, struct rpc_msg, queue_node);
 
@@ -105,10 +101,6 @@ void poll_rpc_msg(struct protocol_stack *stack)
         } else {
             rpc_msg_free(stack->rpc_pool, msg);
         }
-
-        if (first_node == node) {
-            break;
-        }
     }
 }
 
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 1020e83..f289465 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -72,7 +72,6 @@ struct protocol_stack_group {
     /* dfx stats */
     bool latency_start;
     uint64_t call_alloc_fail;
-    uint64_t event_null;
 };
 
 long get_stack_tid(void);
diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c
index 796a46d..026f545 100644
--- a/src/lstack/netif/lstack_ethdev.c
+++ b/src/lstack/netif/lstack_ethdev.c
@@ -97,8 +97,8 @@ int32_t eth_dev_poll(void)
     }
 
     if (get_protocol_stack_group()->latency_start) {
-            uint64_t time_stamp = get_current_time();
-            time_stamp_into_mbuf(nr_pkts, pkts, time_stamp);
+        uint64_t time_stamp = get_current_time();
+        time_stamp_into_mbuf(nr_pkts, pkts, time_stamp);
         }
 
     for (uint32_t i = 0; i < nr_pkts; i++) {
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 4b46ac9..5b3a95c 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -390,7 +390,7 @@ static void gazelle_print_ltran_start_latency(void *buf, const struct gazelle_st
 static void gazelle_print_ltran_stat_latency(void *buf, const struct gazelle_stat_msg_request *req_msg)
 {
     struct in_addr *ip_addr = (struct in_addr *)buf;
-    struct gazelle_stat_lstack_total *stat = (struct gazelle_stat_lstack_total *)(buf + sizeof(*ip_addr));
+    struct gazelle_stat_lstack_total *stat = (struct gazelle_stat_lstack_total *)((char *)buf + sizeof(*ip_addr));
     uint64_t total_rx = 0;
     double total_latency = 0;
     uint64_t max = 0;
@@ -574,9 +574,8 @@ static void show_lstack_stats(struct gazelle_stack_dfx_data *lstack_stat)
     printf("read_null: %-18"PRIu64" ", lstack_stat->data.pkts.read_null);
     printf("recv_empty: %-17"PRIu64" \n", lstack_stat->data.pkts.recv_empty);
     printf("call_alloc_fail: %-12"PRIu64" ", lstack_stat->data.pkts.call_alloc_fail);
-    printf("event_null: %-17"PRIu64" ", lstack_stat->data.pkts.event_null);
-    printf("remove_event: %-15"PRIu64" \n", lstack_stat->data.pkts.remove_event);
-    printf("send_self_rpc: %-14"PRIu64" ", lstack_stat->data.pkts.send_self_rpc);
+    printf("remove_event: %-15"PRIu64" ", lstack_stat->data.pkts.remove_event);
+    printf("send_self_rpc: %-14"PRIu64" \n", lstack_stat->data.pkts.send_self_rpc);
     printf("call_null: %-18"PRIu64" \n", lstack_stat->data.pkts.call_null);
 }
 
-- 
1.8.3.1