From 2c3000bba21c4c636860498530b57e842f56480f Mon Sep 17 00:00:00 2001
From: Lemmy Huang <huangliming5@huawei.com>
Date: Thu, 16 Mar 2023 11:40:29 +0800
Subject: [PATCH] refactor mbuf private data

Signed-off-by: Lemmy Huang <huangliming5@huawei.com>
---
 src/common/dpdk_common.h            | 49 ++++++++++++++++++-----------
 src/lstack/core/lstack_dpdk.c       |  2 +-
 src/lstack/core/lstack_stack_stat.c | 12 ++++---
 src/ltran/CMakeLists.txt            |  3 +-
 src/ltran/ltran_ethdev.c            |  4 +--
 src/ltran/ltran_forward.c           | 14 ++++-----
 6 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/src/common/dpdk_common.h b/src/common/dpdk_common.h
index 1305819..6b107ae 100644
--- a/src/common/dpdk_common.h
+++ b/src/common/dpdk_common.h
@@ -16,32 +16,44 @@
 #include <stdbool.h>
 #include <rte_mbuf.h>
 #include <rte_ring.h>
+#include <lwip/pbuf.h>
 
 #include "gazelle_opt.h"
 
 #define GAZELLE_KNI_NAME                     "kni"   // will be removed during dpdk update
 
+
 /* Layout:
- * | rte_mbuf | pbuf_custom| tcp_seg | gazelle_prive  | payload |
- * |   128    |     64     |   32    |       16       |
- * rte_prefetch0 in lwip project,tcp_out.c,tcp_output_segment use constants
- * cacheline is 64, make sure pbuf_custom in same cacheline
+ * | rte_mbuf | mbuf_private | payload |
+ * |   128    |              |         |
  **/
-struct pbuf;
-#define LATENCY_TIMESTAMP_SIZE  (sizeof(uint64_t) * 2)
-#define MBUF_PRIVATE_SIZE       128
-#define LATENCY_OFFSET          96
-static __rte_always_inline uint64_t *mbuf_to_private(struct rte_mbuf *mbuf)
+struct latency_timestamp {
+        uint64_t stamp; // time stamp
+        uint64_t check; // just for later vaild check
+};
+struct mbuf_private {
+    /* struct pbuf_custom must at first */
+    struct pbuf_custom pc;
+    /* don't use `struct tcp_seg` directly to avoid conflicts by include lwip tcp header */
+    char ts[32]; // 32 > sizeof(struct tcp_seg)
+    struct latency_timestamp lt;
+};
+
+static __rte_always_inline struct mbuf_private *mbuf_to_private(const struct rte_mbuf *m)
+{
+    return (struct mbuf_private *)RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
+}
+static __rte_always_inline struct pbuf_custom *mbuf_to_pbuf(const struct rte_mbuf *m)
 {
-    return (uint64_t *)((uint8_t *)(mbuf) + sizeof(struct rte_mbuf) + LATENCY_OFFSET);
+    return &mbuf_to_private(m)->pc;
 }
-static __rte_always_inline struct rte_mbuf *pbuf_to_mbuf(struct pbuf *p)
+static __rte_always_inline struct rte_mbuf *pbuf_to_mbuf(const struct pbuf *p)
 {
-    return ((struct rte_mbuf *)(void *)((uint8_t *)(p) - sizeof(struct rte_mbuf)));
+    return (struct rte_mbuf *)RTE_PTR_SUB(p, sizeof(struct rte_mbuf));
 }
-static __rte_always_inline struct pbuf_custom *mbuf_to_pbuf(struct rte_mbuf *m)
+static __rte_always_inline struct mbuf_private *pbuf_to_private(const struct pbuf *p)
 {
-    return ((struct pbuf_custom *)((uint8_t *)(m) + sizeof(struct rte_mbuf)));
+    return mbuf_to_private(pbuf_to_mbuf(p));
 }
 
 /* NOTE!!! magic code, even the order.
@@ -69,15 +81,16 @@ static __rte_always_inline void copy_mbuf(struct rte_mbuf *dst, struct rte_mbuf
     // copy private date.
     dst_data = (uint8_t *)mbuf_to_private(dst);
     src_data = (uint8_t *)mbuf_to_private(src);
-    rte_memcpy(dst_data, src_data, LATENCY_TIMESTAMP_SIZE);
+    rte_memcpy(dst_data, src_data, sizeof(struct mbuf_private));
 }
 
 static __rte_always_inline void time_stamp_into_mbuf(uint32_t rx_count, struct rte_mbuf *buf[], uint64_t time_stamp)
 {
+    struct latency_timestamp *lt;
     for (uint32_t i = 0; i < rx_count; i++) {
-        uint64_t *priv = mbuf_to_private(buf[i]);
-        *priv = time_stamp; // time stamp
-        *(priv + 1) = ~(*priv); // just for later vaid check
+        lt = &mbuf_to_private(buf[i])->lt;
+        lt->stamp = time_stamp;
+        lt->check = ~(time_stamp);
     }
 }
 
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 2ecfd1d..7ded9e9 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -146,7 +146,7 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
     }
 
     /* time stamp before pbuf_custom as priv_data */
-    uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE);
+    uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE);
     pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, rte_socket_id());
     if (pool == NULL) {
         LSTACK_LOG(ERR, LSTACK, "cannot create %s pool rte_err=%d\n", pool_name, rte_errno);
diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c
index 489b267..eed7fbf 100644
--- a/src/lstack/core/lstack_stack_stat.c
+++ b/src/lstack/core/lstack_stack_stat.c
@@ -51,16 +51,18 @@ uint64_t get_current_time(void)
 void calculate_lstack_latency(struct gazelle_stack_latency *stack_latency, const struct pbuf *pbuf,
     enum GAZELLE_LATENCY_TYPE type)
 {
+    uint64_t latency;
+    const struct latency_timestamp *lt;
+
     if (pbuf == NULL) {
        return;
     }
-    const uint64_t *priv = (uint64_t *)((uint8_t *)(pbuf) + LATENCY_OFFSET);
-    if (*priv != ~(*(priv + 1)) || *priv < stack_latency->start_time) {
+
+    lt = &pbuf_to_private(pbuf)->lt;
+    if (lt->stamp != ~(lt->check) || lt->stamp < stack_latency->start_time) {
         return;
     }
-
-    uint64_t latency = get_current_time();
-    latency = latency - *priv;
+    latency = get_current_time() - lt->stamp;
 
     struct stack_latency *latency_stat = (type == GAZELLE_LATENCY_LWIP) ?
         &stack_latency->lwip_latency : &stack_latency->read_latency;
diff --git a/src/ltran/CMakeLists.txt b/src/ltran/CMakeLists.txt
index 1c82dae..f37a232 100644
--- a/src/ltran/CMakeLists.txt
+++ b/src/ltran/CMakeLists.txt
@@ -12,6 +12,7 @@ cmake_minimum_required(VERSION 3.12.1)
 project(ltran)
 
 set(COMMON_DIR ${PROJECT_SOURCE_DIR}/../common)
+set(LWIP_DIR /usr/include/lwip)
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
@@ -31,7 +32,7 @@ add_executable(ltran main.c ltran_param.c ltran_config.c ltran_ethdev.c ltran_st
 				ltran_forward.c ltran_timer.c ${COMMON_DIR}/gazelle_dfx_msg.c ${COMMON_DIR}/dpdk_common.c
                 ${COMMON_DIR}/gazelle_parse_config.c)
 
-target_include_directories(ltran PRIVATE ${COMMON_DIR} ${PROJECT_SOURCE_DIR})
+target_include_directories(ltran PRIVATE ${COMMON_DIR} ${PROJECT_SOURCE_DIR} ${LWIP_DIR})
 target_compile_options(ltran PRIVATE -march=native -fno-strict-aliasing -D__ARM_FEATURE_CRC32=1 -DRTE_MACHINE_CPUFLAG_NEON
                -DRTE_MACHINE_CPUFLAG_CRC32 -DRTE_MACHINE_CPUFLAG_PMULL -DRTE_MACHINE_CPUFLAG_AES
                -DRTE_MACHINE_CPUFLAG_SHA1 -DRTE_MACHINE_CPUFLAG_SHA2 -include rte_config.h
diff --git a/src/ltran/ltran_ethdev.c b/src/ltran/ltran_ethdev.c
index e0c824a..e2eb4a8 100644
--- a/src/ltran/ltran_ethdev.c
+++ b/src/ltran/ltran_ethdev.c
@@ -147,7 +147,7 @@ static struct rte_mempool *ltran_create_rx_mbuf_pool(uint32_t bond_port_index)
         return NULL;
     }
 
-    uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE);
+    uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE);
     return rte_pktmbuf_pool_create(mbuf_pool_name, num_mbufs, GAZELLE_MBUFS_CACHE_SIZE, private_size,
                                    RTE_MBUF_DEFAULT_BUF_SIZE, (int32_t)rte_socket_id());
 }
@@ -166,7 +166,7 @@ static struct rte_mempool *ltran_create_tx_mbuf_pool(uint32_t bond_port_index)
         return NULL;
     }
 
-    uint16_t private_size = RTE_ALIGN(MBUF_PRIVATE_SIZE, RTE_CACHE_LINE_SIZE);
+    uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE);
     return rte_pktmbuf_pool_create(mbuf_pool_name, num_mbufs, GAZELLE_MBUFS_CACHE_SIZE, private_size,
                                    RTE_MBUF_DEFAULT_BUF_SIZE, (int32_t)rte_socket_id());
 }
diff --git a/src/ltran/ltran_forward.c b/src/ltran/ltran_forward.c
index 4d9c1bb..8629acb 100644
--- a/src/ltran/ltran_forward.c
+++ b/src/ltran/ltran_forward.c
@@ -50,22 +50,22 @@ static __rte_always_inline struct gazelle_stack *get_kni_stack(void)
 
 static void calculate_ltran_latency(struct gazelle_stack *stack, const struct rte_mbuf *mbuf)
 {
+    struct latency_timestamp *lt;
     uint64_t latency;
-    uint64_t *priv = NULL;
 
-    priv = (uint64_t *)RTE_PTR_ADD(mbuf, sizeof(struct rte_mbuf) + LATENCY_OFFSET);
-    // priv--time stamp   priv+1 --- vaild check
-    if (*priv != ~(*(priv + 1))) {
+    lt = &mbuf_to_private(mbuf)->lt;
+    // vaild check
+    if (lt->stamp != ~(lt->check)) {
         return;
     }
 
     // time stamp must > start time
-    if (*priv < get_start_time_stamp()) {
-        *priv = 0;
+    if (lt->stamp < get_start_time_stamp()) {
+        lt->stamp = 0;
         return;
     }
 
-    latency = get_current_time() - *priv;
+    latency = get_current_time() - lt->stamp;
 
     stack->stack_stats.latency_total += latency;
     stack->stack_stats.latency_pkts++;
-- 
2.23.0