From 1929cc2a07a8e6d6a81b13a8338d816d7f09e709 Mon Sep 17 00:00:00 2001
From: yinbin6 <yinbin8@huawei.com>
Date: Wed, 6 Mar 2024 10:51:42 +0800
Subject: [PATCH] FAULT INJECT: add duplicate and reorder methods

---
 src/common/gazelle_fault_inject_common.h |   4 +-
 src/lstack/netif/lstack_fault_inject.c   | 162 ++++++++++++++++++++++-
 src/ltran/ltran_dfx.c                    |  20 ++-
 3 files changed, 174 insertions(+), 12 deletions(-)

diff --git a/src/common/gazelle_fault_inject_common.h b/src/common/gazelle_fault_inject_common.h
index 3a77f39..72d778c 100644
--- a/src/common/gazelle_fault_inject_common.h
+++ b/src/common/gazelle_fault_inject_common.h
@@ -17,10 +17,10 @@
 
 enum GAZELLE_FAULT_INJECT_TYPE {
     GAZELLE_FAULT_INJECT_TYPE_ERR = 0,
-    GAZELLE_FAULT_INJECT_PACKET_DELAY,
     GAZELLE_FAULT_INJECT_PACKET_LOSS,
-    GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
     GAZELLE_FAULT_INJECT_PACKET_REORDER,
+    GAZELLE_FAULT_INJECT_PACKET_DELAY,
+    GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
     GAZELLE_FAULT_INJECT_TYPE_MAX,
 };
 
diff --git a/src/lstack/netif/lstack_fault_inject.c b/src/lstack/netif/lstack_fault_inject.c
index 41e7d95..4edc6cc 100644
--- a/src/lstack/netif/lstack_fault_inject.c
+++ b/src/lstack/netif/lstack_fault_inject.c
@@ -9,7 +9,7 @@
 * PURPOSE.
 * See the Mulan PSL v2 for more details.
 */
-
+#include <pthread.h>
 #include <securec.h>
 #include <rte_gro.h>
 #include <rte_net.h>
@@ -31,6 +31,15 @@ struct inject_tbl {
 };
 static struct inject_tbl g_inject_tbl[GAZELLE_FAULT_INJECT_TYPE_MAX];
 
+struct reorder_stat {
+    int32_t enable;
+    int32_t arr_size;
+    int32_t cur_cnt;
+    struct rte_mbuf **array;
+};
+
+static struct reorder_stat g_reorder[PROTOCOL_STACK_MAX];
+
 struct inject_func_tbl {
     enum GAZELLE_FAULT_INJECT_TYPE type;
     enum GAZELLE_FAULT_INJECT_RULE rule;
@@ -41,10 +50,16 @@ static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct r
                                           uint32_t nr_pkts, struct gazelle_fault_inject_data data);
 static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
                                          uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+static int32_t inject_packet_duplicate_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                              uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+static int32_t inject_packet_reorder_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                            uint32_t nr_pkts, struct gazelle_fault_inject_data data);
 
 static struct inject_func_tbl g_inject_func_tbl[] = {
     {GAZELLE_FAULT_INJECT_PACKET_LOSS,  INJECT_LOSS_RANDOM, inject_packet_loss_random},
     {GAZELLE_FAULT_INJECT_PACKET_DELAY, INJECT_DELAY_RANDOM, inject_packet_delay_random},
+    {GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, INJECT_DUPLICATE_RANDOM, inject_packet_duplicate_random},
+    {GAZELLE_FAULT_INJECT_PACKET_REORDER, INJECT_REORDER_RANDOM, inject_packet_reorder_random},
 };
 
 static int32_t inject_func_tbl_update()
@@ -68,16 +83,16 @@ static int32_t inject_func_tbl_update()
 static uint32_t inject_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
 {
     for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
-        if (g_inject_tbl[i].inject_data.fault_inject_on) {
+        if (g_inject_tbl[i].inject_data.fault_inject_on && g_inject_tbl[i].inject_func) {
             int32_t xmit_pkts = 0;
-            xmit_pkts =  g_inject_tbl[i].inject_func(stack, pkts, nr_pkts,
-                                                     g_inject_tbl[i].inject_data);
+            xmit_pkts = g_inject_tbl[i].inject_func(stack, pkts, nr_pkts, g_inject_tbl[i].inject_data);
             if (xmit_pkts == nr_pkts) {
                 continue;
             }
             return xmit_pkts;
         }
     }
+    
     if (rte_mbuf_refcnt_read(*pkts) == 1) {
         return nr_pkts;
     }
@@ -105,7 +120,7 @@ static int32_t inject_strategy_update()
         }
         return 0;
     }
-    
+
     for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
         struct protocol_stack *stack = stack_group->stacks[i];
         vdev_dev_ops_init(&stack->dev_ops);
@@ -150,6 +165,132 @@ static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rt
     return nr_pkts;
 }
 
+static int32_t inject_packet_duplicate_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                              uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+    if (rte_mbuf_refcnt_read(*pkts) == 1) {
+        return nr_pkts;
+    }
+    
+    double duplicate_rate = data.inject_data.duplicate.duplicate_rate;
+    int32_t boundary = (int32_t) (duplicate_rate * INJECT_MODULO);
+    int32_t count_max = data.inject_data.duplicate.duplicate_sigle_count;
+    
+    uint32_t rand_num = rte_rand() % INJECT_MODULO;
+    if (rand_num > boundary) {
+        return nr_pkts;
+    }
+
+    struct rte_mempool *mp = stack->rxtx_mbuf_pool;
+    struct rte_mbuf *mbuf_clone = NULL;
+    int32_t ret = 0;
+    
+    for (int32_t i = 0; i < nr_pkts; ++i) {
+        int32_t count = count_max;
+        while (count--) {
+            mbuf_clone = rte_pktmbuf_clone(pkts[i], mp);
+            rte_pktmbuf_free(pkts[i]);
+            if (mbuf_clone == NULL) {
+                LSTACK_LOG(ERR, LSTACK, "fault inject mbuf_clone fail.\n");
+                return 0;
+            }
+            ret = vdev_tx_xmit(stack, &mbuf_clone, 1);
+            if (ret < 1) {
+                rte_pktmbuf_free(mbuf_clone);
+                return ret;
+            }
+        }
+    }
+    return nr_pkts;
+}
+
+static int32_t send_reorder_array(struct protocol_stack *stack)
+{
+    int32_t idx = stack->stack_idx;
+    int32_t ret = 0;
+
+    for (int32_t i = 0; i < g_reorder[idx].cur_cnt; ++i) {
+        ret = vdev_tx_xmit(stack, g_reorder[idx].array + i, 1);
+        if (ret < 1) {
+            rte_pktmbuf_free(*(g_reorder[idx].array + i));
+        }
+    }
+    g_reorder[idx].cur_cnt = 0;
+
+    return 0;
+}
+
+static int32_t inject_packet_reorder_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                            uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+    if (rte_mbuf_refcnt_read(*pkts) == 1) {
+        return nr_pkts;
+    }
+    
+    double reorder_rate = data.inject_data.reorder.reorder_rate;
+    int32_t boundary = (int32_t) (reorder_rate * INJECT_MODULO);
+    int32_t count_max = data.inject_data.reorder.reorder_sigle_count;
+
+    uint32_t rand_num = rte_rand() % INJECT_MODULO;
+    if (rand_num > boundary) {
+        return nr_pkts;
+    }
+
+    struct rte_mempool *mp = stack->rxtx_mbuf_pool;
+    struct rte_mbuf *mbuf_clone = NULL;
+    int32_t idx = stack->stack_idx;
+    for (int32_t i = 0; i < nr_pkts; ++i) {
+        if (g_reorder[idx].cur_cnt < count_max) {
+            mbuf_clone = rte_pktmbuf_clone(pkts[i], mp);
+            if (mbuf_clone == NULL) {
+                LSTACK_LOG(ERR, LSTACK, "fault inject mbuf_clone fail.\n");
+                return 0;
+            }
+            *(g_reorder[idx].array + g_reorder[idx].cur_cnt++) = mbuf_clone;
+            /* func rte_pktmbuf_clone will add refcnt of mbuf, so following operation will free mbuf double */
+            rte_pktmbuf_free(pkts[i]);
+            rte_pktmbuf_free(pkts[i]);
+        } else {
+            send_reorder_array(stack);
+        }
+    }
+    
+    return nr_pkts;
+}
+
+static int32_t inject_reorder_mem_release()
+{
+    struct protocol_stack_group *stack_group = get_protocol_stack_group();
+    
+    for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+        struct protocol_stack *stack = stack_group->stacks[i];
+        if (!g_reorder[i].enable) {
+            return 0;
+        }
+        send_reorder_array(stack);
+        free(g_reorder[i].array);
+        g_reorder[i].enable = 0;
+    }
+
+    return 0;
+}
+
+static int32_t inject_reorder_mem_alloc()
+{
+    struct protocol_stack_group *stack_group = get_protocol_stack_group();
+    
+    for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+        g_reorder[i].enable = 1;
+    
+        g_reorder[i].arr_size =
+            g_inject_tbl[GAZELLE_FAULT_INJECT_PACKET_REORDER].inject_data.inject_data.reorder.reorder_sigle_count;
+        g_reorder[i].cur_cnt = 0;
+        g_reorder[i].array =
+            (struct rte_mbuf**) malloc(sizeof(struct rte_mbuf**) * (g_reorder[i].arr_size));
+    }
+    return 0;
+}
+
 static int32_t inject_respond_msg(int32_t sockfd)
 {
     struct gazelle_stack_dfx_data rsp = {0};
@@ -192,6 +333,10 @@ static int32_t inject_unset_cmd(int32_t sockfd, struct gazelle_fault_inject_data
         }
     }
 
+    if (!g_inject_tbl[GAZELLE_FAULT_INJECT_PACKET_REORDER].inject_data.fault_inject_on) {
+        inject_reorder_mem_release();
+    }
+
     inject_strategy_update();
 
     return inject_respond_msg(sockfd);
@@ -208,8 +353,13 @@ static int32_t inject_set_cmd(int32_t sockfd, struct gazelle_fault_inject_data i
         return -1;
     }
 
+    if (inject.inject_type == GAZELLE_FAULT_INJECT_PACKET_REORDER) {
+        inject_reorder_mem_release();
+        inject_reorder_mem_alloc();
+    }
+
     inject_strategy_update();
-    
+
     return inject_respond_msg(sockfd);
 }
 
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 96509e5..2cc3504 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -77,10 +77,10 @@ struct gazelle_fault_inject_type_list {
 };
 
 static struct gazelle_fault_inject_type_list inject_type_list[] = {
-    {"delay",     GAZELLE_FAULT_INJECT_PACKET_DELAY,      parse_inject_packet_delay_digit},
     {"loss",      GAZELLE_FAULT_INJECT_PACKET_LOSS,       parse_inject_packet_loss_digit},
-    {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
     {"reorder",  GAZELLE_FAULT_INJECT_PACKET_REORDER,   parse_inject_packet_reorder_digit},
+    {"delay",     GAZELLE_FAULT_INJECT_PACKET_DELAY,      parse_inject_packet_delay_digit},
+    {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
 };
 
 struct gazelle_fault_inject_rule_list {
@@ -1514,7 +1514,7 @@ static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_dat
     }
     
     if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_DELAY) {
-        printf("\t| inject_type: delay   | delay_time: %-7d | delay_range: %-3d             | "
+        printf("\t| inject_type: delay     | delay_time: %-7d   | delay_range: %-3d             | "
                "inject_rule: random |\n", inject->inject_data.delay.delay_time,
                inject->inject_data.delay.delay_range);
     }
@@ -1522,10 +1522,22 @@ static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_dat
 #define INJECT_PERCENT                          100
     
     if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_LOSS) {
-        printf("\t| inject_type: loss    | loss_rate: %-3.1f%%    | loss_single_count: %-3d       | "
+        printf("\t| inject_type: loss      | loss_rate: %-4.1f%%      | loss_single_count: %-3d       | "
                "inject_rule: random |\n", inject->inject_data.loss.loss_rate * INJECT_PERCENT,
                inject->inject_data.loss.loss_sigle_count);
     }
+    
+    if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE) {
+        printf("\t| inject_type: duplicate | duplicate_rate: %-4.1f%% | duplicate_single_count: %-3d  | "
+               "inject_rule: random |\n", inject->inject_data.duplicate.duplicate_rate * INJECT_PERCENT,
+               inject->inject_data.duplicate.duplicate_sigle_count);
+    }
+
+    if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_REORDER) {
+        printf("\t| inject_type: reorder   | reorder_rate: %-4.1f%%   | reorder_sigle_count: %-3d     | "
+               "inject_rule: random |\n", inject->inject_data.reorder.reorder_rate * INJECT_PERCENT,
+               inject->inject_data.reorder.reorder_sigle_count);
+    }
     printf("\n");
 }
 
-- 
2.33.0