From f28b880f3be68377003c60005011a835eb18e105 Mon Sep 17 00:00:00 2001
From: yinbin6 <yinbin8@huawei.com>
Date: Sun, 18 Feb 2024 17:11:05 +0800
Subject: [PATCH] FAULT INJECT: gazelle add packet delay and packet drop

---
 src/common/gazelle_dfx_msg.h             |  15 ++
 src/common/gazelle_fault_inject_common.h |  75 +++++++
 src/lstack/Makefile                      |   5 +
 src/lstack/core/lstack_control_plane.c   |  13 +-
 src/lstack/include/lstack_fault_inject.h |  20 ++
 src/lstack/include/lstack_vdev.h         |   1 +
 src/lstack/netif/dir.mk                  |   3 +
 src/lstack/netif/lstack_fault_inject.c   | 187 ++++++++++++++++
 src/lstack/netif/lstack_vdev.c           |   2 +-
 src/ltran/CMakeLists.txt                 |   5 +
 src/ltran/ltran_dfx.c                    | 274 ++++++++++++++++++++++-
 11 files changed, 595 insertions(+), 5 deletions(-)
 create mode 100644 src/common/gazelle_fault_inject_common.h
 create mode 100644 src/lstack/include/lstack_fault_inject.h
 create mode 100644 src/lstack/netif/lstack_fault_inject.c

diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 7f8422a..8d528d6 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -18,6 +18,10 @@
 
 #include <lwip/reg_sock.h>
 
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#include "gazelle_fault_inject_common.h"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
 #define GAZELLE_CLIENT_NUM_MIN           1
 #define GAZELLE_LOG_LEVEL_MAX            10
 #define GAZELLECTL_TIMEOUT               5000 // millisecond
@@ -54,6 +58,11 @@ enum GAZELLE_STAT_MODE {
     GAZELLE_STAT_LSTACK_SHOW_AGGREGATE,
     GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES,
 
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+    GAZELLE_STAT_FAULT_INJECT_SET,
+    GAZELLE_STAT_FAULT_INJECT_UNSET,
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+    
     GAZELLE_STAT_MODE_MAX,
 };
 
@@ -277,6 +286,9 @@ struct gazelle_stack_dfx_data {
         struct gazelle_stat_lstack_snmp snmp;
         struct nic_eth_xstats nic_xstats;
         struct nic_eth_features nic_features;
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+        struct gazelle_fault_inject_data inject;
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
     } data;
 };
 
@@ -307,6 +319,9 @@ struct gazelle_stat_msg_request {
     union stat_param {
         char log_level[GAZELLE_LOG_LEVEL_MAX];
         uint16_t low_power_mod;
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+        struct gazelle_fault_inject_data inject;
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
     } data;
 };
 
diff --git a/src/common/gazelle_fault_inject_common.h b/src/common/gazelle_fault_inject_common.h
new file mode 100644
index 0000000..3a77f39
--- /dev/null
+++ b/src/common/gazelle_fault_inject_common.h
@@ -0,0 +1,75 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef __GAZELLE_INJECT_COMMON_H__
+#define __GAZELLE_INJECT_COMMON_H__
+
+#include <stdint.h>
+
+enum GAZELLE_FAULT_INJECT_TYPE {
+    GAZELLE_FAULT_INJECT_TYPE_ERR = 0,
+    GAZELLE_FAULT_INJECT_PACKET_DELAY,
+    GAZELLE_FAULT_INJECT_PACKET_LOSS,
+    GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE,
+    GAZELLE_FAULT_INJECT_PACKET_REORDER,
+    GAZELLE_FAULT_INJECT_TYPE_MAX,
+};
+
+enum GAZELLE_FAULT_INJECT_RULE {
+    INJECT_RULE_ERR = 0,
+    /* packet delay rule */
+    INJECT_DELAY_RANDOM,
+    /* packet loss rule */
+    INJECT_LOSS_RANDOM,
+    /* packet duplicate */
+    INJECT_DUPLICATE_RANDOM,
+    /* packet reorder */
+    INJECT_REORDER_RANDOM,
+};
+
+/* fault inject delay: packet delay's time and range, time unit is "ms" */
+struct delay_data {
+    uint32_t delay_time;
+    uint32_t delay_range;
+};
+
+/* fault inject loss: packet loss rate */
+struct loss_data {
+    double loss_rate;
+    uint32_t loss_sigle_count;
+};
+
+/* fault inject duplicate: packet duplicate rate and duplicate count */
+struct duplicate_data {
+    double duplicate_rate;
+    uint32_t duplicate_sigle_count;
+};
+
+/* fault inject reorder: packet reorder rate and reorder count */
+struct reorder_data {
+    double reorder_rate;
+    uint32_t reorder_sigle_count;
+};
+
+struct gazelle_fault_inject_data {
+    int32_t fault_inject_on;
+    enum GAZELLE_FAULT_INJECT_TYPE inject_type;
+    enum GAZELLE_FAULT_INJECT_RULE inject_rule;
+    union {
+        struct delay_data delay;
+        struct loss_data loss;
+        struct duplicate_data duplicate;
+        struct reorder_data reorder;
+    } inject_data;
+};
+
+#endif /* __GAZELLE_INJECT_COMMON_H__ */
diff --git a/src/lstack/Makefile b/src/lstack/Makefile
index df1ddaa..d2c039d 100644
--- a/src/lstack/Makefile
+++ b/src/lstack/Makefile
@@ -47,6 +47,11 @@ ifeq ($(GAZELLE_COVERAGE_ENABLE), 1)
 	CFLAGS += -fprofile-arcs -ftest-coverage
 endif
 
+ifeq ($(GAZELLE_FAULT_INJECT_ENABLE), 1)
+	LDFLAGS += -DGAZELLE_FAULT_INJECT_ENABLE
+	CFLAGS += -DGAZELLE_FAULT_INJECT_ENABLE
+endif
+
 ifeq ($(shell $(CC) -dumpmachine | cut -d"-" -f1), x86_64)
 	CFLAGS += -mssse3
 endif
diff --git a/src/lstack/core/lstack_control_plane.c b/src/lstack/core/lstack_control_plane.c
index 025291d..783f21c 100644
--- a/src/lstack/core/lstack_control_plane.c
+++ b/src/lstack/core/lstack_control_plane.c
@@ -34,6 +34,10 @@
 #include "lstack_protocol_stack.h"
 #include "lstack_control_plane.h"
 
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#include "lstack_fault_inject.h"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
 /* intervals between two connection attempts and two registration attempts, in second */
 #define CONNECT_TO_LTRAN_INFINITE       (-1)
 #define CONNECT_TO_LTRAN_RETRY_INTERVAL 1
@@ -582,7 +586,14 @@ static int32_t handle_stat_request(int32_t sockfd)
         LSTACK_LOG(ERR, LSTACK, "recv wrong stat mode %d\n", (int32_t)msg.stat_mode);
         return 0;
     }
-
+    
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+    if (msg.stat_mode == GAZELLE_STAT_FAULT_INJECT_SET ||
+        msg.stat_mode == GAZELLE_STAT_FAULT_INJECT_UNSET) {
+        return handle_fault_inject_cmd(sockfd, &msg);
+    }
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+    
     if (msg.stat_mode == GAZELLE_STAT_LSTACK_LOG_LEVEL_SET ||
         msg.stat_mode == GAZELLE_STAT_LSTACK_LOW_POWER_MDF) {
         return handle_proc_cmd(sockfd, &msg);
diff --git a/src/lstack/include/lstack_fault_inject.h b/src/lstack/include/lstack_fault_inject.h
new file mode 100644
index 0000000..d54f1f2
--- /dev/null
+++ b/src/lstack/include/lstack_fault_inject.h
@@ -0,0 +1,20 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef __GAZELLE_INJECT_H__
+#define __GAZELLE_INJECT_H__
+
+#include <stdbool.h>
+
+int32_t handle_fault_inject_cmd(int32_t sockfd, struct gazelle_stat_msg_request *msg);
+
+#endif /* __GAZELLE_INJECT_H__ */
diff --git a/src/lstack/include/lstack_vdev.h b/src/lstack/include/lstack_vdev.h
index 4e5d191..1a8342a 100644
--- a/src/lstack/include/lstack_vdev.h
+++ b/src/lstack/include/lstack_vdev.h
@@ -18,5 +18,6 @@ struct gazelle_quintuple;
 enum reg_ring_type;
 void vdev_dev_ops_init(struct lstack_dev_ops *dev_ops);
 int vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple);
+uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts);
 
 #endif /* _GAZELLE_VDEV_H_ */
diff --git a/src/lstack/netif/dir.mk b/src/lstack/netif/dir.mk
index 20fb5d6..1e67734 100644
--- a/src/lstack/netif/dir.mk
+++ b/src/lstack/netif/dir.mk
@@ -9,4 +9,7 @@
 # See the Mulan PSL v2 for more details.
 
 SRC = lstack_ethdev.c lstack_vdev.c lstack_flow.c
+ifeq ($(GAZELLE_FAULT_INJECT_ENABLE), 1)
+	SRC +=  lstack_fault_inject.c
+endif
 $(eval $(call register_dir, netif, $(SRC)))
diff --git a/src/lstack/netif/lstack_fault_inject.c b/src/lstack/netif/lstack_fault_inject.c
new file mode 100644
index 0000000..4daad10
--- /dev/null
+++ b/src/lstack/netif/lstack_fault_inject.c
@@ -0,0 +1,187 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#include <securec.h>
+#include <rte_gro.h>
+#include <rte_net.h>
+
+#include <lwip/posix_api.h>
+
+#include "lstack_cfg.h"
+#include "lstack_log.h"
+#include "lstack_vdev.h"
+
+#define INJECT_MODULO 1000  /* used in modulus operator */
+#define INJECT_US_TO_MS 1000  /* transefer time unit us to ms */
+
+typedef int32_t (*inject_xmit_func)(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                    uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+struct inject_tbl {
+    struct gazelle_fault_inject_data inject_data;
+    inject_xmit_func inject_func;
+};
+static struct inject_tbl g_inject_tbl[GAZELLE_FAULT_INJECT_TYPE_MAX];
+
+struct inject_func_tbl {
+    enum GAZELLE_FAULT_INJECT_TYPE type;
+    enum GAZELLE_FAULT_INJECT_RULE rule;
+    inject_xmit_func inject_func;
+};
+
+static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                          uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                         uint32_t nr_pkts, struct gazelle_fault_inject_data data);
+
+static struct inject_func_tbl g_inject_func_tbl[] = {
+    {GAZELLE_FAULT_INJECT_PACKET_LOSS,  INJECT_LOSS_RANDOM, inject_packet_loss_random},
+    {GAZELLE_FAULT_INJECT_PACKET_DELAY, INJECT_DELAY_RANDOM, inject_packet_delay_random},
+};
+
+static int32_t inject_func_tbl_update()
+{
+    int32_t func_count = sizeof(g_inject_func_tbl) / sizeof(g_inject_func_tbl[0]);
+    
+    for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+        if (!g_inject_tbl[i].inject_data.fault_inject_on) {
+            continue;
+        }
+        for (int32_t j = 0; j < func_count; ++j) {
+            if (g_inject_func_tbl[j].type == g_inject_tbl[i].inject_data.inject_type &&
+                g_inject_func_tbl[j].rule == g_inject_tbl[i].inject_data.inject_rule) {
+                g_inject_tbl[i].inject_func = g_inject_func_tbl[j].inject_func;
+            }
+        }
+    }
+    return 0;
+}
+
+static uint32_t inject_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
+{
+    for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+        if (g_inject_tbl[i].inject_data.fault_inject_on) {
+            int32_t xmit_pkts = 0;
+            xmit_pkts =  g_inject_tbl[i].inject_func(stack, pkts, nr_pkts,
+                                                     g_inject_tbl[i].inject_data);
+            if (xmit_pkts == nr_pkts) {
+                continue;
+            }
+            return xmit_pkts;
+        }
+    }
+    if (rte_mbuf_refcnt_read(*pkts) == 1) {
+        return nr_pkts;
+    }
+    return vdev_tx_xmit(stack, pkts, nr_pkts);
+}
+
+static int32_t inject_strategy_update()
+{
+    inject_func_tbl_update();
+    
+    int32_t inject_on = 0;
+    for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+        if (g_inject_tbl[i].inject_data.fault_inject_on) {
+            inject_on = 1;
+            break;
+        }
+    }
+
+    struct protocol_stack_group *stack_group = get_protocol_stack_group();
+    
+    if (inject_on) {
+        for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+            struct protocol_stack *stack = stack_group->stacks[i];
+            stack->dev_ops.tx_xmit = inject_tx_xmit;
+        }
+        return 0;
+    }
+    
+    for (uint32_t i = 0; i < stack_group->stack_num; ++i) {
+        struct protocol_stack *stack = stack_group->stacks[i];
+        vdev_dev_ops_init(&stack->dev_ops);
+    }
+
+    return 0;
+}
+
+static int32_t inject_packet_delay_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                          uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+    /* while *pkts->refcnt == 1, other inject type is on, and the packets have been loss. */
+    if (rte_mbuf_refcnt_read(*pkts) == 1) {
+        return nr_pkts;
+    }
+    int32_t delay_time = data.inject_data.delay.delay_time;
+    int32_t delay_range = data.inject_data.delay.delay_range;
+    int32_t rand_num = rte_rand();
+    rand_num %= INJECT_MODULO;
+    delay_time += delay_range * rand_num / INJECT_MODULO;
+    rte_delay_us(delay_time * INJECT_US_TO_MS);
+
+    return nr_pkts;
+}
+
+static int32_t inject_packet_loss_random(struct protocol_stack *stack, struct rte_mbuf **pkts,
+                                         uint32_t nr_pkts, struct gazelle_fault_inject_data data)
+{
+    double loss_rate = data.inject_data.loss.loss_rate;
+    int32_t boundary = (int32_t) (loss_rate * INJECT_MODULO);
+    
+    uint32_t rand_num = rte_rand();
+    rand_num %= INJECT_MODULO;
+    
+    if (rand_num > boundary) {
+        return nr_pkts;
+    }
+    
+    for (int32_t i = 0; i < nr_pkts; ++i) {
+        rte_pktmbuf_free(pkts[i]);
+    }
+    return nr_pkts;
+}
+
+int32_t handle_fault_inject_cmd(int32_t sockfd, struct gazelle_stat_msg_request *msg)
+{
+    struct gazelle_stack_dfx_data rsp = {0};
+    int32_t ret = 0;
+    
+    ret = memcpy_s(&g_inject_tbl[msg->data.inject.inject_type].inject_data,
+                   sizeof(struct gazelle_fault_inject_data),
+                   &msg->data.inject, sizeof(struct gazelle_fault_inject_data));
+    if (ret != EOK) {
+        LSTACK_LOG(ERR, LSTACK, "fault inject memcpy_s error, ret = %d", ret);
+        return -1;
+    }
+
+    inject_strategy_update();
+
+    for (int32_t i = 0; i < GAZELLE_FAULT_INJECT_TYPE_MAX; ++i) {
+        ret = memcpy_s(&rsp.data.inject, sizeof(struct gazelle_fault_inject_data),
+                       &g_inject_tbl[i].inject_data, sizeof(struct gazelle_fault_inject_data));
+        if (ret != EOK) {
+            LSTACK_LOG(ERR, LSTACK, "fault inject memcpy_s error, ret = %d", ret);
+            return -1;
+        }
+        if (i == GAZELLE_FAULT_INJECT_TYPE_MAX -1) {
+            rsp.eof = 1;
+        }
+        ret = (int32_t) posix_api->write_fn(sockfd, (void *)&rsp, sizeof(rsp));
+        if (ret <= 0) {
+            LSTACK_LOG(ERR, LSTACK, "write msg from peer failed, errno %d. ret=%d\n", errno, ret);
+            return -1;
+        }
+    }
+    
+    return 0;
+}
+
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index 3703092..107ee8c 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -150,7 +150,7 @@ static uint32_t ltran_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pk
     return sent_pkts;
 }
 
-static uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
+uint32_t vdev_tx_xmit(struct protocol_stack *stack, struct rte_mbuf **pkts, uint32_t nr_pkts)
 {
     uint32_t sent_pkts = 0;
 
diff --git a/src/ltran/CMakeLists.txt b/src/ltran/CMakeLists.txt
index f739ceb..a484ae3 100644
--- a/src/ltran/CMakeLists.txt
+++ b/src/ltran/CMakeLists.txt
@@ -27,6 +27,11 @@ if($ENV{GAZELLE_COVERAGE_ENABLE})
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftest-coverage -fprofile-arcs")
 endif($ENV{GAZELLE_COVERAGE_ENABLE})
 
+if($ENV{GAZELLE_FAULT_INJECT_ENABLE})
+        message("Enable Fault inject option")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DGAZELLE_FAULT_INJECT_ENABLE")
+endif($ENV{GAZELLE_FAULT_INJECT_ENABLE})
+
 execute_process(
     COMMAND rpm -qa dpdk
     OUTPUT_VARIABLE DPDK_VERSION_FULL
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 344afb2..413bf89 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -60,6 +60,46 @@ static struct gazelle_stat_ltran_total g_last_ltran_total;
 static struct gazelle_stat_lstack_total g_last_lstack_total[GAZELLE_MAX_STACK_ARRAY_SIZE];
 static struct gazelle_stack_dfx_data g_last_lstack_data[GAZELLE_MAX_STACK_ARRAY_SIZE];
 
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+#define INJECT_NAME_SIZE 32
+#define INJECT_RULE_SIZE 32
+
+typedef int32_t  (*inject_parse_digit_fun)(char*, char*, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_delay_digit(char *time, char *range, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_loss_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_duplicate_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+static int32_t parse_inject_packet_reorder_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg);
+
+struct gazelle_fault_inject_type_list {
+    char inject_type_item[INJECT_NAME_SIZE];
+    enum GAZELLE_FAULT_INJECT_TYPE inject_type_parsed;
+    inject_parse_digit_fun parse_digit_func;
+};
+
+static struct gazelle_fault_inject_type_list inject_type_list[] = {
+    {"delay",     GAZELLE_FAULT_INJECT_PACKET_DELAY,      parse_inject_packet_delay_digit},
+    {"loss",      GAZELLE_FAULT_INJECT_PACKET_LOSS,       parse_inject_packet_loss_digit},
+    {"duplicate", GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE, parse_inject_packet_duplicate_digit},
+    {"reorder",  GAZELLE_FAULT_INJECT_PACKET_REORDER,   parse_inject_packet_reorder_digit},
+};
+
+struct gazelle_fault_inject_rule_list {
+    char inject_rule_item[INJECT_RULE_SIZE];
+    enum GAZELLE_FAULT_INJECT_RULE inject_rule_parsed;
+    enum GAZELLE_FAULT_INJECT_TYPE rule_parse_assit;
+};
+
+static struct gazelle_fault_inject_rule_list g_gazelle_fault_inject_rule_list[] = {
+    {"random",     INJECT_DELAY_RANDOM,         GAZELLE_FAULT_INJECT_PACKET_DELAY},
+    {"random",     INJECT_LOSS_RANDOM,          GAZELLE_FAULT_INJECT_PACKET_LOSS},
+    {"random",      INJECT_DUPLICATE_RANDOM,     GAZELLE_FAULT_INJECT_PACKAET_DUPLICATE},
+    {"random",      INJECT_REORDER_RANDOM,      GAZELLE_FAULT_INJECT_PACKET_REORDER},
+};
+
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
 static bool g_use_ltran = false;
 
 static char* g_unix_prefix;
@@ -95,6 +135,11 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg
 static void gazelle_print_lstack_aggregate(void *buf, const struct gazelle_stat_msg_request *req_msg);
 static void gazelle_print_lstack_nic_features(void *buf, const struct gazelle_stat_msg_request *req_msg);
 
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+static void gazelle_print_fault_inject_unset_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
 static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = {
     {GAZELLE_STAT_LTRAN_SHOW,          sizeof(struct gazelle_stat_ltran_total),  gazelle_print_ltran_stat_total},
     {GAZELLE_STAT_LTRAN_SHOW_RATE,     sizeof(struct gazelle_stat_ltran_total),  gazelle_print_ltran_stat_rate},
@@ -121,6 +166,11 @@ static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = {
     {GAZELLE_STAT_LSTACK_SHOW_XSTATS, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_xstats},
     {GAZELLE_STAT_LSTACK_SHOW_AGGREGATE, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_aggregate},
     {GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_nic_features},
+
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+    {GAZELLE_STAT_FAULT_INJECT_SET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_set_status},
+    {GAZELLE_STAT_FAULT_INJECT_UNSET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_unset_status},
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
 };
 
 static int32_t g_wait_reply = 1;
@@ -1121,7 +1171,10 @@ static void show_usage(void)
 {
     printf("Usage: gazellectl [-h | help] \n"
            "  or:  gazellectl ltran  {quit | show | set} [LTRAN_OPTIONS] [-u UNIX_PREFIX]\n"
-           "  or:  gazellectl lstack {show | set} ip [LSTACK_OPTIONS] [-u UNIX_PREFIX]\n \n"
+           "  or:  gazellectl lstack {show | set} ip [LSTACK_OPTIONS] [-u UNIX_PREFIX]\n\n"
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+           "  or:  gazellectl inject [inject_type] [digit_param_1] [digit_param_2] [inject_rule]\n\n"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
            "  quit            ltran process exit \n \n"
            "  where  LTRAN_OPTIONS := \n"
            "  show: \n"
@@ -1146,7 +1199,13 @@ static void show_usage(void)
            "  set: \n"
            "  loglevel        {error | info | debug}  set lstack loglevel \n"
            "  lowpower        {0 | 1}  set lowpower enable \n"
-           "  [time]          measure latency time default 1S, maximum 30mins \n");
+           "  [time]          measure latency time default 1S, maximum 30mins \n\n"
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+           "                                     *inject params*\n"
+           " |inject_type    |     digit_param_1      |      digit_param_2     |     inject_rule   |\n"
+           " |  delay        |   delay_time(unit: ms) |  delay_range(unit: ms) |       random      |\n"
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+           );
 }
 
 static int32_t parse_dfx_ltran_set_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
@@ -1436,6 +1495,211 @@ static int32_t parse_dfx_lstack_args(int32_t argc, char *argv[], struct gazelle_
     return num_cmd;
 }
 
+
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+
+#define GAZELLE_SET_FAULT_INJECT_PARAM_COUNT    6
+#define GAZELLE_UNSET_FAULT_INJECT_PARAM_COUNT  4
+#define INJECT_TYPE_INDEX                       2
+#define INJECT_DIGITAL_FIRST_INDEX              3
+#define INJECT_DIGITAL_SECOND_INDEX             4
+#define INJECT_RULE_INDEX                       5
+
+
+static void gazelle_print_fault_inject_type_info(struct gazelle_fault_inject_data *inject)
+{
+    if (!inject->fault_inject_on) {
+        return;
+    }
+    
+    if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_DELAY) {
+        printf("\t| inject_type: delay   | delay_time: %-7d | delay_range: %-3d             | "
+               "inject_rule: random |\n", inject->inject_data.delay.delay_time,
+               inject->inject_data.delay.delay_range);
+    }
+    
+#define INJECT_PERCENT                          100
+    
+    if (inject->inject_type == GAZELLE_FAULT_INJECT_PACKET_LOSS) {
+        printf("\t| inject_type: loss    | loss_rate: %-3.1f%%    | loss_single_count: %-3d       | "
+               "inject_rule: random |\n", inject->inject_data.loss.loss_rate * INJECT_PERCENT,
+               inject->inject_data.loss.loss_sigle_count);
+    }
+    printf("\n");
+}
+
+static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t ret;
+    struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf;
+    struct gazelle_fault_inject_data *inject = &stat->data.inject;
+
+    printf("\n\n\t\t\t\t\t **** FAULT INJECT INFO **** \n\n");
+    do {
+        gazelle_print_fault_inject_type_info(inject);
+        if (stat->eof != 0) {
+            break;
+        }
+        ret = dfx_stat_read_from_ltran(buf, sizeof(struct gazelle_stack_dfx_data), req_msg->stat_mode);
+        if (ret != GAZELLE_OK) {
+            return;
+        }
+    } while (true);
+}
+
+static void gazelle_print_fault_inject_unset_status(void *buf, const struct gazelle_stat_msg_request *req_msg)
+{
+    return;
+}
+
+static int32_t parse_inject_packet_delay_digit(char* time, char* range, struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t parse_success = 0;
+    int32_t delay_time = atoi(time);
+    if (delay_time <= 0) {
+        printf("FAULT INJECT error: delay time error -- %d, need positive integer.\n", delay_time);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.delay.delay_time = (uint32_t) delay_time;
+
+    int32_t delay_range = atoi(range);
+    if (delay_range < 0) {
+        printf("FAULT INJECT error: delay range error -- %d, need positive integer.\n", delay_range);
+        return parse_success;
+    }
+    if (delay_time - delay_range <= 0) {
+        printf("FAULT INJECT error: delay range should lower than delay time.\n");
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.delay.delay_range = delay_range;
+
+    return ++parse_success;
+}
+
+#define INJECT_RATE_LOWER                    0.001
+
+static int32_t parse_inject_packet_loss_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t parse_success = 0;
+    double loss_rate = atof(rate);
+    if (loss_rate < INJECT_RATE_LOWER || loss_rate >= 1) {
+        printf("FAULT INJECT error: loss rate error, range in [0.001, 1), now is %f\n", loss_rate);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.loss.loss_rate = loss_rate;
+
+    int32_t loss_counts = atoi(count);
+    if (loss_counts <= 0) {
+        printf("FAULT INJECT error: single loss counts wrong --%d, need positive integer.", loss_counts);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.loss.loss_sigle_count = loss_counts;
+
+    return ++parse_success;
+}
+
+static int32_t parse_inject_packet_duplicate_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t parse_success = 0;
+    double duplicate_rate = atof(rate);
+    if (duplicate_rate < INJECT_RATE_LOWER || duplicate_rate >= 1) {
+        printf("FAULT INJECT error: duplicate rate error, range in [0.001, 1), now is %f\n", duplicate_rate);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.duplicate.duplicate_rate = duplicate_rate;
+
+    int32_t duplicate_counts = atoi(count);
+    if (duplicate_counts <= 0) {
+        printf("FAULT INJECT error: single duplicate counts wrong --%d, need positive integer.", duplicate_counts);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.duplicate.duplicate_sigle_count = duplicate_counts;
+
+    return ++parse_success;
+}
+
+static int32_t parse_inject_packet_reorder_digit(char *rate, char *count, struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t parse_success = 0;
+    double reorder_rate = atof(rate);
+    if (reorder_rate < INJECT_RATE_LOWER || reorder_rate >= 1) {
+        printf("FAULT INJECT error: reorder rate error, range in [0.001, 1), now is %f\n", reorder_rate);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.reorder.reorder_rate = reorder_rate;
+
+    int32_t reorder_counts = atoi(count);
+    if (reorder_counts <= 0) {
+        printf("FAULT INJECT error: single duplicate counts wrong --%d, need positive integer.", reorder_counts);
+        return parse_success;
+    }
+    req_msg->data.inject.inject_data.reorder.reorder_sigle_count = reorder_counts;
+
+    return ++parse_success;
+}
+
+static int32_t parse_fault_inject_digital_data(char *argv[], struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t parse_success = 0;
+    int32_t func_count = sizeof(inject_type_list) / sizeof(inject_type_list[0]);
+    for (int32_t i = 0; i < func_count; ++i) {
+        if (inject_type_list[i].inject_type_parsed == req_msg->data.inject.inject_type) {
+            parse_success = inject_type_list[i].parse_digit_func(argv[INJECT_DIGITAL_FIRST_INDEX],
+                                                                 argv[INJECT_DIGITAL_SECOND_INDEX], req_msg);
+            break;
+        }
+    }
+
+    return parse_success;
+}
+
+static int32_t parse_dfx_fault_inject_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
+{
+    int32_t num_cmd = 0; /* while parse error, num_cmd will return as 0, or num_cmd should be returned as 1. */
+
+    req_msg->data.inject.fault_inject_on = 1; /* set fault inject on */
+
+    if (argc == GAZELLE_SET_FAULT_INJECT_PARAM_COUNT) {
+        req_msg->stat_mode = GAZELLE_STAT_FAULT_INJECT_SET;
+    } else if (argc == GAZELLE_UNSET_FAULT_INJECT_PARAM_COUNT) {
+        req_msg->stat_mode = GAZELLE_STAT_FAULT_INJECT_UNSET;
+    } else {
+        printf("FAULT_INJECT error: Count of params wrong , correct count is 6 or 4, now is %d\n", argc);
+        return num_cmd;
+    }
+
+    int32_t inject_type_count = sizeof(inject_type_list) / sizeof(inject_type_list[0]);
+    for (int32_t i = 0; i < inject_type_count; ++i) {
+        if (strcmp(inject_type_list[i].inject_type_item, argv[INJECT_TYPE_INDEX]) == 0) {
+            req_msg->data.inject.inject_type = inject_type_list[i].inject_type_parsed;
+            break;
+        }
+    }
+    if (req_msg->data.inject.inject_type == GAZELLE_FAULT_INJECT_TYPE_ERR) {
+        printf("FAULT_INJECT error: input inject type is wrong -- %s\n", argv[INJECT_TYPE_INDEX]);
+        return num_cmd;
+    }
+
+    int32_t inject_rule_count = sizeof(g_gazelle_fault_inject_rule_list) / sizeof(g_gazelle_fault_inject_rule_list[0]);
+    for (int32_t i = 0; i < inject_rule_count; ++i) {
+        if (strcmp(g_gazelle_fault_inject_rule_list[i].inject_rule_item, argv[INJECT_RULE_INDEX]) == 0 &&
+            g_gazelle_fault_inject_rule_list[i].rule_parse_assit == req_msg->data.inject.inject_type) {
+            req_msg->data.inject.inject_rule = g_gazelle_fault_inject_rule_list[i].inject_rule_parsed;
+            break;
+        }
+    }
+    if (req_msg->data.inject.inject_rule == INJECT_RULE_ERR) {
+        printf("FAULT_INJECT error: input inject rule is wrong -- %s\n", argv[INJECT_RULE_INDEX]);
+        return num_cmd;
+    }
+
+    num_cmd = parse_fault_inject_digital_data(argv, req_msg);
+    
+    return num_cmd;
+}
+
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
+
 static int32_t parse_dfx_cmd_args(int32_t argc, char *argv[], struct gazelle_stat_msg_request *req_msg)
 {
     int32_t num_cmd = 0;
@@ -1451,7 +1715,11 @@ static int32_t parse_dfx_cmd_args(int32_t argc, char *argv[], struct gazelle_sta
     if (strcmp(param, "lstack") == 0) {
         num_cmd = parse_dfx_lstack_args(argc, argv, req_msg);
     }
-
+#ifdef GAZELLE_FAULT_INJECT_ENABLE
+    if (strcmp(param, "inject") == 0) {
+        num_cmd = parse_dfx_fault_inject_args(argc, argv, req_msg);
+    }
+#endif /* GAZELLE_FAULT_INJECT_ENABLE */
     return num_cmd;
 }
 
-- 
2.27.0