代码拉取完成，页面将自动刷新
From bbb97803a36a5de0bb6ca6ce0518a6ef0666a22b Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Mon, 15 Jan 2024 09:37:20 +0800
Subject: [PATCH] adpat dpdk 23.11
---
src/common/dpdk_common.c | 171 +++++++++++----------
src/common/gazelle_parse_config.c | 1 +
src/lstack/Makefile | 50 +++---
src/lstack/core/lstack_dpdk.c | 34 ++--
src/lstack/core/lstack_init.c | 2 +
src/lstack/core/lstack_protocol_stack.c | 12 +-
src/lstack/include/lstack_ethdev.h | 8 +
src/lstack/include/lstack_protocol_stack.h | 1 -
src/lstack/netif/lstack_ethdev.c | 8 +
src/lstack/netif/lstack_vdev.c | 4 +-
src/ltran/CMakeLists.txt | 33 +++-
src/ltran/ltran_dfx.c | 20 +--
src/ltran/ltran_ethdev.c | 29 +++-
src/ltran/ltran_forward.c | 14 ++
src/ltran/ltran_stack.c | 1 +
src/ltran/ltran_timer.c | 1 +
src/ltran/main.c | 6 +
17 files changed, 254 insertions(+), 141 deletions(-)
diff --git a/src/common/dpdk_common.c b/src/common/dpdk_common.c
index 62278c2..c0c4f63 100644
--- a/src/common/dpdk_common.c
+++ b/src/common/dpdk_common.c
@@ -10,18 +10,21 @@
* See the Mulan PSL v2 for more details.
*/
-#include <rte_kni.h>
+#include <securec.h>
#include <rte_bus_pci.h>
#include <rte_ethdev.h>
#include <rte_bus_pci.h>
#include <rte_mbuf.h>
#include <rte_ethdev.h>
-#include <securec.h>
+#include <rte_version.h>
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
+#include <rte_kni.h>
+#endif
+
+#include <lwip/dpdk_version.h>
#include "dpdk_common.h"
-#define GAZELLE_KNI_IFACES_NUM 1
-#define GAZELLE_KNI_READ_SIZE 32
#define GAZELLE_MAX_PKT_SZ 2048
#ifdef LTRAN_COMPILE
@@ -34,6 +37,9 @@
#define COMMON_INFO(fmt, ...) LSTACK_LOG(INFO, LSTACK, fmt, ##__VA_ARGS__)
#endif
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
+#define GAZELLE_KNI_IFACES_NUM 1
+#define GAZELLE_KNI_READ_SIZE 32
struct rte_kni *g_pkni = NULL;
static volatile bool g_kni_started = false;
@@ -79,84 +85,6 @@ static int32_t kni_config_network_interface(uint16_t port_id, uint8_t if_up)
return ret;
}
-void eth_params_checksum(struct rte_eth_conf *conf, struct rte_eth_dev_info *dev_info)
-{
- uint64_t rx_ol = 0;
- uint64_t tx_ol = 0;
- uint64_t rx_ol_capa = dev_info->rx_offload_capa;
- uint64_t tx_ol_capa = dev_info->tx_offload_capa;
-
- // rx ip
- if (rx_ol_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) {
- rx_ol |= DEV_RX_OFFLOAD_IPV4_CKSUM;
- COMMON_INFO("DEV_RX_OFFLOAD_IPV4_CKSUM\n");
- }
-
- // rx tcp
- if (rx_ol_capa & DEV_RX_OFFLOAD_TCP_CKSUM) {
- rx_ol |= DEV_RX_OFFLOAD_TCP_CKSUM;
- COMMON_INFO("DEV_RX_OFFLOAD_TCP_CKSUM\n");
- }
-
- // rx udp
- if (rx_ol_capa & DEV_RX_OFFLOAD_UDP_CKSUM) {
- rx_ol |= DEV_RX_OFFLOAD_UDP_CKSUM;
- COMMON_INFO("DEV_RX_OFFLOAD_UDP_CKSUM\n");
- }
-
- // rx vlan
- if (rx_ol_capa & DEV_RX_OFFLOAD_VLAN_STRIP) {
- rx_ol |= DEV_RX_OFFLOAD_VLAN_STRIP;
- COMMON_INFO("DEV_RX_OFFLOAD_VLAN_STRIP\n");
- }
-
- // tx ip
- if (tx_ol_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) {
- tx_ol |= DEV_TX_OFFLOAD_IPV4_CKSUM;
- COMMON_INFO("DEV_TX_OFFLOAD_IPV4_CKSUM\n");
- }
-
- // tx tcp
- if (tx_ol_capa & DEV_TX_OFFLOAD_TCP_CKSUM) {
- tx_ol |= DEV_TX_OFFLOAD_TCP_CKSUM;
- COMMON_INFO("DEV_TX_OFFLOAD_TCP_CKSUM\n");
- }
-
- // tx udp
- if (tx_ol_capa & DEV_TX_OFFLOAD_UDP_CKSUM) {
- tx_ol |= DEV_TX_OFFLOAD_UDP_CKSUM;
- COMMON_INFO("DEV_TX_OFFLOAD_UDP_CKSUM\n");
- }
-
- // tx tso
- if (tx_ol_capa & DEV_TX_OFFLOAD_TCP_TSO) {
- tx_ol |= (DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_MULTI_SEGS);
- COMMON_INFO("DEV_TX_OFFLOAD_TCP_TSO\n");
- }
-
- // tx vlan
- if (tx_ol_capa & DEV_TX_OFFLOAD_VLAN_INSERT) {
- tx_ol |= DEV_TX_OFFLOAD_VLAN_INSERT;
- COMMON_INFO("DEV_TX_OFFLOAD_VLAN_INSERT\n");
- }
-
- if (!(rx_ol & DEV_RX_OFFLOAD_UDP_CKSUM) ||
- !(rx_ol & DEV_RX_OFFLOAD_TCP_CKSUM) ||
- !(rx_ol & DEV_RX_OFFLOAD_IPV4_CKSUM)) {
- rx_ol = 0;
- }
- if (!(tx_ol & DEV_TX_OFFLOAD_UDP_CKSUM) ||
- !(tx_ol & DEV_TX_OFFLOAD_TCP_CKSUM) ||
- !(tx_ol & DEV_TX_OFFLOAD_IPV4_CKSUM)) {
- tx_ol = 0;
- }
-
- conf->rxmode.offloads = rx_ol;
- conf->txmode.offloads = tx_ol;
-
- COMMON_INFO("Set checksum offloads\n");
-}
-
int32_t dpdk_kni_init(uint16_t port, struct rte_mempool *pool)
{
int32_t ret;
@@ -262,3 +190,82 @@ void kni_process_rx(uint16_t port)
}
}
}
+#endif
+
+void eth_params_checksum(struct rte_eth_conf *conf, struct rte_eth_dev_info *dev_info)
+{
+ uint64_t rx_ol = 0;
+ uint64_t tx_ol = 0;
+ uint64_t rx_ol_capa = dev_info->rx_offload_capa;
+ uint64_t tx_ol_capa = dev_info->tx_offload_capa;
+
+ // rx ip
+ if (rx_ol_capa & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM) {
+ rx_ol |= RTE_ETH_RX_OFFLOAD_IPV4_CKSUM;
+ COMMON_INFO("RTE_ETH_RX_OFFLOAD_IPV4_CKSUM\n");
+ }
+
+ // rx tcp
+ if (rx_ol_capa & RTE_ETH_RX_OFFLOAD_TCP_CKSUM) {
+ rx_ol |= RTE_ETH_RX_OFFLOAD_TCP_CKSUM;
+ COMMON_INFO("RTE_ETH_RX_OFFLOAD_TCP_CKSUM\n");
+ }
+
+ // rx udp
+ if (rx_ol_capa & RTE_ETH_RX_OFFLOAD_UDP_CKSUM) {
+ rx_ol |= RTE_ETH_RX_OFFLOAD_UDP_CKSUM;
+ COMMON_INFO("RTE_ETH_RX_OFFLOAD_UDP_CKSUM\n");
+ }
+
+ // rx vlan
+ if (rx_ol_capa & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
+ rx_ol |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
+ COMMON_INFO("RTE_ETH_RX_OFFLOAD_VLAN_STRIP\n");
+ }
+
+ // tx ip
+ if (tx_ol_capa & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) {
+ tx_ol |= RTE_ETH_TX_OFFLOAD_IPV4_CKSUM;
+ COMMON_INFO("RTE_ETH_TX_OFFLOAD_IPV4_CKSUM\n");
+ }
+
+ // tx tcp
+ if (tx_ol_capa & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) {
+ tx_ol |= RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
+ COMMON_INFO("RTE_ETH_TX_OFFLOAD_TCP_CKSUM\n");
+ }
+
+ // tx udp
+ if (tx_ol_capa & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) {
+ tx_ol |= RTE_ETH_TX_OFFLOAD_UDP_CKSUM;
+ COMMON_INFO("RTE_ETH_TX_OFFLOAD_UDP_CKSUM\n");
+ }
+
+ // tx tso
+ if (tx_ol_capa & RTE_ETH_TX_OFFLOAD_TCP_TSO) {
+ tx_ol |= (RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_MULTI_SEGS);
+ COMMON_INFO("RTE_ETH_TX_OFFLOAD_TCP_TSO\n");
+ }
+
+ // tx vlan
+ if (tx_ol_capa & RTE_ETH_TX_OFFLOAD_VLAN_INSERT) {
+ tx_ol |= RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
+ COMMON_INFO("RTE_ETH_TX_OFFLOAD_VLAN_INSERT\n");
+ }
+
+ if (!(rx_ol & RTE_ETH_RX_OFFLOAD_UDP_CKSUM) ||
+ !(rx_ol & RTE_ETH_RX_OFFLOAD_TCP_CKSUM) ||
+ !(rx_ol & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM)) {
+ rx_ol = 0;
+ }
+ if (!(tx_ol & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) ||
+ !(tx_ol & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) ||
+ !(tx_ol & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM)) {
+ tx_ol = 0;
+ }
+
+ conf->rxmode.offloads = rx_ol;
+ conf->txmode.offloads = tx_ol;
+
+ COMMON_INFO("Set checksum offloads\n");
+}
diff --git a/src/common/gazelle_parse_config.c b/src/common/gazelle_parse_config.c
index 1adce78..0553f0d 100644
--- a/src/common/gazelle_parse_config.c
+++ b/src/common/gazelle_parse_config.c
@@ -15,6 +15,7 @@
#include <limits.h>
#include <stdint.h>
#include <unistd.h>
+#include <ctype.h>
#include <sys/stat.h>
#include <sys/types.h>
diff --git a/src/lstack/Makefile b/src/lstack/Makefile
index d585040..df1ddaa 100644
--- a/src/lstack/Makefile
+++ b/src/lstack/Makefile
@@ -7,17 +7,17 @@
# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
# PURPOSE.
# See the Mulan PSL v2 for more details.
-
LSTACK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
ROOT_DIR := $(dir $(abspath $(LSTACK_DIR)))
LWIP_INCLUDE_FILE ?= /usr/include/lwip
LIB_PATH ?= /usr/lib64
-ifeq ($(DPDK_VERSION_1911), 1)
- DPDK_INCLUDE_FILE ?= /usr/include/dpdk
+DPDK_VERSION := $(shell rpm -q --queryformat '%{VERSION}' dpdk)
+ifeq ($(DPDK_VERSION),21.11)
+ DPDK_INCLUDE_FILE := /usr/local/include
else
- DPDK_INCLUDE_FILE ?= /usr/local/include/
+ DPDK_INCLUDE_FILE := /usr/include/dpdk
endif
AR = ar
@@ -77,34 +77,36 @@ LIBRTE_LIB = $(LIB_PATH)/librte_pci.so \
$(LIB_PATH)/librte_gro.so \
$(LIB_PATH)/librte_ring.so \
$(LIB_PATH)/librte_mbuf.so \
- $(LIB_PATH)/librte_kni.so \
$(LIB_PATH)/librte_kvargs.so \
$(LIB_PATH)/librte_bus_vdev.so \
$(LIB_PATH)/librte_net.so \
$(LIB_PATH)/librte_ethdev.so \
- $(LIB_PATH)/librte_pdump.so \
+ $(LIB_PATH)/librte_pdump.so
+
+ifeq ($(shell expr $(DPDK_VERSION) \< 23.11), 1)
+ LIBRTE_LIB += $(LIB_PATH)/librte_kni.so
+endif
-ifeq ($(DPDK_VERSION_1911), 1)
- CFLAGS += -DDPDK_VERSION_1911=1
+ifeq ($(DPDK_VERSION), 19.11)
LIBRTE_LIB += $(LIB_PATH)/librte_pmd_pcap.so \
- $(LIB_PATH)/librte_pmd_bond.so \
- $(LIB_PATH)/librte_pmd_hinic.so \
- $(LIB_PATH)/librte_pmd_ixgbe.so \
- $(LIB_PATH)/librte_pmd_virtio.so
- ifneq ($(ARCH), loongarch64)
- LIBRTE_LIB += $(LIB_PATH)/librte_pmd_i40e.so
- endif
+ $(LIB_PATH)/librte_pmd_bond.so \
+ $(LIB_PATH)/librte_pmd_hinic.so \
+ $(LIB_PATH)/librte_pmd_ixgbe.so \
+ $(LIB_PATH)/librte_pmd_virtio.so
+ ifneq ($(ARCH), loongarch64)
+ LIBRTE_LIB += $(LIB_PATH)/librte_pmd_i40e.so
+ endif
else
LIBRTE_LIB += $(LIB_PATH)/librte_net_pcap.so \
- $(LIB_PATH)/librte_net_bond.so \
- $(LIB_PATH)/librte_net_hinic.so \
- $(LIB_PATH)/librte_net_ixgbe.so \
- $(LIB_PATH)/librte_net_virtio.so \
- $(LIB_PATH)/librte_telemetry.so \
- $(LIB_PATH)/librte_pcapng.so
- ifneq ($(ARCH), loongarch64)
- LIBRTE_LIB += $(LIB_PATH)/librte_net_i40e.so
- endif
+ $(LIB_PATH)/librte_net_bond.so \
+ $(LIB_PATH)/librte_net_hinic.so \
+ $(LIB_PATH)/librte_net_ixgbe.so \
+ $(LIB_PATH)/librte_net_virtio.so \
+ $(LIB_PATH)/librte_telemetry.so \
+ $(LIB_PATH)/librte_pcapng.so
+ ifneq ($(ARCH), loongarch64)
+ LIBRTE_LIB += $(LIB_PATH)/librte_net_i40e.so
+ endif
endif
DEP_LIBS = $(LWIP_LIB) $(LIBRTE_LIB)
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 630b043..7f1ceb8 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -27,7 +27,10 @@
#include <rte_malloc.h>
#include <rte_bus.h>
#include <rte_errno.h>
+#include <rte_version.h>
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
#include <rte_kni.h>
+#endif
#include <rte_pdump.h>
#include <rte_thash.h>
#include <lwip/posix_api.h>
@@ -35,6 +38,7 @@
#include <lwip/pbuf.h>
#include <lwip/reg_sock.h>
#include <lwip/priv/tcp_priv.h>
+
#include <rte_eth_bond_8023ad.h>
#include <rte_eth_bond.h>
#include <rte_ethdev.h>
@@ -59,8 +63,10 @@ struct eth_params {
struct rte_eth_rxconf rx_conf;
struct rte_eth_txconf tx_conf;
};
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
struct rte_kni;
static struct rte_bus *g_pci_bus = NULL;
+#endif
#define RSS_HASH_KEY_LEN 40
static uint8_t g_default_rss_key[] = {
@@ -368,9 +374,9 @@ static struct eth_params *alloc_eth_params(uint16_t port_id, uint16_t nb_queues)
eth_params->nb_queues = nb_queues;
eth_params->nb_rx_desc = get_global_cfg_params()->nic.rxqueue_size;
eth_params->nb_tx_desc = get_global_cfg_params()->nic.txqueue_size;
- eth_params->conf.link_speeds = ETH_LINK_SPEED_AUTONEG;
- eth_params->conf.txmode.mq_mode = ETH_MQ_TX_NONE;
- eth_params->conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
+ eth_params->conf.link_speeds = RTE_ETH_LINK_SPEED_AUTONEG;
+ eth_params->conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE;
+ eth_params->conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
return eth_params;
}
@@ -388,7 +394,7 @@ uint64_t get_eth_params_tx_ol(void)
static int eth_params_rss(struct rte_eth_conf *conf, struct rte_eth_dev_info *dev_info)
{
int rss_enable = 0;
- uint64_t def_rss_hf = ETH_RSS_TCP | ETH_RSS_UDP | ETH_RSS_IP;
+ uint64_t def_rss_hf = RTE_ETH_RSS_TCP | RTE_ETH_RSS_UDP | RTE_ETH_RSS_IP;
struct rte_eth_rss_conf rss_conf = {
g_default_rss_key,
RSS_HASH_KEY_LEN,
@@ -404,7 +410,7 @@ static int eth_params_rss(struct rte_eth_conf *conf, struct rte_eth_dev_info *de
if (rss_conf.rss_hf) {
rss_enable = 1;
conf->rx_adv_conf.rss_conf = rss_conf;
- conf->rxmode.mq_mode = ETH_MQ_RX_RSS;
+ conf->rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;
LSTACK_LOG(INFO, LSTACK, "set rss_hf: %lx\n", rss_conf.rss_hf);
}
@@ -427,8 +433,8 @@ static void rss_setup(const int port_id, const uint16_t nb_queues)
return;
}
- reta_conf_size = dev_info.reta_size / RTE_RETA_GROUP_SIZE;
- if (dev_info.reta_size % RTE_RETA_GROUP_SIZE) {
+ reta_conf_size = dev_info.reta_size / RTE_ETH_RETA_GROUP_SIZE;
+ if (dev_info.reta_size % RTE_ETH_RETA_GROUP_SIZE) {
reta_conf_size += 1;
}
@@ -438,8 +444,8 @@ static void rss_setup(const int port_id, const uint16_t nb_queues)
}
for (i = 0; i < dev_info.reta_size; i++) {
struct rte_eth_rss_reta_entry64 *one_reta_conf =
- &reta_conf[i / RTE_RETA_GROUP_SIZE];
- one_reta_conf->reta[i % RTE_RETA_GROUP_SIZE] = i % nb_queues;
+ &reta_conf[i / RTE_ETH_RETA_GROUP_SIZE];
+ one_reta_conf->reta[i % RTE_ETH_RETA_GROUP_SIZE] = i % nb_queues;
}
for (i = 0; i < reta_conf_size; i++) {
@@ -518,7 +524,11 @@ int32_t dpdk_ethdev_init(int port_id, bool bond_port)
return -1;
}
+#if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0)
+ ret = rte_eth_bond_member_add(port_id, slave_port_id[i]);
+#else
ret = rte_eth_bond_slave_add(port_id, slave_port_id[i]);
+#endif
if (ret != 0) {
LSTACK_LOG(ERR, LSTACK, "dpdk add slave port failed ret = %d\n", ret);
return -1;
@@ -671,6 +681,7 @@ int32_t dpdk_init_lstack_kni(void)
return 0;
}
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
void dpdk_skip_nic_init(void)
{
/* when lstack init nic again, ltran can't read pkts from nic. unregister pci_bus to avoid init nic in lstack */
@@ -686,6 +697,7 @@ void dpdk_restore_pci(void)
rte_bus_register(g_pci_bus);
}
}
+#endif
int32_t init_dpdk_ethdev(void)
{
@@ -849,7 +861,11 @@ void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id)
if (strcmp(dev_info.driver_name, "net_bonding") == 0) {
uint16_t slaves[RTE_MAX_ETHPORTS];
int slave_count;
+#if RTE_VERSION >= RTE_VERSION_NUM(23, 11, 0, 0)
+ slave_count = rte_eth_bond_members_get(port_id, slaves, RTE_MAX_ETHPORTS);
+#else
slave_count = rte_eth_bond_slaves_get(port_id, slaves, RTE_MAX_ETHPORTS);
+#endif
if (slave_count <= 0) {
LSTACK_LOG(ERR, LSTACK, "rte_eth_bond_slaves_get failed.\n");
return;
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index fef2942..31fd91d 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -116,7 +116,9 @@ void gazelle_exit(void)
stack_group_exit();
}
if (!use_ltran()) {
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
dpdk_kni_release();
+#endif
}
}
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 1fa7373..a24fc4c 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -13,8 +13,6 @@
#include <pthread.h>
#include <stdatomic.h>
-#include <rte_kni.h>
-
#include <lwip/sockets.h>
#include <lwip/tcpip.h>
#include <lwip/tcp.h>
@@ -38,6 +36,10 @@
#include "lstack_stack_stat.h"
#include "lstack_protocol_stack.h"
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
+#include <rte_kni.h>
+#endif
+
#define KERNEL_EVENT_10us 10
static PER_THREAD struct protocol_stack *g_stack_p = NULL;
@@ -453,7 +455,9 @@ int stack_polling(uint32_t wakeup_tick)
int force_quit;
struct cfg_params *cfg = get_global_cfg_params();
uint8_t use_ltran_flag = cfg->use_ltran;
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
bool kni_switch = cfg->kni_switch;
+#endif
bool use_sockmap = cfg->use_sockmap;
bool stack_mode_rtc = cfg->stack_mode_rtc;
uint32_t rpc_number = cfg->rpc_number;
@@ -486,6 +490,7 @@ int stack_polling(uint32_t wakeup_tick)
}
}
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
/* run to completion mode currently does not support kni */
/* KNI requests are generally low-rate I/Os,
* so processing KNI requests only in the thread with queue_id No.0 is sufficient. */
@@ -495,6 +500,7 @@ int stack_polling(uint32_t wakeup_tick)
kni_handle_rx(stack->port_id);
}
}
+#endif
return force_quit;
}
@@ -950,7 +956,9 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack
return;
}
copy_mbuf(mbuf_copy, mbuf);
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
kni_handle_tx(mbuf_copy);
+#endif
return;
}
diff --git a/src/lstack/include/lstack_ethdev.h b/src/lstack/include/lstack_ethdev.h
index 326bd1b..3252906 100644
--- a/src/lstack/include/lstack_ethdev.h
+++ b/src/lstack/include/lstack_ethdev.h
@@ -13,6 +13,9 @@
#ifndef __GAZELLE_ETHDEV_H__
#define __GAZELLE_ETHDEV_H__
+#include <rte_eal.h>
+#include <rte_version.h>
+
#define INVAILD_PROCESS_IDX 255
enum port_type {
@@ -46,7 +49,12 @@ void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack);
int recv_pkts_from_other_process(int process_index, void* arg);
int32_t check_params_from_primary(void);
+
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
void kni_handle_rx(uint16_t port_id);
+void kni_handle_tx(struct rte_mbuf *mbuf);
+#endif
+
void delete_user_process_port(uint16_t dst_port, enum port_type type);
void add_user_process_port(uint16_t dst_port, uint8_t process_idx, enum port_type type);
void delete_flow_director(uint32_t dst_ip, uint16_t src_port, uint16_t dst_port);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index 8e2e807..c681547 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -181,5 +181,4 @@ void stack_recvlist_count(struct rpc_msg *msg);
void stack_exit_by_rpc(struct rpc_msg *msg);
int stack_polling(uint32_t wakeup_tick);
-void kni_handle_tx(struct rte_mbuf *mbuf);
#endif
diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c
index 2e11670..5b0f83e 100644
--- a/src/lstack/netif/lstack_ethdev.c
+++ b/src/lstack/netif/lstack_ethdev.c
@@ -13,7 +13,11 @@
#include <sys/socket.h>
#include <sys/un.h>
+#include <rte_eal.h>
+#include <rte_version.h>
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
#include <rte_kni.h>
+#endif
#include <rte_ethdev.h>
#include <rte_malloc.h>
#include <rte_ether.h>
@@ -755,6 +759,7 @@ int distribute_pakages(struct rte_mbuf *mbuf)
return TRANSFER_KERNEL;
}
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
void kni_handle_rx(uint16_t port_id)
{
struct rte_mbuf *pkts_burst[PACKET_READ_SIZE];
@@ -793,6 +798,7 @@ void kni_handle_tx(struct rte_mbuf *mbuf)
rte_pktmbuf_free(mbuf);
}
}
+#endif
/* optimized eth_dev_poll() in lstack */
int32_t gazelle_eth_dev_poll(struct protocol_stack *stack, uint8_t use_ltran_flag, uint32_t nic_read_number)
@@ -835,7 +841,9 @@ int32_t gazelle_eth_dev_poll(struct protocol_stack *stack, uint8_t use_ltran_fla
if (likely(transfer_type == TRANSFER_CURRENT_THREAD)) {
eth_dev_recv(stack->pkts[i], stack);
} else if (transfer_type == TRANSFER_KERNEL) {
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
kni_handle_tx(stack->pkts[i]);
+#endif
} else {
/* transfer to other thread */
}
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index c845f7a..fe17f59 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -22,6 +22,8 @@
#include <rte_net.h>
#include <netif/ethernet.h>
+#include <lwip/dpdk_version.h>
+
#include "lstack_cfg.h"
#include "lstack_dpdk.h"
#include "lstack_ethdev.h"
@@ -85,7 +87,7 @@ static uint32_t vdev_rx_poll(struct protocol_stack *stack, struct rte_mbuf **pkt
/* skip gro when tcp/ip cksum offloads disable */
if (get_protocol_stack_group()->rx_offload == 0 || (get_global_cfg_params()->nic.vlan_mode >= 0
- && !(get_protocol_stack_group()->rx_offload & DEV_RX_OFFLOAD_VLAN_STRIP))) {
+ && !(get_protocol_stack_group()->rx_offload & RTE_ETH_RX_OFFLOAD_VLAN_STRIP))) {
return pkt_num;
}
diff --git a/src/ltran/CMakeLists.txt b/src/ltran/CMakeLists.txt
index e098a77..f739ceb 100644
--- a/src/ltran/CMakeLists.txt
+++ b/src/ltran/CMakeLists.txt
@@ -27,22 +27,38 @@ if($ENV{GAZELLE_COVERAGE_ENABLE})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftest-coverage -fprofile-arcs")
endif($ENV{GAZELLE_COVERAGE_ENABLE})
-if($ENV{DPDK_VERSION_1911})
+execute_process(
+ COMMAND rpm -qa dpdk
+ OUTPUT_VARIABLE DPDK_VERSION_FULL
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+string(REGEX MATCH "[0-9]+\\.[0-9]+" DPDK_VERSION ${DPDK_VERSION_FULL})
+
+if (DPDK_VERSION STREQUAL "21.11")
+ set(DPDK_DIR /usr/local/include/)
+else()
set(DPDK_DIR /usr/include/dpdk)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DDPDK_VERSION_1911=1")
+endif()
+
+if (DPDK_VERSION STREQUAL "19.11")
set(DPDK_LINK_FLAGS "-Wl,-lrte_pmd_bond -Wl,-lrte_pmd_hinic -Wl,-lrte_pmd_ixgbe \
-Wl,-lrte_pmd_pcap -Wl,-lrte_pmd_virtio")
if(NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "loongarch64")
set(DPDK_LINK_FLAGS "${DPDK_LINK_FLAGS} -Wl,-lrte_pmd_i40e")
endif()
else()
- set(DPDK_DIR /usr/local/include/)
set(DPDK_LINK_FLAGS "-Wl,-lrte_net_bond -Wl,-lrte_net_hinic -Wl,-lrte_net_ixgbe \
-Wl,-lpcap -Wl,-lrte_net_pcap -Wl,-lrte_net_virtio -Wl,-lrte_pcapng -Wl,-lrte_telemetry")
if(NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "loongarch64")
set(DPDK_LINK_FLAGS "${DPDK_LINK_FLAGS} -Wl,-lrte_net_i40e")
endif()
-endif($ENV{DPDK_VERSION_1911})
+endif()
+
+if (DPDK_VERSION GREATER_EQUAL 23.11)
+ set(DPDK_LINK_FLAGS "${DPDK_LINK_FLAGS} -lrte_log -larchive -Wl,-lrte_eal")
+else()
+ set(DPDK_LINK_FLAGS "${DPDK_LINK_FLAGS} -lrte_kni -Wl,-Bstatic -lrte_eal -Wl,-Bdynamic")
+endif()
add_executable(ltran main.c ltran_param.c ltran_config.c ltran_ethdev.c ltran_stat.c ltran_errno.c
ltran_monitor.c ltran_instance.c ltran_stack.c ltran_tcp_conn.c ltran_tcp_sock.c
@@ -55,16 +71,19 @@ target_compile_options(ltran PRIVATE -fno-strict-aliasing -D__ARM_FEATURE_CRC32=
-DRTE_MACHINE_CPUFLAG_SHA1 -DRTE_MACHINE_CPUFLAG_SHA2 -include rte_config.h
-D_GNU_SOURCE -W -Wall -Wstrict-prototypes -Wmissing-declarations -Wmissing-prototypes -Wold-style-definition
-Wpointer-arith -Wcast-qual -Wnested-externs -Wformat-nonliteral -Wformat-security -Wundef
- -Wdeprecated -Wwrite-strings -Wno-implicit-fallthrough -D_FORTIFY_SOURCE=2)
-if(NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "sw_64")
+ -Wdeprecated -Wwrite-strings -Wno-implicit-fallthrough -D_FORTIFY_SOURCE=2 -Wno-error=deprecated-declarations)
+
+# gcc coredump in openEuler-20.03-LTS-XX aarch64 when add -march=native
+if(NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "sw_64|aarch64" )
target_compile_options(ltran PRIVATE -march=native -Wcast-align)
endif()
target_link_libraries(ltran PRIVATE config boundscheck rte_pdump -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines)
+
set_target_properties(ltran PROPERTIES LINK_FLAGS "-L$ENV{DPDK_LIB_PATH} -Wl,--no-whole-archive \
-Wl,-lrte_meter -Wl,--whole-archive -Wl,-lrte_gro -Wl,-lrte_hash -Wl,-lrte_kvargs -Wl,-lrte_mbuf -Wl,-lrte_ethdev \
-Wl,-lrte_net -Wl,-lrte_timer -Wl,-lrte_mempool -Wl,-lrte_mempool_ring -Wl,-lrte_ring -Wl,-lrte_pci \
- -Wl,-Bstatic -lrte_eal -Wl,-Bdynamic -Wl,-lrte_cmdline -Wl,-lrte_kni -Wl,-lrte_bus_pci \
+ -Wl,-lrte_cmdline -Wl,-lrte_bus_pci\
-Wl,-lrte_bus_vdev ${DPDK_LINK_FLAGS} \
-Wl,--no-whole-archive -Wl,-lm -Wl,-lrt -Wl,-lnuma -Wl,-ldl -Wl,-export-dynamic -Wl,-export-dynamic \
-Wl,--as-needed -Wl,-export-dynamic -Wl,-Map=ltran.map -Wl,--cref")
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 0d7ea7b..4b246a1 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -26,6 +26,8 @@
#include <rte_log.h>
#include <rte_ethdev.h>
+#include <lwip/dpdk_version.h>
+
#include "ltran_stat.h"
#include "ltran_base.h"
#include "gazelle_dfx_msg.h"
@@ -168,16 +170,16 @@ static void gazelle_print_lstack_nic_features(void *buf, const struct gazelle_st
struct nic_eth_features *f = &(((struct gazelle_stack_dfx_data *)buf)->data.nic_features);
printf("###### NIC offload and other features for port %-2d #########\n", f->port_id);
- printf("tx-ipv4-checksum: %s\n", (f->tx_offload & DEV_TX_OFFLOAD_IPV4_CKSUM) ? "on" : "off");
- printf("tx-tcp-checksum: %s\n", (f->tx_offload & DEV_TX_OFFLOAD_TCP_CKSUM) ? "on" : "off");
- printf("tx-tcp-tso: %s\n", (f->tx_offload & DEV_TX_OFFLOAD_TCP_TSO) ? "on" : "off");
- printf("tx-udp-checksum: %s\n", (f->tx_offload & DEV_TX_OFFLOAD_UDP_CKSUM) ? "on" : "off");
- printf("tx-vlan-insert: %s\n", (f->tx_offload & DEV_TX_OFFLOAD_VLAN_INSERT) ? "on" : "off");
+ printf("tx-ipv4-checksum: %s\n", (f->tx_offload & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) ? "on" : "off");
+ printf("tx-tcp-checksum: %s\n", (f->tx_offload & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) ? "on" : "off");
+ printf("tx-tcp-tso: %s\n", (f->tx_offload & RTE_ETH_TX_OFFLOAD_TCP_TSO) ? "on" : "off");
+ printf("tx-udp-checksum: %s\n", (f->tx_offload & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) ? "on" : "off");
+ printf("tx-vlan-insert: %s\n", (f->tx_offload & RTE_ETH_TX_OFFLOAD_VLAN_INSERT) ? "on" : "off");
- printf("rx-ipv4-checksum: %s\n", (f->rx_offload & DEV_RX_OFFLOAD_IPV4_CKSUM) ? "on" : "off");
- printf("rx-tcp-checksum: %s\n", (f->rx_offload & DEV_RX_OFFLOAD_TCP_CKSUM) ? "on" : "off");
- printf("rx-udp-checksum: %s\n", (f->rx_offload & DEV_RX_OFFLOAD_UDP_CKSUM) ? "on" : "off");
- printf("rx-vlan-strip: %s\n", (f->rx_offload & DEV_RX_OFFLOAD_VLAN_STRIP) ? "on" : "off");
+ printf("rx-ipv4-checksum: %s\n", (f->rx_offload & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM) ? "on" : "off");
+ printf("rx-tcp-checksum: %s\n", (f->rx_offload & RTE_ETH_RX_OFFLOAD_TCP_CKSUM) ? "on" : "off");
+ printf("rx-udp-checksum: %s\n", (f->rx_offload & RTE_ETH_RX_OFFLOAD_UDP_CKSUM) ? "on" : "off");
+ printf("rx-vlan-strip: %s\n", (f->rx_offload & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) ? "on" : "off");
}
static void gazelle_print_ltran_conn(void *buf, const struct gazelle_stat_msg_request *req_msg)
diff --git a/src/ltran/ltran_ethdev.c b/src/ltran/ltran_ethdev.c
index e2eb4a8..2fb7fab 100644
--- a/src/ltran/ltran_ethdev.c
+++ b/src/ltran/ltran_ethdev.c
@@ -21,10 +21,15 @@
#include <rte_eth_bond.h>
#include <rte_ethdev.h>
#include <rte_errno.h>
+#include <rte_version.h>
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
#include <rte_kni.h>
+#endif
#include <syslog.h>
#include <securec.h>
+#include <lwip/dpdk_version.h>
+
#include "dpdk_common.h"
#include "ltran_param.h"
#include "ltran_log.h"
@@ -85,7 +90,11 @@ static int32_t ltran_mbuf_pool_init(void);
static int32_t ltran_single_slave_port_init(uint16_t port_num, struct rte_mempool *pktmbuf_rxpool);
static int32_t ltran_single_bond_port_init(uint16_t port_num, struct rte_mempool *pktmbuf_rxpool);
static int32_t ltran_slave_port_init(void);
+
+
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
static int32_t ltran_kni_init(void);
+#endif
static int32_t ltran_bond_port_init(void);
static int32_t ltran_eal_init(void)
@@ -248,10 +257,10 @@ static int32_t ltran_single_slave_port_init(uint16_t port_num, struct rte_mempoo
}
struct rte_eth_conf port_conf = {0};
- port_conf.txmode.mq_mode = ETH_MQ_TX_NONE;
- port_conf.link_speeds = ETH_LINK_SPEED_AUTONEG;
+ port_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE;
+ port_conf.link_speeds = RTE_ETH_LINK_SPEED_AUTONEG;
eth_params_checksum(&port_conf, &dev_info);
- port_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
+ port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
struct ltran_config *ltran_config = get_ltran_config();
ltran_config->dpdk.rx_offload = port_conf.rxmode.offloads;
@@ -311,7 +320,11 @@ static int32_t ltran_slave_port_init(void)
static int32_t ltran_eth_bond_slave(const struct port_info *port_info, uint16_t port_num, uint16_t bond_port_id)
{
for (uint32_t i = 0; i < port_info[port_num].num_ports; i++) {
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
int32_t ret = rte_eth_bond_slave_add(bond_port_id, port_info[port_num].id[i]);
+#else
+ int32_t ret = rte_eth_bond_member_add(bond_port_id, port_info[port_num].id[i]);
+#endif
if (ret < 0) {
return ret;
}
@@ -370,9 +383,9 @@ static int32_t ltran_bond_port_attr_set(uint16_t port_num, uint16_t bond_port_id
}
struct rte_eth_conf port_conf = {0};
- port_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
- port_conf.txmode.mq_mode = ETH_MQ_TX_NONE;
- port_conf.link_speeds = ETH_LINK_SPEED_AUTONEG;
+ port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
+ port_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE;
+ port_conf.link_speeds = RTE_ETH_LINK_SPEED_AUTONEG;
eth_params_checksum(&port_conf, &dev_info);
ret = rte_eth_dev_configure(bond_port_id, rx_queue_num, tx_queue_num, &port_conf);
@@ -469,6 +482,7 @@ static int32_t ltran_bond_port_init(void)
return GAZELLE_OK;
}
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
static int32_t ltran_kni_init(void)
{
// if not use kni. skip kni init and return
@@ -481,6 +495,7 @@ static int32_t ltran_kni_init(void)
return dpdk_kni_init(bond_port[0], txpool[0]);
}
+#endif
typedef int32_t (*ethdev_init_func)(void);
@@ -492,7 +507,9 @@ static ethdev_init_func g_ltran_ethdev_init_tbl[] = {
ltran_mbuf_pool_init,
ltran_slave_port_init,
ltran_bond_port_init,
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
ltran_kni_init,
+#endif
};
int32_t ltran_ethdev_init(void)
diff --git a/src/ltran/ltran_forward.c b/src/ltran/ltran_forward.c
index ee379b5..a5756d7 100644
--- a/src/ltran/ltran_forward.c
+++ b/src/ltran/ltran_forward.c
@@ -13,7 +13,12 @@
#include <rte_arp.h>
#include <rte_eal.h>
#include <rte_common.h>
+#include <rte_version.h>
+
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
#include <rte_kni.h>
+#endif
+
#include <rte_malloc.h>
#include <rte_ethdev.h>
#include <rte_mempool.h>
@@ -195,6 +200,7 @@ static __rte_always_inline uint32_t pkt_bufs_enque_rx_ring(struct gazelle_stack
static __rte_always_inline void flush_rx_ring(struct gazelle_stack *stack)
{
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
if (unlikely(stack == get_kni_stack())) {
// if fail, free mbuf inside
kni_process_tx(stack->pkt_buf, stack->pkt_cnt);
@@ -202,6 +208,7 @@ static __rte_always_inline void flush_rx_ring(struct gazelle_stack *stack)
stack->pkt_cnt = 0;
return;
}
+#endif
/* first flush backup mbuf pointer avoid packet disorder */
if (unlikely(stack->backup_pkt_cnt > 0)) {
@@ -414,9 +421,12 @@ static __rte_always_inline void upstream_forward_one(struct rte_mbuf *m)
}
forward_to_kni:
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
if (get_ltran_config()->dpdk.kni_switch == GAZELLE_ON) {
enqueue_rx_packet(get_kni_stack(), m);
}
+#endif
+ return;
}
static __rte_always_inline void msg_to_quintuple(struct gazelle_quintuple *transfer_qtuple,
@@ -653,10 +663,12 @@ void upstream_forward(const uint16_t *port)
upstream_forward_loop(port_id, queue_id);
}
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
if (get_ltran_config()->dpdk.kni_switch == GAZELLE_ON) {
flush_rx_ring(get_kni_stack());
rte_kni_handle_request(get_gazelle_kni());
}
+#endif
now_time = get_current_time();
if (now_time - aging_conn_last_time > GAZELLE_CONN_INTERVAL) {
@@ -755,11 +767,13 @@ int32_t downstream_forward(uint16_t *port)
uint32_t queue_num = get_ltran_config()->bond.tx_queue_num;
while (get_ltran_stop_flag() != GAZELLE_TRUE) {
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
/* kni rx means read from kni and send to nic */
if (get_ltran_config()->dpdk.kni_switch == GAZELLE_ON &&
get_kni_started()) {
kni_process_rx(g_port_index);
}
+#endif
for (uint32_t queue_id = 0; queue_id < queue_num; queue_id++) {
downstream_forward_loop(port_id, queue_id);
diff --git a/src/ltran/ltran_stack.c b/src/ltran/ltran_stack.c
index d4e935e..2aed6c9 100644
--- a/src/ltran/ltran_stack.c
+++ b/src/ltran/ltran_stack.c
@@ -10,6 +10,7 @@
* See the Mulan PSL v2 for more details.
*/
+#include <malloc.h>
#include <rte_mbuf.h>
#include "ltran_instance.h"
diff --git a/src/ltran/ltran_timer.c b/src/ltran/ltran_timer.c
index 96c622e..0832c5f 100644
--- a/src/ltran/ltran_timer.c
+++ b/src/ltran/ltran_timer.c
@@ -10,6 +10,7 @@
* See the Mulan PSL v2 for more details.
*/
+#include <malloc.h>
#include <sys/time.h>
#include <pthread.h>
diff --git a/src/ltran/main.c b/src/ltran/main.c
index c2598a7..6cc8bc9 100644
--- a/src/ltran/main.c
+++ b/src/ltran/main.c
@@ -17,6 +17,8 @@
#include <syslog.h>
#include <sys/types.h>
#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_version.h>
#include "dpdk_common.h"
#include "ltran_log.h"
@@ -57,7 +59,9 @@ static void sig_default_handler(int32_t sig)
{
LTRAN_ERR("ltran dumped, caught signal: %d.\n", sig);
print_stack();
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
dpdk_kni_release();
+#endif
int ret = 0;
ret = unlink(get_ltran_config()->unix_socket_filename);
if (ret) {
@@ -143,7 +147,9 @@ static void ltran_core_destroy(void)
gazelle_stack_htable_destroy();
gazelle_tcp_conn_htable_destroy();
gazelle_tcp_sock_htable_destroy();
+#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
dpdk_kni_release();
+#endif
}
static void wait_thread_finish(pthread_t ctrl_thread, uint32_t next_core)
--
2.33.0
此处可能存在不合适展示的内容，页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容，可点击提交进行申诉，我们将尽快为您处理。
马建仓 AI 助手
