From c3ee7375c80c7a8f0a943679566c87f17f87aa17 Mon Sep 17 00:00:00 2001
From: Guofeng Yue <yueguofeng@hisilicon.com>
Date: Mon, 9 May 2022 16:03:38 +0800
Subject: [PATCH rdma-core 2/2] libhns: Add reset stop flow mechanism

driver inclusion
category: bugfix
bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I65WI7

------------------------------------------------------------------

Add an interface to the user space, which is used to receive
the kernel reset state. After receiving the reset flag, the
user space stops sending db.

Signed-off-by: Yixing Liu <liuyixing1@huawei.com>
Signed-off-by: Guofeng Yue <yueguofeng@hisilicon.com>
Reviewed-by: Yangyang Li <liyangyang20@huawei.com>
---
 providers/hns/hns_roce_u.c       | 30 ++++++++++++++++++++++++++++--
 providers/hns/hns_roce_u.h       |  5 +++++
 providers/hns/hns_roce_u_db.h    |  8 +++++++-
 providers/hns/hns_roce_u_hw_v2.c | 19 ++++++++++++++-----
 4 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 0cf6d4b..3d29838 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -221,6 +221,24 @@ static void uninit_dca_context(struct hns_roce_context *ctx)
 	pthread_spin_destroy(&dca_ctx->lock);
 }
 
+static int init_reset_context(struct hns_roce_context *ctx, int cmd_fd,
+			struct hns_roce_alloc_ucontext_resp *resp,
+			int page_size)
+{
+	uint64_t reset_mmap_key = resp->reset_mmap_key;
+
+	/* The reset mmap key is 0, which means it is not supported. */
+	if (reset_mmap_key == 0)
+		return 0;
+
+	ctx->reset_state = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
+				cmd_fd, reset_mmap_key);
+	if (ctx->reset_state == MAP_FAILED)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int hns_roce_mmap(struct hns_roce_device *hr_dev,
 			 struct hns_roce_context *context, int cmd_fd)
 {
@@ -325,8 +343,11 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 			     &resp, ctx_attr, hr_dev->page_size))
 		goto err_free;
 
+	if (init_reset_context(context, cmd_fd, &resp, hr_dev->page_size))
+		goto reset_free;
+
 	if (hns_roce_mmap(hr_dev, context, cmd_fd))
-		goto dca_free;
+		goto uar_free;
 
 	pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
 
@@ -335,7 +356,10 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 
 	return &context->ibv_ctx;
 
-dca_free:
+uar_free:
+	if (context->reset_state)
+		munmap(context->reset_state, hr_dev->page_size);
+reset_free:
 	uninit_dca_context(context);
 err_free:
 	verbs_uninit_context(&context->ibv_ctx);
@@ -349,6 +373,8 @@ static void hns_roce_free_context(struct ibv_context *ibctx)
 	struct hns_roce_context *context = to_hr_ctx(ibctx);
 
 	munmap(context->uar, hr_dev->page_size);
+	if (context->reset_state)
+		munmap(context->reset_state, hr_dev->page_size);
 	uninit_dca_context(context);
 	verbs_uninit_context(&context->ibv_ctx);
 	free(context);
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 71c35c5..76c7adb 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -226,9 +226,14 @@ struct hns_roce_dca_ctx {
 	atomic_bitmap_t *sync_status;
 };
 
+struct hns_roce_v2_reset_state {
+	uint32_t is_reset;
+};
+
 struct hns_roce_context {
 	struct verbs_context		ibv_ctx;
 	void				*uar;
+	void				*reset_state;
 	pthread_spinlock_t		uar_lock;
 
 	struct {
diff --git a/providers/hns/hns_roce_u_db.h b/providers/hns/hns_roce_u_db.h
index 8c47a53..de288de 100644
--- a/providers/hns/hns_roce_u_db.h
+++ b/providers/hns/hns_roce_u_db.h
@@ -40,8 +40,14 @@
 
 #define HNS_ROCE_WORD_NUM	2
 
-static inline void hns_roce_write64(void *dest, __le32 val[HNS_ROCE_WORD_NUM])
+static inline void hns_roce_write64(struct hns_roce_context *ctx, void *dest,
+				    __le32 val[HNS_ROCE_WORD_NUM])
 {
+	struct hns_roce_v2_reset_state *state = ctx->reset_state;
+
+	if (state && state->is_reset)
+		return;
+
 	mmio_write64_le(dest, *(__le64 *)val);
 }
 
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 7661863..d0067d3 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -298,7 +298,8 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx,
 	hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);
 	hr_reg_write(&rq_db, DB_PI, rq_head);
 
-	hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&rq_db);
+	hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+			 (__le32 *)&rq_db);
 }
 
 static void hns_roce_update_sq_db(struct hns_roce_context *ctx,
@@ -312,7 +313,7 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx,
 	hr_reg_write(&sq_db, DB_PI, qp->sq.head);
 	hr_reg_write(&sq_db, DB_SL, qp->sl);
 
-	hns_roce_write64(qp->sq.db_reg, (__le32 *)&sq_db);
+	hns_roce_write64(ctx, qp->sq.db_reg, (__le32 *)&sq_db);
 }
 
 static void hns_roce_write512(uint64_t *dest, uint64_t *val)
@@ -323,6 +324,12 @@ static void hns_roce_write512(uint64_t *dest, uint64_t *val)
 static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe)
 {
 	struct hns_roce_rc_sq_wqe *rc_sq_wqe = wqe;
+	struct ibv_qp *ibvqp = &qp->verbs_qp.qp;
+	struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context);
+	struct hns_roce_v2_reset_state *state = ctx->reset_state;
+
+	if (state && state->is_reset)
+		return;
 
 	/* All kinds of DirectWQE have the same header field layout */
 	hr_reg_enable(rc_sq_wqe, RCWQE_FLAG);
@@ -342,7 +349,8 @@ static void update_cq_db(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
 	hr_reg_write(&cq_db, DB_CQ_CI, cq->cons_index);
 	hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);
 
-	hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db);
+	hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+			 (__le32 *)&cq_db);
 }
 
 static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx,
@@ -857,7 +865,8 @@ static int hns_roce_u_v2_arm_cq(struct ibv_cq *ibvcq, int solicited)
 	hr_reg_write(&cq_db, DB_CQ_CMD_SN, cq->arm_sn);
 	hr_reg_write(&cq_db, DB_CQ_NOTIFY, solicited_flag);
 
-	hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db);
+	hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+			 (__le32 *)&cq_db);
 
 	return 0;
 }
@@ -1934,7 +1943,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
 
 		update_srq_db(&srq_db, srq);
 
-		hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+		hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
 				 (__le32 *)&srq_db);
 	}
 
-- 
2.30.0