From ccacbfd79f17b3955acff68135e029433b2072d6 Mon Sep 17 00:00:00 2001
Message-Id: <ccacbfd79f17b3955acff68135e029433b2072d6.1669880730.git.chengyou@linux.alibaba.com>
In-Reply-To: <b8fdbf0cd29d6630ad53e78ec4724bb85f19611d.1669880730.git.chengyou@linux.alibaba.com>
References: <b8fdbf0cd29d6630ad53e78ec4724bb85f19611d.1669880730.git.chengyou@linux.alibaba.com>
From: Cheng Xu <chengyou@linux.alibaba.com>
Date: Thu, 1 Dec 2022 15:40:48 +0800
Subject: [PATCH 3/4] providers/erdma: Add the main module of erdma

Add the definitions of erdma provider driver, and add the application
interface to core, so that core can recognize erdma provider.

Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
---
 libibverbs/verbs.h          |   8 +-
 providers/erdma/erdma.c     | 149 ++++++++++++++++++++++++++++++++++++
 providers/erdma/erdma.h     |  79 +++++++++++++++++++
 providers/erdma/erdma_abi.h |  21 +++++
 4 files changed, 254 insertions(+), 3 deletions(-)
 create mode 100644 providers/erdma/erdma.c
 create mode 100644 providers/erdma/erdma.h
 create mode 100644 providers/erdma/erdma_abi.h

diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 36b4142517c0..b2f2b4b07be6 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -2190,7 +2190,7 @@ struct ibv_device **ibv_get_device_list(int *num_devices);
  */
 #ifdef RDMA_STATIC_PROVIDERS
 #define _RDMA_STATIC_PREFIX_(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11,     \
-			     _12, _13, _14, _15, _16, _17, ...)                \
+			     _12, _13, _14, _15, _16, _17, _18, ...)           \
 	&verbs_provider_##_1, &verbs_provider_##_2, &verbs_provider_##_3,      \
 		&verbs_provider_##_4, &verbs_provider_##_5,                    \
 		&verbs_provider_##_6, &verbs_provider_##_7,                    \
@@ -2198,16 +2198,18 @@ struct ibv_device **ibv_get_device_list(int *num_devices);
 		&verbs_provider_##_10, &verbs_provider_##_11,                  \
 		&verbs_provider_##_12, &verbs_provider_##_13,                  \
 		&verbs_provider_##_14, &verbs_provider_##_15,                  \
-		&verbs_provider_##_16, &verbs_provider_##_17
+		&verbs_provider_##_16, &verbs_provider_##_17,                  \
+		&verbs_provider_##_18
 #define _RDMA_STATIC_PREFIX(arg)                                               \
 	_RDMA_STATIC_PREFIX_(arg, none, none, none, none, none, none, none,    \
 			     none, none, none, none, none, none, none, none,   \
-			     none)
+			     none, none)
 
 struct verbs_devices_ops;
 extern const struct verbs_device_ops verbs_provider_bnxt_re;
 extern const struct verbs_device_ops verbs_provider_cxgb4;
 extern const struct verbs_device_ops verbs_provider_efa;
+extern const struct verbs_device_ops verbs_provider_erdma;
 extern const struct verbs_device_ops verbs_provider_hfi1verbs;
 extern const struct verbs_device_ops verbs_provider_hns;
 extern const struct verbs_device_ops verbs_provider_ipathverbs;
diff --git a/providers/erdma/erdma.c b/providers/erdma/erdma.c
new file mode 100644
index 000000000000..440314599cf1
--- /dev/null
+++ b/providers/erdma/erdma.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file
+
+// Authors: Cheng Xu <chengyou@linux.alibaba.com>
+// Copyright (c) 2020-2021, Alibaba Group.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <util/mmio.h>
+#include <util/udma_barrier.h>
+#include <util/util.h>
+
+#include "erdma.h"
+#include "erdma_abi.h"
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static const struct verbs_context_ops erdma_context_ops = {
+	.alloc_pd = erdma_alloc_pd,
+	.cq_event = erdma_cq_event,
+	.create_cq = erdma_create_cq,
+	.create_qp = erdma_create_qp,
+	.create_qp_ex = erdma_create_qp_ex,
+	.dealloc_pd = erdma_free_pd,
+	.dereg_mr = erdma_dereg_mr,
+	.destroy_cq = erdma_destroy_cq,
+	.destroy_qp = erdma_destroy_qp,
+	.free_context = erdma_free_context,
+	.modify_qp = erdma_modify_qp,
+	.poll_cq = erdma_poll_cq,
+	.post_recv = erdma_post_recv,
+	.post_send = erdma_post_send,
+	.query_device_ex = erdma_query_device,
+	.query_port = erdma_query_port,
+	.query_qp = erdma_query_qp,
+	.reg_mr = erdma_reg_mr,
+	.req_notify_cq = erdma_notify_cq,
+};
+
+static struct verbs_context *erdma_alloc_context(struct ibv_device *device,
+						 int cmd_fd, void *private_data)
+{
+	struct erdma_cmd_alloc_context_resp resp = {};
+	struct ibv_get_context cmd = {};
+	struct erdma_context *ctx;
+	int i;
+
+	ctx = verbs_init_and_alloc_context(device, cmd_fd, ctx, ibv_ctx,
+					   RDMA_DRIVER_ERDMA);
+	if (!ctx)
+		return NULL;
+
+	pthread_mutex_init(&ctx->qp_table_mutex, NULL);
+	for (i = 0; i < ERDMA_QP_TABLE_SIZE; ++i)
+		ctx->qp_table[i].refcnt = 0;
+
+	if (ibv_cmd_get_context(&ctx->ibv_ctx, &cmd, sizeof(cmd),
+				&resp.ibv_resp, sizeof(resp)))
+		goto err_out;
+
+	verbs_set_ops(&ctx->ibv_ctx, &erdma_context_ops);
+	ctx->page_size = to_edev(device)->page_size;
+	ctx->dev_id = resp.dev_id;
+
+	ctx->sdb_type = resp.sdb_type;
+	ctx->sdb_entid = resp.sdb_entid;
+
+	ctx->sdb = mmap(NULL, ctx->page_size, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.sdb);
+	if (ctx->sdb == MAP_FAILED)
+		goto err_out;
+
+	ctx->sdb += resp.sdb_off;
+
+	ctx->rdb = mmap(NULL, ctx->page_size, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.rdb);
+	if (ctx->rdb == MAP_FAILED)
+		goto err_rdb_map;
+
+	ctx->rdb += resp.rdb_off;
+
+	ctx->cdb = mmap(NULL, ctx->page_size, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.cdb);
+	if (ctx->cdb == MAP_FAILED)
+		goto err_cdb_map;
+
+	ctx->cdb += resp.cdb_off;
+
+	ctx->dbrecord_pages = NULL;
+	pthread_mutex_init(&ctx->dbrecord_pages_mutex, NULL);
+
+	for (i = 0; i < 32; i++)
+		pthread_spin_init(&ctx->sdb_lock[i].lock,
+				  PTHREAD_PROCESS_PRIVATE);
+
+	return &ctx->ibv_ctx;
+
+err_cdb_map:
+	munmap(aligned_address(ctx->rdb, ctx->page_size), ctx->page_size);
+err_rdb_map:
+	munmap(aligned_address(ctx->sdb, ctx->page_size), ctx->page_size);
+err_out:
+	verbs_uninit_context(&ctx->ibv_ctx);
+	free(ctx);
+
+	return NULL;
+}
+
+static struct verbs_device *
+erdma_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
+{
+	struct erdma_device *dev;
+
+	dev = calloc(1, sizeof(*dev));
+	if (!dev)
+		return NULL;
+
+	dev->page_size = sysconf(_SC_PAGESIZE);
+
+	return &dev->ibv_dev;
+}
+
+static void erdma_device_free(struct verbs_device *vdev)
+{
+	struct erdma_device *dev =
+		container_of(vdev, struct erdma_device, ibv_dev);
+
+	free(dev);
+}
+
+static const struct verbs_match_ent match_table[] = {
+	VERBS_DRIVER_ID(RDMA_DRIVER_ERDMA),
+	VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x107f, NULL),
+	{},
+};
+
+static const struct verbs_device_ops erdma_dev_ops = {
+	.name = "erdma",
+	.match_min_abi_version = 0,
+	.match_max_abi_version = ERDMA_ABI_VERSION,
+	.match_table = match_table,
+	.alloc_device = erdma_device_alloc,
+	.uninit_device = erdma_device_free,
+	.alloc_context = erdma_alloc_context,
+};
+
+PROVIDER_DRIVER(erdma, erdma_dev_ops);
diff --git a/providers/erdma/erdma.h b/providers/erdma/erdma.h
new file mode 100644
index 000000000000..41bca6e7915f
--- /dev/null
+++ b/providers/erdma/erdma.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */
+/*
+ * Authors: Cheng Xu <chengyou@linux.alibaba.com>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_H__
+#define __ERDMA_H__
+
+#include <inttypes.h>
+#include <pthread.h>
+#include <stddef.h>
+
+#include <infiniband/driver.h>
+#include <infiniband/kern-abi.h>
+#include <sys/param.h>
+
+#ifndef PCI_VENDOR_ID_ALIBABA
+#define PCI_VENDOR_ID_ALIBABA 0x1ded
+#endif
+
+#define ERDMA_PAGE_SIZE 4096
+#define ERDMA_PAGE_SHIFT 12
+#define ERDMA_SIZE_TO_NPAGE(size)                                              \
+	(((size) + ERDMA_PAGE_SIZE - 1) >> ERDMA_PAGE_SHIFT)
+
+struct erdma_device {
+	struct verbs_device ibv_dev;
+	uint32_t page_size;
+};
+
+struct erdma_aligned_lock_t {
+	pthread_spinlock_t lock;
+} __attribute__((__packed__)) __attribute__((__aligned__(64)));
+
+#define ERDMA_QP_TABLE_SIZE 4096
+#define ERDMA_QP_TABLE_SHIFT 12
+#define ERDMA_QP_TABLE_MASK 0xFFF
+
+struct erdma_context {
+	struct verbs_context ibv_ctx;
+	uint32_t dev_id;
+
+	struct {
+		struct erdma_qp **table;
+		int refcnt;
+	} qp_table[ERDMA_QP_TABLE_SIZE];
+	pthread_mutex_t qp_table_mutex;
+
+	uint8_t sdb_type;
+	uint32_t sdb_entid;
+
+	void *sdb;
+	void *rdb;
+	void *cdb;
+
+	uint32_t page_size;
+	pthread_mutex_t dbrecord_pages_mutex;
+	struct erdma_dbrecord_page *dbrecord_pages;
+
+	struct erdma_aligned_lock_t sdb_lock[32];
+};
+
+static inline struct erdma_context *to_ectx(struct ibv_context *base)
+{
+	return container_of(base, struct erdma_context, ibv_ctx.context);
+}
+
+static inline struct erdma_device *to_edev(struct ibv_device *ibv_dev)
+{
+	return container_of(ibv_dev, struct erdma_device, ibv_dev.device);
+}
+
+static inline void *aligned_address(void *addr, uint32_t align)
+{
+	return addr - ((uintptr_t)addr & (align - 1));
+}
+
+#endif
diff --git a/providers/erdma/erdma_abi.h b/providers/erdma/erdma_abi.h
new file mode 100644
index 000000000000..184804711d9b
--- /dev/null
+++ b/providers/erdma/erdma_abi.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */
+/*
+ * Authors: Cheng Xu <chengyou@linux.alibaba.com>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_ABI_H__
+#define __ERDMA_ABI_H__
+
+#include <infiniband/kern-abi.h>
+#include <rdma/erdma-abi.h>
+#include <kernel-abi/erdma-abi.h>
+
+DECLARE_DRV_CMD(erdma_cmd_alloc_context, IB_USER_VERBS_CMD_GET_CONTEXT, empty,
+		erdma_uresp_alloc_ctx);
+DECLARE_DRV_CMD(erdma_cmd_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+		erdma_ureq_create_cq, erdma_uresp_create_cq);
+DECLARE_DRV_CMD(erdma_cmd_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
+		erdma_ureq_create_qp, erdma_uresp_create_qp);
+
+#endif
-- 
2.37.0