diff --git a/hw/hns3/hns3_udma_u_common.h b/hw/hns3/hns3_udma_u_common.h index a701ac4c7c6d0ac4d9574ed1d5c1faf5ede9fc7f..3050ab0238bfea3870310f7716df104ea07120cb 100644 --- a/hw/hns3/hns3_udma_u_common.h +++ b/hw/hns3/hns3_udma_u_common.h @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -71,7 +70,11 @@ #define __bf_shf(x) (__builtin_ffsll(x) - 1) -#define BUILD_ASSERT(cond) ((void)sizeof(char[1 - 2 * !(cond)])) +#define BUILD_ASSERT(cond) \ + ((void)sizeof(char[1 - 2 * !(cond)])) + +#define BUILD_ASSERT_OR_ZERO(cond) \ + (sizeof(char [1 - 2 * !(cond)]) - 1) #define GENMASK(h, l) \ (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) @@ -91,7 +94,7 @@ #define udma_reg_enable(ptr, field) \ ({ \ const uint32_t *_ptr = (uint32_t *)(ptr); \ - BUILD_ASSERT((((field) >> 32) / 32) == \ + BUILD_ASSERT_OR_ZERO((((field) >> 32) / 32) == \ ((((field) << 32) >> 32) / 32)); \ BUILD_ASSERT(((field) >> 32) == (((field) << 32) >> 32)); \ *((uint32_t *)_ptr + ((field) >> 32) / 32) |= \ @@ -101,7 +104,7 @@ #define udma_reg_clear(ptr, field) \ ({ \ const uint32_t *_ptr = (uint32_t *)(ptr); \ - BUILD_ASSERT((((field) >> 32) / 32) == \ + BUILD_ASSERT_OR_ZERO((((field) >> 32) / 32) == \ ((((field) << 32) >> 32) / 32)); \ BUILD_ASSERT(((field) >> 32) >= (((field) << 32) >> 32)); \ *((uint32_t *)_ptr + ((field) >> 32) / 32) &= \ @@ -127,7 +130,7 @@ #define udma_reg_read(ptr, field) \ ({ \ const uint32_t *_ptr = (uint32_t *)(ptr); \ - BUILD_ASSERT((((field) >> 32) / 32) == \ + BUILD_ASSERT_OR_ZERO((((field) >> 32) / 32) == \ ((((field) << 32) >> 32) / 32)); \ BUILD_ASSERT(((field) >> 32) >= (((field) << 32) >> 32)); \ FIELD_GET(GENMASK(((field) >> 32) % 32, \ @@ -274,6 +277,10 @@ static inline off_t get_mmap_offset(uint32_t idx, int page_size, int cmd) return offset * page_size; } +/* + * Hmap uses list to resolve hash conflicts in a bucket + * NOT multi-thread safe in the current version + */ struct udma_hmap_node { struct udma_hmap_node *next; uint32_t hash; @@ -289,15 +296,14 @@ struct udma_hmap { struct udma_hmap_head *bucket; }; +void udma_hmap_remove(struct udma_hmap *hmap, const struct udma_hmap_node *node); + static inline struct udma_hmap_node *udma_hmap_first_with_hash(const struct udma_hmap *hmap, uint32_t hash) { struct udma_hmap_head *head = &hmap->bucket[hash & hmap->mask]; struct udma_hmap_node *node; - if (head == NULL) - return NULL; - node = head->next; while ((node != NULL) && node->hash != hash) @@ -320,7 +326,7 @@ static inline struct udma_hmap_node *udma_table_first_with_hash(const struct udm } static inline bool udma_hmap_insert(struct udma_hmap *hmap, struct udma_hmap_node *node, - uint32_t hash) + uint32_t hash) { struct udma_hmap_head *head = &hmap->bucket[hash & hmap->mask]; @@ -528,8 +534,6 @@ static inline uint32_t udma_get_tgt_hash(const urma_jetty_id_t *id) return udma_hash_uint64(idx); } -void udma_hmap_remove(struct udma_hmap *hmap, const struct udma_hmap_node *node); - uint64_t *udma_bitmap_alloc(uint32_t n_bits, uint32_t *bitmap_cnt); void udma_bitmap_free(uint64_t *bitmap); int udma_bitmap_use_idx(uint64_t *bitmap, uint32_t bitmap_cnt, diff --git a/hw/hns3/hns3_udma_u_db.h b/hw/hns3/hns3_udma_u_db.h index 5726f49a9537916952adbe546e599fea60dfafe9..087691c0f521b330191277b4aa313123d233ff31 100644 --- a/hw/hns3/hns3_udma_u_db.h +++ b/hw/hns3/hns3_udma_u_db.h @@ -16,7 +16,6 @@ #ifndef _UDMA_U_DB_H #define _UDMA_U_DB_H -#include "hns3_udma_u_buf.h" #include "hns3_udma_u_common.h" #include "hns3_udma_u_provider_ops.h" diff --git a/hw/hns3/hns3_udma_u_jetty.c b/hw/hns3/hns3_udma_u_jetty.c index 2d195ee1c7bb99f53d26ddaeaf7df5cd6f905b1b..57e48da88eca03d4c00554eef7a0229b2a8e73b9 100644 --- a/hw/hns3/hns3_udma_u_jetty.c +++ b/hw/hns3/hns3_udma_u_jetty.c @@ -19,8 +19,7 @@ #include "hns3_udma_u_db.h" #include "hns3_udma_u_jetty.h" -urma_status_t verify_jfs_init_attr(urma_context_t *ctx, - urma_jfs_cfg_t *cfg) +urma_status_t verify_jfs_init_attr(urma_context_t *ctx, urma_jfs_cfg_t *cfg) { struct udma_u_context *udma_ctx = to_udma_ctx(ctx); @@ -69,8 +68,8 @@ static urma_status_t alloc_qp_node_table(struct udma_u_jetty *jetty, return ret; if (jetty->tp_mode == URMA_TM_UM) { - jetty->um_qp = udma_alloc_qp(udma_ctx, jetty_cfg->jfs_cfg, NULL, - jetty->urma_jetty.jetty_id.id, true); + jetty->um_qp = udma_alloc_qp(udma_ctx, true, &jetty_cfg->jfs_cfg, + NULL); if (!jetty->um_qp) { URMA_LOG_ERR("UM qp alloc failed, jetty_id = %u.\n", jetty->urma_jetty.jetty_id.id); @@ -85,14 +84,12 @@ static urma_status_t alloc_qp_node_table(struct udma_u_jetty *jetty, } if (jetty->udma_jfr == NULL) - jetty->rc_node->qp = udma_alloc_qp(udma_ctx, jetty_cfg->jfs_cfg, - jetty_cfg->jfr_cfg, - jetty->urma_jetty.jetty_id.id, - true); + jetty->rc_node->qp = udma_alloc_qp(udma_ctx, true, + &jetty_cfg->jfs_cfg, + jetty_cfg->jfr_cfg); else - jetty->rc_node->qp = udma_alloc_qp(udma_ctx, jetty_cfg->jfs_cfg, - NULL, jetty->urma_jetty.jetty_id.id, - true); + jetty->rc_node->qp = udma_alloc_qp(udma_ctx, true, + &jetty_cfg->jfs_cfg, NULL); if (!jetty->rc_node->qp) { URMA_LOG_ERR("alloc rc_node failed.\n"); goto err_alloc_rc_node_qp; @@ -107,9 +104,8 @@ err_alloc_rc_node_qp: return URMA_ENOMEM; } -static urma_status_t udma_add_to_qp_table(struct udma_u_context *ctx, - urma_jetty_t *jetty, struct udma_qp *qp, - uint32_t qpn) +urma_status_t udma_add_to_qp_table(struct udma_u_context *ctx, struct udma_qp *qp, + uint32_t qpn) { struct udma_jfs_qp_node *qp_node; @@ -132,7 +128,7 @@ static urma_status_t udma_add_to_qp_table(struct udma_u_context *ctx, return URMA_SUCCESS; } -static void udma_remove_from_qp_table(struct udma_u_context *ctx, uint32_t qpn) +void udma_remove_from_qp_table(struct udma_u_context *ctx, uint32_t qpn) { struct udma_jfs_qp_node *qp_node; struct udma_hmap_node *node; @@ -168,7 +164,7 @@ static urma_status_t exec_jetty_create_cmd(urma_context_t *ctx, if (jetty->tp_mode == URMA_TM_UM) { cmd.create_tp_ucmd.buf_addr = (uint64_t)jetty->um_qp->buf.buf; cmd.create_tp_ucmd.sdb_addr = (uint64_t)jetty->um_qp->sdb; - } else if (jetty->tp_mode == URMA_TM_RC) { + } else { cmd.buf_addr = (uint64_t)jetty->rc_node->qp->buf.buf; cmd.sdb_addr = (uint64_t)jetty->rc_node->qp->sdb; } @@ -180,72 +176,67 @@ static urma_status_t exec_jetty_create_cmd(urma_context_t *ctx, } if (jetty->tp_mode == URMA_TM_UM) { - jetty->um_qp->qp_num = resp.create_tp_resp.qpn; - jetty->um_qp->flags = resp.create_tp_resp.cap_flags; + fill_um_jetty_qp(jetty, resp); if (resp.create_tp_resp.cap_flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) { - if (mmap_dwqe(ctx, jetty->um_qp)) { - urma_cmd_delete_jetty(&jetty->urma_jetty); + ret = mmap_dwqe(ctx, jetty->um_qp); + if (ret) { URMA_LOG_ERR("mmap dwqe failed\n"); - return URMA_FAIL; + goto err_mmap_dwqe; } } - jetty->um_qp->path_mtu = (urma_mtu_t)resp.create_tp_resp.path_mtu; - jetty->um_qp->um_srcport = resp.create_tp_resp.um_srcport; - jetty->um_qp->sq.priority = resp.create_tp_resp.priority; + memcpy(&jetty->um_qp->um_srcport, &resp.create_tp_resp.um_srcport, sizeof(struct udp_srcport)); - ret = udma_add_to_qp_table(udma_ctx, &jetty->urma_jetty, - jetty->um_qp, jetty->um_qp->qp_num); - if (ret) + ret = udma_add_to_qp_table(udma_ctx, jetty->um_qp, jetty->um_qp->qp_num); + if (ret) { URMA_LOG_ERR("add to qp table failed for um jetty, ret = %d.\n", ret); + goto err_add_qp_table; + } + } else { + jetty->rc_node->qp->jetty_id = jetty->urma_jetty.jetty_id.id; } return ret; + +err_add_qp_table: + if (resp.create_tp_resp.cap_flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(jetty->um_qp); +err_mmap_dwqe: + urma_cmd_delete_jetty(&jetty->urma_jetty); + return ret; } static void rc_free_node(struct udma_u_context *udma_ctx, struct udma_u_jetty *udma_jetty) { - struct udma_qp *qp; + struct udma_qp *qp = udma_jetty->rc_node->qp; - if (udma_jetty->rc_node) { - qp = udma_jetty->rc_node->qp; - if (qp) { - if (udma_jetty->rc_node->tjetty) - udma_u_unbind_jetty(&udma_jetty->urma_jetty); - udma_free_sw_db(udma_ctx, qp->sdb, - UDMA_JETTY_TYPE_DB); - free(qp->sq.wrid); - qp->sq.wrid = NULL; - if (qp->dca_wqe.bufs) - free(qp->dca_wqe.bufs); - else - udma_free_buf(&qp->buf); - free(qp); - qp = NULL; - } + if (udma_jetty->rc_node->tjetty) + udma_u_unbind_jetty(&udma_jetty->urma_jetty); - free(udma_jetty->rc_node); - udma_jetty->rc_node = NULL; - } + udma_free_sw_db(udma_ctx, qp->sdb, UDMA_JETTY_TYPE_DB); + + if (qp->dca_wqe.bufs) + free(qp->dca_wqe.bufs); + else + udma_free_buf(&qp->buf); + + free(qp->sq.wrid); + free(qp); + free(udma_jetty->rc_node); } static void um_free_qp(struct udma_u_jetty *udma_jetty) { - struct udma_u_context *udma_ctx; - - if (udma_jetty->um_qp) { - udma_ctx = to_udma_ctx(udma_jetty->urma_jetty.urma_ctx); - udma_free_sw_db(udma_ctx, udma_jetty->um_qp->sdb, - UDMA_JETTY_TYPE_DB); - free(udma_jetty->um_qp->sq.wrid); - udma_jetty->um_qp->sq.wrid = NULL; - udma_free_buf(&udma_jetty->um_qp->buf); - free(udma_jetty->um_qp); - udma_jetty->um_qp = NULL; - } + struct udma_u_context *udma_ctx = to_udma_ctx(udma_jetty->urma_jetty.urma_ctx); + + udma_free_sw_db(udma_ctx, udma_jetty->um_qp->sdb, UDMA_JETTY_TYPE_DB); + udma_free_buf(&udma_jetty->um_qp->buf); + free(udma_jetty->um_qp->sq.wrid); + free(udma_jetty->um_qp); } -static void delete_qp_node_table(struct udma_u_context *udma_ctx, struct udma_u_jetty *udma_jetty) +static void delete_jetty_qp_node(struct udma_u_context *udma_ctx, + struct udma_u_jetty *udma_jetty) { if (udma_jetty->tp_mode == URMA_TM_UM) um_free_qp(udma_jetty); @@ -319,8 +310,8 @@ urma_status_t verify_jetty_trans_mode(urma_jetty_cfg_t *jetty_cfg) return URMA_SUCCESS; } -static void init_jetty_param(struct udma_u_jetty *udma_jetty, - urma_jetty_cfg_t *jetty_cfg) +static inline void init_jetty_param(struct udma_u_jetty *udma_jetty, + urma_jetty_cfg_t *jetty_cfg) { udma_jetty->tp_mode = jetty_cfg->jfs_cfg->trans_mode; udma_jetty->share_jfr = jetty_cfg->flag.bs.share_jfr; @@ -328,6 +319,17 @@ static void init_jetty_param(struct udma_u_jetty *udma_jetty, udma_jetty->urma_jetty.jetty_cfg = *jetty_cfg; } +static int exec_jetty_delete_cmd(struct udma_u_context *ctx, struct udma_u_jetty *jetty) +{ + if (jetty->tp_mode == URMA_TM_UM) { + udma_remove_from_qp_table(ctx, jetty->um_qp->qp_num); + if (jetty->um_qp->flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(jetty->um_qp); + } + + return urma_cmd_delete_jetty(&jetty->urma_jetty); +} + static urma_jetty_t *udma_u_create_jetty_rq(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg) { @@ -378,12 +380,12 @@ static urma_jetty_t *udma_u_create_jetty_rq(urma_context_t *ctx, err_insert_jetty_node: pthread_spin_destroy(&udma_jetty->lock); err_init_lock: - urma_cmd_delete_jetty(&udma_jetty->urma_jetty); + (void)exec_jetty_delete_cmd(udma_ctx, udma_jetty); err_exec_jetty_create_cmd: if (!udma_jetty->share_jfr) udma_u_delete_jfr(&udma_jetty->udma_jfr->urma_jfr); err_alloc_qp_node_table: - delete_qp_node_table(udma_ctx, udma_jetty); + delete_jetty_qp_node(udma_ctx, udma_jetty); err_alloc_jetty: free(udma_jetty); @@ -444,9 +446,9 @@ urma_jetty_t *udma_u_create_jetty(urma_context_t *ctx, err_insert_jetty_node: pthread_spin_destroy(&udma_jetty->lock); err_init_lock: - urma_cmd_delete_jetty(&udma_jetty->urma_jetty); + (void)exec_jetty_delete_cmd(udma_ctx, udma_jetty); err_exec_jetty_create_cmd: - delete_qp_node_table(udma_ctx, udma_jetty); + delete_jetty_qp_node(udma_ctx, udma_jetty); err_alloc_qp_node_table: if (!udma_jetty->share_jfr) udma_u_delete_jfr(&udma_jetty->udma_jfr->urma_jfr); @@ -460,28 +462,22 @@ urma_status_t udma_u_delete_jetty(urma_jetty_t *jetty) { struct udma_u_context *udma_ctx = to_udma_ctx(jetty->urma_ctx); struct udma_u_jetty *udma_jetty = to_udma_jetty(jetty); - int ret; - - if (udma_jetty->tp_mode == URMA_TM_UM) - udma_remove_from_qp_table(udma_ctx, udma_jetty->um_qp->qp_num); - delete_qp_node_table(udma_ctx, udma_jetty); delete_jetty_node(udma_ctx, udma_jetty->urma_jetty.jetty_id.id); pthread_spin_destroy(&udma_jetty->lock); - ret = urma_cmd_delete_jetty(jetty); - if (ret) { + if (exec_jetty_delete_cmd(udma_ctx, udma_jetty)) { URMA_LOG_ERR("jetty delete failed!\n"); return URMA_FAIL; } if (!udma_jetty->share_jfr) { - if (udma_jetty->udma_jfr) { - udma_u_delete_jfr(&udma_jetty->udma_jfr->urma_jfr); - udma_jetty->udma_jfr = NULL; - } + udma_u_delete_jfr(&udma_jetty->udma_jfr->urma_jfr); + udma_jetty->udma_jfr = NULL; } + delete_jetty_qp_node(udma_ctx, udma_jetty); + free(udma_jetty); return URMA_SUCCESS; @@ -603,8 +599,7 @@ static urma_status_t exec_jetty_bind_cmd(urma_jetty_t *jetty, return URMA_SUCCESS; } -urma_status_t udma_u_bind_jetty(urma_jetty_t *jetty, - urma_target_jetty_t *tjetty) +urma_status_t udma_u_bind_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty) { struct udma_u_target_jetty *udma_target_jetty = to_udma_target_jetty(tjetty); struct udma_u_context *udma_ctx = to_udma_ctx(jetty->urma_ctx); @@ -634,7 +629,7 @@ urma_status_t udma_u_bind_jetty(urma_jetty_t *jetty, return URMA_FAIL; } - ret = udma_add_to_qp_table(udma_ctx, jetty, udma_jetty->rc_node->qp, + ret = udma_add_to_qp_table(udma_ctx, udma_jetty->rc_node->qp, udma_jetty->rc_node->qp->qp_num); if (ret) { URMA_LOG_ERR("add to qp table failed when bind jetty, ret = %d.\n", ret); @@ -647,6 +642,8 @@ urma_status_t udma_u_bind_jetty(urma_jetty_t *jetty, return URMA_SUCCESS; err_add_to_qp_table: + if (udma_jetty->rc_node->qp->flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(udma_jetty->rc_node->qp); urma_cmd_unbind_jetty(jetty); return URMA_FAIL; @@ -673,6 +670,9 @@ urma_status_t udma_u_unbind_jetty(urma_jetty_t *jetty) udma_target_jetty = to_udma_target_jetty(udma_jetty->rc_node->tjetty); udma_remove_from_qp_table(udma_ctx, udma_jetty->rc_node->qp->qp_num); + if (udma_jetty->rc_node->qp->flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(udma_jetty->rc_node->qp); + ret = urma_cmd_unbind_jetty(jetty); if (ret) { URMA_LOG_ERR("urma_cmd_unbind_jetty failed.\n"); @@ -769,7 +769,7 @@ static urma_status_t udma_u_post_jetty_rc_wr(struct udma_u_context *udma_ctx, return ret; } -static urma_status_t udma_u_post_jetty_qp_wr(struct udma_u_context *udma_ctx, +static urma_status_t udma_u_post_jetty_um_wr(struct udma_u_context *udma_ctx, struct udma_u_jetty *udma_jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr) @@ -780,17 +780,14 @@ static urma_status_t udma_u_post_jetty_qp_wr(struct udma_u_context *udma_ctx, urma_jfs_wr_t *it; void *wqe; - for (it = wr; it != NULL; it = it->next) { - udma_qp = get_qp_of_jetty(udma_jetty, it); - if (!udma_qp) { - URMA_LOG_ERR("failed to find qp for target jetty"); - ret = URMA_EINVAL; - *bad_wr = it; - break; - } + udma_qp = udma_jetty->um_qp; + + ret = check_dca_valid(udma_ctx, udma_qp); + if (ret) + return ret; - ret = udma_u_post_qp_wr(udma_ctx, udma_qp, it, &wqe, - udma_jetty->tp_mode); + for (it = wr; it != NULL; it = it->next) { + ret = udma_u_post_umqp_wr(udma_ctx, udma_qp, it, &wqe); if (ret) { *bad_wr = it; break; @@ -798,6 +795,12 @@ static urma_status_t udma_u_post_jetty_qp_wr(struct udma_u_context *udma_ctx, wr_cnt++; } + if (likely(wr_cnt)) + udma_u_ring_sq_doorbell(udma_ctx, udma_qp, wqe, wr_cnt); + + if (udma_qp->flush_status == UDMA_FLUSH_STATU_ERR) + exec_jfs_flush_cqe_cmd(udma_ctx, udma_qp); + return ret; } @@ -815,7 +818,7 @@ urma_status_t udma_u_post_jetty_send_wr(urma_jetty_t *jetty, if (udma_jetty->tp_mode == URMA_TM_RC) ret = udma_u_post_jetty_rc_wr(udma_ctx, udma_jetty, wr, bad_wr); else - ret = udma_u_post_jetty_qp_wr(udma_ctx, udma_jetty, wr, bad_wr); + ret = udma_u_post_jetty_um_wr(udma_ctx, udma_jetty, wr, bad_wr); if (!udma_jetty->jfs_lock_free) (void)pthread_spin_unlock(&udma_jetty->lock); @@ -827,12 +830,43 @@ urma_status_t udma_u_post_jetty_recv_wr(urma_jetty_t *jetty, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr) { + struct udma_u_context *ctx = to_udma_ctx(jetty->urma_ctx); struct udma_u_jetty *udma_jetty = to_udma_jetty(jetty); + struct udma_u_jfr *udma_jfr = udma_jetty->udma_jfr; urma_status_t ret = URMA_SUCCESS; + uint32_t nreq; - ret = udma_u_post_jfr_wr(&udma_jetty->udma_jfr->urma_jfr, wr, bad_wr); - if (ret) - URMA_LOG_ERR("post jfr wr failed, ret = %d.\n", ret); + if (!udma_jfr->lock_free) + (void)pthread_spin_lock(&udma_jfr->lock); + + if (udma_jfr->rq_en) { + for (nreq = 0; wr; ++nreq, wr = wr->next) { + ret = post_recv_one_rq(udma_jfr, wr); + if (ret) { + *bad_wr = wr; + break; + } + } + } else { + for (nreq = 0; wr; ++nreq, wr = wr->next) { + ret = post_recv_one(udma_jfr, wr); + if (ret) { + *bad_wr = wr; + break; + } + } + } + + if (nreq) { + udma_to_device_barrier(); + if (udma_jfr->cap_flags & HNS3_UDMA_JFR_CAP_RECORD_DB) + *udma_jfr->db = udma_jfr->idx_que.head & UDMA_DB_PROD_IDX_M; + else + update_srq_db(ctx, udma_jfr); + } + + if (!udma_jfr->lock_free) + (void)pthread_spin_unlock(&udma_jfr->lock); return ret; } diff --git a/hw/hns3/hns3_udma_u_jetty.h b/hw/hns3/hns3_udma_u_jetty.h index 838a05a11bbbbaf39854eb20023c3bcf5af8cea2..55fa3e8ca66a4d3b32d8c542a1b556167c13fb5b 100644 --- a/hw/hns3/hns3_udma_u_jetty.h +++ b/hw/hns3/hns3_udma_u_jetty.h @@ -99,6 +99,17 @@ static inline bool is_jetty(struct udma_u_context *udma_ctx, uint32_t qpn) return qpn_prefix == HNS3_UDMA_JETTY_QPN_PREFIX; } +static inline void fill_um_jetty_qp(struct udma_u_jetty *jetty, + struct hns3_udma_create_jetty_resp resp) +{ + jetty->um_qp->qp_num = resp.create_tp_resp.qpn; + jetty->um_qp->flags = resp.create_tp_resp.cap_flags; + jetty->um_qp->path_mtu = (urma_mtu_t)resp.create_tp_resp.path_mtu; + jetty->um_qp->um_srcport = resp.create_tp_resp.um_srcport; + jetty->um_qp->sq.priority = resp.create_tp_resp.priority; + jetty->um_qp->jetty_id = jetty->urma_jetty.jetty_id.id; +} + urma_jetty_t *udma_u_create_jetty(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg); urma_status_t udma_u_delete_jetty(urma_jetty_t *jetty); @@ -121,6 +132,9 @@ urma_status_t verify_jfs_init_attr(urma_context_t *ctx, urma_jfs_cfg_t *cfg); urma_status_t udma_u_modify_jetty(urma_jetty_t *jetty, urma_jetty_attr_t *jetty_attr); +urma_status_t udma_add_to_qp_table(struct udma_u_context *ctx, struct udma_qp *qp, + uint32_t qpn); +void udma_remove_from_qp_table(struct udma_u_context *ctx, uint32_t qpn); void delete_jetty_node(struct udma_u_context *udma_ctx, uint32_t id); urma_status_t insert_jetty_node(struct udma_u_context *udma_ctx, void *pointer, bool is_jetty, uint32_t id); diff --git a/hw/hns3/hns3_udma_u_jfc.c b/hw/hns3/hns3_udma_u_jfc.c index 7f1ee27523ac6ef5f5cdffb17cb053e0b3a00cd7..530c3036d8e55f429ff28ef509cbbc42285e628d 100644 --- a/hw/hns3/hns3_udma_u_jfc.c +++ b/hw/hns3/hns3_udma_u_jfc.c @@ -298,9 +298,6 @@ static struct udma_u_jfr *get_common_jfr(struct udma_u_context *udma_ctx, if (udma_ctx->jetty_table[table_id].refcnt) { jetty_table = udma_ctx->jetty_table[table_id].table; is_jetty = jetty_table[qpn & mask].is_jetty; - } else { - URMA_LOG_ERR("Failed to get jfr. QP 0x%x not found.\n", qpn); - return NULL; } if (is_jetty) { jetty = (struct udma_u_jetty *)jetty_table[qpn & mask].jetty; @@ -350,10 +347,8 @@ static int parse_cqe_for_res(struct udma_u_context *udma_ctx, pthread_spin_unlock(&jfr->lock); } - if (jfr->trans_mode == URMA_TM_UM) { - wqe_idx = udma_reg_read(cqe, CQE_WQE_IDX); + if (jfr->trans_mode == URMA_TM_UM) handle_um_header(jfr, wqe_idx, cr); - } udma_parse_opcode_for_res(cqe, cr); cr->flag.bs.s_r = 1; @@ -384,24 +379,18 @@ static struct udma_qp *get_qp_from_cqe(struct udma_u_context *udma_ctx, { struct common_jetty *jetty_table; struct udma_u_jetty *jetty; - static struct udma_qp *sqp; + struct udma_qp *sqp = NULL; struct udma_u_jfs *jfs; uint32_t qpn = cr->tpn; uint32_t table_id; uint32_t mask; bool is_jetty; - if ((sqp != NULL) && (sqp->qp_num == qpn)) - return sqp; - table_id = qpn >> udma_ctx->jettys_in_tbl_shift; mask = (1 << udma_ctx->jettys_in_tbl_shift) - 1; if (udma_ctx->jetty_table[table_id].refcnt) { jetty_table = udma_ctx->jetty_table[table_id].table; is_jetty = jetty_table[qpn & mask].is_jetty; - } else { - URMA_LOG_ERR("Failed get qp. QP 0x%x been destroyed.\n", qpn); - return NULL; } if (is_jetty) { @@ -430,13 +419,11 @@ static int parse_cqe_for_req(struct udma_u_context *udma_ctx, struct udma_jfc_cqe *cqe, urma_cr_t *cr) { - static struct udma_qp *sqp; + struct udma_qp *sqp = NULL; - if ((sqp == NULL) || (sqp->qp_num != cr->tpn)) { - sqp = get_qp_from_cqe(udma_ctx, cqe, cr); - if (sqp == NULL) - return JFC_POLL_ERR; - } + sqp = get_qp_from_cqe(udma_ctx, cqe, cr); + if (sqp == NULL) + return JFC_POLL_ERR; sqp->sq.tail += (cqe->wqe_idx - sqp->sq.tail) & (sqp->sq.wqe_cnt - 1); @@ -594,7 +581,7 @@ urma_status_t udma_u_modify_jfc(urma_jfc_t *jfc, urma_jfc_attr_t *attr) ret = urma_cmd_modify_jfc(jfc, attr, NULL); if (ret != 0) { - URMA_LOG_ERR("modify jfc failed.\n"); + URMA_LOG_ERR("modify jfc failed. ret = %d\n", ret); return URMA_FAIL; } diff --git a/hw/hns3/hns3_udma_u_jfc.h b/hw/hns3/hns3_udma_u_jfc.h index c052b9715b194ceb5bf7c1939dcc8c132bc1338e..ab9b59f9d1bfe836432645fe708e0cf33bd67ce6 100644 --- a/hw/hns3/hns3_udma_u_jfc.h +++ b/hw/hns3/hns3_udma_u_jfc.h @@ -20,6 +20,7 @@ #include "hns3_udma_u_common.h" #include "hns3_udma_u_provider_ops.h" #include "hns3_udma_u_jfs.h" +#include "hns3_udma_u_jfr.h" #define CQE_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) diff --git a/hw/hns3/hns3_udma_u_jfr.c b/hw/hns3/hns3_udma_u_jfr.c index 9f485778e8ab600f5b23accc8ef0bd8b2317eef5..bffc129b9db32205d42c8c3b7d064594eea5ab22 100644 --- a/hw/hns3/hns3_udma_u_jfr.c +++ b/hw/hns3/hns3_udma_u_jfr.c @@ -13,6 +13,7 @@ * */ +#include #include "hns3_udma_u_common.h" #include "hns3_udma_u_db.h" #include "hns3_udma_u_jfs.h" @@ -53,7 +54,7 @@ static void init_jfr_param(struct udma_u_jfr *jfr, urma_jfr_cfg_t *cfg) } jfr->wqe_shift = udma_ilog32(roundup_pow_of_two(UDMA_HW_SGE_SIZE * - jfr->max_sge)); + jfr->max_sge)); jfr->trans_mode = cfg->trans_mode; } @@ -92,17 +93,13 @@ static int alloc_jfr_wqe_buf(struct udma_u_jfr *jfr) static int alloc_jfr_buf(struct udma_u_jfr *jfr, struct udma_u_jetty *jetty) { - int ret; - - ret = alloc_jfr_idx_que(jfr); - if (ret) { + if (alloc_jfr_idx_que(jfr)) { URMA_LOG_ERR("failed to alloc jfr idx que.\n"); return ENOMEM; } if (jetty == NULL) { - ret = alloc_jfr_wqe_buf(jfr); - if (ret) { + if (alloc_jfr_wqe_buf(jfr)) { URMA_LOG_ERR("failed to alloc jfr wqe buf.\n"); goto err_alloc_buf; } @@ -149,6 +146,7 @@ static int exec_jfr_create_cmd(urma_context_t *ctx, struct udma_u_jfr *jfr, cmd.buf_addr = (uintptr_t)jfr->wqe_buf.buf; cmd.idx_addr = (uintptr_t)jfr->idx_que.idx_buf.buf; cmd.db_addr = (uintptr_t)jfr->db; + cmd.share_jfr = !jfr->rq_en; if (jetty != NULL) { cmd.wqe_buf_addr = (uintptr_t)jetty->rc_node->qp->buf.buf; cmd.sqe_cnt = jetty->rc_node->qp->sq.wqe_cnt; @@ -310,10 +308,10 @@ urma_jfr_t *udma_u_create_jfr(urma_context_t *ctx, urma_jfr_cfg_t *cfg) if (verify_jfr_init_attr(udma_ctx, cfg)) return NULL; - jfr = (struct udma_u_jfr *)calloc(1, sizeof(*jfr)); + jfr = (struct udma_u_jfr *)memalign(UDMA_HW_PAGE_SIZE, sizeof(*jfr)); if (!jfr) return NULL; - + memset(jfr, 0, sizeof(*jfr)); jfr->lock_free = cfg->flag.bs.lock_free; if (pthread_spin_init(&jfr->lock, PTHREAD_PROCESS_PRIVATE)) goto err_init_lock; @@ -394,63 +392,6 @@ urma_status_t udma_u_delete_jfr(urma_jfr_t *jfr) return URMA_SUCCESS; } -urma_target_jetty_t *udma_u_import_jfr(urma_context_t *ctx, - urma_rjfr_t *rjfr, - urma_token_t *token) -{ - struct udma_u_target_jetty *udma_target_jfr; - urma_cmd_udrv_priv_t udata = {}; - urma_target_jetty_t *tjfr; - urma_tjfr_cfg_t cfg = {}; - int ret; - - udma_target_jfr = (struct udma_u_target_jetty *) - calloc(1, sizeof(struct udma_u_target_jetty)); - if (!udma_target_jfr) { - URMA_LOG_ERR("udma_target_jfr alloc failed.\n"); - return NULL; - } - - tjfr = &udma_target_jfr->urma_target_jetty; - tjfr->urma_ctx = ctx; - tjfr->id = rjfr->jfr_id; - tjfr->trans_mode = rjfr->trans_mode; - cfg.jfr_id = rjfr->jfr_id; - cfg.token = token; - cfg.trans_mode = rjfr->trans_mode; - udma_set_udata(&udata, NULL, 0, NULL, 0); - ret = urma_cmd_import_jfr(ctx, tjfr, &cfg, &udata); - if (ret) { - URMA_LOG_ERR("import jfr failed.\n"); - free(udma_target_jfr); - return NULL; - } - - atomic_init(&udma_target_jfr->refcnt, 1); - return tjfr; -} - -urma_status_t udma_u_unimport_jfr(urma_target_jetty_t *target_jfr) -{ - struct udma_u_target_jetty *udma_target_jfr = to_udma_target_jetty(target_jfr); - int ret; - - if (udma_target_jfr->refcnt > 1) { - URMA_LOG_ERR("the target jfr is still being used, id = %d.\n", - target_jfr->id.id); - return URMA_FAIL; - } - - ret = urma_cmd_unimport_jfr(target_jfr); - if (ret != 0) { - URMA_LOG_ERR("unimport jfr failed.\n"); - return URMA_FAIL; - } - free(udma_target_jfr); - - return URMA_SUCCESS; -} - static inline bool udma_jfrwq_overflow(struct udma_u_jfr *jfr) { struct udma_u_jfr_idx_que *idx_que = &jfr->idx_que; @@ -458,9 +399,9 @@ static inline bool udma_jfrwq_overflow(struct udma_u_jfr *jfr) return (idx_que->head - idx_que->tail) >= jfr->wqe_cnt; } -static urma_status_t check_post_jfr_valid(struct udma_u_jfr *jfr, - urma_jfr_wr_t *wr, - uint32_t max_sge) +static inline urma_status_t check_post_jfr_valid(struct udma_u_jfr *jfr, + urma_jfr_wr_t *wr, + uint32_t max_sge) { if (udma_jfrwq_overflow(jfr)) { URMA_LOG_ERR("failed to check jfrwq status, jfrwq is full.\n"); @@ -500,7 +441,7 @@ static void *get_idx_buf(struct udma_u_jfr_idx_que *idx_que, uint32_t n) return (char *)idx_que->idx_buf.buf + (n << idx_que->entry_shift); } -static void fill_wqe_idx(struct udma_u_jfr *jfr, uint32_t wqe_idx) +static inline void fill_wqe_idx(struct udma_u_jfr *jfr, uint32_t wqe_idx) { struct udma_u_jfr_idx_que *idx_que = &jfr->idx_que; uint32_t *idx_buf; @@ -527,8 +468,8 @@ static void *set_um_header_sge(struct udma_u_jfr *jfr, return dseg; } -static void fill_recv_sge_to_wqe(urma_jfr_wr_t *wr, void *wqe, - uint32_t max_sge) +static inline void fill_recv_sge_to_wqe(urma_jfr_wr_t *wr, void *wqe, + uint32_t max_sge) { struct udma_wqe_data_seg *dseg = (struct udma_wqe_data_seg *)wqe; uint32_t i, cnt; @@ -545,8 +486,7 @@ static void fill_recv_sge_to_wqe(urma_jfr_wr_t *wr, void *wqe, memset(dseg + cnt, 0, (max_sge - cnt) * UDMA_HW_SGE_SIZE); } -static urma_status_t post_recv_one_rq(struct udma_u_jfr *udma_jfr, - urma_jfr_wr_t *wr) +urma_status_t post_recv_one_rq(struct udma_u_jfr *udma_jfr, urma_jfr_wr_t *wr) { uint32_t wqe_idx, max_sge; urma_status_t ret; @@ -572,8 +512,7 @@ static urma_status_t post_recv_one_rq(struct udma_u_jfr *udma_jfr, return ret; } -static urma_status_t post_recv_one(struct udma_u_jfr *udma_jfr, - urma_jfr_wr_t *wr) +urma_status_t post_recv_one(struct udma_u_jfr *udma_jfr, urma_jfr_wr_t *wr) { urma_status_t ret = URMA_SUCCESS; uint32_t wqe_idx, max_sge; @@ -581,19 +520,19 @@ static urma_status_t post_recv_one(struct udma_u_jfr *udma_jfr, max_sge = udma_jfr->user_max_sge; ret = check_post_jfr_valid(udma_jfr, wr, max_sge); - if (ret) { + if (unlikely(ret)) { URMA_LOG_ERR("failed to check post, jfrn = %u.\n", udma_jfr->urma_jfr.jfr_id.id); return ret; } ret = get_wqe_idx(udma_jfr, &wqe_idx); - if (ret) { + if (unlikely(ret)) { URMA_LOG_ERR("failed to get jfr wqe idx.\n"); return ret; } wqe = get_jfr_wqe(udma_jfr, wqe_idx); - if (udma_jfr->trans_mode == URMA_TM_UM) + if (unlikely(udma_jfr->trans_mode == URMA_TM_UM)) wqe = set_um_header_sge(udma_jfr, wqe_idx, wqe); fill_recv_sge_to_wqe(wr, wqe, max_sge); @@ -604,7 +543,7 @@ static urma_status_t post_recv_one(struct udma_u_jfr *udma_jfr, return ret; } -static void update_srq_db(struct udma_u_context *ctx, struct udma_u_jfr *jfr) +void update_srq_db(struct udma_u_context *ctx, struct udma_u_jfr *jfr) { struct udma_u_db db = {}; @@ -659,6 +598,63 @@ urma_status_t udma_u_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, return ret; } +urma_target_jetty_t *udma_u_import_jfr(urma_context_t *ctx, + urma_rjfr_t *rjfr, + urma_token_t *token) +{ + struct udma_u_target_jetty *udma_target_jfr; + urma_cmd_udrv_priv_t udata = {}; + urma_target_jetty_t *tjfr; + urma_tjfr_cfg_t cfg = {}; + int ret; + + udma_target_jfr = (struct udma_u_target_jetty *) + calloc(1, sizeof(struct udma_u_target_jetty)); + if (!udma_target_jfr) { + URMA_LOG_ERR("udma_target_jfr alloc failed.\n"); + return NULL; + } + + tjfr = &udma_target_jfr->urma_target_jetty; + tjfr->urma_ctx = ctx; + tjfr->id = rjfr->jfr_id; + tjfr->trans_mode = rjfr->trans_mode; + cfg.jfr_id = rjfr->jfr_id; + cfg.token = token; + cfg.trans_mode = rjfr->trans_mode; + udma_set_udata(&udata, NULL, 0, NULL, 0); + ret = urma_cmd_import_jfr(ctx, tjfr, &cfg, &udata); + if (ret) { + URMA_LOG_ERR("import jfr failed.\n"); + free(udma_target_jfr); + return NULL; + } + + atomic_init(&udma_target_jfr->refcnt, 1); + return tjfr; +} + +urma_status_t udma_u_unimport_jfr(urma_target_jetty_t *target_jfr) +{ + struct udma_u_target_jetty *udma_target_jfr = to_udma_target_jetty(target_jfr); + int ret; + + if (udma_target_jfr->refcnt > 1) { + URMA_LOG_ERR("the target jfr is still being used, id = %d.\n", + target_jfr->id.id); + return URMA_FAIL; + } + + ret = urma_cmd_unimport_jfr(target_jfr); + if (ret != 0) { + URMA_LOG_ERR("unimport jfr failed.\n"); + return URMA_FAIL; + } + free(udma_target_jfr); + + return URMA_SUCCESS; +} + urma_status_t udma_u_modify_jfr(urma_jfr_t *jfr, urma_jfr_attr_t *attr) { struct udma_u_jfr *udma_jfr = to_udma_jfr(jfr); diff --git a/hw/hns3/hns3_udma_u_jfr.h b/hw/hns3/hns3_udma_u_jfr.h index f69d7bbeff621d7eb73be3e935e800d53fca52bd..fa75e1e06df3d202d5824be78262eaf63e86c76d 100644 --- a/hw/hns3/hns3_udma_u_jfr.h +++ b/hw/hns3/hns3_udma_u_jfr.h @@ -48,22 +48,22 @@ struct um_header { struct udma_u_jfr { urma_jfr_t urma_jfr; - pthread_spinlock_t lock; + urma_transport_mode_t trans_mode; uint32_t lock_free; - uint32_t wqe_cnt; - uint32_t wqe_shift; + pthread_spinlock_t lock; uint32_t max_sge; uint32_t user_max_sge; /* max sge allow user assign */ uint64_t *wrid; - struct udma_u_jfr_idx_que idx_que; - struct udma_buf wqe_buf; uint32_t *db; uint32_t jfrn; uint32_t cap_flags; - urma_transport_mode_t trans_mode; struct um_header *um_header_que; urma_target_seg_t *um_header_seg; uint32_t srqn; + struct udma_u_jfr_idx_que idx_que; + struct udma_buf wqe_buf; + uint32_t wqe_cnt; + uint32_t wqe_shift; bool rq_en; }; @@ -101,5 +101,10 @@ urma_status_t udma_u_unimport_jfr(urma_target_jetty_t *target_jfr); urma_status_t udma_u_modify_jfr(urma_jfr_t *jfr, urma_jfr_attr_t *attr); urma_status_t udma_u_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); +urma_status_t post_recv_one_rq(struct udma_u_jfr *udma_jfr, + urma_jfr_wr_t *wr); +urma_status_t post_recv_one(struct udma_u_jfr *udma_jfr, + urma_jfr_wr_t *wr); +void update_srq_db(struct udma_u_context *ctx, struct udma_u_jfr *jfr); #endif /* _UDMA_U_JFR_H */ diff --git a/hw/hns3/hns3_udma_u_jfs.c b/hw/hns3/hns3_udma_u_jfs.c index d99595d0863588edd81f4826e72c1d07df4f5a3f..3b547cef3aa10a32d74e412ecb0d2e2ed73f31f3 100644 --- a/hw/hns3/hns3_udma_u_jfs.c +++ b/hw/hns3/hns3_udma_u_jfs.c @@ -126,10 +126,8 @@ static urma_status_t alloc_qp_wqe(struct udma_u_context *udma_ctx, return ret; } -struct udma_qp *udma_alloc_qp(struct udma_u_context *udma_ctx, - urma_jfs_cfg_t *jfs_cfg, - urma_jfr_cfg_t *jfr_cfg, - uint32_t jetty_id, bool is_jetty) +struct udma_qp *udma_alloc_qp(struct udma_u_context *udma_ctx, bool is_jetty, + urma_jfs_cfg_t *jfs_cfg, urma_jfr_cfg_t *jfr_cfg) { enum udma_db_type db_type; struct udma_qp *qp; @@ -153,7 +151,6 @@ struct udma_qp *udma_alloc_qp(struct udma_u_context *udma_ctx, URMA_LOG_ERR("alloc_qp_wqe failed.\n"); goto err_alloc_sw_db; } - qp->jetty_id = jetty_id; qp->is_jetty = is_jetty; return qp; @@ -166,8 +163,8 @@ err_alloc_qp: return NULL; } -static int alloc_table_qp(struct udma_u_jfs *jfs, - struct udma_u_context *udma_ctx, urma_jfs_cfg_t *cfg) +static int alloc_table_qp(struct udma_u_jfs *jfs, struct udma_u_context *udma_ctx, + urma_jfs_cfg_t *cfg) { int ret; @@ -175,68 +172,15 @@ static int alloc_table_qp(struct udma_u_jfs *jfs, if (ret) return ret; - if (jfs->tp_mode == URMA_TM_UM) { - jfs->um_qp = udma_alloc_qp(udma_ctx, cfg, NULL, jfs->jfs_id, false); - if (!jfs->um_qp) { - URMA_LOG_ERR("alloc qp failed.\n"); - return ENOMEM; - } - } else { - URMA_LOG_ERR("do not support this tp_mode.\n"); + jfs->um_qp = udma_alloc_qp(udma_ctx, false, cfg, NULL); + if (!jfs->um_qp) { + URMA_LOG_ERR("alloc qp failed.\n"); return ENOMEM; } return ret; } -static urma_status_t udma_add_to_qp_table(struct udma_u_context *ctx, - urma_jfs_t *jfs, struct udma_qp *qp, - uint32_t qpn) -{ - struct udma_jfs_qp_node *qp_node; - - if (!ctx || !qp) { - URMA_LOG_ERR("ctx or qp is NULL.\n"); - return URMA_EINVAL; - } - - qp_node = (struct udma_jfs_qp_node *)calloc(1, sizeof(*qp_node)); - if (!qp_node) { - URMA_LOG_ERR("failed to calloc qp_node.\n"); - return URMA_ENOMEM; - } - qp_node->jfs_qp = qp; - - (void)pthread_rwlock_wrlock(&ctx->jfs_qp_table_lock); - if (!udma_hmap_insert(&ctx->jfs_qp_table, &qp_node->node, qpn)) { - free(qp_node); - qp_node = NULL; - (void)pthread_rwlock_unlock(&ctx->jfs_qp_table_lock); - URMA_LOG_ERR("failed to insert qp_node into jfs qp table.\n"); - return URMA_EINVAL; - } - (void)pthread_rwlock_unlock(&ctx->jfs_qp_table_lock); - - return URMA_SUCCESS; -} - -static void udma_remove_from_qp_table(struct udma_u_context *ctx, uint32_t qpn) -{ - struct udma_jfs_qp_node *qp_node; - struct udma_hmap_node *node; - - (void)pthread_rwlock_wrlock(&ctx->jfs_qp_table_lock); - node = udma_hmap_first_with_hash(&ctx->jfs_qp_table, qpn); - if (node) { - qp_node = to_udma_jfs_qp_node(node); - udma_hmap_remove(&ctx->jfs_qp_table, node); - (void)pthread_rwlock_unlock(&ctx->jfs_qp_table_lock); - free(qp_node); - return; - } - (void)pthread_rwlock_unlock(&ctx->jfs_qp_table_lock); -} - static int exec_jfs_create_cmd(urma_context_t *ctx, struct udma_u_jfs *jfs, urma_jfs_cfg_t *cfg) { @@ -246,10 +190,8 @@ static int exec_jfs_create_cmd(urma_context_t *ctx, struct udma_u_jfs *jfs, urma_cmd_udrv_priv_t udata = {}; int ret; - if (jfs->tp_mode == URMA_TM_UM) { - cmd.create_tp_ucmd.buf_addr = (uintptr_t)jfs->um_qp->buf.buf; - cmd.create_tp_ucmd.sdb_addr = (uintptr_t)jfs->um_qp->sdb; - } + cmd.create_tp_ucmd.buf_addr = (uintptr_t)jfs->um_qp->buf.buf; + cmd.create_tp_ucmd.sdb_addr = (uintptr_t)jfs->um_qp->sdb; udma_set_udata(&udata, &cmd, sizeof(cmd), &resp, sizeof(resp)); ret = urma_cmd_create_jfs(ctx, &jfs->base, cfg, &udata); @@ -259,47 +201,55 @@ static int exec_jfs_create_cmd(urma_context_t *ctx, struct udma_u_jfs *jfs, } jfs->jfs_id = jfs->base.jfs_id.id; + jfs->um_qp->qp_num = resp.create_tp_resp.qpn; + jfs->um_qp->flags = resp.create_tp_resp.cap_flags; + jfs->um_qp->path_mtu = (urma_mtu_t)resp.create_tp_resp.path_mtu; + jfs->um_qp->sq.priority = resp.create_tp_resp.priority; - if (jfs->tp_mode == URMA_TM_UM) { - jfs->um_qp->qp_num = resp.create_tp_resp.qpn; - jfs->um_qp->flags = resp.create_tp_resp.cap_flags; - if (resp.create_tp_resp.cap_flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) { - ret = mmap_dwqe(ctx, jfs->um_qp); - if (ret) { - urma_cmd_delete_jfs(&jfs->base); - URMA_LOG_ERR("mmap dwqe failed\n"); - return URMA_FAIL; - } + if (resp.create_tp_resp.cap_flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) { + ret = mmap_dwqe(ctx, jfs->um_qp); + if (ret) { + URMA_LOG_ERR("mmap dwqe failed\n"); + goto err_mmap_dwqe; } - jfs->um_qp->path_mtu = (urma_mtu_t)resp.create_tp_resp.path_mtu; - jfs->um_qp->sq.priority = resp.create_tp_resp.priority; - memcpy(&jfs->um_qp->um_srcport, &resp.create_tp_resp.um_srcport, - sizeof(struct udp_srcport)); - ret = udma_add_to_qp_table(udma_ctx, &jfs->base, jfs->um_qp, - jfs->um_qp->qp_num); - if (ret) - URMA_LOG_ERR("add to qp table failed for um jfs, ret = %d.\n", ret); + } + + memcpy(&jfs->um_qp->um_srcport, &resp.create_tp_resp.um_srcport, + sizeof(struct udp_srcport)); + ret = udma_add_to_qp_table(udma_ctx, jfs->um_qp, jfs->um_qp->qp_num); + if (ret) { + URMA_LOG_ERR("add to qp table failed for um jfs, ret = %d.\n", ret); + goto err_add_qp_table; } return ret; + +err_add_qp_table: + if (resp.create_tp_resp.cap_flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(jfs->um_qp); +err_mmap_dwqe: + urma_cmd_delete_jfs(&jfs->base); + return ret; } -static void um_free_qp(struct udma_u_context *udma_ctx, struct udma_u_jfs *jfs) +static void delete_jfs_qp_node(struct udma_u_context *udma_ctx, struct udma_u_jfs *jfs) { - if (jfs->um_qp) { - udma_free_sw_db(udma_ctx, jfs->um_qp->sdb, UDMA_JFS_TYPE_DB); - free(jfs->um_qp->sq.wrid); - jfs->um_qp->sq.wrid = NULL; - udma_free_buf(&jfs->um_qp->buf); - free(jfs->um_qp); - jfs->um_qp = NULL; - } + udma_free_sw_db(udma_ctx, jfs->um_qp->sdb, UDMA_JFS_TYPE_DB); + free(jfs->um_qp->sq.wrid); + jfs->um_qp->sq.wrid = NULL; + udma_free_buf(&jfs->um_qp->buf); + free(jfs->um_qp); + jfs->um_qp = NULL; } -void delete_qp_node_table(struct udma_u_context *udma_ctx, struct udma_u_jfs *jfs) +static int exec_jfs_delete_cmd(struct udma_u_context *ctx, struct udma_u_jfs *jfs) { - if (jfs->tp_mode == URMA_TM_UM) - um_free_qp(udma_ctx, jfs); + udma_remove_from_qp_table(ctx, jfs->um_qp->qp_num); + + if (jfs->um_qp->flags & HNS3_UDMA_QP_CAP_DIRECT_WQE) + munmap_dwqe(jfs->um_qp); + + return urma_cmd_delete_jfs(&jfs->base); } urma_jfs_t *udma_u_create_jfs(urma_context_t *ctx, urma_jfs_cfg_t *cfg) @@ -308,7 +258,7 @@ urma_jfs_t *udma_u_create_jfs(urma_context_t *ctx, urma_jfs_cfg_t *cfg) struct udma_u_jfs *jfs; int ret; - if (!ctx) { + if (!ctx || cfg->trans_mode != URMA_TM_UM) { URMA_LOG_ERR("Invalid parameter.\n"); return NULL; } @@ -340,8 +290,7 @@ urma_jfs_t *udma_u_create_jfs(urma_context_t *ctx, urma_jfs_cfg_t *cfg) ret = insert_jetty_node(udma_ctx, jfs, false, jfs->jfs_id); if (ret) { - URMA_LOG_ERR("failed to insert jetty node, ret = %d.\n", - ret); + URMA_LOG_ERR("failed to insert jetty node, ret = %d.\n", ret); goto error_insert; } @@ -353,9 +302,9 @@ urma_jfs_t *udma_u_create_jfs(urma_context_t *ctx, urma_jfs_cfg_t *cfg) error_init_lock: delete_jetty_node(udma_ctx, jfs->jfs_id); error_insert: - urma_cmd_delete_jfs(&jfs->base); + (void)exec_jfs_delete_cmd(udma_ctx, jfs); error_create_jfs: - delete_qp_node_table(udma_ctx, jfs); + delete_jfs_qp_node(udma_ctx, jfs); error: free(jfs); @@ -366,9 +315,8 @@ urma_status_t udma_u_delete_jfs(urma_jfs_t *jfs) { struct udma_u_context *udma_ctx; struct udma_u_jfs *udma_jfs; - int ret; - if (jfs == NULL || jfs->urma_ctx == NULL) { + if (jfs == NULL || jfs->urma_ctx == NULL || jfs->jfs_cfg.trans_mode != URMA_TM_UM) { URMA_LOG_ERR("Invalid parameter.\n"); return URMA_EINVAL; } @@ -376,19 +324,15 @@ urma_status_t udma_u_delete_jfs(urma_jfs_t *jfs) udma_jfs = to_udma_jfs(jfs); udma_ctx = to_udma_ctx(jfs->urma_ctx); - if (udma_jfs->tp_mode == URMA_TM_UM) - udma_remove_from_qp_table(udma_ctx, udma_jfs->um_qp->qp_num); - + pthread_spin_destroy(&udma_jfs->lock); delete_jetty_node(udma_ctx, udma_jfs->jfs_id); - delete_qp_node_table(udma_ctx, udma_jfs); - ret = urma_cmd_delete_jfs(jfs); - if (ret) { - URMA_LOG_ERR("jfs delete failed!\n"); + if (exec_jfs_delete_cmd(udma_ctx, udma_jfs)) { + URMA_LOG_ERR("delete jfs cmd failed!\n"); return URMA_FAIL; } - pthread_spin_destroy(&udma_jfs->lock); + delete_jfs_qp_node(udma_ctx, udma_jfs); free(udma_jfs); @@ -406,12 +350,12 @@ static inline void *get_wqe(struct udma_qp *qp, uint32_t offset) return NULL; } -static void *get_send_wqe(struct udma_qp *qp, uint32_t n) +void *get_send_wqe(struct udma_qp *qp, uint32_t n) { return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); } -static void *get_send_sge_ex(struct udma_qp *qp, uint32_t n) +void *get_send_sge_ex(struct udma_qp *qp, uint32_t n) { return get_wqe(qp, qp->ex_sge.offset + (n << qp->ex_sge.sge_shift)); } @@ -945,51 +889,7 @@ static void udma_update_sq_db(struct udma_u_context *ctx, struct udma_qp *qp) udma_reg_write(&sq_db, UDMA_DB_PI, qp->sq.head); udma_reg_write(&sq_db, UDMA_DB_SL, qp->sq.priority); - udma_write64(ctx, (uint64_t *)(ctx->uar + UDMA_DB_CFG0_OFFSET), - (uint64_t *)&sq_db); -} - -struct udma_qp *get_qp(struct udma_u_jfs *udma_jfs, urma_jfs_wr_t *wr) -{ - struct udma_hmap_node *hmap_node = NULL; - struct connect_node *udma_connect_node; - struct udma_qp *udma_qp = NULL; - uint32_t tjfr_index; - - if (!wr->tjetty) { - URMA_LOG_ERR("Failed to get jfs qp, tjetty of wr is null.\n"); - return NULL; - } - - if (udma_jfs->tp_mode == URMA_TM_UM) - return udma_jfs->um_qp; - - switch (wr->opcode) { - case URMA_OPC_SEND: - case URMA_OPC_SEND_IMM: - case URMA_OPC_SEND_INVALIDATE: - case URMA_OPC_WRITE: - case URMA_OPC_WRITE_IMM: - case URMA_OPC_WRITE_NOTIFY: - tjfr_index = udma_get_tgt_hash(&wr->tjetty->id); - hmap_node = udma_table_first_with_hash(&udma_jfs->tjfr_tbl.hmap, - &udma_jfs->tjfr_tbl.rwlock, - tjfr_index); - break; - default: - URMA_LOG_ERR("Unsupported or invalid opcode: %u.\n", - (uint32_t)wr->opcode); - return NULL; - } - - if (!hmap_node) - return NULL; - - udma_connect_node = CONTAINER_OF_FIELD(hmap_node, struct connect_node, - hmap_node); - udma_qp = udma_connect_node->qp; - - return udma_qp; + udma_write64(ctx, (uint64_t *)(ctx->uar + UDMA_DB_CFG0_OFFSET), (uint64_t *)&sq_db); } int exec_jfs_flush_cqe_cmd(struct udma_u_context *udma_ctx, @@ -1113,20 +1013,13 @@ out: return ret; } -urma_status_t udma_u_post_qp_wr(struct udma_u_context *udma_ctx, - struct udma_qp *udma_qp, - urma_jfs_wr_t *wr, void **wqe, - urma_transport_mode_t tp_mode) +urma_status_t udma_u_post_umqp_wr(struct udma_u_context *udma_ctx, + struct udma_qp *udma_qp, + urma_jfs_wr_t *wr, void **wqe) { urma_status_t ret = URMA_SUCCESS; uint32_t wqe_idx; - ret = check_dca_valid(udma_ctx, udma_qp); - if (ret) { - URMA_LOG_ERR("Failed to check send, qpn = %lu.\n", udma_qp->qp_num); - goto out; - } - if (wr->send.src.num_sge > udma_qp->sq.max_gs) { ret = udma_qp->sq.max_gs > 0 ? URMA_EINVAL : URMA_ENOPERM; URMA_LOG_ERR("Invalid wr sge num, ret = 0x%x.\n", ret); @@ -1151,13 +1044,7 @@ urma_status_t udma_u_post_qp_wr(struct udma_u_context *udma_ctx, if (udma_qp->flags & HNS3_UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) udma_write_dca_wqe(udma_qp, *wqe); - udma_to_device_barrier(); - - udma_u_ring_sq_doorbell(udma_ctx, udma_qp, *wqe, 1); - *udma_qp->sdb = udma_qp->sq.head; - if (udma_qp->flush_status == UDMA_FLUSH_STATU_ERR) - exec_jfs_flush_cqe_cmd(udma_ctx, udma_qp); out: if (udma_qp->flags & HNS3_UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) @@ -1172,6 +1059,7 @@ urma_status_t udma_u_post_jfs_wr(urma_jfs_t *jfs, urma_jfs_wr_t *wr, struct udma_u_context *udma_ctx; struct udma_u_jfs *udma_jfs; struct udma_qp *udma_qp; + uint32_t wr_cnt = 0; urma_status_t ret; urma_jfs_wr_t *it; void *wqe; @@ -1179,27 +1067,30 @@ urma_status_t udma_u_post_jfs_wr(urma_jfs_t *jfs, urma_jfs_wr_t *wr, udma_jfs = to_udma_jfs(jfs); udma_ctx = to_udma_ctx(jfs->urma_ctx); + if (udma_jfs->tp_mode != URMA_TM_UM) + return URMA_EINVAL; + if (!udma_jfs->lock_free) (void)pthread_spin_lock(&udma_jfs->lock); - for (it = wr; it != NULL; it = (urma_jfs_wr_t *)(void *)it->next) { - udma_qp = get_qp(udma_jfs, it); - if (!udma_qp) { - URMA_LOG_ERR("failed to get qp, opcode = 0x%x.\n", - it->opcode); - ret = URMA_EINVAL; - *bad_wr = (urma_jfs_wr_t *)it; - goto out; - } + udma_qp = udma_jfs->um_qp; + + ret = check_dca_valid(udma_ctx, udma_qp); + if (ret) + return ret; - ret = udma_u_post_qp_wr(udma_ctx, udma_qp, it, &wqe, - udma_jfs->tp_mode); + for (it = wr; it != NULL; it = (urma_jfs_wr_t *)(void *)it->next) { + ret = udma_u_post_umqp_wr(udma_ctx, udma_qp, it, &wqe); if (ret) { *bad_wr = (urma_jfs_wr_t *)it; - goto out; + break; } + wr_cnt++; } out: + if (likely(wr_cnt)) + udma_u_ring_sq_doorbell(udma_ctx, udma_qp, wqe, wr_cnt); + if (!udma_jfs->lock_free) (void)pthread_spin_unlock(&udma_jfs->lock); diff --git a/hw/hns3/hns3_udma_u_jfs.h b/hw/hns3/hns3_udma_u_jfs.h index 440c2cbeba8a0d7e36da1968a7c69440f69a1061..4e049f5e324c9e7862268baaa39f51c5c0a73ab3 100644 --- a/hw/hns3/hns3_udma_u_jfs.h +++ b/hw/hns3/hns3_udma_u_jfs.h @@ -157,7 +157,7 @@ struct udma_u_jfs { struct udma_jfs_node { struct udma_hmap_node node; - struct udma_u_jfs *jfs; + struct udma_u_jfs *jfs; }; enum udma_jfs_opcode { @@ -363,22 +363,21 @@ urma_status_t udma_u_post_jfs_wr(urma_jfs_t *jfs, urma_jfs_wr_t *wr, urma_status_t udma_u_post_rcqp_wr(struct udma_u_context *udma_ctx, struct udma_qp *udma_qp, urma_jfs_wr_t *wr, void **wqe, uint32_t nreq); -urma_status_t udma_u_post_qp_wr(struct udma_u_context *udma_ctx, - struct udma_qp *udma_qp, urma_jfs_wr_t *wr, - void **wqe, urma_transport_mode_t tp_mode); +urma_status_t udma_u_post_umqp_wr(struct udma_u_context *udma_ctx, + struct udma_qp *udma_qp, urma_jfs_wr_t *wr, + void **wqe_addr); urma_status_t udma_u_post_qp_wr_ex(struct udma_u_context *udma_ctx, struct udma_qp *udma_qp, urma_jfs_wr_t *wr, urma_transport_mode_t tp_mode); -struct udma_qp *udma_alloc_qp(struct udma_u_context *udma_ctx, - urma_jfs_cfg_t *jfs_cfg, - urma_jfr_cfg_t *jfr_cfg, - uint32_t jetty_id, bool is_jetty); -struct udma_qp *get_qp(struct udma_u_jfs *udma_jfs, urma_jfs_wr_t *wr); +struct udma_qp *udma_alloc_qp(struct udma_u_context *udma_ctx, bool is_jetty, + urma_jfs_cfg_t *jfs_cfg, urma_jfr_cfg_t *jfr_cfg); void udma_u_ring_sq_doorbell(struct udma_u_context *udma_ctx, struct udma_qp *udma_qp, void *wqe, uint32_t num); int exec_jfs_flush_cqe_cmd(struct udma_u_context *udma_ctx, struct udma_qp *qp); urma_status_t check_dca_valid(struct udma_u_context *udma_ctx, struct udma_qp *qp); int udma_u_flush_jfs(urma_jfs_t *jfs, int cr_cnt, urma_cr_t *cr); +void *get_send_wqe(struct udma_qp *qp, uint32_t n); +void *get_send_sge_ex(struct udma_qp *qp, uint32_t n); #endif /* _UDMA_JFS_H */ diff --git a/hw/hns3/hns3_udma_u_segment.h b/hw/hns3/hns3_udma_u_segment.h index d1e574778825cd8a40c989c6b344cd887eb477c3..9b91622d3a42790588db836e90b7465c4374e2bc 100644 --- a/hw/hns3/hns3_udma_u_segment.h +++ b/hw/hns3/hns3_udma_u_segment.h @@ -18,7 +18,6 @@ #include "urma_types.h" #include "hns3_udma_u_common.h" -#include "hns3_udma_u_jfc.h" #define UDMA_RESERVED_JFR_SGE 1 diff --git a/hw/hns3/hns3_udma_u_tp.c b/hw/hns3/hns3_udma_u_tp.c index 9c79085e6c95c26a78cd67abc3f3a31900b188ff..39bb69496339b3dffff127c65c9a029acc2259d4 100644 --- a/hw/hns3/hns3_udma_u_tp.c +++ b/hw/hns3/hns3_udma_u_tp.c @@ -16,8 +16,6 @@ #include #include "hns3_udma_u_provider_ops.h" #include "hns3_udma_u_common.h" -#include "hns3_udma_u_jfs.h" -#include "hns3_udma_u_db.h" #include "hns3_udma_u_tp.h" int mmap_dwqe(struct urma_context *urma_ctx, struct udma_qp *qp) @@ -38,3 +36,12 @@ int mmap_dwqe(struct urma_context *urma_ctx, struct udma_qp *qp) return 0; } +void munmap_dwqe(struct udma_qp *qp) +{ + if (qp->dwqe_page) { + if (!(munmap(qp->dwqe_page, HNS3_UDMA_DWQE_PAGE_SIZE))) + URMA_LOG_ERR("failed to munmap direct wqe page, QPN = %lu.\n", + qp->qp_num); + qp->dwqe_page = NULL; + } +} diff --git a/hw/hns3/hns3_udma_u_tp.h b/hw/hns3/hns3_udma_u_tp.h index 89a3c6620d99046822c4e49b7a0fd4e6024655b0..b2452d1ae14713df9c5db8129c1af9eaad01aede 100644 --- a/hw/hns3/hns3_udma_u_tp.h +++ b/hw/hns3/hns3_udma_u_tp.h @@ -17,15 +17,11 @@ #define _UDMA_U_TP_H #include "hns3_udma_u_jfs.h" -#include "hns3_udma_u_jetty.h" #include "urma_provider.h" #define max(a, b) ((a) > (b) ? (a) : (b)) -struct jfs_conn_node { - struct connect_node *tgt_conn_node; - struct udma_jfs_qp_node *qp_conn_node; -}; - int mmap_dwqe(struct urma_context *urma_ctx, struct udma_qp *qp); +void munmap_dwqe(struct udma_qp *qp); + #endif /* _UDMA_U_TP_H */ diff --git a/hw/hns3/hns3_udma_u_user_ctl.c b/hw/hns3/hns3_udma_u_user_ctl.c index 5502a95e173d5d965256e7a9f50015c11ba2fa28..c45e7b08bdb67d598f956bca1b996a0658acc4e2 100644 --- a/hw/hns3/hns3_udma_u_user_ctl.c +++ b/hw/hns3/hns3_udma_u_user_ctl.c @@ -35,11 +35,14 @@ static int udma_u_post_jfs_ex(urma_jfs_t *jfs, urma_jfs_wr_t *wr, udma_jfs = to_udma_jfs(jfs); udma_ctx = to_udma_ctx(jfs->urma_ctx); + if (udma_jfs->tp_mode != URMA_TM_UM) + return -URMA_EINVAL; + if (!udma_jfs->lock_free) (void)pthread_spin_lock(&udma_jfs->lock); for (it = wr; it != NULL; it = it->next) { - udma_qp = get_qp(udma_jfs, it); + udma_qp = udma_jfs->um_qp; if (!udma_qp) { URMA_LOG_ERR("failed to get qp, opcode = 0x%x.\n", it->opcode); @@ -494,6 +497,54 @@ static int udma_u_query_hw_id(urma_context_t *ctx, urma_user_ctl_in_t *in, return 0; } +static void udma_u_get_jetty_info_set_info_out(struct hns3_u_udma_get_jetty_info_out *info_out, + struct udma_qp *qp, + struct udma_u_context *udma_ctx) +{ + info_out->queue_addr = qp->buf.buf + qp->sq.offset; + info_out->ext_sge_addr = qp->buf.buf + qp->ex_sge.offset; + info_out->user_ctx_addr = (void *)qp->sq.wrid; + info_out->head_idx = &qp->sq.head; + info_out->sl = qp->sq.priority; + info_out->queue_length = qp->sq.wqe_cnt; + info_out->ext_sge_length = qp->ex_sge.sge_cnt; + info_out->user_ctx_length = qp->sq.wqe_cnt; + info_out->db_addr = udma_ctx->uar + UDMA_DB_CFG0_OFFSET; + info_out->dwqe_addr = qp->dwqe_page; + info_out->ext_sge_tail_addr = get_send_sge_ex(qp, qp->ex_sge.sge_cnt); +} + +int udma_u_get_jetty_info(urma_context_t *ctx, urma_user_ctl_in_t *in, + urma_user_ctl_out_t *out) +{ + struct hns3_u_udma_get_jetty_info_out *info_out; + struct hns3_u_udma_get_jetty_info_in *info_in; + struct udma_u_jetty *udma_jetty; + struct udma_u_context *udma_ctx; + struct udma_qp *qp; + + if (in->len != sizeof(struct hns3_u_udma_get_jetty_info_in)) { + URMA_LOG_ERR("Invalid buffer size(%u) for getting jetty info.\n", in->len); + return EINVAL; + } + + info_in = (struct hns3_u_udma_get_jetty_info_in *)in->addr; + + if (!info_in->jetty) { + URMA_LOG_ERR("Jetty is null.\n"); + return EINVAL; + } + + udma_jetty = to_udma_jetty(info_in->jetty); + udma_ctx = to_udma_ctx(ctx); + qp = udma_jetty->rc_node->qp; + + udma_u_get_jetty_info_set_info_out((struct hns3_u_udma_get_jetty_info_out *)out->addr, + qp, udma_ctx); + + return 0; +} + typedef int (*udma_u_user_ctl_ops)(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out); @@ -505,6 +556,7 @@ static udma_u_user_ctl_ops g_udma_u_user_ctl_ops[] = { [HNS3_UDMA_U_USER_CTL_DELETE_JFC_EX] = udma_u_delete_jfc_ex, [HNS3_UDMA_U_USER_CTL_UPDATE_QUEUE_CI] = udma_u_update_queue_ci, [HNS3_UDMA_U_USER_CTL_QUERY_HW_ID] = udma_u_query_hw_id, + [HNS3_UDMA_U_USER_CTL_GET_JETTY_INFO] = udma_u_get_jetty_info, }; int udma_u_user_ctl(urma_context_t *ctx, urma_user_ctl_in_t *in, diff --git a/hw/hns3/hns3_udma_u_user_ctl_api.h b/hw/hns3/hns3_udma_u_user_ctl_api.h index 8036a90dcd9d4af16916a67319c637b25dd9b4c2..0402661161c4d9d7b3a1658a053773b8575d045d 100644 --- a/hw/hns3/hns3_udma_u_user_ctl_api.h +++ b/hw/hns3/hns3_udma_u_user_ctl_api.h @@ -100,6 +100,28 @@ struct hns3_udma_query_hw_id_out { uint32_t reserved; }; +struct hns3_u_udma_get_jetty_info_in { + enum hns3_queue_type type; + union { + urma_jfs_t *jfs; + urma_jetty_t *jetty; + }; +}; + +struct hns3_u_udma_get_jetty_info_out { + void *queue_addr; + uint32_t queue_length; + void *ext_sge_addr; + uint32_t ext_sge_length; + void *user_ctx_addr; + uint32_t user_ctx_length; + void *db_addr; + void *dwqe_addr; + void *ext_sge_tail_addr; + uint32_t sl; + void *head_idx; +}; + enum hns3_udma_u_user_ctl_opcode { HNS3_UDMA_U_USER_CTL_POST_SEND_AND_RET_DB, HNS3_UDMA_U_USER_CTL_CONFIG_POE_CHANNEL, @@ -108,6 +130,7 @@ enum hns3_udma_u_user_ctl_opcode { HNS3_UDMA_U_USER_CTL_DELETE_JFC_EX, HNS3_UDMA_U_USER_CTL_UPDATE_QUEUE_CI, HNS3_UDMA_U_USER_CTL_QUERY_HW_ID, + HNS3_UDMA_U_USER_CTL_GET_JETTY_INFO, HNS3_UDMA_U_USER_CTL_MAX, };