From 14f16323afc4fe3872048f18a344c58a1e4f71a0 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Sun, 8 Sep 2024 23:08:41 +0000 Subject: [PATCH 01/13] uadk: use wd_find_msg_in_pool to replace wd_xxx_get_msg Use the common func wd_find_msg_in_pool to replace the different wd_xxx_get_msg via passing the pool to the drv.priv->config.poll Signed-off-by: Zhangfei Gao --- drv/hisi_comp.c | 9 ++++--- drv/hisi_dae.c | 3 ++- drv/hisi_hpre.c | 38 +++++++++++++------------- drv/hisi_sec.c | 54 ++++++++++++++++++++----------------- include/drv/wd_aead_drv.h | 2 -- include/drv/wd_agg_drv.h | 2 -- include/drv/wd_cipher_drv.h | 2 -- include/drv/wd_comp_drv.h | 2 -- include/drv/wd_dh_drv.h | 2 -- include/drv/wd_digest_drv.h | 2 -- include/drv/wd_ecc_drv.h | 2 -- include/drv/wd_rsa_drv.h | 2 -- include/wd_alg_common.h | 1 + libwd_comp.map | 2 +- libwd_crypto.map | 7 +---- libwd_dae.map | 2 +- wd_aead.c | 7 ++--- wd_agg.c | 7 ++--- wd_cipher.c | 7 ++--- wd_comp.c | 7 ++--- wd_dh.c | 7 ++--- wd_digest.c | 7 ++--- wd_ecc.c | 7 ++--- wd_rsa.c | 7 ++--- 24 files changed, 77 insertions(+), 111 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 07d875488..d30da966a 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -1003,9 +1003,10 @@ static void get_ctx_buf(struct hisi_zip_sqe *sqe, } } -static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg) +static int parse_zip_sqe(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { + struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; __u32 buf_type = (sqe->dw9 & HZ_BUF_TYPE_MASK) >> BUF_TYPE_SHIFT; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; @@ -1030,7 +1031,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, recv_msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - recv_msg = wd_comp_get_msg(qp->q_info.idx, tag); + recv_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (unlikely(!recv_msg)) { WD_ERR("failed to get send msg! idx = %u, tag = %u!\n", qp->q_info.idx, tag); @@ -1091,7 +1092,7 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com if (unlikely(ret < 0)) return ret; - return parse_zip_sqe(qp, &sqe, recv_msg); + return parse_zip_sqe(drv, qp, &sqe, recv_msg); } #define GEN_ZIP_ALG_DRIVER(zip_alg_name) \ diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index 864d21bae..aa5da9072 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -653,6 +653,7 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) { + struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct dae_extend_addr *ext_addr = qp->priv; @@ -673,7 +674,7 @@ static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_m msg->tag = sqe.low_tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_agg_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { msg->result = WD_AGG_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 313cdcc8f..e9d077573 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -659,6 +659,7 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct hisi_hpre_sqe hw_msg = {0}; @@ -677,7 +678,7 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) msg->tag = LW_U16(hw_msg.low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -802,6 +803,7 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_dh_msg *msg = dh_msg; @@ -820,7 +822,7 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) msg->tag = LW_U16(hw_msg.low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -2037,7 +2039,6 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, return ret; } - static __u32 get_hash_bytes(__u8 type) { __u32 val = 0; @@ -2303,15 +2304,16 @@ static int sm2_convert_dec_out(struct wd_ecc_msg *src, return ret; } -static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int ecc_sqe_parse(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; struct wd_ecc_msg *temp_msg; int ret; msg->tag = LW_U16(hw_msg->low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -2342,7 +2344,7 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, return ret; } -static int parse_second_sqe(handle_t h_qp, +static int parse_second_sqe(struct wd_alg_driver *drv, handle_t h_qp, struct wd_ecc_msg *msg, struct wd_ecc_msg **second) { @@ -2371,15 +2373,15 @@ static int parse_second_sqe(handle_t h_qp, hsz = (hw_msg.task_len1 + 1) * BYTE_BITS; dst = *(struct wd_ecc_msg **)((uintptr_t)data + hsz * ECDH_OUT_PARAM_NUM); - ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, dst, &hw_msg); msg->result = dst->result; *second = dst; return ret; } -static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_enc_parse(struct wd_alg_driver *drv, handle_t h_qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *second = NULL; @@ -2397,14 +2399,14 @@ static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, memcpy(&src, first + 1, sizeof(src)); /* parse first sqe */ - ret = ecc_sqe_parse((struct hisi_qp *)h_qp, first, hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, first, hw_msg); if (ret) { WD_ERR("failed to parse first BD, ret = %d!\n", ret); goto free_first; } /* parse second sqe */ - ret = parse_second_sqe(h_qp, msg, &second); + ret = parse_second_sqe(drv, h_qp, msg, &second); if (unlikely(ret)) { WD_ERR("failed to parse second BD, ret = %d!\n", ret); goto free_first; @@ -2424,8 +2426,8 @@ static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, return ret; } -static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_dec_parse(struct wd_alg_driver *drv, handle_t ctx, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *dst; @@ -2441,7 +2443,7 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, memcpy(&src, dst + 1, sizeof(src)); /* parse first sqe */ - ret = ecc_sqe_parse((struct hisi_qp *)ctx, dst, hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)ctx, dst, hw_msg); if (ret) { WD_ERR("failed to parse decode BD, ret = %d!\n", ret); goto fail; @@ -2480,12 +2482,12 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_ENC) - return sm2_enc_parse(h_qp, msg, &hw_msg); + return sm2_enc_parse(drv, h_qp, msg, &hw_msg); else if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_DEC) - return sm2_dec_parse(h_qp, msg, &hw_msg); + return sm2_dec_parse(drv, h_qp, msg, &hw_msg); - return ecc_sqe_parse((struct hisi_qp *)h_qp, msg, &hw_msg); + return ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, msg, &hw_msg); } static int hpre_get_usage(void *param) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 10db124b7..d422a4a3c 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -920,9 +920,10 @@ static void fill_cipher_bd2_addr(struct wd_cipher_msg *msg, sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->key; } -static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_cipher_msg *recv_msg) +static void parse_cipher_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_cipher_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_cipher_msg *temp_msg; __u16 done; __u32 tag; @@ -944,7 +945,7 @@ static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1218,7 +1219,7 @@ static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (ret) return ret; - parse_cipher_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_cipher_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -1405,9 +1406,10 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void return 0; } -static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_cipher_msg *recv_msg) +static void parse_cipher_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_cipher_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_cipher_msg *temp_msg; __u16 done; __u32 tag; @@ -1429,7 +1431,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1466,7 +1468,7 @@ static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (ret) return ret; - parse_cipher_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_cipher_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -1576,9 +1578,10 @@ static int fill_digest_long_hash(handle_t h_qp, struct wd_digest_msg *msg, return 0; } -static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_digest_msg *recv_msg) +static void parse_digest_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_digest_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_digest_msg *temp_msg; __u16 done; @@ -1597,7 +1600,7 @@ static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1787,7 +1790,7 @@ static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (ret) return ret; - parse_digest_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_digest_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -2010,9 +2013,10 @@ static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void return ret; } -static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_digest_msg *recv_msg) +static void parse_digest_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_digest_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_digest_msg *temp_msg; __u16 done; @@ -2031,7 +2035,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2063,7 +2067,7 @@ static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (ret) return ret; - parse_digest_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_digest_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -2618,9 +2622,10 @@ static void update_stream_counter(struct wd_aead_msg *recv_msg) } } -static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_aead_msg *recv_msg) +static void parse_aead_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_aead_msg *temp_msg; __u16 done, icv; @@ -2642,7 +2647,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2697,7 +2702,7 @@ static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_ if (ret) return ret; - parse_aead_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_aead_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -3006,9 +3011,10 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * return ret; } -static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_aead_msg *recv_msg) +static void parse_aead_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_aead_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_aead_msg *temp_msg; __u16 done, icv; @@ -3031,7 +3037,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -3068,7 +3074,7 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * if (ret) return ret; - parse_aead_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_aead_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index a9c0e7c09..18696e238 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -70,8 +70,6 @@ struct wd_aead_msg { enum wd_aead_msg_state msg_state; }; -struct wd_aead_msg *wd_aead_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_agg_drv.h b/include/drv/wd_agg_drv.h index 3d26eeb3c..ea7134399 100644 --- a/include/drv/wd_agg_drv.h +++ b/include/drv/wd_agg_drv.h @@ -48,8 +48,6 @@ struct wd_agg_ops { int (*hash_table_init)(struct wd_dae_hash_table *hash_table, void *priv); }; -struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_cipher_drv.h b/include/drv/wd_cipher_drv.h index c6d8ddfbc..15529e190 100644 --- a/include/drv/wd_cipher_drv.h +++ b/include/drv/wd_cipher_drv.h @@ -50,8 +50,6 @@ struct wd_cipher_msg { __u8 *out; }; -struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d59..3628ae690 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -55,8 +55,6 @@ struct wd_comp_msg { __u32 tag; }; -struct wd_comp_msg *wd_comp_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h index d205dc45b..de8b06832 100644 --- a/include/drv/wd_dh_drv.h +++ b/include/drv/wd_dh_drv.h @@ -24,8 +24,6 @@ struct wd_dh_msg { __u8 result; /* Data format, denoted by WD error code */ }; -struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index a55ef5bb1..4cea1cf78 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -80,8 +80,6 @@ static inline enum hash_block_type get_hash_block_type(struct wd_digest_msg *msg return HASH_SINGLE_BLOCK; } -struct wd_digest_msg *wd_digest_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index f5805cdc0..e9624ef9e 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -175,8 +175,6 @@ struct wd_ecc_out { char data[]; }; -struct wd_ecc_msg *wd_ecc_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_rsa_drv.h b/include/drv/wd_rsa_drv.h index d231ecfb8..77406fc68 100644 --- a/include/drv/wd_rsa_drv.h +++ b/include/drv/wd_rsa_drv.h @@ -49,8 +49,6 @@ struct wd_rsa_msg { __u8 *key; /* Input key VA pointer, should be DMA buffer */ }; -struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index fd7742698..e1c2bfb25 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -151,6 +151,7 @@ struct wd_ctx_config_internal { void *priv; bool epoll_en; unsigned long *msg_cnt; + struct wd_async_msg_pool *pool; }; /* diff --git a/libwd_comp.map b/libwd_comp.map index 6b1f8c25e..efc449aaf 100644 --- a/libwd_comp.map +++ b/libwd_comp.map @@ -20,7 +20,6 @@ global: wd_comp_get_env_param; wd_comp_set_driver; wd_comp_get_driver; - wd_comp_get_msg; wd_comp_reset_sess; wd_sched_rr_instance; @@ -37,5 +36,6 @@ global: wd_inflate_reset; wd_inflate_end; + wd_find_msg_in_pool; local: *; }; diff --git a/libwd_crypto.map b/libwd_crypto.map index e8555c982..cf23c5680 100644 --- a/libwd_crypto.map +++ b/libwd_crypto.map @@ -19,7 +19,6 @@ global: wd_cipher_get_env_param; wd_cipher_set_driver; wd_cipher_get_driver; - wd_cipher_get_msg; wd_aead_init; wd_aead_uninit; @@ -44,7 +43,6 @@ global: wd_aead_get_env_param; wd_aead_set_driver; wd_aead_get_driver; - wd_aead_get_msg; wd_digest_init; wd_digest_uninit; @@ -65,7 +63,6 @@ global: wd_digest_get_env_param; wd_digest_set_driver; wd_digest_get_driver; - wd_digest_get_msg; wd_rsa_is_crt; wd_rsa_get_key_bits; @@ -107,7 +104,6 @@ global: wd_rsa_get_env_param; wd_rsa_set_driver; wd_rsa_get_driver; - wd_rsa_get_msg; wd_dh_get_mode; wd_dh_key_bits; @@ -131,7 +127,6 @@ global: wd_dh_get_env_param; wd_dh_set_driver; wd_dh_get_driver; - wd_dh_get_msg; wd_ecc_get_key_bits; wd_ecc_get_key; @@ -148,7 +143,6 @@ global: wd_ecxdh_get_out_params; wd_ecc_set_driver; wd_ecc_get_driver; - wd_ecc_get_msg; wd_sm2_new_sign_in; wd_sm2_new_verf_in; @@ -187,5 +181,6 @@ global: wd_sched_rr_instance; wd_sched_rr_alloc; wd_sched_rr_release; + wd_find_msg_in_pool; local: *; }; diff --git a/libwd_dae.map b/libwd_dae.map index 4c51b856f..660794c8b 100644 --- a/libwd_dae.map +++ b/libwd_dae.map @@ -11,11 +11,11 @@ global: wd_agg_get_output_sync; wd_agg_get_output_async; wd_agg_rehash_sync; - wd_agg_get_msg; wd_agg_poll; wd_sched_rr_instance; wd_sched_rr_alloc; wd_sched_rr_release; + wd_find_msg_in_pool; local: *; }; diff --git a/wd_aead.c b/wd_aead.c index 608f0e6d7..8157e1989 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -464,6 +464,8 @@ static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; + wd_aead_setting.config.pool = &wd_aead_setting.pool; + ret = wd_alg_init_driver(&wd_aead_setting.config, wd_aead_setting.driver); if (ret) @@ -831,11 +833,6 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return ret; } -struct wd_aead_msg *wd_aead_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_aead_setting.pool, idx, tag); -} - int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_aead_setting.config; diff --git a/wd_agg.c b/wd_agg.c index b536efe58..faafafb18 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -592,6 +592,8 @@ static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret < 0) goto out_clear_sched; + wd_agg_setting.config.pool = &wd_agg_setting.pool; + ret = wd_alg_init_driver(&wd_agg_setting.config, wd_agg_setting.driver); if (ret) goto out_clear_pool; @@ -1504,11 +1506,6 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) return WD_SUCCESS; } -struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_agg_setting.pool, idx, tag); -} - static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_agg_setting.config; diff --git a/wd_cipher.c b/wd_cipher.c index ec6fb1562..996f867ac 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -345,6 +345,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched; + wd_cipher_setting.config.pool = &wd_cipher_setting.pool; + ret = wd_alg_init_driver(&wd_cipher_setting.config, wd_cipher_setting.driver); if (ret) @@ -782,11 +784,6 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } -struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_cipher_setting.pool, idx, tag); -} - int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_cipher_setting.config; diff --git a/wd_comp.c b/wd_comp.c index 0fa5f9262..4abb64d27 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -165,6 +165,8 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; + wd_comp_setting.config.pool = &wd_comp_setting.pool; + ret = wd_alg_init_driver(&wd_comp_setting.config, wd_comp_setting.driver); if (ret) @@ -347,11 +349,6 @@ void wd_comp_uninit2(void) wd_alg_clear_init(&wd_comp_setting.status); } -struct wd_comp_msg *wd_comp_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_comp_setting.pool, idx, tag); -} - int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_comp_setting.config; diff --git a/wd_dh.c b/wd_dh.c index 043c3be95..2686dda66 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -139,6 +139,8 @@ static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sche if (ret) goto out_clear_sched; + wd_dh_setting.config.pool = &wd_dh_setting.pool; + ret = wd_alg_init_driver(&wd_dh_setting.config, wd_dh_setting.driver); if (ret) @@ -453,11 +455,6 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return ret; } -struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_dh_setting.pool, idx, tag); -} - int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_dh_setting.config; diff --git a/wd_digest.c b/wd_digest.c index 58f621af6..85b722f3d 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -276,6 +276,8 @@ static int wd_digest_init_nolock(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched; + wd_digest_setting.config.pool = &wd_digest_setting.pool; + ret = wd_alg_init_driver(&wd_digest_setting.config, wd_digest_setting.driver); if (ret) @@ -733,11 +735,6 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return ret; } -struct wd_digest_msg *wd_digest_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_digest_setting.pool, idx, tag); -} - int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_digest_setting.config; diff --git a/wd_ecc.c b/wd_ecc.c index 292338a6e..5588e577b 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -202,6 +202,8 @@ static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; + wd_ecc_setting.config.pool = &wd_ecc_setting.pool; + ret = wd_alg_init_driver(&wd_ecc_setting.config, wd_ecc_setting.driver); if (ret) @@ -2298,11 +2300,6 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return ret; } -struct wd_ecc_msg *wd_ecc_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_ecc_setting.pool, idx, tag); -} - int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_ecc_setting.config; diff --git a/wd_rsa.c b/wd_rsa.c index 366e76603..ea340c076 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -179,6 +179,8 @@ static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; + wd_rsa_setting.config.pool = &wd_rsa_setting.pool; + ret = wd_alg_init_driver(&wd_rsa_setting.config, wd_rsa_setting.driver); if (ret) @@ -513,11 +515,6 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return ret; } -struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_rsa_setting.pool, idx, tag); -} - int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_rsa_setting.config; From 22f4e3002b08a59b6fe8d687427c34b72712e6f8 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Sun, 8 Sep 2024 23:38:14 +0000 Subject: [PATCH 02/13] uadk: move wd_util and wd_shced to libwd The wd_util.c and wd_shced.c should be in libwd.so No need to export in every algo.so Signed-off-by: Zhangfei Gao --- Makefile.am | 1 + libwd.map | 5 +++++ libwd_comp.map | 6 ------ libwd_crypto.map | 5 ----- libwd_dae.map | 5 ----- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/Makefile.am b/Makefile.am index d671d09d8..c0bdc8653 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,6 +48,7 @@ uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ libisa_ce.la libisa_sve.la libhisi_dae.la libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ + wd_sched.c wd_util.c \ v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ v1/wd_rng.c v1/wd_rng.h \ v1/wd_rsa.c v1/wd_rsa.h \ diff --git a/libwd.map b/libwd.map index 5522ec07f..132fd044d 100644 --- a/libwd.map +++ b/libwd.map @@ -49,5 +49,10 @@ global: wd_enable_drv; wd_disable_drv; wd_get_alg_head; + + wd_sched_rr_instance; + wd_sched_rr_alloc; + wd_sched_rr_release; + wd_find_msg_in_pool; local: *; }; diff --git a/libwd_comp.map b/libwd_comp.map index efc449aaf..336831989 100644 --- a/libwd_comp.map +++ b/libwd_comp.map @@ -22,10 +22,6 @@ global: wd_comp_get_driver; wd_comp_reset_sess; - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; - wd_deflate_init; wd_deflate; wd_deflate_reset; @@ -35,7 +31,5 @@ global: wd_inflate; wd_inflate_reset; wd_inflate_end; - - wd_find_msg_in_pool; local: *; }; diff --git a/libwd_crypto.map b/libwd_crypto.map index cf23c5680..33de0bf7b 100644 --- a/libwd_crypto.map +++ b/libwd_crypto.map @@ -177,10 +177,5 @@ global: wd_ecc_ctx_num_init; wd_ecc_ctx_num_uninit; wd_ecc_get_env_param; - - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; - wd_find_msg_in_pool; local: *; }; diff --git a/libwd_dae.map b/libwd_dae.map index 660794c8b..1243c0616 100644 --- a/libwd_dae.map +++ b/libwd_dae.map @@ -12,10 +12,5 @@ global: wd_agg_get_output_async; wd_agg_rehash_sync; wd_agg_poll; - - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; - wd_find_msg_in_pool; local: *; }; From c46e341489d4b0f36225b048c4bdeb6434130849 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Tue, 12 Nov 2024 02:28:53 +0000 Subject: [PATCH 03/13] uadk_tool: use static wd_sched instead of temp var Now wd_init_sched will copy sched memory, later the &sched will be directly used without copy. So use static var instead of temp var, which will be freed. Signed-off-by: Zhangfei Gao --- uadk_tool/test/test_sec.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/uadk_tool/test/test_sec.c b/uadk_tool/test/test_sec.c index 87fc71840..2e52916ce 100644 --- a/uadk_tool/test/test_sec.c +++ b/uadk_tool/test/test_sec.c @@ -42,6 +42,7 @@ static struct wd_ctx_config g_ctx_cfg; static struct wd_sched *g_sched; +static struct wd_sched g_sched_test; static long long int g_times; static unsigned int g_thread_num; @@ -1416,7 +1417,6 @@ static __u32 sched_digest_pick_next_ctx(handle_t h_sched_ctx, static int digest_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; int ret; if (g_use_env) @@ -1443,12 +1443,12 @@ static int digest_init1(int type, int mode) g_ctx_cfg.ctxs[0].op_type = type; g_ctx_cfg.ctxs[0].ctx_mode = mode; - sched.name = SCHED_SINGLE; - sched.pick_next_ctx = sched_digest_pick_next_ctx; - sched.poll_policy = sched_single_poll_policy; - sched.sched_init = sched_digest_init; + g_sched_test.name = SCHED_SINGLE; + g_sched_test.pick_next_ctx = sched_digest_pick_next_ctx; + g_sched_test.poll_policy = sched_single_poll_policy; + g_sched_test.sched_init = sched_digest_init; /* digest init */ - ret = wd_digest_init(&g_ctx_cfg, &sched); + ret = wd_digest_init(&g_ctx_cfg, &g_sched_test); if (ret) { SEC_TST_PRT("Fail to digest ctx!\n"); goto out; @@ -2685,7 +2685,6 @@ static __u32 sched_aead_pick_next_ctx(handle_t h_sched_ctx, static int aead_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; int ret; if (g_use_env) @@ -2712,12 +2711,12 @@ static int aead_init1(int type, int mode) g_ctx_cfg.ctxs[0].op_type = type; g_ctx_cfg.ctxs[0].ctx_mode = mode; - sched.name = SCHED_SINGLE; - sched.pick_next_ctx = sched_aead_pick_next_ctx; - sched.poll_policy = sched_single_poll_policy; - sched.sched_init = sched_aead_init; + g_sched_test.name = SCHED_SINGLE; + g_sched_test.pick_next_ctx = sched_aead_pick_next_ctx; + g_sched_test.poll_policy = sched_single_poll_policy; + g_sched_test.sched_init = sched_aead_init; /* aead init*/ - ret = wd_aead_init(&g_ctx_cfg, &sched); + ret = wd_aead_init(&g_ctx_cfg, &g_sched_test); if (ret) { SEC_TST_PRT("Fail to aead ctx!\n"); goto out; From ff39c2bb04b02d332671f1b02d95c39b1ba89e95 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Tue, 12 Nov 2024 11:39:08 +0000 Subject: [PATCH 04/13] uadk: poll_ctx adds para wd_sched poll_ctx adds para wd_sched to transfer para, since different driver may use different pool. Signed-off-by: Zhangfei Gao --- include/wd_aead.h | 1 + include/wd_alg_common.h | 4 +- include/wd_cipher.h | 1 + include/wd_comp.h | 1 + include/wd_dh.h | 1 + include/wd_digest.h | 1 + include/wd_ecc.h | 1 + include/wd_rsa.h | 1 + include/wd_sched.h | 3 +- libwd_comp.map | 1 + libwd_crypto.map | 6 ++ sample/uadk_comp.c | 2 +- test/hisi_hpre_test/test_hisi_hpre.c | 6 +- test/hisi_zip_test/test_lib.c | 2 +- test/hisi_zip_test/test_lib.h | 2 +- test/wd_mempool_test.c | 2 +- uadk_tool/benchmark/hpre_uadk_benchmark.c | 6 +- uadk_tool/benchmark/sec_uadk_benchmark.c | 6 +- uadk_tool/benchmark/zip_uadk_benchmark.c | 2 +- uadk_tool/test/comp_lib.c | 3 +- uadk_tool/test/test_sec.c | 6 +- wd_aead.c | 12 ++- wd_agg.c | 7 +- wd_cipher.c | 12 ++- wd_comp.c | 17 ++-- wd_dh.c | 13 +-- wd_digest.c | 12 ++- wd_ecc.c | 13 +-- wd_rsa.c | 13 +-- wd_sched.c | 104 +++++++++------------- 30 files changed, 138 insertions(+), 123 deletions(-) diff --git a/include/wd_aead.h b/include/wd_aead.h index 01f6980f7..6c1b37f21 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -207,6 +207,7 @@ int wd_aead_get_maxauthsize(handle_t h_sess); * @count: how many respondences this poll has to get. */ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_aead_poll() Poll finished request. diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index e1c2bfb25..3b240dffe 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -175,12 +175,12 @@ struct wd_sched { __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); - int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + int (*poll_policy)(struct wd_sched *sched, __u32 expect, __u32 *count); handle_t h_sched_ctx; }; typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); -typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); +typedef int (*wd_alg_poll_ctx)(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); struct wd_datalist { void *data; diff --git a/include/wd_cipher.h b/include/wd_cipher.h index d54f7fef2..94e91eaaa 100644 --- a/include/wd_cipher.h +++ b/include/wd_cipher.h @@ -171,6 +171,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req); * @count: how many respondences this poll has to get. */ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_cipher_poll() Poll finished request. * this function will call poll_policy function which is registered to wd cipher diff --git a/include/wd_comp.h b/include/wd_comp.h index 45994ff69..5fa2b8423 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -199,6 +199,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req); * specific ctx, this function should be used. */ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); int wd_comp_poll(__u32 expt, __u32 *count); diff --git a/include/wd_dh.h b/include/wd_dh.h index afc2f7c08..b0f2dd3da 100644 --- a/include/wd_dh.h +++ b/include/wd_dh.h @@ -57,6 +57,7 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup); void wd_dh_free_sess(handle_t sess); int wd_do_dh_async(handle_t sess, struct wd_dh_req *req); int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req); +int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count); int wd_dh_poll(__u32 expt, __u32 *count); int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched); diff --git a/include/wd_digest.h b/include/wd_digest.h index f0916c362..69ffdf576 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -230,6 +230,7 @@ int wd_digest_set_key(handle_t h_sess, const __u8 *key, __u32 key_len); * @count: recv poll nums. */ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_digest_poll() - Poll operation for asynchronous operation. diff --git a/include/wd_ecc.h b/include/wd_ecc.h index 6f670e2d0..e981920c7 100644 --- a/include/wd_ecc.h +++ b/include/wd_ecc.h @@ -509,6 +509,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req); * specific ctx, this function should be used. */ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_ecc_env_init() - Init ctx and schedule resources according to wd ecc diff --git a/include/wd_rsa.h b/include/wd_rsa.h index 733d0b74f..b1092c037 100644 --- a/include/wd_rsa.h +++ b/include/wd_rsa.h @@ -202,6 +202,7 @@ __u32 wd_rsa_poll(void); * specific ctx, this function should be used. */ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_rsa_env_init() - Init ctx and schedule resources according to wd rsa diff --git a/include/wd_sched.h b/include/wd_sched.h index be541c6dd..553f63d61 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -34,7 +34,8 @@ struct sched_params { __u32 end; }; -typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); +typedef int (*user_poll_func)(struct wd_sched *sched, __u32 pos, + __u32 expect, __u32 *count); /* * wd_sched_rr_instance - Instante the schedule min region. diff --git a/libwd_comp.map b/libwd_comp.map index 336831989..dba9536d3 100644 --- a/libwd_comp.map +++ b/libwd_comp.map @@ -11,6 +11,7 @@ global: wd_do_comp_strm; wd_do_comp_async; wd_comp_poll_ctx; + wd_comp_poll_ctx_; wd_comp_poll; wd_do_comp_sync2; wd_comp_env_init; diff --git a/libwd_crypto.map b/libwd_crypto.map index 33de0bf7b..450b2bd3e 100644 --- a/libwd_crypto.map +++ b/libwd_crypto.map @@ -11,6 +11,7 @@ global: wd_do_cipher_sync; wd_do_cipher_async; wd_cipher_poll_ctx; + wd_cipher_poll_ctx_; wd_cipher_poll; wd_cipher_env_init; wd_cipher_env_uninit; @@ -35,6 +36,7 @@ global: wd_aead_get_authsize; wd_aead_get_maxauthsize; wd_aead_poll_ctx; + wd_aead_poll_ctx_; wd_aead_poll; wd_aead_env_init; wd_aead_env_uninit; @@ -55,6 +57,7 @@ global: wd_do_digest_async; wd_digest_set_key; wd_digest_poll_ctx; + wd_digest_poll_ctx_; wd_digest_poll; wd_digest_env_init; wd_digest_env_uninit; @@ -97,6 +100,7 @@ global: wd_do_rsa_sync; wd_do_rsa_async; wd_rsa_poll_ctx; + wd_rsa_poll_ctx_; wd_rsa_env_init; wd_rsa_env_uninit; wd_rsa_ctx_num_init; @@ -114,6 +118,7 @@ global: wd_do_dh_async; wd_do_dh_sync; wd_dh_poll_ctx; + wd_dh_poll_ctx_; wd_dh_poll; wd_dh_init; wd_dh_uninit; @@ -172,6 +177,7 @@ global: wd_do_ecc_sync; wd_do_ecc_async; wd_ecc_poll_ctx; + wd_ecc_poll_ctx_; wd_ecc_env_init; wd_ecc_env_uninit; wd_ecc_ctx_num_init; diff --git a/sample/uadk_comp.c b/sample/uadk_comp.c index 908c7bcb3..e3243631c 100644 --- a/sample/uadk_comp.c +++ b/sample/uadk_comp.c @@ -127,7 +127,7 @@ static struct uacce_dev_list* get_dev_list(char *alg_name) return NULL; } -static int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +static int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count) { int ret; diff --git a/test/hisi_hpre_test/test_hisi_hpre.c b/test/hisi_hpre_test/test_hisi_hpre.c index fbefef87c..1c7351432 100644 --- a/test/hisi_hpre_test/test_hisi_hpre.c +++ b/test/hisi_hpre_test/test_hisi_hpre.c @@ -1789,7 +1789,7 @@ static __u32 hpre_pick_next_ctx(handle_t sched_ctx, return last_ctx; } -int rsa_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +int rsa_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; @@ -1813,7 +1813,7 @@ int rsa_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) return 0; } -static int dh_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int dh_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; @@ -1836,7 +1836,7 @@ static int dh_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) return 0; } -static int ecc_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int ecc_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; diff --git a/test/hisi_zip_test/test_lib.c b/test/hisi_zip_test/test_lib.c index 384be4964..2ab9807e4 100644 --- a/test/hisi_zip_test/test_lib.c +++ b/test/hisi_zip_test/test_lib.c @@ -590,7 +590,7 @@ void *send_thread_func(void *arg) return NULL; } -int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count) { int ret; diff --git a/test/hisi_zip_test/test_lib.h b/test/hisi_zip_test/test_lib.h index 70d2c9d69..994ae888f 100644 --- a/test/hisi_zip_test/test_lib.h +++ b/test/hisi_zip_test/test_lib.h @@ -267,7 +267,7 @@ int hizip_verify_random_output(struct test_options *opts, void *mmap_alloc(size_t len); int mmap_free(void *addr, size_t len); -int lib_poll_func(__u32 pos, __u32 expect, __u32 *count); +int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count); typedef int (*check_output_fn)(unsigned char *buf, unsigned int size, void *opaque); /* for block interface */ diff --git a/test/wd_mempool_test.c b/test/wd_mempool_test.c index 6e28b462a..0b1e4e1a2 100644 --- a/test/wd_mempool_test.c +++ b/test/wd_mempool_test.c @@ -436,7 +436,7 @@ static __u32 sva_sched_pick_next_ctx(handle_t h_sched_ctx, return index; } -static int sva_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, +static int sva_sched_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int recv = 0; diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c index bc0687da5..0000d72d3 100644 --- a/uadk_tool/benchmark/hpre_uadk_benchmark.c +++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c @@ -486,17 +486,17 @@ static int init_hpre_ctx_config(struct acc_option *options) switch(subtype) { case RSA_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_rsa_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_rsa_poll_ctx_); break; case DH_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_dh_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_dh_poll_ctx_); break; case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_ecc_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_ecc_poll_ctx_); break; default: HPRE_TST_PRT("failed to parse alg subtype!\n"); diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index 411d0c47c..68e7928d8 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -686,13 +686,13 @@ static int init_ctx_config(struct acc_option *options) switch(subtype) { case CIPHER_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_cipher_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_cipher_poll_ctx_); break; case AEAD_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_aead_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_aead_poll_ctx_); break; case DIGEST_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_digest_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_digest_poll_ctx_); break; default: SEC_TST_PRT("failed to parse alg subtype!\n"); diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c index 2133297f7..5adf6e874 100644 --- a/uadk_tool/benchmark/zip_uadk_benchmark.c +++ b/uadk_tool/benchmark/zip_uadk_benchmark.c @@ -490,7 +490,7 @@ static int init_ctx_config(struct acc_option *options) goto free_ctxs; } - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 2, max_node, wd_comp_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 2, max_node, wd_comp_poll_ctx_); if (!g_sched) { ZIP_TST_PRT("failed to alloc sched!\n"); ret = -ENOMEM; diff --git a/uadk_tool/test/comp_lib.c b/uadk_tool/test/comp_lib.c index 014aa9895..6cb494a68 100644 --- a/uadk_tool/test/comp_lib.c +++ b/uadk_tool/test/comp_lib.c @@ -229,7 +229,8 @@ int hw_stream_compress(struct test_options *opts, return ret; } -static int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +static int lib_poll_func(struct wd_sched *sched, __u32 pos, + __u32 expect, __u32 *count) { int ret; diff --git a/uadk_tool/test/test_sec.c b/uadk_tool/test/test_sec.c index 2e52916ce..9ee3ad419 100644 --- a/uadk_tool/test/test_sec.c +++ b/uadk_tool/test/test_sec.c @@ -594,7 +594,7 @@ int get_cipher_resource(struct cipher_testvec **alg_tv, int* alg, int* mode) return 0; } -static int sched_single_poll_policy(handle_t h_sched_ctx, +static int sched_single_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { return 0; @@ -619,7 +619,7 @@ static int init_ctx_config(int type, int mode) g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, numa_max_node() + 1, - wd_cipher_poll_ctx); + wd_cipher_poll_ctx_); if (!g_sched) { printf("Fail to alloc sched!\n"); goto out; @@ -1227,7 +1227,7 @@ static void *poll_func(void *arg) int expt = g_times * g_thread_num; while (1) { - ret = g_sched->poll_policy(g_sched->h_sched_ctx, 1, &count); + ret = g_sched->poll_policy(g_sched, 1, &count); if (ret != -EAGAIN && ret < 0) { SEC_TST_PRT("poll ctx is error----------->\n"); break; diff --git a/wd_aead.c b/wd_aead.c index 8157e1989..e014b2790 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -614,7 +614,7 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, wd_aead_init_attrs.driver = wd_aead_setting.driver; wd_aead_init_attrs.ctx_params = &aead_ctx_params; wd_aead_init_attrs.alg_init = wd_aead_init_nolock; - wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx; + wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx_; ret = wd_alg_attrs_init(&wd_aead_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -833,7 +833,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return ret; } -int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_aead_setting.config; struct wd_ctx_internal *ctx; @@ -885,9 +885,13 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } +int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_aead_poll_ctx_(NULL, idx, expt, count); +} + int wd_aead_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_aead_setting.sched.h_sched_ctx; struct wd_sched *sched = &wd_aead_setting.sched; if (unlikely(!count)) { @@ -895,7 +899,7 @@ int wd_aead_poll(__u32 expt, __u32 *count) return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + return sched->poll_policy(sched, expt, count); } static const struct wd_config_variable table[] = { diff --git a/wd_agg.c b/wd_agg.c index faafafb18..6fce1274f 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -71,7 +71,7 @@ struct wd_agg_sess { static char *wd_agg_alg_name = "hashagg"; static struct wd_init_attrs wd_agg_init_attrs; -static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); static void wd_agg_close_driver(void) { @@ -1506,7 +1506,7 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) return WD_SUCCESS; } -static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_agg_setting.config; struct wd_agg_msg resp_msg, *msg; @@ -1557,7 +1557,6 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_agg_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_agg_setting.sched.h_sched_ctx; struct wd_sched *sched = &wd_agg_setting.sched; if (unlikely(!expt || !count)) { @@ -1565,5 +1564,5 @@ int wd_agg_poll(__u32 expt, __u32 *count) return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + return sched->poll_policy(sched, expt, count); } diff --git a/wd_cipher.c b/wd_cipher.c index 996f867ac..e7d55b707 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -485,7 +485,7 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p wd_cipher_init_attrs.driver = wd_cipher_setting.driver; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; - wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; + wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx_; ret = wd_alg_attrs_init(&wd_cipher_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -784,7 +784,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } -int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_cipher_setting.config; struct wd_ctx_internal *ctx; @@ -837,9 +837,13 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } +int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_cipher_poll_ctx_(NULL, idx, expt, count); +} + int wd_cipher_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_cipher_setting.sched.h_sched_ctx; struct wd_sched *sched = &wd_cipher_setting.sched; if (unlikely(!count)) { @@ -847,7 +851,7 @@ int wd_cipher_poll(__u32 expt, __u32 *count) return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + return sched->poll_policy(sched, expt, count); } static const struct wd_config_variable table[] = { diff --git a/wd_comp.c b/wd_comp.c index 4abb64d27..fc2cb5174 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -304,7 +304,7 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par wd_comp_init_attrs.driver = wd_comp_setting.driver; wd_comp_init_attrs.ctx_params = &comp_ctx_params; wd_comp_init_attrs.alg_init = wd_comp_init_nolock; - wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx; + wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx_; ret = wd_alg_attrs_init(&wd_comp_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -349,7 +349,7 @@ void wd_comp_uninit2(void) wd_alg_clear_init(&wd_comp_setting.status); } -int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_comp_setting.config; struct wd_ctx_internal *ctx; @@ -403,6 +403,11 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } +int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_comp_poll_ctx_(NULL, idx, expt, count); +} + static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) { if (setup->alg_type >= WD_COMP_ALG_MAX) { @@ -884,18 +889,14 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) int wd_comp_poll(__u32 expt, __u32 *count) { - handle_t h_sched_ctx; - struct wd_sched *sched; + struct wd_sched *sched = &wd_comp_setting.sched; if (unlikely(!count)) { WD_ERR("invalid: comp poll count is NULL!\n"); return -WD_EINVAL; } - h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; - sched = &wd_comp_setting.sched; - - return sched->poll_policy(h_sched_ctx, expt, count); + return sched->poll_policy(sched, expt, count); } static const struct wd_config_variable table[] = { diff --git a/wd_dh.c b/wd_dh.c index 2686dda66..3a99caec2 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -276,7 +276,7 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param wd_dh_init_attrs.driver = wd_dh_setting.driver; wd_dh_init_attrs.ctx_params = &dh_ctx_params; wd_dh_init_attrs.alg_init = wd_dh_common_init; - wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx; + wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx_; ret = wd_alg_attrs_init(&wd_dh_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -455,7 +455,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return ret; } -int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_dh_setting.config; struct wd_ctx_internal *ctx; @@ -509,16 +509,19 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } -int wd_dh_poll(__u32 expt, __u32 *count) +int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; + return wd_dh_poll_ctx_(NULL, idx, expt, count); +} +int wd_dh_poll(__u32 expt, __u32 *count) +{ if (unlikely(!count)) { WD_ERR("invalid: dh poll count is NULL!\n"); return -WD_EINVAL; } - return wd_dh_setting.sched.poll_policy(h_sched_ctx, expt, count); + return wd_dh_setting.sched.poll_policy(&wd_dh_setting.sched, expt, count); } int wd_dh_get_mode(handle_t sess, __u8 *alg_mode) diff --git a/wd_digest.c b/wd_digest.c index 85b722f3d..5875a24de 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -421,7 +421,7 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, wd_digest_init_attrs.driver = wd_digest_setting.driver; wd_digest_init_attrs.ctx_params = &digest_ctx_params; wd_digest_init_attrs.alg_init = wd_digest_init_nolock; - wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx; + wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx_; ret = wd_alg_attrs_init(&wd_digest_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -735,7 +735,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return ret; } -int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_digest_setting.config; struct wd_ctx_internal *ctx; @@ -789,9 +789,13 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } +int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_digest_poll_ctx_(NULL, idx, expt, count); +} + int wd_digest_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_digest_setting.sched.h_sched_ctx; struct wd_sched *sched = &wd_digest_setting.sched; if (unlikely(!count)) { @@ -799,7 +803,7 @@ int wd_digest_poll(__u32 expt, __u32 *count) return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + return sched->poll_policy(sched, expt, count); } static const struct wd_config_variable table[] = { diff --git a/wd_ecc.c b/wd_ecc.c index 5588e577b..b2a8f853a 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -341,7 +341,7 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para wd_ecc_init_attrs.driver = wd_ecc_setting.driver; wd_ecc_init_attrs.ctx_params = &ecc_ctx_params; wd_ecc_init_attrs.alg_init = wd_ecc_common_init; - wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx; + wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx_; ret = wd_alg_attrs_init(&wd_ecc_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -2300,7 +2300,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return ret; } -int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_ecc_setting.config; struct wd_ecc_msg recv_msg, *msg; @@ -2353,16 +2353,19 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } -int wd_ecc_poll(__u32 expt, __u32 *count) +int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { - handle_t h_sched_sess = wd_ecc_setting.sched.h_sched_ctx; + return wd_ecc_poll_ctx_(NULL, idx, expt, count); +} +int wd_ecc_poll(__u32 expt, __u32 *count) +{ if (unlikely(!count)) { WD_ERR("invalid: ecc poll param count is NULL!\n"); return -WD_EINVAL; } - return wd_ecc_setting.sched.poll_policy(h_sched_sess, expt, count); + return wd_ecc_setting.sched.poll_policy(&wd_ecc_setting.sched, expt, count); } static const struct wd_config_variable table[] = { diff --git a/wd_rsa.c b/wd_rsa.c index ea340c076..a25b8ffc5 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -316,7 +316,7 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para wd_rsa_init_attrs.driver = wd_rsa_setting.driver; wd_rsa_init_attrs.ctx_params = &rsa_ctx_params; wd_rsa_init_attrs.alg_init = wd_rsa_common_init; - wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx; + wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx_; ret = wd_alg_attrs_init(&wd_rsa_init_attrs); if (ret) { if (ret == -WD_ENODEV) { @@ -515,7 +515,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return ret; } -int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { struct wd_ctx_config_internal *config = &wd_rsa_setting.config; struct wd_ctx_internal *ctx; @@ -567,16 +567,19 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } -int wd_rsa_poll(__u32 expt, __u32 *count) +int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) { - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; + return wd_rsa_poll_ctx_(NULL, idx, expt, count); +} +int wd_rsa_poll(__u32 expt, __u32 *count) +{ if (unlikely(!count)) { WD_ERR("invalid: rsa poll count is NULL!\n"); return -WD_EINVAL; } - return wd_rsa_setting.sched.poll_policy(h_sched_ctx, expt, count); + return wd_rsa_setting.sched.poll_policy(&wd_rsa_setting.sched, expt, count); } int wd_rsa_kg_in_data(struct wd_rsa_kg_in *ki, char **data) diff --git a/wd_sched.c b/wd_sched.c index aa6c91ec2..28cae6ec8 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -244,9 +244,10 @@ static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, return key->async_ctxid; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, +static int session_poll_region(struct wd_sched *sched, __u32 begin, __u32 end, __u32 expect, __u32 *count) { + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; __u32 i; int ret; @@ -257,7 +258,7 @@ static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, * RR schedule, one time poll one package, * poll_num is always not more than one here. */ - ret = sched_ctx->poll_func(i, 1, &poll_num); + ret = sched_ctx->poll_func(sched, i, 1, &poll_num); if ((ret < 0) && (ret != -EAGAIN)) return ret; else if (ret == -EAGAIN) @@ -270,9 +271,10 @@ static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, return 0; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, +static int session_poll_policy_rr(struct wd_sched *sched, int numa_id, __u32 expect, __u32 *count) { + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; __u32 begin, end; __u32 i; @@ -284,7 +286,7 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, begin = region[SCHED_MODE_ASYNC][i].begin; end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); + ret = session_poll_region(sched, begin, end, expect, count); if (unlikely(ret)) return ret; } @@ -294,7 +296,7 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, /* * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. + * @sched: Schedule. * @cfg: The global resoure info. * @expect: User expect poll msg num. * @count: The actually poll num. @@ -302,11 +304,10 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, * The user must init the schedule info through wd_sched_rr_instance, the * func interval will not check the valid, becouse it will affect performance. */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int session_sched_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; struct wd_sched_info *sched_info; - __u32 loop_time = 0; __u32 last_count = 0; __u16 i; int ret; @@ -323,34 +324,27 @@ static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 * sched_info = sched_ctx->sched_info; - /* - * Try different numa's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < sched_ctx->numa_num;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + for (i = 0; i < sched_ctx->numa_num;) { + /* If current numa is not valid, find next. */ + if (!sched_info[i].valid) { + i++; + continue; + } - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + last_count = *count; + ret = session_poll_policy_rr(sched, i, expect, count); + if (unlikely(ret)) + return ret; - if (expect == *count) - return 0; + if (expect == *count) + return 0; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; - } + /* + * If no package is received, find next numa, + * otherwise, keep receiving packets at this node. + */ + if (last_count == *count) + i++; } return 0; @@ -367,11 +361,10 @@ static __u32 sched_none_pick_next_ctx(handle_t sched_ctx, return 0; } -static int sched_none_poll_policy(handle_t h_sched_ctx, +static int sched_none_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - __u32 loop_times = MAX_POLL_TIMES + expect; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; int ret; @@ -380,19 +373,12 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } - while (loop_times > 0) { - /* Default use ctx 0 */ - loop_times--; - ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) - continue; + /* Default use ctx 0 */ + ret = sched_ctx->poll_func(sched, 0, 1, &poll_num); + if (ret < 0) + return ret; - *count += poll_num; - if (*count == expect) - break; - } + *count += poll_num; return 0; } @@ -414,11 +400,10 @@ static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, return CTX_SYNC; } -static int sched_single_poll_policy(handle_t h_sched_ctx, +static int sched_single_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - __u32 loop_times = MAX_POLL_TIMES + expect; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; int ret; @@ -427,19 +412,12 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } - while (loop_times > 0) { - /* Default async mode use ctx 1 */ - loop_times--; - ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) - continue; + /* Default async mode use ctx 1 */ + ret = sched_ctx->poll_func(sched, 1, 1, &poll_num); + if (ret < 0) + return ret; - *count += poll_num; - if (*count == expect) - break; - } + *count += poll_num; return 0; } From bc628a084599b44bf2ccd752433177f92e57a889 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Wed, 6 Nov 2024 18:14:39 +0000 Subject: [PATCH 05/13] wd: add wd_find_drv wd_find_drv is used to find drv with name and alg, also takes idx to search the idx'th drv Signed-off-by: Zhangfei Gao --- include/wd_alg.h | 1 + libwd.map | 1 + wd_alg.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/include/wd_alg.h b/include/wd_alg.h index aba855d65..307ff0951 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -193,6 +193,7 @@ bool wd_drv_alg_support(const char *alg_name, */ void wd_enable_drv(struct wd_alg_driver *drv); void wd_disable_drv(struct wd_alg_driver *drv); +struct wd_alg_driver *wd_find_drv(char *drv_name, char *alg_name, int idx); struct wd_alg_list *wd_get_alg_head(void); diff --git a/libwd.map b/libwd.map index 132fd044d..3dccef4b7 100644 --- a/libwd.map +++ b/libwd.map @@ -45,6 +45,7 @@ global: wd_alg_driver_unregister; wd_request_drv; wd_release_drv; + wd_find_drv; wd_drv_alg_support; wd_enable_drv; wd_disable_drv; diff --git a/wd_alg.c b/wd_alg.c index 3d014e18e..2692fc7ce 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -395,3 +395,44 @@ void wd_release_drv(struct wd_alg_driver *drv) select_node->refcnt--; pthread_mutex_unlock(&mutex); } + +struct wd_alg_driver *wd_find_drv(char *drv_name, char *alg_name, int idx) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + struct wd_alg_driver *drv = NULL; + + if (!pnext || !alg_name) { + WD_ERR("invalid: request alg param is error!\n"); + return NULL; + } + + pthread_mutex_lock(&mutex); + + if (drv_name) { + while (pnext) { + if (!strcmp(alg_name, pnext->alg_name) && + !strcmp(drv_name, pnext->drv_name)) { + drv = pnext->drv; + break; + } + pnext = pnext->next; + } + } else { + int i = 0; + + while (pnext) { + if (!strcmp(alg_name, pnext->alg_name)) { + if (i++ == idx) { + drv = pnext->drv; + break; + } + } + pnext = pnext->next; + } + } + + pthread_mutex_unlock(&mutex); + + return drv; +} From 0e3f35f71f804e8ece52fcde55797597656193e2 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Wed, 6 Nov 2024 18:03:35 +0000 Subject: [PATCH 06/13] uadk: add adapter.c Using uadk_adapter_worker to record driver and resources. So adding multi driver is adding workers. The worker priority and mode can be set by env UADK_CONF. For export UADK_CONF=TO_PATH/uadk.conf Example: uadk.conf mode=1 driver_name=isa_ce_sm3 driver_name=hisi_sec2 If env UADK_CONF exist, only parse UADK_CONF and add to adapter->workers[] one by one. So the first driver has higher priority Otherwise, parse all drivers and choose the top two priorities drivers to adapter->workers[] Signed-off-by: Zhangfei Gao --- Makefile.am | 3 +- adapter.c | 174 ++++++++++++++++++++++++++++++++++++++++++++++ include/adapter.h | 50 +++++++++++++ libwd.map | 4 ++ 4 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 adapter.c create mode 100644 include/adapter.h diff --git a/Makefile.am b/Makefile.am index c0bdc8653..612f45d8d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -65,7 +65,8 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ - v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h + v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h \ + adapter.c libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h diff --git a/adapter.c b/adapter.c new file mode 100644 index 000000000..a1cbfbae7 --- /dev/null +++ b/adapter.c @@ -0,0 +1,174 @@ + +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2024-2025 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2024-2025 Linaro ltd. + */ + +#include "adapter.h" + +#define CONFIG_FILE_ENV "UADK_CONF" +#define DRIVER_NAME_KEY "driver_name" +#define MODE_KEY "mode" +#define WORKERS_NB 8 + +static int read_value_int(char *conf, const char *key) +{ + FILE *fp = fopen(conf, "r"); + char line[1024]; + int ret = 0; + + if (fp == NULL) + return 0; + + while (fgets(line, sizeof(line), fp) != NULL) { + char *key_value = strtok(line, "="); + if (key_value && strcmp(key_value, key) == 0) { + char *value = strtok(NULL, "\n"); + + if (value) { + ret = atoi(value); + goto exit; + } + } + } +exit: + fclose(fp); + return ret; +} + +static void read_config_entries(char *conf, struct uadk_adapter *adapter, char *alg_name) +{ + struct uadk_adapter_worker *worker; + FILE *fp = fopen(conf, "r"); + struct wd_alg_driver *drv; + char *drv_name = NULL; + char line[1024]; + int i = 0; + + if (fp == NULL) + return; + + while (fgets(line, sizeof(line), fp) != NULL) { + char *key_value = strtok(line, "="); + + if (key_value && strcmp(key_value, DRIVER_NAME_KEY) == 0) + drv_name = strdup(strtok(NULL, "\n")); + + if ((drv_name != NULL) && (alg_name != NULL)) { + drv = wd_find_drv(drv_name, alg_name, 0); + if (!drv) + continue; + + worker = &adapter->workers[i]; + worker->driver = drv; + worker->idx = i; + adapter->workers_nb++; + if (drv_name) { + free(drv_name); + drv_name = NULL; + } + + if (++i >= UADK_MAX_NB_WORKERS) + break; + } + } + + if (drv_name) + free(drv_name); + + fclose(fp); +} + +int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) +{ + char *conf = getenv(CONFIG_FILE_ENV); + struct uadk_adapter_worker workers[WORKERS_NB]; + struct uadk_adapter_worker worker; + struct wd_alg_driver *drv; + int idx = 0, i, j; + + adapter->looptime = UADK_WORKER_LOOPTIME; + + if (conf != NULL) { + /* if env UADK_CONF exist, parse config first */ + adapter->mode = read_value_int(conf, MODE_KEY); + read_config_entries(conf, adapter, alg); + if (adapter->workers_nb != 0) + return 0; + } + + /* Then parse all system drivers to workers */ + do { + drv = wd_find_drv(NULL, alg, idx); + if (!drv) + break; + + workers[idx++].driver = drv; + + if (idx >= WORKERS_NB) + break; + } while (drv); + + /* Sorted as priority */ + for (i = 0; i < idx; i++) { + for (j = i; j < idx; j++) { + if (workers[i].driver->priority < + workers[j].driver->priority) { + worker.driver = workers[i].driver; + workers[i].driver = workers[j].driver; + workers[j].driver = worker.driver; + } + } + } + + for (i = 0; i < idx; i++) { + adapter->workers[i].driver = workers[i].driver; + adapter->workers[i].idx = i; + adapter->workers_nb++; + + if (adapter->workers_nb >= UADK_MAX_NB_WORKERS) + break; + } + + return (adapter->workers_nb == 0); +} + +struct uadk_adapter_worker *uadk_adapter_choose_worker( + struct uadk_adapter *adapter, + enum alg_task_type type) +{ + struct uadk_adapter_worker *worker; + + /* use worker[0] for simplicity now */ + worker = &adapter->workers[0]; + worker->valid = true; + + return worker; +} + +struct uadk_adapter_worker *uadk_adapter_switch_worker( + struct uadk_adapter *adapter, + struct uadk_adapter_worker *worker, + int para) +{ + struct uadk_adapter_worker *new_worker; + int idx = worker->idx; + + if (adapter->workers_nb == 1) + return worker; + + if (para) { + idx += 1; + } else { + if (idx == 0) + idx = adapter->workers_nb - 1; + else + idx -= 1; + } + + new_worker = &adapter->workers[idx]; + new_worker->valid = true; + + return new_worker; +} diff --git a/include/adapter.h b/include/adapter.h new file mode 100644 index 000000000..b206bf750 --- /dev/null +++ b/include/adapter.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2024-2025 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2024-2025 Linaro ltd. + */ + +#ifndef __ADAPTER_H +#define __ADAPTER_H + +#include "wd_alg.h" +#include "wd_util.h" + +#define UADK_MAX_NB_WORKERS (2) +#define UADK_WORKER_LOOPTIME (10) + +enum uadk_adapter_mode { + UADK_ADAPT_MODE_PRIMARY, + UADK_ADAPT_MODE_ROUNDROBIN, +}; + +struct uadk_adapter_worker { + struct wd_alg_driver *driver; + struct wd_ctx_config *ctx_config; + struct wd_sched *sched; + struct wd_ctx_config_internal config; + struct wd_async_msg_pool pool; + bool valid; + int idx; +}; + +struct uadk_adapter { + unsigned int workers_nb; + unsigned int looptime; + enum uadk_adapter_mode mode; + struct uadk_adapter_worker workers[UADK_MAX_NB_WORKERS]; +}; + +int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg); + +struct uadk_adapter_worker *uadk_adapter_choose_worker( + struct uadk_adapter *adapter, + enum alg_task_type type +); + +struct uadk_adapter_worker *uadk_adapter_switch_worker( + struct uadk_adapter *adapter, + struct uadk_adapter_worker *worker, + int para +); +#endif diff --git a/libwd.map b/libwd.map index 3dccef4b7..c6e1dbdeb 100644 --- a/libwd.map +++ b/libwd.map @@ -55,5 +55,9 @@ global: wd_sched_rr_alloc; wd_sched_rr_release; wd_find_msg_in_pool; + + uadk_adapter_add_workers; + uadk_adapter_choose_worker; + uadk_adapter_switch_worker; local: *; }; From 744cf2454172a682e2bf3b85f30c3b147d9fc46a Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Mon, 18 Nov 2024 02:58:35 +0000 Subject: [PATCH 07/13] uadk: make hw sw co-work Using uadk_adapter_worker to record driver and resources. so adding multi driver is adding workers. To make sure different worker can work. Signed-off-by: Zhangfei Gao --- include/wd_alg_common.h | 3 +- include/wd_util.h | 21 +-- wd_aead.c | 278 ++++++++++++++++++++++---------------- wd_agg.c | 270 ++++++++++++++++++++----------------- wd_cipher.c | 281 ++++++++++++++++++++++---------------- wd_comp.c | 273 +++++++++++++++++++++---------------- wd_dh.c | 284 ++++++++++++++++++++++---------------- wd_digest.c | 286 +++++++++++++++++++++++---------------- wd_ecc.c | 292 +++++++++++++++++++++++----------------- wd_rsa.c | 284 +++++++++++++++++++++----------------- wd_util.c | 75 ++++------- 11 files changed, 1328 insertions(+), 1019 deletions(-) diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index 3b240dffe..00370535c 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -171,6 +171,7 @@ struct wd_ctx_config_internal { struct wd_sched { const char *name; int sched_policy; + struct uadk_adapter_worker *worker; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, @@ -179,7 +180,7 @@ struct wd_sched { handle_t h_sched_ctx; }; -typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); +typedef int (*wd_alg_init)(struct uadk_adapter_worker *worker, struct wd_sched *sched); typedef int (*wd_alg_poll_ctx)(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); struct wd_datalist { diff --git a/include/wd_util.h b/include/wd_util.h index dc1e41fa4..23961c25e 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -143,21 +143,6 @@ struct wd_init_attrs { int wd_init_ctx_config(struct wd_ctx_config_internal *in, struct wd_ctx_config *cfg); -/* - * wd_init_sched() - Init internal scheduler configuration. - * @in: Scheduler configuration in global setting. - * @from: Scheduler configuration input by user. - * - * Return 0 if successful or less than 0 otherwise. - */ -int wd_init_sched(struct wd_sched *in, struct wd_sched *from); - -/* - * wd_clear_sched() - Clear internal scheduler configuration. - * @in: Scheduler configuration in global setting. - */ -void wd_clear_sched(struct wd_sched *in); - /* * wd_clear_ctx_config() - Clear internal ctx configuration. * @in: ctx configuration in global setting. @@ -461,12 +446,14 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); /** * wd_alg_attrs_init() - Request the ctxs and initialize the sched_domain * with the given devices list, ctxs number and numa mask. + * @worker: uadk_adapter_worker. * @attrs: the algorithm initialization parameters. * * Return device if succeed and other error number if fail. */ -int wd_alg_attrs_init(struct wd_init_attrs *attrs); -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); +int wd_alg_attrs_init(struct uadk_adapter_worker *worker, + struct wd_init_attrs *attrs); +void wd_alg_attrs_uninit(struct uadk_adapter_worker *worker); /** * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain diff --git a/wd_aead.c b/wd_aead.c index e014b2790..6da6456e8 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -9,6 +9,7 @@ #include #include "include/drv/wd_aead_drv.h" #include "wd_aead.h" +#include "adapter.h" #define WD_AEAD_CCM_GCM_MIN 4U #define WD_AEAD_CCM_GCM_MAX 16 @@ -30,12 +31,9 @@ static char *wd_aead_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] = struct wd_aead_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_alg_driver *driver; - struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_aead_setting; struct wd_aead_sess { @@ -52,11 +50,14 @@ struct wd_aead_sess { __u16 akey_bytes; __u16 auth_bytes; void *priv; - void *sched_key; + void **sched_key; /* Stored the counter for gcm stream mode */ __u8 iv[MAX_IV_SIZE]; /* Total of data for stream mode */ __u64 long_data_len; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_env_config wd_aead_env_config; @@ -71,20 +72,16 @@ static void wd_aead_close_driver(int init_type) } if (wd_aead_setting.dlhandle) { - wd_release_drv(wd_aead_setting.driver); dlclose(wd_aead_setting.dlhandle); wd_aead_setting.dlhandle = NULL; } #else - wd_release_drv(wd_aead_setting.driver); hisi_sec2_remove(); #endif } static int wd_aead_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "gcm(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -118,15 +115,6 @@ static int wd_aead_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_aead_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_aead_setting.driver = driver; - return WD_SUCCESS; } @@ -307,7 +295,9 @@ int wd_aead_get_maxauthsize(handle_t h_sess) handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) { struct wd_aead_sess *sess = NULL; - bool ret; + struct uadk_adapter_worker *worker; + int nb = wd_aead_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("failed to check session input parameter!\n"); @@ -320,6 +310,10 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) return (handle_t)0; } + worker = sess->worker = &wd_aead_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + sess = malloc(sizeof(struct wd_aead_sess)); if (!sess) { WD_ERR("failed to alloc session memory!\n"); @@ -332,24 +326,31 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) sess->cmode = setup->cmode; sess->dalg = setup->dalg; sess->dmode = setup->dmode; - ret = wd_drv_alg_support(sess->alg_name, wd_aead_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_aead_setting.sched.sched_init( - wd_aead_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -366,8 +367,11 @@ void wd_aead_free_sess(handle_t h_sess) wd_memset_zero(sess->ckey, sess->ckey_bytes); wd_memset_zero(sess->akey, sess->akey_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -440,51 +444,49 @@ static void wd_aead_clear_status(void) wd_alg_clear_init(&wd_aead_setting.status); } -static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_aead_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_AEAD_EPOLL_EN", - &wd_aead_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_aead_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret) return ret; - ret = wd_init_sched(&wd_aead_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* init async request pool */ - ret = wd_init_async_request_pool(&wd_aead_setting.pool, - config, WD_POOL_MAX_ENTRIES, - sizeof(struct wd_aead_msg)); + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, + sizeof(struct wd_aead_msg)); if (ret < 0) - goto out_clear_sched; - - wd_aead_setting.config.pool = &wd_aead_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_aead_setting.config, - wd_aead_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_aead_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_aead_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_aead_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "gcm(aes)"; int ret; pthread_atfork(NULL, NULL, wd_aead_clear_status); @@ -497,11 +499,24 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_aead_setting.adapter = adapter; + ret = wd_aead_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_aead_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_aead_init_nolock(worker, sched); if (ret) goto out_close_driver; @@ -512,22 +527,26 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_aead_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_aead_setting.status); return ret; } static int wd_aead_uninit_nolock(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_aead_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_aead_setting.pool); - wd_clear_sched(&wd_aead_setting.sched); - wd_alg_uninit_driver(&wd_aead_setting.config, - wd_aead_setting.driver); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } @@ -540,6 +559,7 @@ void wd_aead_uninit(void) if (ret) return; + free(wd_aead_setting.adapter); wd_aead_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_aead_setting.status); } @@ -563,7 +583,10 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, { struct wd_ctx_nums aead_ctx_num[WD_DIGEST_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params aead_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_aead_clear_status); @@ -582,77 +605,70 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_aead_setting.adapter = adapter; + state = wd_aead_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_aead_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_aead_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_aead_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; aead_ctx_params.ctx_set_num = aead_ctx_num; ret = wd_ctx_param_init(&aead_ctx_params, ctx_params, - wd_aead_setting.driver, WD_AEAD_TYPE, + worker->driver, WD_AEAD_TYPE, WD_DIGEST_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_aead_init_attrs.alg = alg; - wd_aead_init_attrs.sched_type = sched_type; - wd_aead_init_attrs.driver = wd_aead_setting.driver; wd_aead_init_attrs.ctx_params = &aead_ctx_params; wd_aead_init_attrs.alg_init = wd_aead_init_nolock; wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx_; - ret = wd_alg_attrs_init(&wd_aead_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_aead_init_attrs); + wd_ctx_param_uninit(&aead_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); - wd_ctx_param_uninit(&aead_ctx_params); - continue; - } WD_ERR("failed to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlopen; } } + wd_alg_set_init(&wd_aead_setting.status); - wd_ctx_param_uninit(&aead_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&aead_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_aead_setting.driver); out_dlopen: wd_aead_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_aead_setting.status); return ret; } void wd_aead_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_aead_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_aead_init_attrs); - wd_alg_drv_unbind(wd_aead_setting.driver); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_aead_setting.adapter); wd_aead_close_driver(WD_TYPE_V2); wd_aead_setting.dlh_list = NULL; wd_alg_clear_init(&wd_aead_setting.status); @@ -727,18 +743,18 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, fill_stream_msg(msg, req, sess); } -static int send_recv_sync(struct wd_ctx_internal *ctx, +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, struct wd_aead_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_aead_setting.driver->send; - msg_handle.recv = wd_aead_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_aead_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_aead_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -746,8 +762,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; struct wd_aead_sess *sess = (struct wd_aead_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg msg; __u32 idx; @@ -757,20 +773,24 @@ int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) if (unlikely(ret)) return -WD_EINVAL; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_aead_msg)); fill_request_msg(&msg, req, sess); req->state = 0; - idx = wd_aead_setting.sched.pick_next_ctx( - wd_aead_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; - ret = send_recv_sync(ctx, &msg); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; + ret = send_recv_sync(worker, ctx, &msg); req->state = msg.result; return ret; @@ -778,8 +798,8 @@ int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; struct wd_aead_sess *sess = (struct wd_aead_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg *msg; int msg_id, ret; @@ -794,16 +814,20 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return -WD_EINVAL; } - idx = wd_aead_setting.sched.pick_next_ctx( - wd_aead_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_aead_setting.pool, + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); @@ -813,7 +837,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_aead_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -821,7 +845,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_aead_env_config, idx); if (ret) goto fail_with_msg; @@ -829,13 +853,13 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_aead_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg resp_msg, *msg; struct wd_aead_req *req; @@ -848,16 +872,22 @@ int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_aead_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_aead_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -866,7 +896,7 @@ int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun } recv_count++; - msg = wd_find_msg_in_pool(&wd_aead_setting.pool, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -877,8 +907,7 @@ int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun msg->req.state = resp_msg.result; req = &msg->req; req->cb(req, req->cb_param); - wd_put_msg_to_pool(&wd_aead_setting.pool, - idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); @@ -892,14 +921,33 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_aead_poll(__u32 expt, __u32 *count) { - struct wd_sched *sched = &wd_aead_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: aead poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(sched, expt, count); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_agg.c b/wd_agg.c index 6fce1274f..c90dd530a 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -8,6 +8,7 @@ #include #include #include "include/drv/wd_agg_drv.h" +#include "adapter.h" #include "wd_agg.h" #define DECIMAL_PRECISION_OFFSET 8 @@ -29,13 +30,10 @@ enum wd_agg_sess_state { struct wd_agg_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *priv; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_agg_setting; struct wd_agg_sess_key_conf { @@ -59,7 +57,7 @@ struct wd_agg_sess { wd_dev_mask_t *dev_mask; struct wd_alg_agg *drv; void *priv; - void *sched_key; + void **sched_key; enum wd_agg_sess_state state; struct wd_agg_ops ops; struct wd_agg_sess_key_conf key_conf; @@ -67,6 +65,9 @@ struct wd_agg_sess { struct wd_dae_charset charset_info; struct wd_dae_hash_table hash_table; struct wd_dae_hash_table rehash_table; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static char *wd_agg_alg_name = "hashagg"; @@ -78,7 +79,6 @@ static void wd_agg_close_driver(void) #ifndef WD_STATIC_DRV wd_dlclose_drv(wd_agg_setting.dlh_list); #else - wd_release_drv(wd_agg_setting.driver); hisi_dae_remove(); #endif } @@ -384,9 +384,11 @@ static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_se handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { + struct uadk_adapter_worker *worker; + int nb = wd_agg_setting.adapter->workers_nb; __u32 out_agg_cols_num = 0; struct wd_agg_sess *sess; - int ret; + int ret, i; ret = wd_agg_check_sess_params(setup, &out_agg_cols_num); if (ret) @@ -400,22 +402,30 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) memset(sess, 0, sizeof(struct wd_agg_sess)); sess->agg_conf.out_cols_num = out_agg_cols_num; + worker = sess->worker = &wd_agg_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + sess->alg_name = wd_agg_alg_name; - ret = wd_drv_alg_support(sess->alg_name, wd_agg_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support agg algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_agg_setting.sched.sched_init( - wd_agg_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init agg session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } - ret = wd_agg_setting.driver->get_extend_ops(&sess->ops); + ret = worker->driver->get_extend_ops(&sess->ops); if (ret) { WD_ERR("failed to get agg extend ops!\n"); goto err_sess; @@ -437,8 +447,11 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) if (sess->ops.sess_uninit) sess->ops.sess_uninit(sess->priv); err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -458,8 +471,11 @@ void wd_agg_free_sess(handle_t h_sess) if (sess->ops.sess_uninit) sess->ops.sess_uninit(sess->priv); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -570,69 +586,50 @@ static void wd_agg_clear_status(void) wd_alg_clear_init(&wd_agg_setting.status); } -static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_agg_alg_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_AGG_EPOLL_EN", &wd_agg_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_AGG_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_agg_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_agg_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* Allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_agg_setting.pool, config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_agg_msg)); if (ret < 0) - goto out_clear_sched; - - wd_agg_setting.config.pool = &wd_agg_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_agg_setting.config, wd_agg_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_agg_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_agg_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_agg_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } -static int wd_agg_alg_uninit(void) -{ - void *priv = wd_agg_setting.priv; - - if (!priv) - return -WD_EINVAL; - - /* Uninit async request pool */ - wd_uninit_async_request_pool(&wd_agg_setting.pool); - - /* Unset config, sched, driver */ - wd_clear_sched(&wd_agg_setting.sched); - - wd_alg_uninit_driver(&wd_agg_setting.config, wd_agg_setting.driver); - - return WD_SUCCESS; -} - int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) { struct wd_ctx_params agg_ctx_params = {0}; struct wd_ctx_nums agg_ctx_num = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int ret = -WD_EINVAL; - int state; + int state, i; bool flag; pthread_atfork(NULL, NULL, wd_agg_clear_status); @@ -653,60 +650,46 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_agg_setting.adapter = adapter; + state = wd_agg_open_driver(); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_agg_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_agg_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_agg_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; agg_ctx_params.ctx_set_num = &agg_ctx_num; - ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, wd_agg_setting.driver, + ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, worker->driver, WD_AGG_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_agg_init_attrs.alg = alg; - wd_agg_init_attrs.sched_type = sched_type; - wd_agg_init_attrs.driver = wd_agg_setting.driver; wd_agg_init_attrs.ctx_params = &agg_ctx_params; wd_agg_init_attrs.alg_init = wd_agg_alg_init; wd_agg_init_attrs.alg_poll_ctx = wd_agg_poll_ctx; - ret = wd_alg_attrs_init(&wd_agg_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_agg_init_attrs); + wd_ctx_param_uninit(&agg_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); - wd_ctx_param_uninit(&agg_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_agg_setting.status); - wd_ctx_param_uninit(&agg_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&agg_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_agg_setting.driver); out_dlopen: wd_agg_close_driver(); out_uninit: @@ -716,14 +699,15 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params void wd_agg_uninit(void) { - int ret; + struct uadk_adapter_worker *worker; - ret = wd_agg_alg_uninit(); - if (ret) - return; + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } - wd_alg_attrs_uninit(&wd_agg_init_attrs); - wd_alg_drv_unbind(wd_agg_setting.driver); wd_agg_close_driver(); wd_agg_setting.dlh_list = NULL; wd_alg_clear_init(&wd_agg_setting.status); @@ -1078,30 +1062,30 @@ static int wd_agg_check_rehash_params(struct wd_agg_sess *sess, struct wd_agg_re return ret; } -static int wd_agg_sync_job(struct wd_agg_sess *sess, struct wd_agg_req *req, - struct wd_agg_msg *msg) +static int wd_agg_sync_job(struct uadk_adapter_worker *worker, struct wd_agg_sess *sess, + struct wd_agg_req *req, struct wd_agg_msg *msg) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; __u32 idx; int ret; - idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx(worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], + CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - msg_handle.send = wd_agg_setting.driver->send; - msg_handle.recv = wd_agg_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_agg_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -1146,6 +1130,7 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_INIT; + struct uadk_adapter_worker *worker; struct wd_agg_msg msg; int ret; @@ -1159,11 +1144,15 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_input(&msg, req, sess, false); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) { if (expected == WD_AGG_SESS_INIT) __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); @@ -1179,20 +1168,26 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bool is_input) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; __u32 idx; int msg_id, ret; struct wd_agg_msg *msg; - idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx(worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_agg_setting.pool, idx, (void **)&msg); + ctx = worker->config.ctxs + idx; + + msg_id = wd_get_msg_from_pool(&worker->pool, idx, + (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get agg msg from pool!\n"); return msg_id; @@ -1203,7 +1198,7 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo else fill_request_msg_output(msg, req, sess, false); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_agg_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd agg async send err!\n"); @@ -1211,12 +1206,12 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_agg_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } @@ -1265,6 +1260,7 @@ int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_INPUT; + struct uadk_adapter_worker *worker; struct wd_agg_msg msg; int ret; @@ -1278,11 +1274,15 @@ int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_output(&msg, req, sess, false); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) { if (expected == WD_AGG_SESS_INPUT) __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); @@ -1407,7 +1407,9 @@ static int wd_agg_set_col_size(struct wd_agg_sess *sess, struct wd_agg_req *req, return WD_SUCCESS; } -static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req *req) +static int wd_agg_rehash_sync_inner(struct uadk_adapter_worker *worker, + struct wd_agg_sess *sess, + struct wd_agg_req *req) { struct wd_agg_msg msg = {0}; bool output_done; @@ -1416,7 +1418,7 @@ static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req fill_request_msg_output(&msg, req, sess, true); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) return ret; @@ -1441,7 +1443,7 @@ static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_input(&msg, req, sess, true); - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) return ret; @@ -1473,6 +1475,7 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_RESET; + struct uadk_adapter_worker *worker; struct wd_agg_req src_req; __u64 cnt = 0; __u64 max_cnt; @@ -1488,10 +1491,14 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memcpy(&src_req, req, sizeof(struct wd_agg_req)); max_cnt = MAX_HASH_TABLE_ROW_NUM / req->out_row_count; while (cnt < max_cnt) { - ret = wd_agg_rehash_sync_inner(sess, &src_req); + ret = wd_agg_rehash_sync_inner(worker, sess, &src_req); if (ret) { __atomic_store_n(&sess->state, WD_AGG_SESS_RESET, __ATOMIC_RELEASE); WD_ERR("failed to do agg rehash task!\n"); @@ -1508,7 +1515,7 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct uadk_adapter_worker *worker = sched->worker; struct wd_agg_msg resp_msg, *msg; struct wd_ctx_internal *ctx; struct wd_agg_req *req; @@ -1518,14 +1525,14 @@ static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_agg_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret < 0)) { @@ -1533,7 +1540,7 @@ static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 return ret; } recv_count++; - msg = wd_find_msg_in_pool(&wd_agg_setting.pool, idx, resp_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (unlikely(!msg)) { WD_ERR("failed to get agg msg from pool!\n"); return -WD_EINVAL; @@ -1548,7 +1555,7 @@ static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 req->cb(req, req->cb_param); /* Free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_agg_setting.pool, idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); @@ -1557,12 +1564,31 @@ static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 int wd_agg_poll(__u32 expt, __u32 *count) { - struct wd_sched *sched = &wd_agg_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!expt || !count)) { WD_ERR("invalid: agg poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(sched, expt, count); + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } diff --git a/wd_cipher.c b/wd_cipher.c index e7d55b707..7d7525e2d 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -10,6 +10,7 @@ #include #include "include/drv/wd_cipher_drv.h" #include "wd_cipher.h" +#include "adapter.h" #define XTS_MODE_KEY_SHIFT 1 #define XTS_MODE_KEY_LEN_MASK 0x1 @@ -49,12 +50,9 @@ static char *wd_cipher_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] struct wd_cipher_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_cipher_setting; struct wd_cipher_sess { @@ -66,7 +64,10 @@ struct wd_cipher_sess { void *priv; unsigned char key[MAX_CIPHER_KEY_SIZE]; __u32 key_bytes; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_env_config wd_cipher_env_config; @@ -81,20 +82,16 @@ static void wd_cipher_close_driver(int init_type) } if (wd_cipher_setting.dlhandle) { - wd_release_drv(wd_cipher_setting.driver); dlclose(wd_cipher_setting.dlhandle); wd_cipher_setting.dlhandle = NULL; } #else - wd_release_drv(wd_cipher_setting.driver); hisi_sec2_remove(); #endif } static int wd_cipher_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "cbc(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -128,14 +125,6 @@ static int wd_cipher_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_cipher_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_cipher_setting.driver = driver; return WD_SUCCESS; } @@ -252,8 +241,10 @@ int wd_cipher_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_cipher_sess *sess = NULL; - bool ret; + int nb = wd_cipher_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("invalid: cipher input setup is NULL!\n"); @@ -273,8 +264,12 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) goto err_sess; } + worker = sess->worker = &wd_cipher_setting.adapter->workers[0]; + worker->valid = true; + + sess->worker_looptime = 0; sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; - ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; @@ -282,19 +277,26 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) sess->alg = setup->alg; sess->mode = setup->mode; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_cipher_setting.sched.sched_init( - wd_cipher_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -310,8 +312,11 @@ void wd_cipher_free_sess(handle_t h_sess) wd_memset_zero(sess->key, sess->key_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -320,51 +325,47 @@ static void wd_cipher_clear_status(void) wd_alg_clear_init(&wd_cipher_setting.status); } -static int wd_cipher_common_init(struct wd_ctx_config *config, +static int wd_cipher_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_CIPHER_EPOLL_EN", - &wd_cipher_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_cipher_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_cipher_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_cipher_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_cipher_msg)); if (ret < 0) - goto out_clear_sched; - - wd_cipher_setting.config.pool = &wd_cipher_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_cipher_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_cipher_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_cipher_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_cipher_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_cipher_setting.status, &status); @@ -372,19 +373,20 @@ static int wd_cipher_common_uninit(void) return -WD_EINVAL; /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_cipher_setting.pool); - - /* unset config, sched, driver */ - wd_clear_sched(&wd_cipher_setting.sched); - - wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "cbc(aes)"; int ret; pthread_atfork(NULL, NULL, wd_cipher_clear_status); @@ -397,11 +399,24 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_cipher_setting.adapter = adapter; + ret = wd_cipher_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_cipher_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_cipher_common_init(worker, sched); if (ret) goto out_close_driver; @@ -412,6 +427,7 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_cipher_setting.status); return ret; } @@ -424,6 +440,7 @@ void wd_cipher_uninit(void) if (ret) return; + free(wd_cipher_setting.adapter); wd_cipher_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_cipher_setting.status); } @@ -432,8 +449,11 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_cipher_clear_status); @@ -453,78 +473,69 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_cipher_setting.adapter = adapter; + state = wd_cipher_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_driver; - /* Get alg driver and dev name */ - wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_cipher_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; cipher_ctx_params.ctx_set_num = cipher_ctx_num; - ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - wd_cipher_setting.driver, + ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, worker->driver, WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); - continue; - } + WD_ERR("fail to init ctx param\n"); goto out_driver; } wd_cipher_init_attrs.alg = alg; - wd_cipher_init_attrs.sched_type = sched_type; - wd_cipher_init_attrs.driver = wd_cipher_setting.driver; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx_; - ret = wd_alg_attrs_init(&wd_cipher_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_cipher_init_attrs); + wd_ctx_param_uninit(&cipher_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); - wd_ctx_param_uninit(&cipher_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_driver; } } wd_alg_set_init(&wd_cipher_setting.status); - wd_ctx_param_uninit(&cipher_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&cipher_ctx_params); out_driver: - wd_alg_drv_unbind(wd_cipher_setting.driver); -out_dlopen: wd_cipher_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_cipher_setting.status); return ret; } void wd_cipher_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_cipher_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_cipher_init_attrs); - wd_alg_drv_unbind(wd_cipher_setting.driver); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_cipher_setting.adapter); wd_cipher_close_driver(WD_TYPE_V2); wd_cipher_setting.dlh_list = NULL; wd_alg_clear_init(&wd_cipher_setting.status); @@ -678,27 +689,27 @@ static int wd_cipher_check_params(handle_t h_sess, return cipher_iv_len_check(req, sess); } -static int send_recv_sync(struct wd_ctx_internal *ctx, +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, struct wd_cipher_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_cipher_setting.driver->send; - msg_handle.recv = wd_cipher_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; - wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, worker->driver->calc_type); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); + wd_ctx_spin_unlock(ctx, worker->driver->calc_type); return ret; } int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg msg; __u32 idx; @@ -710,21 +721,25 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) return ret; } + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_cipher_msg)); fill_request_msg(&msg, req, sess); req->state = 0; - idx = wd_cipher_setting.sched.pick_next_ctx( - wd_cipher_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - ret = send_recv_sync(ctx, &msg); + ret = send_recv_sync(worker, ctx, &msg); req->state = msg.result; return ret; @@ -732,8 +747,8 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg *msg; int msg_id, ret; @@ -745,17 +760,20 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } - idx = wd_cipher_setting.sched.pick_next_ctx( - wd_cipher_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, - (void **)&msg); + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); return msg_id; @@ -764,7 +782,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -772,7 +790,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_cipher_env_config, idx); if (ret) goto fail_with_msg; @@ -780,13 +798,13 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_cipher_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg resp_msg, *msg; struct wd_cipher_req *req; @@ -799,16 +817,22 @@ int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_cipher_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { @@ -816,7 +840,7 @@ int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co return ret; } recv_count++; - msg = wd_find_msg_in_pool(&wd_cipher_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -829,8 +853,7 @@ int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co req->cb(req, req->cb_param); /* free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_cipher_setting.pool, idx, - resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); @@ -844,14 +867,34 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_cipher_poll(__u32 expt, __u32 *count) { - struct wd_sched *sched = &wd_cipher_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: cipher poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(sched, expt, count); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_comp.c b/wd_comp.c index fc2cb5174..9d4de40a4 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -14,6 +14,7 @@ #include "drv/wd_comp_drv.h" #include "wd_comp.h" +#include "adapter.h" #define HW_CTX_SIZE (64 * 1024) #define STREAM_CHUNK (128 * 1024) @@ -38,17 +39,17 @@ struct wd_comp_sess { __u32 isize; __u32 checksum; __u8 *ctx_buf; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_comp_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_comp_setting; struct wd_env_config wd_comp_env_config; @@ -63,20 +64,16 @@ static void wd_comp_close_driver(int init_type) } if (wd_comp_setting.dlhandle) { - wd_release_drv(wd_comp_setting.driver); dlclose(wd_comp_setting.dlhandle); wd_comp_setting.dlhandle = NULL; } #else - wd_release_drv(wd_comp_setting.driver); hisi_zip_remove(); #endif } static int wd_comp_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "zlib"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -110,15 +107,6 @@ static int wd_comp_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_comp_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_comp_setting.driver = driver; - return WD_SUCCESS; } @@ -142,69 +130,66 @@ static bool wd_comp_alg_check(const char *alg_name) return false; } -static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_comp_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", - &wd_comp_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_comp_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_comp_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_comp_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_comp_msg)); if (ret < 0) - goto out_clear_sched; - - wd_comp_setting.config.pool = &wd_comp_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_comp_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_comp_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_comp_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_comp_uninit_nolock(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_comp_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* Uninit async request pool */ - wd_uninit_async_request_pool(&wd_comp_setting.pool); - - /* Unset config, sched, driver */ - wd_clear_sched(&wd_comp_setting.sched); + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; - wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "zlib"; int ret; pthread_atfork(NULL, NULL, wd_comp_clear_status); @@ -217,11 +202,24 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_comp_setting.adapter = adapter; + ret = wd_comp_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_comp_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_clear_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_comp_init_nolock(worker, sched); if (ret) goto out_clear_driver; @@ -232,6 +230,7 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) out_clear_driver: wd_comp_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_comp_setting.status); return ret; } @@ -244,6 +243,7 @@ void wd_comp_uninit(void) if (ret) return; + free(wd_comp_setting.adapter); wd_comp_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_comp_setting.status); } @@ -252,8 +252,11 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par { struct wd_ctx_nums comp_ctx_num[WD_DIR_MAX] = {0}; struct wd_ctx_params comp_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_comp_clear_status); @@ -273,77 +276,70 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_comp_setting.adapter = adapter; + state = wd_comp_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlclose; - /* Get alg driver and dev name */ - wd_comp_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_comp_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlclose; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; comp_ctx_params.ctx_set_num = comp_ctx_num; ret = wd_ctx_param_init(&comp_ctx_params, ctx_params, - wd_comp_setting.driver, WD_COMP_TYPE, WD_DIR_MAX); + worker->driver, + WD_COMP_TYPE, WD_DIR_MAX); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); - continue; - } - goto out_unbind_drv; + WD_ERR("fail to init ctx param\n"); + goto out_dlclose; } wd_comp_init_attrs.alg = alg; - wd_comp_init_attrs.sched_type = sched_type; - wd_comp_init_attrs.driver = wd_comp_setting.driver; wd_comp_init_attrs.ctx_params = &comp_ctx_params; wd_comp_init_attrs.alg_init = wd_comp_init_nolock; wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx_; - ret = wd_alg_attrs_init(&wd_comp_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_comp_init_attrs); + wd_ctx_param_uninit(&comp_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); - wd_ctx_param_uninit(&comp_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlclose; } } wd_alg_set_init(&wd_comp_setting.status); - wd_ctx_param_uninit(&comp_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&comp_ctx_params); -out_unbind_drv: - wd_alg_drv_unbind(wd_comp_setting.driver); out_dlclose: wd_comp_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_comp_setting.status); return ret; } void wd_comp_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_comp_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_comp_init_attrs); - wd_alg_drv_unbind(wd_comp_setting.driver); + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_comp_setting.adapter); wd_comp_close_driver(WD_TYPE_V2); wd_comp_setting.dlh_list = NULL; wd_alg_clear_init(&wd_comp_setting.status); @@ -351,7 +347,7 @@ void wd_comp_uninit2(void) int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_comp_msg resp_msg; struct wd_comp_msg *msg; @@ -365,16 +361,22 @@ int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_comp_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_comp_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -383,7 +385,7 @@ int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun recv_count++; - msg = wd_find_msg_in_pool(&wd_comp_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (unlikely(!msg)) { WD_ERR("failed to find msg from pool!\n"); @@ -396,7 +398,7 @@ int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun req->cb(req, req->cb_param); /* free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_comp_setting.pool, idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); @@ -438,8 +440,10 @@ static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_comp_sess *sess; - int ret; + int nb = wd_comp_setting.adapter->workers_nb; + int ret, i; if (!setup) return (handle_t)0; @@ -456,22 +460,34 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) if (!sess->ctx_buf) goto sess_err; + worker = sess->worker = &wd_comp_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; sess->alg_type = setup->alg_type; sess->comp_lv = setup->comp_lv; sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_comp_setting.sched.sched_init( - wd_comp_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; sched_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess->ctx_buf); sess_err: free(sess); @@ -488,8 +504,11 @@ void wd_comp_free_sess(handle_t h_sess) if (sess->ctx_buf) free(sess->ctx_buf); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -592,29 +611,32 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, struct wd_comp_req *req, struct wd_comp_msg *msg) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; - handle_t h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; __u32 idx; int ret; - idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - msg_handle.send = wd_comp_setting.driver->send; - msg_handle.recv = wd_comp_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_comp_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -833,9 +855,8 @@ int wd_do_comp_strm(handle_t h_sess, struct wd_comp_req *req) int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; struct wd_comp_sess *sess = (struct wd_comp_sess *)h_sess; - handle_t h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_comp_msg *msg; int tag, ret; @@ -850,16 +871,20 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) return -WD_EINVAL; } - idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - tag = wd_get_msg_from_pool(&wd_comp_setting.pool, idx, (void **)&msg); + tag = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(tag < 0)) { WD_ERR("failed to get msg from pool!\n"); return tag; @@ -868,13 +893,13 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS; - ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { WD_ERR("wd comp send error, ret = %d!\n", ret); goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_comp_env_config, idx); if (unlikely(ret)) goto fail_with_msg; @@ -882,21 +907,41 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_comp_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_comp_poll(__u32 expt, __u32 *count) { - struct wd_sched *sched = &wd_comp_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: comp poll count is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(sched, expt, count); + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_dh.c b/wd_dh.c index 3a99caec2..c3209c6fb 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -13,6 +13,7 @@ #include #include "include/drv/wd_dh_drv.h" +#include "adapter.h" #include "wd_util.h" #define DH_MAX_KEY_SIZE 512 @@ -25,17 +26,17 @@ struct wd_dh_sess { __u32 key_size; struct wd_dtb g; struct wd_dh_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static struct wd_dh_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_dh_setting; struct wd_env_config wd_dh_env_config; @@ -52,19 +53,15 @@ static void wd_dh_close_driver(int init_type) if (!wd_dh_setting.dlhandle) return; - wd_release_drv(wd_dh_setting.driver); dlclose(wd_dh_setting.dlhandle); wd_dh_setting.dlhandle = NULL; #else - wd_release_drv(wd_dh_setting.driver); hisi_hpre_remove(); #endif } static int wd_dh_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "dh"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -98,15 +95,6 @@ static int wd_dh_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_dh_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_dh_setting.driver = driver; - return WD_SUCCESS; } @@ -115,69 +103,67 @@ static void wd_dh_clear_status(void) wd_alg_clear_init(&wd_dh_setting.status); } -static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_dh_common_init(struct uadk_adapter_worker *worker, + struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_DH_EPOLL_EN", - &wd_dh_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_DH_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_dh_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret) return ret; - ret = wd_init_sched(&wd_dh_setting.sched, sched); - if (ret) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* initialize async request pool */ - ret = wd_init_async_request_pool(&wd_dh_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_dh_msg)); if (ret) - goto out_clear_sched; - - wd_dh_setting.config.pool = &wd_dh_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_dh_setting.config, - wd_dh_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_dh_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_dh_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_dh_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_dh_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_dh_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_dh_setting.pool); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_dh_setting.sched); - wd_alg_uninit_driver(&wd_dh_setting.config, - wd_dh_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "dh"; int ret; pthread_atfork(NULL, NULL, wd_dh_clear_status); @@ -190,11 +176,24 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_dh_setting.adapter = adapter; + ret = wd_dh_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_dh_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_dh_common_init(worker, sched); if (ret) goto out_close_driver; @@ -205,6 +204,7 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_dh_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_dh_setting.status); return ret; } @@ -217,6 +217,7 @@ void wd_dh_uninit(void) if (ret) return; + free(wd_dh_setting.adapter); wd_dh_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_dh_setting.status); } @@ -224,8 +225,11 @@ void wd_dh_uninit(void) int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) { struct wd_ctx_nums dh_ctx_num[WD_DH_PHASE2] = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; struct wd_ctx_params dh_ctx_params = {0}; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_dh_clear_status); @@ -244,78 +248,69 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_dh_setting.adapter = adapter; + state = wd_dh_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_dh_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_dh_setting.driver) { - WD_ERR("fail to bind a valid driver.\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; dh_ctx_params.ctx_set_num = dh_ctx_num; ret = wd_ctx_param_init(&dh_ctx_params, ctx_params, - wd_dh_setting.driver, WD_DH_TYPE, WD_DH_PHASE2); + worker->driver, WD_DH_TYPE, WD_DH_PHASE2); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_dh_init_attrs.alg = alg; - wd_dh_init_attrs.sched_type = sched_type; - wd_dh_init_attrs.driver = wd_dh_setting.driver; wd_dh_init_attrs.ctx_params = &dh_ctx_params; wd_dh_init_attrs.alg_init = wd_dh_common_init; wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx_; - ret = wd_alg_attrs_init(&wd_dh_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_dh_init_attrs); + wd_ctx_param_uninit(&dh_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); - wd_ctx_param_uninit(&dh_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_dh_setting.status); - wd_ctx_param_uninit(&dh_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&dh_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_dh_setting.driver); out_dlopen: wd_dh_close_driver(WD_TYPE_V2); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_dh_setting.status); return ret; } void wd_dh_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_dh_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_dh_init_attrs); - wd_alg_drv_unbind(wd_dh_setting.driver); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_dh_setting.adapter); wd_dh_close_driver(WD_TYPE_V2); wd_dh_setting.dlh_list = NULL; wd_alg_clear_init(&wd_dh_setting.status); @@ -352,11 +347,10 @@ static int fill_dh_msg(struct wd_dh_msg *msg, struct wd_dh_req *req, return WD_SUCCESS; } -int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) +int wd_do_dh_sync(handle_t h_sess, struct wd_dh_req *req) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; - struct wd_dh_sess *sess_t = (struct wd_dh_sess *)sess; + struct wd_dh_sess *sess = (struct wd_dh_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_dh_msg msg; @@ -368,27 +362,31 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) return -WD_EINVAL; } - idx = wd_dh_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_dh_msg)); - ret = fill_dh_msg(&msg, req, sess_t); + ret = fill_dh_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_dh_setting.driver->send; - msg_handle.recv = wd_dh_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_dh_setting.driver, &msg_handle, ctx->ctx, - &msg, &balance, wd_dh_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + &msg, &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -399,11 +397,10 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) return GET_NEGATIVE(msg.result); } -int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) +int wd_do_dh_async(handle_t h_sess, struct wd_dh_req *req) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; - struct wd_dh_sess *sess_t = (struct wd_dh_sess *)sess; + struct wd_dh_sess *sess = (struct wd_dh_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_dh_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -414,16 +411,20 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return -WD_EINVAL; } - idx = wd_dh_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_dh_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -434,7 +435,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_dh_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send dh BD, hw is err!\n"); @@ -442,7 +443,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_dh_env_config, idx); if (ret) goto fail_with_msg; @@ -450,14 +451,14 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_dh_msg rcv_msg; struct wd_dh_req *req; @@ -471,27 +472,33 @@ int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_dh_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_dh_setting.driver, ctx->ctx, &rcv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &rcv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret)) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, rcv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_dh_setting.pool, + msg = wd_find_msg_in_pool(&worker->pool, idx, rcv_msg.tag); if (!msg) { WD_ERR("failed to find msg!\n"); @@ -502,7 +509,7 @@ int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) msg->req.status = rcv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, rcv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, rcv_msg.tag); *count = rcv_cnt; } while (--tmp); @@ -516,12 +523,33 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_dh_poll(__u32 expt, __u32 *count) { + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; + if (unlikely(!count)) { WD_ERR("invalid: dh poll count is NULL!\n"); return -WD_EINVAL; } - return wd_dh_setting.sched.poll_policy(&wd_dh_setting.sched, expt, count); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } int wd_dh_get_mode(handle_t sess, __u8 *alg_mode) @@ -581,7 +609,10 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g) handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_dh_sess *sess; + int nb = wd_dh_setting.adapter->workers_nb; + int i; if (!setup) { WD_ERR("invalid: alloc dh sess setup NULL!\n"); @@ -604,6 +635,9 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) return (handle_t)0; memset(sess, 0, sizeof(struct wd_dh_sess)); + worker = sess->worker = &wd_dh_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; @@ -612,12 +646,16 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) goto sess_err; sess->g.bsize = sess->key_size; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_dh_setting.sched.sched_init( - wd_dh_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -625,6 +663,11 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) sched_err: free(sess->g.data); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -641,8 +684,11 @@ void wd_dh_free_sess(handle_t sess) if (sess_t->g.data) free(sess_t->g.data); - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } free(sess_t); } diff --git a/wd_digest.c b/wd_digest.c index 5875a24de..c2d0fcca8 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -9,6 +9,7 @@ #include #include "include/drv/wd_digest_drv.h" #include "wd_digest.h" +#include "adapter.h" #define GMAC_IV_LEN 16 #define MAX_BLOCK_SIZE 128 @@ -38,12 +39,9 @@ static char *wd_digest_alg_name[WD_DIGEST_TYPE_MAX] = { struct wd_digest_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_alg_driver *driver; - struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_digest_setting; struct wd_digest_stream_data { @@ -66,8 +64,11 @@ struct wd_digest_sess { void *priv; unsigned char key[MAX_HMAC_KEY_SIZE]; __u32 key_bytes; - void *sched_key; + void **sched_key; struct wd_digest_stream_data stream_data; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_env_config wd_digest_env_config; @@ -82,20 +83,16 @@ static void wd_digest_close_driver(int init_type) } if (wd_digest_setting.dlhandle) { - wd_release_drv(wd_digest_setting.driver); dlclose(wd_digest_setting.dlhandle); wd_digest_setting.dlhandle = NULL; } #else - wd_release_drv(wd_digest_setting.driver); hisi_sec2_remove(); #endif } static int wd_digest_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm3"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -129,14 +126,6 @@ static int wd_digest_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_digest_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_digest_setting.driver = driver; return WD_SUCCESS; } @@ -189,7 +178,9 @@ int wd_digest_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) { struct wd_digest_sess *sess = NULL; - bool ret; + struct uadk_adapter_worker *worker; + int nb = wd_digest_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("failed to check alloc sess param!\n"); @@ -206,27 +197,39 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) return (handle_t)0; memset(sess, 0, sizeof(struct wd_digest_sess)); + worker = sess->worker = &wd_digest_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; sess->alg_name = wd_digest_alg_name[setup->alg]; sess->alg = setup->alg; sess->mode = setup->mode; - ret = wd_drv_alg_support(sess->alg_name, wd_digest_setting.driver); + + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_digest_setting.sched.sched_init( - wd_digest_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -241,8 +244,11 @@ void wd_digest_free_sess(handle_t h_sess) } wd_memset_zero(sess->key, sess->key_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -251,52 +257,50 @@ static void wd_digest_clear_status(void) wd_alg_clear_init(&wd_digest_setting.status); } -static int wd_digest_init_nolock(struct wd_ctx_config *config, +static int wd_digest_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_DIGEST_EPOLL_EN", - &wd_digest_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_digest_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_digest_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_digest_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_digest_msg)); if (ret < 0) - goto out_clear_sched; - - wd_digest_setting.config.pool = &wd_digest_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_digest_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_digest_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_digest_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "sm3"; int ret; pthread_atfork(NULL, NULL, wd_digest_clear_status); @@ -309,11 +313,23 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_digest_setting.adapter = adapter; + ret = wd_digest_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_digest_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + ret = wd_digest_init_nolock(worker, sched); if (ret) goto out_close_driver; @@ -324,6 +340,7 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_digest_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_digest_setting.status); return ret; } @@ -336,10 +353,13 @@ static int wd_digest_uninit_nolock(void) if (status == WD_UNINIT) return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_digest_setting.pool); - wd_clear_sched(&wd_digest_setting.sched); - wd_alg_uninit_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + struct uadk_adapter_worker *worker = &wd_digest_setting.adapter->workers[i]; + + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } + return 0; } @@ -351,6 +371,7 @@ void wd_digest_uninit(void) if (ret) return; + free(wd_digest_setting.adapter); wd_digest_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_digest_setting.status); } @@ -370,7 +391,10 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, { struct wd_ctx_params digest_ctx_params = {0}; struct wd_ctx_nums digest_ctx_num = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_digest_clear_status); @@ -389,77 +413,68 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_digest_setting.adapter = adapter; + state = wd_digest_open_driver(WD_TYPE_V2); if (state) goto out_uninit; + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlclose; - while (ret != 0) { - memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_digest_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_digest_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; digest_ctx_params.ctx_set_num = &digest_ctx_num; ret = wd_ctx_param_init(&digest_ctx_params, ctx_params, - wd_digest_setting.driver, WD_DIGEST_TYPE, 1); + worker->driver, WD_DIGEST_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlclose; } wd_digest_init_attrs.alg = alg; - wd_digest_init_attrs.sched_type = sched_type; - wd_digest_init_attrs.driver = wd_digest_setting.driver; wd_digest_init_attrs.ctx_params = &digest_ctx_params; wd_digest_init_attrs.alg_init = wd_digest_init_nolock; wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx_; - ret = wd_alg_attrs_init(&wd_digest_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_digest_init_attrs); + wd_ctx_param_uninit(&digest_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); - wd_ctx_param_uninit(&digest_ctx_params); - continue; - } - WD_ERR("failed to init alg attrs.\n"); - goto out_params_uninit; + WD_ERR("fail to init alg attrs.\n"); + goto out_dlclose; } } - wd_alg_set_init(&wd_digest_setting.status); - wd_ctx_param_uninit(&digest_ctx_params); + wd_alg_set_init(&wd_digest_setting.status); return 0; -out_params_uninit: - wd_ctx_param_uninit(&digest_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_digest_setting.driver); -out_dlopen: +out_dlclose: wd_digest_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_digest_setting.status); return ret; } void wd_digest_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_digest_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_digest_init_attrs); - wd_alg_drv_unbind(wd_digest_setting.driver); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_digest_setting.adapter); wd_digest_close_driver(WD_TYPE_V2); wd_digest_setting.dlh_list = NULL; wd_alg_clear_init(&wd_digest_setting.status); @@ -610,19 +625,19 @@ static void fill_request_msg(struct wd_digest_msg *msg, msg->iv_bytes = sess->stream_data.msg_state; } -static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *dsess, - struct wd_digest_msg *msg) +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, + struct wd_digest_sess *dsess, struct wd_digest_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_digest_setting.driver->send; - msg_handle.recv = wd_digest_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; - wd_ctx_spin_lock(ctx, wd_digest_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_digest_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_digest_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, worker->driver->calc_type); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); + wd_ctx_spin_unlock(ctx, worker->driver->calc_type); if (unlikely(ret)) return ret; @@ -648,8 +663,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; struct wd_digest_sess *dsess = (struct wd_digest_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg msg; __u32 idx; @@ -659,20 +674,24 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) if (unlikely(ret)) return -WD_EINVAL; + pthread_spin_lock(&dsess->worker_lock); + worker = dsess->worker; + pthread_spin_unlock(&dsess->worker_lock); + memset(&msg, 0, sizeof(struct wd_digest_msg)); fill_request_msg(&msg, req, dsess); req->state = 0; - idx = wd_digest_setting.sched.pick_next_ctx( - wd_digest_setting.sched.h_sched_ctx, - dsess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + dsess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; - ret = send_recv_sync(ctx, dsess, &msg); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; + ret = send_recv_sync(worker, ctx, dsess, &msg); req->state = msg.result; return ret; @@ -680,8 +699,8 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; struct wd_digest_sess *dsess = (struct wd_digest_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg *msg; int msg_id, ret; @@ -696,16 +715,20 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return -WD_EINVAL; } - idx = wd_digest_setting.sched.pick_next_ctx( - wd_digest_setting.sched.h_sched_ctx, - dsess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&dsess->worker_lock); + worker = dsess->worker; + pthread_spin_unlock(&dsess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + dsess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_digest_setting.pool, idx, + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); @@ -715,7 +738,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) fill_request_msg(msg, req, dsess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_digest_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -723,7 +746,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_digest_env_config, idx); if (ret) goto fail_with_msg; @@ -731,13 +754,13 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_digest_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg recv_msg, *msg; struct wd_digest_req *req; @@ -745,6 +768,12 @@ int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co __u32 tmp = expt; int ret; + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_digest_setting.adapter->workers[0]; + else + worker = sched->worker; + if (unlikely(!count || !expt)) { WD_ERR("invalid: digest poll ctx input param is NULL!\n"); return -WD_EINVAL; @@ -752,14 +781,14 @@ int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_digest_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -769,7 +798,7 @@ int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co recv_cnt++; - msg = wd_find_msg_in_pool(&wd_digest_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -781,7 +810,7 @@ int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co if (likely(req)) req->cb(req); - wd_put_msg_to_pool(&wd_digest_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = recv_cnt; } while (--tmp); @@ -796,14 +825,33 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_digest_poll(__u32 expt, __u32 *count) { - struct wd_sched *sched = &wd_digest_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: digest poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(sched, expt, count); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_ecc.c b/wd_ecc.c index b2a8f853a..25738350b 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -15,6 +15,7 @@ #include "include/drv/wd_ecc_drv.h" #include "include/wd_ecc_curve.h" +#include "adapter.h" #include "wd_ecc.h" #define ECC_MAX_HW_BITS 521 @@ -49,7 +50,10 @@ struct wd_ecc_sess { __u32 key_size; struct wd_ecc_key key; struct wd_ecc_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_ecc_curve_list { @@ -61,12 +65,9 @@ struct wd_ecc_curve_list { static struct wd_ecc_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_ecc_setting; struct wd_env_config wd_ecc_env_config; @@ -106,19 +107,15 @@ static void wd_ecc_close_driver(int init_type) if (!wd_ecc_setting.dlhandle) return; - wd_release_drv(wd_ecc_setting.driver); dlclose(wd_ecc_setting.dlhandle); wd_ecc_setting.dlhandle = NULL; #else - wd_release_drv(wd_ecc_setting.driver); hisi_hpre_remove(); #endif } static int wd_ecc_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm2"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -152,15 +149,6 @@ static int wd_ecc_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_ecc_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_ecc_setting.driver = driver; - return WD_SUCCESS; } @@ -179,68 +167,66 @@ static void wd_ecc_clear_status(void) wd_alg_clear_init(&wd_ecc_setting.status); } -static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_ecc_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_ECC_EPOLL_EN", - &wd_ecc_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_ecc_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_ecc_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_ecc_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_ecc_msg)); if (ret < 0) - goto out_clear_sched; - - wd_ecc_setting.config.pool = &wd_ecc_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_ecc_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_ecc_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_ecc_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_ecc_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_ecc_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_ecc_setting.pool); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_ecc_setting.sched); - wd_alg_uninit_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "sm2"; int ret; pthread_atfork(NULL, NULL, wd_ecc_clear_status); @@ -253,11 +239,24 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_ecc_setting.adapter = adapter; + ret = wd_ecc_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_ecc_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_ecc_common_init(worker, sched); if (ret) goto out_close_driver; @@ -268,6 +267,7 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_ecc_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_ecc_setting.status); return ret; } @@ -280,6 +280,7 @@ void wd_ecc_uninit(void) if (ret) return; + free(wd_ecc_setting.adapter); wd_ecc_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -288,8 +289,11 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums ecc_ctx_num[WD_EC_OP_MAX] = {0}; struct wd_ctx_params ecc_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_ecc_clear_status); @@ -309,61 +313,46 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_ecc_setting.adapter = adapter; + state = wd_ecc_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_ecc_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_ecc_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; ecc_ctx_params.ctx_set_num = ecc_ctx_num; - ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, - wd_ecc_setting.driver, WD_ECC_TYPE, WD_EC_OP_MAX); + ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, worker->driver, + WD_ECC_TYPE, WD_EC_OP_MAX); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_ecc_init_attrs.alg = alg; - wd_ecc_init_attrs.sched_type = sched_type; - wd_ecc_init_attrs.driver = wd_ecc_setting.driver; wd_ecc_init_attrs.ctx_params = &ecc_ctx_params; wd_ecc_init_attrs.alg_init = wd_ecc_common_init; wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx_; - ret = wd_alg_attrs_init(&wd_ecc_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_ecc_init_attrs); + wd_ctx_param_uninit(&ecc_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); - wd_ctx_param_uninit(&ecc_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_ecc_setting.status); - wd_ctx_param_uninit(&ecc_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&ecc_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_ecc_setting.driver); out_dlopen: wd_ecc_close_driver(WD_TYPE_V2); out_clear_init: @@ -373,14 +362,19 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para void wd_ecc_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_ecc_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_ecc_init_attrs); - wd_alg_drv_unbind(wd_ecc_setting.driver); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_ecc_setting.adapter); wd_ecc_close_driver(WD_TYPE_V2); wd_ecc_setting.dlh_list = NULL; wd_alg_clear_init(&wd_ecc_setting.status); @@ -1170,22 +1164,28 @@ static void del_sess_key(struct wd_ecc_sess *sess) handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_ecc_sess *sess; - int ret; + int nb = wd_ecc_setting.adapter->workers_nb; + int ret, i; if (setup_param_check(setup)) return (handle_t)0; - ret = wd_drv_alg_support(setup->alg, wd_ecc_setting.driver); - if (!ret) { - WD_ERR("failed to support this algorithm: %s!\n", setup->alg); - return (handle_t)0; - } - sess = calloc(1, sizeof(struct wd_ecc_sess)); if (!sess) return (handle_t)0; + worker = sess->worker = &wd_ecc_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + + ret = wd_drv_alg_support(setup->alg, worker->driver); + if (!ret) { + WD_ERR("failed to support this algorithm: %s!\n", setup->alg); + goto sess_err; + } + memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = BITS_TO_BYTES(setup->key_bits); @@ -1195,12 +1195,16 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto sess_err; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_ecc_setting.sched.sched_init( - wd_ecc_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -1208,6 +1212,11 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) sched_err: del_sess_key(sess); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -1221,8 +1230,11 @@ void wd_ecc_free_sess(handle_t sess) return; } - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } del_sess_key(sess_t); free(sess_t); } @@ -1559,9 +1571,8 @@ static void msg_pack(char *dst, __u64 *out_len, int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) { - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; - handle_t h_sched_ctx = wd_ecc_setting.sched.h_sched_ctx; struct wd_ecc_sess *sess = (struct wd_ecc_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_ecc_msg msg; @@ -1573,27 +1584,31 @@ int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) return -WD_EINVAL; } - idx = wd_ecc_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_ecc_msg)); ret = fill_ecc_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_ecc_setting.driver->send; - msg_handle.recv = wd_ecc_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_ecc_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_ecc_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, &msg, + &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -2245,11 +2260,10 @@ struct wd_ecc_in *wd_ecdsa_new_verf_in(handle_t sess, return new_verf_in(sess, dgst, r, s, NULL, 1); } -int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) +int wd_do_ecc_async(handle_t h_sess, struct wd_ecc_req *req) { - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; - handle_t h_sched_ctx = wd_ecc_setting.sched.h_sched_ctx; - struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess; + struct wd_ecc_sess *sess = (struct wd_ecc_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ecc_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -2260,16 +2274,20 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return -WD_EINVAL; } - idx = wd_ecc_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_ecc_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -2280,7 +2298,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_ecc_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send ecc BD, hw is err!\n"); @@ -2288,7 +2306,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_ecc_env_config, idx); if (ret) goto fail_with_msg; @@ -2296,13 +2314,13 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct uadk_adapter_worker *worker; struct wd_ecc_msg recv_msg, *msg; struct wd_ctx_internal *ctx; struct wd_ecc_req *req; @@ -2315,28 +2333,33 @@ int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_ecc_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_ecc_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_ecc_setting.pool, idx, - recv_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); return -WD_EINVAL; @@ -2346,7 +2369,7 @@ int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count msg->req.status = recv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = rcv_cnt; } while (--tmp); @@ -2360,12 +2383,33 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_ecc_poll(__u32 expt, __u32 *count) { + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; + if (unlikely(!count)) { WD_ERR("invalid: ecc poll param count is NULL!\n"); return -WD_EINVAL; } - return wd_ecc_setting.sched.poll_policy(&wd_ecc_setting.sched, expt, count); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_rsa.c b/wd_rsa.c index a25b8ffc5..575244463 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -13,6 +13,7 @@ #include #include "include/drv/wd_rsa_drv.h" +#include "adapter.h" #include "wd_rsa.h" #define RSA_MAX_KEY_SIZE 512 @@ -66,17 +67,17 @@ struct wd_rsa_sess { struct wd_rsa_pubkey *pubkey; struct wd_rsa_prikey *prikey; struct wd_rsa_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static struct wd_rsa_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_rsa_setting; struct wd_env_config wd_rsa_env_config; @@ -93,19 +94,15 @@ static void wd_rsa_close_driver(int init_type) if (!wd_rsa_setting.dlhandle) return; - wd_release_drv(wd_rsa_setting.driver); dlclose(wd_rsa_setting.dlhandle); wd_rsa_setting.dlhandle = NULL; #else - wd_release_drv(wd_rsa_setting.driver); hisi_hpre_remove(); #endif } static int wd_rsa_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "rsa"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -139,15 +136,6 @@ static int wd_rsa_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_rsa_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support!\n", alg_name); - return -WD_EINVAL; - } - - wd_rsa_setting.driver = driver; - return WD_SUCCESS; } @@ -156,68 +144,65 @@ static void wd_rsa_clear_status(void) wd_alg_clear_init(&wd_rsa_setting.status); } -static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_rsa_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_RSA_EPOLL_EN", - &wd_rsa_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_RSA_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_rsa_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_rsa_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_rsa_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_rsa_msg)); if (ret < 0) - goto out_clear_sched; - - wd_rsa_setting.config.pool = &wd_rsa_setting.pool; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_rsa_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_rsa_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_rsa_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_rsa_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_rsa_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_rsa_setting.pool); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_rsa_setting.sched); - wd_alg_uninit_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "rsa"; int ret; pthread_atfork(NULL, NULL, wd_rsa_clear_status); @@ -230,11 +215,24 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_rsa_setting.adapter = adapter; + ret = wd_rsa_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_rsa_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_rsa_common_init(worker, sched); if (ret) goto out_close_driver; @@ -245,6 +243,7 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_rsa_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_rsa_setting.status); return ret; } @@ -257,6 +256,7 @@ void wd_rsa_uninit(void) if (ret) return; + free(wd_rsa_setting.adapter); wd_rsa_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -265,7 +265,10 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums rsa_ctx_num[WD_RSA_GENKEY] = {0}; struct wd_ctx_params rsa_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_rsa_clear_status); @@ -284,78 +287,69 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_rsa_setting.adapter = adapter; + state = wd_rsa_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_rsa_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_rsa_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; rsa_ctx_params.ctx_set_num = rsa_ctx_num; - ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, - wd_rsa_setting.driver, WD_RSA_TYPE, WD_RSA_GENKEY); + ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, worker->driver, + WD_RSA_TYPE, WD_RSA_GENKEY); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_rsa_init_attrs.alg = alg; - wd_rsa_init_attrs.sched_type = sched_type; - wd_rsa_init_attrs.driver = wd_rsa_setting.driver; wd_rsa_init_attrs.ctx_params = &rsa_ctx_params; wd_rsa_init_attrs.alg_init = wd_rsa_common_init; wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx_; - ret = wd_alg_attrs_init(&wd_rsa_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_rsa_init_attrs); + wd_ctx_param_uninit(&rsa_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); - wd_ctx_param_uninit(&rsa_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_rsa_setting.status); - wd_ctx_param_uninit(&rsa_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&rsa_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_rsa_setting.driver); out_dlopen: wd_rsa_close_driver(WD_TYPE_V2); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_rsa_setting.status); return ret; } void wd_rsa_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_rsa_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_rsa_init_attrs); - wd_alg_drv_unbind(wd_rsa_setting.driver); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_rsa_setting.adapter); wd_rsa_close_driver(WD_TYPE_V2); wd_rsa_setting.dlh_list = NULL; wd_alg_clear_init(&wd_rsa_setting.status); @@ -415,9 +409,8 @@ static int fill_rsa_msg(struct wd_rsa_msg *msg, struct wd_rsa_req *req, int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) { - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; struct wd_rsa_sess *sess = (struct wd_rsa_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_rsa_msg msg; @@ -429,27 +422,31 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) return -WD_EINVAL; } - idx = wd_rsa_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_rsa_msg)); ret = fill_rsa_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_rsa_setting.driver->send; - msg_handle.recv = wd_rsa_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_rsa_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_rsa_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, &msg, + &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -460,11 +457,10 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) return GET_NEGATIVE(msg.result); } -int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) +int wd_do_rsa_async(handle_t h_sess, struct wd_rsa_req *req) { - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; - struct wd_rsa_sess *sess_t = (struct wd_rsa_sess *)sess; + struct wd_rsa_sess *sess = (struct wd_rsa_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_rsa_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -475,16 +471,20 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return -WD_EINVAL; } - idx = wd_rsa_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_rsa_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -495,7 +495,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_rsa_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send rsa BD, hw is err!\n"); @@ -503,7 +503,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_rsa_env_config, idx); if (ret) goto fail_with_msg; @@ -511,13 +511,13 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, mid); return ret; } int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_rsa_req *req; struct wd_rsa_msg recv_msg, *msg; @@ -530,27 +530,31 @@ int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_rsa_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_rsa_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, - recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_rsa_setting.pool, idx, - recv_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); return -WD_EINVAL; @@ -560,7 +564,7 @@ int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count msg->req.status = recv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = rcv_cnt; } while (--tmp); @@ -574,12 +578,34 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_rsa_poll(__u32 expt, __u32 *count) { + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; + if (unlikely(!count)) { WD_ERR("invalid: rsa poll count is NULL!\n"); return -WD_EINVAL; } - return wd_rsa_setting.sched.poll_policy(&wd_rsa_setting.sched, expt, count); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } int wd_rsa_kg_in_data(struct wd_rsa_kg_in *ki, char **data) @@ -921,8 +947,10 @@ static void del_sess(struct wd_rsa_sess *c) /* Before initiate this context, we should get a queue from WD */ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) { + struct uadk_adapter_worker *worker; + int nb = wd_rsa_setting.adapter->workers_nb; struct wd_rsa_sess *sess; - int ret; + int ret, i; if (!setup) { WD_ERR("invalid: alloc rsa sess setup NULL!\n"); @@ -944,18 +972,26 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; + worker = sess->worker = &wd_rsa_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + ret = create_sess_key(setup, sess); if (ret) { WD_ERR("failed to create rsa sess keys!\n"); goto sess_err; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_rsa_setting.sched.sched_init( - wd_rsa_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -963,6 +999,11 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) sched_err: del_sess_key(sess); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -976,8 +1017,11 @@ void wd_rsa_free_sess(handle_t sess) return; } - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } del_sess_key(sess_t); del_sess(sess_t); } diff --git a/wd_util.c b/wd_util.c index e908dcb44..0e6d8d57d 100644 --- a/wd_util.c +++ b/wd_util.c @@ -13,6 +13,7 @@ #include #include "wd_sched.h" #include "wd_util.h" +#include "adapter.h" #define WD_ASYNC_DEF_POLL_NUM 1 #define WD_ASYNC_DEF_QUEUE_DEPTH 1024 @@ -263,36 +264,6 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, return ret; } -int wd_init_sched(struct wd_sched *in, struct wd_sched *from) -{ - if (!from->name || !from->sched_init || - !from->pick_next_ctx || !from->poll_policy) { - WD_ERR("invalid: member of wd_sched is NULL!\n"); - return -WD_EINVAL; - } - - in->h_sched_ctx = from->h_sched_ctx; - in->name = strdup(from->name); - in->sched_init = from->sched_init; - in->pick_next_ctx = from->pick_next_ctx; - in->poll_policy = from->poll_policy; - - return 0; -} - -void wd_clear_sched(struct wd_sched *in) -{ - char *name = (char *)in->name; - - if (name) - free(name); - in->h_sched_ctx = 0; - in->name = NULL; - in->sched_init = NULL; - in->pick_next_ctx = NULL; - in->poll_policy = NULL; -} - void wd_clear_ctx_config(struct wd_ctx_config_internal *in) { __u32 i; @@ -2271,6 +2242,9 @@ void *wd_dlopen_drv(const char *cust_lib_dir) node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); if (!node->dlhandle) { + char *error = dlerror(); + + fprintf(stderr, "%s\n", error); free(node); /* there are many other files need to skip */ continue; @@ -2638,9 +2612,9 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs) return ret; } -static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +static int wd_alg_ce_ctx_init(struct uadk_adapter_worker *worker) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; + struct wd_ctx_config *ctx_config = worker->ctx_config; ctx_config->ctx_num = 1; ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); @@ -2727,11 +2701,11 @@ static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) free(ctx_config->ctxs); } -int wd_alg_attrs_init(struct wd_init_attrs *attrs) +int wd_alg_attrs_init(struct uadk_adapter_worker *worker, + struct wd_init_attrs *attrs) { wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; wd_alg_init alg_init_func = attrs->alg_init; - __u32 sched_type = attrs->sched_type; struct wd_ctx_config *ctx_config = NULL; struct wd_sched *alg_sched = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; @@ -2742,8 +2716,8 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) if (!attrs->ctx_params) return -WD_EINVAL; - if (attrs->driver) - driver_type = attrs->driver->calc_type; + if (worker->driver) + driver_type = worker->driver->calc_type; switch (driver_type) { case UADK_ALG_SOFT: @@ -2753,7 +2727,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) WD_ERR("fail to alloc ctx config\n"); return -WD_ENOMEM; } - attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; /* Use default sched_type to alloc scheduler */ alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); @@ -2762,15 +2736,15 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) goto out_ctx_config; } - attrs->sched = alg_sched; + worker->sched = alg_sched; - ret = wd_alg_ce_ctx_init(attrs); + ret = wd_alg_ce_ctx_init(worker); if (ret) { WD_ERR("fail to init ce ctx\n"); goto out_freesched; } - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) goto out_pre_init; @@ -2782,14 +2756,14 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) WD_ERR("fail to alloc scheduler\n"); return -WD_EINVAL; } - attrs->sched = alg_sched; + worker->sched = alg_sched; ctx_config = calloc(1, sizeof(*ctx_config)); if (!ctx_config) { WD_ERR("fail to alloc ctx config\n"); goto out_freesched; } - attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; ret = wd_alg_init_sve_ctx(ctx_config); if (ret) { @@ -2798,7 +2772,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) } ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) { wd_alg_uninit_sve_ctx(ctx_config); goto out_freesched; @@ -2814,14 +2788,17 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) return -WD_ENOMEM; } attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, + /* Use default sched_type to alloc scheduler */ + alg_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, attrs->ctx_params->op_type_num, numa_max_node() + 1, alg_poll_func); if (!alg_sched) { WD_ERR("fail to instance scheduler\n"); goto out_ctx_config; } attrs->sched = alg_sched; + worker->sched = alg_sched; ret = wd_alg_ctx_init(attrs); if (ret) { @@ -2830,7 +2807,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) } ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) goto out_pre_init; break; @@ -2854,11 +2831,11 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) return ret; } -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) +void wd_alg_attrs_uninit(struct uadk_adapter_worker *worker) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type; + struct wd_ctx_config *ctx_config = worker->ctx_config; + struct wd_sched *alg_sched = worker->sched; + int driver_type = worker->driver->calc_type; if (!ctx_config) { wd_sched_rr_release(alg_sched); From f4fc058e8e0aff3a9f00d67a42495df4b51fd984 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Mon, 18 Nov 2024 14:00:15 +0000 Subject: [PATCH 08/13] uadk: support switch between hw and sw case 1: sync or async error, switch worker case 2: roundrobin mode, switch worker after worker->looptime (10) Signed-off-by: Zhangfei Gao --- wd_cipher.c | 36 ++++++++++++++++++++++++++++++++++++ wd_comp.c | 37 ++++++++++++++++++++++++++++++++++++- wd_digest.c | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/wd_cipher.c b/wd_cipher.c index 7d7525e2d..413734c27 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -73,6 +73,19 @@ struct wd_cipher_sess { struct wd_env_config wd_cipher_env_config; static struct wd_init_attrs wd_cipher_init_attrs; +void wd_cipher_switch_worker(struct wd_cipher_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + pthread_spin_lock(&sess->worker_lock); + worker = uadk_adapter_switch_worker(wd_cipher_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; + pthread_spin_unlock(&sess->worker_lock); +} + static void wd_cipher_close_driver(int init_type) { #ifndef WD_STATIC_DRV @@ -742,6 +755,20 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) ret = send_recv_sync(worker, ctx, &msg); req->state = msg.result; + if (ret) { + wd_cipher_switch_worker(sess, 1); + sess->worker_looptime++; + return ret; + } + + if ((sess->worker_looptime != 0) || + (wd_cipher_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) { + sess->worker_looptime++; + } + + if (sess->worker_looptime >= wd_cipher_setting.adapter->looptime) + wd_cipher_switch_worker(sess, 0); + return ret; } @@ -795,10 +822,19 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) if (ret) goto fail_with_msg; + if ((sess->worker_looptime != 0) || + (wd_cipher_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + sess->worker_looptime++; + + if (sess->worker_looptime >= wd_cipher_setting.adapter->looptime) + wd_cipher_switch_worker(sess, 0); + return 0; fail_with_msg: wd_put_msg_to_pool(&worker->pool, idx, msg->tag); + wd_cipher_switch_worker(sess, 1); + sess->worker_looptime++; return ret; } diff --git a/wd_comp.c b/wd_comp.c index 9d4de40a4..826fc2acd 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -55,6 +55,19 @@ struct wd_comp_setting { struct wd_env_config wd_comp_env_config; static struct wd_init_attrs wd_comp_init_attrs; +void wd_comp_switch_worker(struct wd_comp_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + pthread_spin_lock(&sess->worker_lock); + worker = uadk_adapter_switch_worker(wd_comp_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; + pthread_spin_unlock(&sess->worker_lock); +} + static void wd_comp_close_driver(int init_type) { #ifndef WD_STATIC_DRV @@ -639,6 +652,20 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); + if (ret) { + wd_comp_switch_worker(sess, 1); + sess->worker_looptime++; + return ret; + } + + if ((sess->worker_looptime != 0) || + (wd_comp_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) { + sess->worker_looptime++; + } + + if (sess->worker_looptime >= wd_comp_setting.adapter->looptime) + wd_comp_switch_worker(sess, 0); + return ret; } @@ -904,11 +931,19 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) if (unlikely(ret)) goto fail_with_msg; + if ((sess->worker_looptime != 0) || + (wd_comp_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + sess->worker_looptime++; + + if (sess->worker_looptime >= wd_comp_setting.adapter->looptime) + wd_comp_switch_worker(sess, 0); + return 0; fail_with_msg: wd_put_msg_to_pool(&worker->pool, idx, msg->tag); - + wd_comp_switch_worker(sess, 1); + sess->worker_looptime++; return ret; } diff --git a/wd_digest.c b/wd_digest.c index c2d0fcca8..7d45e41b1 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -71,6 +71,19 @@ struct wd_digest_sess { int worker_looptime; }; +void wd_digest_switch_worker(struct wd_digest_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + pthread_spin_lock(&sess->worker_lock); + worker = uadk_adapter_switch_worker(wd_digest_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; + pthread_spin_unlock(&sess->worker_lock); +} + struct wd_env_config wd_digest_env_config; static struct wd_init_attrs wd_digest_init_attrs; @@ -694,6 +707,19 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) ret = send_recv_sync(worker, ctx, dsess, &msg); req->state = msg.result; + if (ret) { + wd_digest_switch_worker(dsess, 1); + dsess->worker_looptime++; + return ret; + } + + if ((dsess->worker_looptime != 0) || + (wd_digest_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + dsess->worker_looptime++; + + if (dsess->worker_looptime >= wd_digest_setting.adapter->looptime) + wd_digest_switch_worker(dsess, 0); + return ret; } @@ -751,10 +777,19 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) if (ret) goto fail_with_msg; + if ((dsess->worker_looptime != 0) || + (wd_digest_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + dsess->worker_looptime++; + + if (dsess->worker_looptime >= wd_digest_setting.adapter->looptime) + wd_digest_switch_worker(dsess, 0); + return 0; fail_with_msg: wd_put_msg_to_pool(&worker->pool, idx, msg->tag); + wd_digest_switch_worker(dsess, 1); + dsess->worker_looptime++; return ret; } From 3fc0fc53806ecc373bcbb7b3fc0ab1390d9e1c9a Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Mon, 18 Nov 2024 14:23:26 +0000 Subject: [PATCH 09/13] adapter: add LOOP_KEY "looptime" By default each worker loop 10 times in roundrobin mode, except looptime is set in UADK_CONF For example: uadk.conf looptime=5 mode=1 Signed-off-by: Zhangfei Gao --- adapter.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/adapter.c b/adapter.c index a1cbfbae7..34ebf219f 100644 --- a/adapter.c +++ b/adapter.c @@ -10,6 +10,7 @@ #define CONFIG_FILE_ENV "UADK_CONF" #define DRIVER_NAME_KEY "driver_name" #define MODE_KEY "mode" +#define LOOP_KEY "looptime" #define WORKERS_NB 8 static int read_value_int(char *conf, const char *key) @@ -91,8 +92,14 @@ int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) adapter->looptime = UADK_WORKER_LOOPTIME; if (conf != NULL) { + int looptime = 0; + /* if env UADK_CONF exist, parse config first */ adapter->mode = read_value_int(conf, MODE_KEY); + looptime = read_value_int(conf, LOOP_KEY); + if (looptime != 0) + adapter->looptime = looptime; + read_config_entries(conf, adapter, alg); if (adapter->workers_nb != 0) return 0; From 212b5ec0a45e993503c2bdb7cb5bff4b4f7764e0 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Sat, 4 Jan 2025 06:51:52 +0000 Subject: [PATCH 10/13] uadk: simulate async mode for driver only support sync There may driver does not support async mode. The operation is finished when send is over. So we can simulate async mode just return the correct number Signed-off-by: Zhangfei Gao --- adapter.c | 9 +++++++++ drv/isa_ce_sm3.c | 1 + drv/isa_ce_sm4.c | 1 + include/adapter.h | 4 ++++ include/wd_alg.h | 6 ++++++ libwd.map | 1 + wd_cipher.c | 21 +++++++++++++++++++++ wd_comp.c | 25 ++++++++++++++++++++----- wd_digest.c | 21 +++++++++++++++++++++ 9 files changed, 84 insertions(+), 5 deletions(-) diff --git a/adapter.c b/adapter.c index 34ebf219f..1bd02e6f4 100644 --- a/adapter.c +++ b/adapter.c @@ -64,6 +64,7 @@ static void read_config_entries(char *conf, struct uadk_adapter *adapter, char * worker = &adapter->workers[i]; worker->driver = drv; worker->idx = i; + pthread_mutex_init(&worker->mutex, NULL); adapter->workers_nb++; if (drv_name) { free(drv_name); @@ -133,6 +134,7 @@ int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) adapter->workers[i].driver = workers[i].driver; adapter->workers[i].idx = i; adapter->workers_nb++; + pthread_mutex_init(&adapter->workers[i].mutex, NULL); if (adapter->workers_nb >= UADK_MAX_NB_WORKERS) break; @@ -179,3 +181,10 @@ struct uadk_adapter_worker *uadk_adapter_switch_worker( return new_worker; } + +void uadk_adapter_free(struct uadk_adapter *adapter) +{ + for (int i = 0; i < adapter->workers_nb; i++) + pthread_mutex_destroy(&adapter->workers[i].mutex); + free(adapter); +} diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 54c2a9e72..88532297a 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -32,6 +32,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .drv_name = "isa_ce_sm3", .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, + .mode = UADK_DRV_SYNCONLY, .priority = 200, .queue_num = 1, .op_type_num = 1, diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 5e448fa0f..60a09f296 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -419,6 +419,7 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .drv_name = "isa_ce_sm4",\ .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ + .mode = UADK_DRV_SYNCONLY,\ .priority = 200,\ .op_type_num = 1,\ .fallback = 0,\ diff --git a/include/adapter.h b/include/adapter.h index b206bf750..bd9d640c3 100644 --- a/include/adapter.h +++ b/include/adapter.h @@ -26,6 +26,8 @@ struct uadk_adapter_worker { struct wd_async_msg_pool pool; bool valid; int idx; + __u64 async_recv; + pthread_mutex_t mutex; }; struct uadk_adapter { @@ -47,4 +49,6 @@ struct uadk_adapter_worker *uadk_adapter_switch_worker( struct uadk_adapter_worker *worker, int para ); + +void uadk_adapter_free(struct uadk_adapter *adapter); #endif diff --git a/include/wd_alg.h b/include/wd_alg.h index 307ff0951..01cb10486 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -69,6 +69,11 @@ enum alg_dev_type { UADK_ALG_HW = 0x3 }; +enum alg_drv_mode { + UADK_DRV_NORMAL = 0x0, + UADK_DRV_SYNCONLY = 0x1, +}; + /* * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver @@ -102,6 +107,7 @@ struct wd_alg_driver { const char *alg_name; int priority; int calc_type; + int mode; int queue_num; int op_type_num; void *priv; diff --git a/libwd.map b/libwd.map index c6e1dbdeb..f9d55207b 100644 --- a/libwd.map +++ b/libwd.map @@ -59,5 +59,6 @@ global: uadk_adapter_add_workers; uadk_adapter_choose_worker; uadk_adapter_switch_worker; + uadk_adapter_free; local: *; }; diff --git a/wd_cipher.c b/wd_cipher.c index 413734c27..769a0f5c3 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -791,6 +791,17 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) worker = sess->worker; pthread_spin_unlock(&sess->worker_lock); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_cipher_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req, req->cb_param); + } + return ret; + } + idx = worker->sched->pick_next_ctx( worker->sched->h_sched_ctx, sess->sched_key[worker->idx], CTX_MODE_ASYNC); @@ -861,6 +872,16 @@ int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co *count = 0; + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; diff --git a/wd_comp.c b/wd_comp.c index 826fc2acd..e5d988981 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -382,6 +382,16 @@ int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *coun *count = 0; + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; @@ -612,11 +622,6 @@ static int wd_comp_check_params(struct wd_comp_sess *sess, return -WD_EINVAL; } - if (unlikely(mode == CTX_MODE_SYNC && req->cb)) { - WD_ERR("invalid: sync comp cb should be NULL!\n"); - return -WD_EINVAL; - } - return 0; } @@ -902,6 +907,16 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) worker = sess->worker; pthread_spin_unlock(&sess->worker_lock); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_comp_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req, req->cb_param); + } + return ret; + } idx = worker->sched->pick_next_ctx( worker->sched->h_sched_ctx, sess->sched_key[worker->idx], CTX_MODE_ASYNC); diff --git a/wd_digest.c b/wd_digest.c index 7d45e41b1..853740296 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -745,6 +745,17 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) worker = dsess->worker; pthread_spin_unlock(&dsess->worker_lock); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_digest_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req); + } + return ret; + } + idx = worker->sched->pick_next_ctx( worker->sched->h_sched_ctx, dsess->sched_key[worker->idx], CTX_MODE_ASYNC); @@ -816,6 +827,16 @@ int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *co *count = 0; + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; From 9598e0ab2aa74f5462f82dd6b5f6f60e318b5fcb Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Mon, 6 Jan 2025 06:11:21 +0000 Subject: [PATCH 11/13] uadk: add test.sh for performance testing Signed-off-by: Zhangfei Gao --- test/test.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 test/test.sh diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 000000000..faab1a0ec --- /dev/null +++ b/test/test.sh @@ -0,0 +1,31 @@ +set -x + +echo "Testing sm3 /sm4, sync & async" +echo "Testing hw, hw+ce, ce" +echo "Testing hw, hw+sve, sve" + +default_size=8192 +values=(1 2 4 8 16 32) + +if [ -n "$1" ]; then + size=$1 +else + size=$default_size +fi + +for x in "${values[@]}" +do + echo $x + #sm4 sync + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm4-128-ecb --mode sva --opt 0 --sync --pktlen $size --seconds 10 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + #sm4 async + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm4-128-ecb --mode sva --opt 0 --async --pktlen $size --seconds 10 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + + #sm3 sync + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm3 --mode sva --opt 0 --sync --pktlen $size --seconds 20 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + #sm3 async + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm3 --mode sva --opt 0 --async --pktlen $size --seconds 20 --multi 1 --thread $x --ctxnum $x --init2 --prefetch +done From 4b09983b53afc9cd299912ef1ac93f5ba9f5a804 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Thu, 9 Jan 2025 01:42:12 +0000 Subject: [PATCH 12/13] uadk: add wd_xx_set_adapter_mode wd_xx_set_adapter_mode can change the cipher apdater mode while system is running Signed-off-by: Zhangfei Gao --- wd_cipher.c | 31 +++++++++++++++++++++---------- wd_comp.c | 31 +++++++++++++++++++++---------- wd_digest.c | 31 +++++++++++++++++++++---------- 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/wd_cipher.c b/wd_cipher.c index 769a0f5c3..15b85a358 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -66,24 +66,33 @@ struct wd_cipher_sess { __u32 key_bytes; void **sched_key; struct uadk_adapter_worker *worker; - pthread_spinlock_t worker_lock; int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; struct wd_env_config wd_cipher_env_config; static struct wd_init_attrs wd_cipher_init_attrs; -void wd_cipher_switch_worker(struct wd_cipher_sess *sess, int para) +static void wd_cipher_switch_worker(struct wd_cipher_sess *sess, int para) { struct uadk_adapter_worker *worker; - pthread_spin_lock(&sess->worker_lock); worker = uadk_adapter_switch_worker(wd_cipher_setting.adapter, sess->worker, para); if (worker) sess->worker = worker; sess->worker_looptime = 0; - pthread_spin_unlock(&sess->worker_lock); +} + +static struct uadk_adapter_worker *wd_cipher_get_worker(struct wd_cipher_sess *sess) +{ + if (sess->adapter_mode != wd_cipher_setting.adapter->mode) { + sess->worker = &wd_cipher_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_cipher_setting.adapter->mode; + } + + return sess->worker; } static void wd_cipher_close_driver(int init_type) @@ -289,6 +298,7 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) } sess->alg = setup->alg; sess->mode = setup->mode; + sess->adapter_mode = wd_cipher_setting.adapter->mode; sess->sched_key = (void **)calloc(nb, sizeof(void *)); for (i = 0; i < nb; i++) { @@ -734,9 +744,7 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) return ret; } - pthread_spin_lock(&sess->worker_lock); - worker = sess->worker; - pthread_spin_unlock(&sess->worker_lock); + worker = wd_cipher_get_worker(sess); memset(&msg, 0, sizeof(struct wd_cipher_msg)); fill_request_msg(&msg, req, sess); @@ -787,9 +795,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } - pthread_spin_lock(&sess->worker_lock); - worker = sess->worker; - pthread_spin_unlock(&sess->worker_lock); + worker = wd_cipher_get_worker(sess); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_cipher_sync(h_sess, req); @@ -1017,3 +1023,8 @@ int wd_cipher_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_cipher_env_config, ctx_attr, num, is_enable); } + +void wd_cipher_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_cipher_setting.adapter->mode = mode; +} diff --git a/wd_comp.c b/wd_comp.c index e5d988981..d7f4c1ae4 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -41,8 +41,8 @@ struct wd_comp_sess { __u8 *ctx_buf; void **sched_key; struct uadk_adapter_worker *worker; - pthread_spinlock_t worker_lock; int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; struct wd_comp_setting { @@ -55,17 +55,26 @@ struct wd_comp_setting { struct wd_env_config wd_comp_env_config; static struct wd_init_attrs wd_comp_init_attrs; -void wd_comp_switch_worker(struct wd_comp_sess *sess, int para) +static void wd_comp_switch_worker(struct wd_comp_sess *sess, int para) { struct uadk_adapter_worker *worker; - pthread_spin_lock(&sess->worker_lock); worker = uadk_adapter_switch_worker(wd_comp_setting.adapter, sess->worker, para); if (worker) sess->worker = worker; sess->worker_looptime = 0; - pthread_spin_unlock(&sess->worker_lock); +} + +static struct uadk_adapter_worker *wd_comp_get_worker(struct wd_comp_sess *sess) +{ + if (sess->adapter_mode != wd_comp_setting.adapter->mode) { + sess->worker = &wd_comp_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_comp_setting.adapter->mode; + } + + return sess->worker; } static void wd_comp_close_driver(int init_type) @@ -490,6 +499,7 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) sess->comp_lv = setup->comp_lv; sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; + sess->adapter_mode = wd_comp_setting.adapter->mode; sess->sched_key = (void **)calloc(nb, sizeof(void *)); for (i = 0; i < nb; i++) { @@ -635,9 +645,7 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, __u32 idx; int ret; - pthread_spin_lock(&sess->worker_lock); - worker = sess->worker; - pthread_spin_unlock(&sess->worker_lock); + worker = wd_comp_get_worker(sess); idx = worker->sched->pick_next_ctx( worker->sched->h_sched_ctx, @@ -903,9 +911,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) return -WD_EINVAL; } - pthread_spin_lock(&sess->worker_lock); - worker = sess->worker; - pthread_spin_unlock(&sess->worker_lock); + worker = wd_comp_get_worker(sess); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_comp_sync(h_sess, req); @@ -1076,3 +1082,8 @@ int wd_comp_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_comp_env_config, ctx_attr, num, is_enable); } + +void wd_comp_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_comp_setting.adapter->mode = mode; +} diff --git a/wd_digest.c b/wd_digest.c index 853740296..f8b1808f8 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -67,21 +67,30 @@ struct wd_digest_sess { void **sched_key; struct wd_digest_stream_data stream_data; struct uadk_adapter_worker *worker; - pthread_spinlock_t worker_lock; int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; -void wd_digest_switch_worker(struct wd_digest_sess *sess, int para) +static void wd_digest_switch_worker(struct wd_digest_sess *sess, int para) { struct uadk_adapter_worker *worker; - pthread_spin_lock(&sess->worker_lock); worker = uadk_adapter_switch_worker(wd_digest_setting.adapter, sess->worker, para); if (worker) sess->worker = worker; sess->worker_looptime = 0; - pthread_spin_unlock(&sess->worker_lock); +} + +static struct uadk_adapter_worker *wd_digest_get_worker(struct wd_digest_sess *sess) +{ + if (sess->adapter_mode != wd_digest_setting.adapter->mode) { + sess->worker = &wd_digest_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_digest_setting.adapter->mode; + } + + return sess->worker; } struct wd_env_config wd_digest_env_config; @@ -216,6 +225,7 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) sess->alg_name = wd_digest_alg_name[setup->alg]; sess->alg = setup->alg; sess->mode = setup->mode; + sess->adapter_mode = wd_digest_setting.adapter->mode; ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { @@ -687,9 +697,7 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) if (unlikely(ret)) return -WD_EINVAL; - pthread_spin_lock(&dsess->worker_lock); - worker = dsess->worker; - pthread_spin_unlock(&dsess->worker_lock); + worker = wd_digest_get_worker(dsess); memset(&msg, 0, sizeof(struct wd_digest_msg)); fill_request_msg(&msg, req, dsess); @@ -741,9 +749,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return -WD_EINVAL; } - pthread_spin_lock(&dsess->worker_lock); - worker = dsess->worker; - pthread_spin_unlock(&dsess->worker_lock); + worker = wd_digest_get_worker(dsess); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_digest_sync(h_sess, req); @@ -973,3 +979,8 @@ int wd_digest_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_digest_env_config, ctx_attr, num, is_enable); } + +void wd_digest_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_digest_setting.adapter->mode = mode; +} From 6ce07bd90fe64295c07e185b59b244f15a57a8b3 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Thu, 9 Jan 2025 03:57:00 +0000 Subject: [PATCH 13/13] uadk: add UADK_ADAPT_MODE_THRESHOLD Add UADK_ADAPT_MODE_THRESHOLD if (pkg_len > threshold) choose hw else choose sw Signed-off-by: Zhangfei Gao --- adapter.c | 7 +++++++ include/adapter.h | 3 +++ wd_cipher.c | 16 ++++++++++++---- wd_comp.c | 15 ++++++++++++--- wd_digest.c | 15 ++++++++++++--- 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/adapter.c b/adapter.c index 1bd02e6f4..58ec63587 100644 --- a/adapter.c +++ b/adapter.c @@ -11,6 +11,7 @@ #define DRIVER_NAME_KEY "driver_name" #define MODE_KEY "mode" #define LOOP_KEY "looptime" +#define THRESHOLD_KEY "threshold" #define WORKERS_NB 8 static int read_value_int(char *conf, const char *key) @@ -91,9 +92,11 @@ int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) int idx = 0, i, j; adapter->looptime = UADK_WORKER_LOOPTIME; + adapter->threshold = UADK_PKT_THRESHOLD; if (conf != NULL) { int looptime = 0; + int threshold = 0; /* if env UADK_CONF exist, parse config first */ adapter->mode = read_value_int(conf, MODE_KEY); @@ -101,6 +104,10 @@ int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) if (looptime != 0) adapter->looptime = looptime; + threshold = read_value_int(conf, THRESHOLD_KEY); + if (threshold != 0) + adapter->threshold = threshold; + read_config_entries(conf, adapter, alg); if (adapter->workers_nb != 0) return 0; diff --git a/include/adapter.h b/include/adapter.h index bd9d640c3..be77f4337 100644 --- a/include/adapter.h +++ b/include/adapter.h @@ -12,10 +12,12 @@ #define UADK_MAX_NB_WORKERS (2) #define UADK_WORKER_LOOPTIME (10) +#define UADK_PKT_THRESHOLD 8192 enum uadk_adapter_mode { UADK_ADAPT_MODE_PRIMARY, UADK_ADAPT_MODE_ROUNDROBIN, + UADK_ADAPT_MODE_THRESHOLD, }; struct uadk_adapter_worker { @@ -33,6 +35,7 @@ struct uadk_adapter_worker { struct uadk_adapter { unsigned int workers_nb; unsigned int looptime; + unsigned int threshold; enum uadk_adapter_mode mode; struct uadk_adapter_worker workers[UADK_MAX_NB_WORKERS]; }; diff --git a/wd_cipher.c b/wd_cipher.c index 15b85a358..486ca47b2 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -84,7 +84,8 @@ static void wd_cipher_switch_worker(struct wd_cipher_sess *sess, int para) sess->worker_looptime = 0; } -static struct uadk_adapter_worker *wd_cipher_get_worker(struct wd_cipher_sess *sess) +static struct uadk_adapter_worker * +wd_cipher_get_worker(struct wd_cipher_sess *sess, struct wd_cipher_req *req) { if (sess->adapter_mode != wd_cipher_setting.adapter->mode) { sess->worker = &wd_cipher_setting.adapter->workers[0]; @@ -92,6 +93,14 @@ static struct uadk_adapter_worker *wd_cipher_get_worker(struct wd_cipher_sess *s sess->adapter_mode = wd_cipher_setting.adapter->mode; } + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_cipher_setting.adapter->workers_nb == 2) { + if (req->in_bytes >= wd_cipher_setting.adapter->threshold) + sess->worker = &wd_cipher_setting.adapter->workers[0]; + else + sess->worker = &wd_cipher_setting.adapter->workers[1]; + } + return sess->worker; } @@ -744,7 +753,7 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) return ret; } - worker = wd_cipher_get_worker(sess); + worker = wd_cipher_get_worker(sess, req); memset(&msg, 0, sizeof(struct wd_cipher_msg)); fill_request_msg(&msg, req, sess); @@ -795,8 +804,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } - worker = wd_cipher_get_worker(sess); - + worker = wd_cipher_get_worker(sess, req); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_cipher_sync(h_sess, req); if (!ret) { diff --git a/wd_comp.c b/wd_comp.c index d7f4c1ae4..0eb084b49 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -66,7 +66,8 @@ static void wd_comp_switch_worker(struct wd_comp_sess *sess, int para) sess->worker_looptime = 0; } -static struct uadk_adapter_worker *wd_comp_get_worker(struct wd_comp_sess *sess) +static struct uadk_adapter_worker * +wd_comp_get_worker(struct wd_comp_sess *sess, struct wd_comp_req *req) { if (sess->adapter_mode != wd_comp_setting.adapter->mode) { sess->worker = &wd_comp_setting.adapter->workers[0]; @@ -74,6 +75,14 @@ static struct uadk_adapter_worker *wd_comp_get_worker(struct wd_comp_sess *sess) sess->adapter_mode = wd_comp_setting.adapter->mode; } + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_comp_setting.adapter->workers_nb == 2) { + if (req->src_len >= wd_comp_setting.adapter->threshold) + sess->worker = &wd_comp_setting.adapter->workers[0]; + else + sess->worker = &wd_comp_setting.adapter->workers[1]; + } + return sess->worker; } @@ -645,7 +654,7 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, __u32 idx; int ret; - worker = wd_comp_get_worker(sess); + worker = wd_comp_get_worker(sess, req); idx = worker->sched->pick_next_ctx( worker->sched->h_sched_ctx, @@ -911,7 +920,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) return -WD_EINVAL; } - worker = wd_comp_get_worker(sess); + worker = wd_comp_get_worker(sess, req); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_comp_sync(h_sess, req); diff --git a/wd_digest.c b/wd_digest.c index f8b1808f8..7948e0a6e 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -82,7 +82,8 @@ static void wd_digest_switch_worker(struct wd_digest_sess *sess, int para) sess->worker_looptime = 0; } -static struct uadk_adapter_worker *wd_digest_get_worker(struct wd_digest_sess *sess) +static struct uadk_adapter_worker * +wd_digest_get_worker(struct wd_digest_sess *sess, struct wd_digest_req *req) { if (sess->adapter_mode != wd_digest_setting.adapter->mode) { sess->worker = &wd_digest_setting.adapter->workers[0]; @@ -90,6 +91,14 @@ static struct uadk_adapter_worker *wd_digest_get_worker(struct wd_digest_sess *s sess->adapter_mode = wd_digest_setting.adapter->mode; } + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_digest_setting.adapter->workers_nb == 2) { + if (req->in_bytes >= wd_digest_setting.adapter->threshold) + sess->worker = &wd_digest_setting.adapter->workers[0]; + else + sess->worker = &wd_digest_setting.adapter->workers[1]; + } + return sess->worker; } @@ -697,7 +706,7 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) if (unlikely(ret)) return -WD_EINVAL; - worker = wd_digest_get_worker(dsess); + worker = wd_digest_get_worker(dsess, req); memset(&msg, 0, sizeof(struct wd_digest_msg)); fill_request_msg(&msg, req, dsess); @@ -749,7 +758,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return -WD_EINVAL; } - worker = wd_digest_get_worker(dsess); + worker = wd_digest_get_worker(dsess, req); if (worker->driver->mode == UADK_DRV_SYNCONLY) { ret = wd_do_digest_sync(h_sess, req);