diff --git a/sha1_mb/Makefile.am b/sha1_mb/Makefile.am index aa5b5b9..0e73273 100644 --- a/sha1_mb/Makefile.am +++ b/sha1_mb/Makefile.am @@ -77,7 +77,13 @@ lsrc_aarch64 += sha1_mb/sha1_ctx_base.c \ sha1_mb/aarch64/sha1_mb_mgr_asimd.c \ sha1_mb/aarch64/sha1_mb_aarch64_dispatcher.c -lsrc_riscv64 += sha1_mb/sha1_ctx_base_aliases.c \ +lsrc_riscv64 += \ + sha1_mb/riscv64/sha1_mb_multibinary.S \ + sha1_mb/riscv64/sha1_ctx_rvv.c \ + sha1_mb/riscv64/sha1_mb_block_rvv.S \ + sha1_mb/riscv64/sha1_riscv64_x1.S \ + sha1_mb/riscv64/sha1_mb_mgr_rvv.c \ + sha1_mb/riscv64/sha1_mb_riscv64_dispatcher.c \ sha1_mb/sha1_ctx_base.c \ sha1_mb/sha1_ref.c diff --git a/sha1_mb/riscv64/sha1_ctx_rvv.c b/sha1_mb/riscv64/sha1_ctx_rvv.c new file mode 100644 index 0000000..ba48690 --- /dev/null +++ b/sha1_mb/riscv64/sha1_ctx_rvv.c @@ -0,0 +1,266 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include +#include "sha1_mb_internal.h" +#include "memcpy_inline.h" +#include "endian_helper.h" +void +sha1_mb_mgr_init_rvv(ISAL_SHA1_MB_JOB_MGR *state); +ISAL_SHA1_JOB * +sha1_mb_mgr_submit_rvv(ISAL_SHA1_MB_JOB_MGR *state, ISAL_SHA1_JOB *job); +ISAL_SHA1_JOB * +sha1_mb_mgr_flush_rvv(ISAL_SHA1_MB_JOB_MGR *state); +static inline void +hash_init_digest(ISAL_SHA1_WORD_T *digest); +static inline uint32_t +hash_pad(uint8_t padblock[ISAL_SHA1_BLOCK_SIZE * 2], uint64_t total_len); +static ISAL_SHA1_HASH_CTX * +sha1_ctx_mgr_resubmit(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx); + +void +sha1_ctx_mgr_init_rvv(ISAL_SHA1_HASH_CTX_MGR *mgr) +{ + sha1_mb_mgr_init_rvv(&mgr->mgr); +} + +ISAL_SHA1_HASH_CTX * +sha1_ctx_mgr_submit_rvv(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx, const void *buffer, + uint32_t len, ISAL_HASH_CTX_FLAG flags) +{ + if (flags & (~ISAL_HASH_ENTIRE)) { + // User should not pass anything other than FIRST, UPDATE, or LAST + ctx->error = ISAL_HASH_CTX_ERROR_INVALID_FLAGS; + return ctx; + } + + if (ctx->status & ISAL_HASH_CTX_STS_PROCESSING) { + // Cannot submit to a currently processing job. + ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_PROCESSING; + return ctx; + } + + if ((ctx->status & ISAL_HASH_CTX_STS_COMPLETE) && !(flags & ISAL_HASH_FIRST)) { + // Cannot update a finished job. + ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_COMPLETED; + return ctx; + } + + if (flags & ISAL_HASH_FIRST) { + // Init digest + hash_init_digest(ctx->job.result_digest); + + // Reset byte counter + ctx->total_length = 0; + + // Clear extra blocks + ctx->partial_block_buffer_length = 0; + } + // If we made it here, there were no errors during this call to submit + ctx->error = ISAL_HASH_CTX_ERROR_NONE; + + // Store buffer ptr info from user + ctx->incoming_buffer = buffer; + ctx->incoming_buffer_length = len; + + // Store the user's request flags and mark this ctx as currently being processed. + ctx->status = (flags & ISAL_HASH_LAST) ? (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING | + ISAL_HASH_CTX_STS_LAST) + : ISAL_HASH_CTX_STS_PROCESSING; + + // Advance byte counter + ctx->total_length += len; + + // If there is anything currently buffered in the extra blocks, append to it until it + // contains a whole block. Or if the user's buffer contains less than a whole block, append + // as much as possible to the extra block. + if ((ctx->partial_block_buffer_length) | (len < ISAL_SHA1_BLOCK_SIZE)) { + // Compute how many bytes to copy from user buffer into extra block + uint32_t copy_len = ISAL_SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length; + if (len < copy_len) + copy_len = len; + + if (copy_len) { + // Copy and update relevant pointers and counters + memcpy_fixedlen( + &ctx->partial_block_buffer[ctx->partial_block_buffer_length], + buffer, copy_len); + + ctx->partial_block_buffer_length += copy_len; + ctx->incoming_buffer = (const void *) ((const char *) buffer + copy_len); + ctx->incoming_buffer_length = len - copy_len; + } + // The extra block should never contain more than 1 block here + assert(ctx->partial_block_buffer_length <= ISAL_SHA1_BLOCK_SIZE); + + // If the extra block buffer contains exactly 1 block, it can be hashed. + if (ctx->partial_block_buffer_length >= ISAL_SHA1_BLOCK_SIZE) { + ctx->partial_block_buffer_length = 0; + + ctx->job.buffer = ctx->partial_block_buffer; + ctx->job.len = 1; + + ctx = (ISAL_SHA1_HASH_CTX *) sha1_mb_mgr_submit_rvv(&mgr->mgr, &ctx->job); + } + } + + return sha1_ctx_mgr_resubmit(mgr, ctx); +} + +ISAL_SHA1_HASH_CTX * +sha1_ctx_mgr_flush_rvv(ISAL_SHA1_HASH_CTX_MGR *mgr) +{ + ISAL_SHA1_HASH_CTX *ctx; + + while (1) { + ctx = (ISAL_SHA1_HASH_CTX *) sha1_mb_mgr_flush_rvv(&mgr->mgr); + + // If flush returned 0, there are no more jobs in flight. + if (!ctx) + return NULL; + + // If flush returned a job, verify that it is safe to return to the user. + // If it is not ready, resubmit the job to finish processing. + ctx = sha1_ctx_mgr_resubmit(mgr, ctx); + + // If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. + if (ctx) + return ctx; + + // Otherwise, all jobs currently being managed by the ISAL_SHA1_HASH_CTX_MGR + // still need processing. Loop. + } +} + +static ISAL_SHA1_HASH_CTX * +sha1_ctx_mgr_resubmit(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx) +{ + while (ctx) { + + if (ctx->status & ISAL_HASH_CTX_STS_COMPLETE) { + ctx->status = ISAL_HASH_CTX_STS_COMPLETE; // Clear PROCESSING bit + return ctx; + } + // If the extra blocks are empty, begin hashing what remains in the user's buffer. + if (ctx->partial_block_buffer_length == 0 && ctx->incoming_buffer_length) { + const void *buffer = ctx->incoming_buffer; + uint32_t len = ctx->incoming_buffer_length; + + // Only entire blocks can be hashed. Copy remainder to extra blocks buffer. + uint32_t copy_len = len & (ISAL_SHA1_BLOCK_SIZE - 1); + + if (copy_len) { + len -= copy_len; + memcpy_fixedlen(ctx->partial_block_buffer, + ((const char *) buffer + len), copy_len); + ctx->partial_block_buffer_length = copy_len; + } + + ctx->incoming_buffer_length = 0; + + // len should be a multiple of the block size now + assert((len % ISAL_SHA1_BLOCK_SIZE) == 0); + + // Set len to the number of blocks to be hashed in the user's buffer + len >>= ISAL_SHA1_LOG2_BLOCK_SIZE; + + if (len) { + ctx->job.buffer = (uint8_t *) buffer; + ctx->job.len = len; + ctx = (ISAL_SHA1_HASH_CTX *) sha1_mb_mgr_submit_rvv(&mgr->mgr, + &ctx->job); + continue; + } + } + // If the extra blocks are not empty, then we are either on the last block(s) + // or we need more user input before continuing. + if (ctx->status & ISAL_HASH_CTX_STS_LAST) { + uint8_t *buf = ctx->partial_block_buffer; + uint32_t n_extra_blocks = hash_pad(buf, ctx->total_length); + + ctx->status = (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING | + ISAL_HASH_CTX_STS_COMPLETE); + ctx->job.buffer = buf; + ctx->job.len = (uint32_t) n_extra_blocks; + ctx = (ISAL_SHA1_HASH_CTX *) sha1_mb_mgr_submit_rvv(&mgr->mgr, &ctx->job); + continue; + } + + if (ctx) + ctx->status = ISAL_HASH_CTX_STS_IDLE; + return ctx; + } + + return NULL; +} + +static inline void +hash_init_digest(ISAL_SHA1_WORD_T *digest) +{ + static const ISAL_SHA1_WORD_T hash_initial_digest[ISAL_SHA1_DIGEST_NWORDS] = { + ISAL_SHA1_INITIAL_DIGEST + }; + memcpy_fixedlen(digest, hash_initial_digest, sizeof(hash_initial_digest)); +} + +static inline uint32_t +hash_pad(uint8_t padblock[ISAL_SHA1_BLOCK_SIZE * 2], uint64_t total_len) +{ + uint32_t i = (uint32_t) (total_len & (ISAL_SHA1_BLOCK_SIZE - 1)); + + memclr_fixedlen(&padblock[i], ISAL_SHA1_BLOCK_SIZE); + padblock[i] = 0x80; + + // Move i to the end of either 1st or 2nd extra block depending on length + i += ((ISAL_SHA1_BLOCK_SIZE - 1) & (0 - (total_len + ISAL_SHA1_PADLENGTHFIELD_SIZE + 1))) + + 1 + ISAL_SHA1_PADLENGTHFIELD_SIZE; + +#if ISAL_SHA1_PADLENGTHFIELD_SIZE == 16 + *((uint64_t *) &padblock[i - 16]) = 0; +#endif + + *((uint64_t *) &padblock[i - 8]) = to_be64((uint64_t) total_len << 3); + + return i >> ISAL_SHA1_LOG2_BLOCK_SIZE; // Number of extra blocks to hash +} + +struct slver { + uint16_t snum; + uint8_t ver; + uint8_t core; +}; +struct slver sha1_ctx_mgr_init_rvv_slver_02020142; +struct slver sha1_ctx_mgr_init_rvv_slver = { 0x0142, 0x02, 0x02 }; + +struct slver sha1_ctx_mgr_submit_rvv_slver_02020143; +struct slver sha1_ctx_mgr_submit_rvv_slver = { 0x0143, 0x02, 0x02 }; + +struct slver sha1_ctx_mgr_flush_rvv_slver_02020144; +struct slver sha1_ctx_mgr_flush_rvv_slver = { 0x0144, 0x02, 0x02 }; diff --git a/sha1_mb/riscv64/sha1_mb_block_rvv.S b/sha1_mb/riscv64/sha1_mb_block_rvv.S new file mode 100644 index 0000000..25e4619 --- /dev/null +++ b/sha1_mb/riscv64/sha1_mb_block_rvv.S @@ -0,0 +1,451 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#if HAVE_RVV + +#define A v10 +#define B v12 +#define C v14 +#define D v16 +#define E v18 +#define F v20 +#define T v22 +#define TT0 v24 +#define TT1 v26 +#define TT2 v28 +#define TT3 v30 + +#define AA v0 +#define BB v2 +#define CC v4 +#define DD v6 +#define EE v8 + +#define key_table t0 +#define P0 t1 +#define P1 t2 +#define P2 t3 +#define P3 t4 +#define P4 t5 +#define P5 t6 + + +#define loops a1 +#define K1 a4 +#define K2 a5 +#define K3 a6 +#define K4 a7 + +#define inp0 s0 +#define inp1 s1 +#define inp2 s2 +#define inp3 s3 +#define transpose_ptr_inp0 s4 +#define transpose_ptr_inp1 s5 +#define transpose_ptr_inp2 s6 +#define transpose_ptr_inp3 s7 +#define transpose_ptr_out1 s8 +#define transpose_ptr_out2 s9 +#define transpose_ptr_out3 s10 +#define ptrdiff_t s11 + +#define FRAMESZ 16*16 +#define SZ 4 +#define SZ4 4*SZ +#define _data_ptr 0x140 + +.macro BACKUP_STACKS + addi sp, sp, -96 + + sd s0, 0(sp) + sd s1, 8(sp) + sd s2, 16(sp) + sd s3, 24(sp) + sd s4, 32(sp) + sd s5, 40(sp) + sd s6, 48(sp) + sd s7, 56(sp) + sd s8, 64(sp) + sd s9, 72(sp) + sd s10, 80(sp) + sd s11, 88(sp) +.endm + +.macro RESTORE_STACKS + ld s0, 0(sp) + ld s1, 8(sp) + ld s2, 16(sp) + ld s3, 24(sp) + ld s4, 32(sp) + ld s5, 40(sp) + ld s6, 48(sp) + ld s7, 56(sp) + ld s8, 64(sp) + ld s9, 72(sp) + ld s10, 80(sp) + ld s11, 88(sp) + + addi sp, sp, 96 +.endm + +.macro TRANSPOSE R0, R1, R2, R3 + vse32.v \R0, (transpose_ptr_inp0) + vse32.v \R1, (transpose_ptr_inp1) + vse32.v \R2, (transpose_ptr_inp2) + vse32.v \R3, (transpose_ptr_inp3) + vlse32.v \R0, (transpose_ptr_inp0), ptrdiff_t + vlse32.v \R1, (transpose_ptr_out1), ptrdiff_t + vlse32.v \R2, (transpose_ptr_out2), ptrdiff_t + vlse32.v \R3, (transpose_ptr_out3), ptrdiff_t +.endm + +.macro MAGIC_F0 B, C, D + vxor.vv F, \C, \D + vand.vv F, F, \B + vxor.vv F, F, \D +.endm + +.macro MAGIC_F1 B, C, D + vxor.vv F, \D, \C + vxor.vv F, F, \B +.endm + +.macro MAGIC_F2 B, C, D + vor.vv F, \B, \C + vand.vv T, \B, \C + vand.vv F, F, \D + vor.vv F, F, T +.endm + +.macro MAGIC_F3 B, C, D + MAGIC_F1 \B, \C, \D +.endm + +.macro PROLD reg, imm, tmp + vsrl.vi \tmp, \reg, (32-(\imm)) + vsll.vi \reg, \reg, \imm + vor.vv \reg, \reg, \tmp +.endm + +.macro PROLD_nd reg, imm, tmp, src + vsrl.vi \tmp, \src, (32-(\imm)) + vsll.vi \reg, \src, \imm + vor.vv \reg, \reg, \tmp +.endm + +.macro SHA1_STEP_00_15 A, B, C, D, E, memW, key, MAGIC + vmv.v.x TT0, \key + vadd.vv \E, \E, TT0 + + addi P0, sp, SZ4*(\memW * 1) + vle32.v TT0, (P0) + vadd.vv \E, \E, TT0 + PROLD_nd T, 5, F, \A + vadd.vv \E, \E, T + \MAGIC \B, \C, \D + PROLD \B, 30, T + vadd.vv \E, \E, F +.endm + +.macro SHA1_STEP_16_79 A, B, C, D, E, memW, key, MAGIC, TMP1, TMP2, TMP3 + vmv.v.x TT0, \key + vadd.vv \E, \E, TT0 + + addi P0, sp, SZ4*((\memW - 14) & 15) + addi P1, sp, SZ4*((\memW - 8) & 15) + addi P2, sp, SZ4*((\memW - 3) & 15) + addi P3, sp, SZ4*((\memW - 0) & 15) + + vle32.v \TMP1, (P0) + vle32.v TT0, (P1) + vle32.v T, (P2) + + vxor.vv \TMP3, \TMP3, \TMP1 + vxor.vv \TMP3, \TMP3, TT0 + vxor.vv \TMP3, \TMP3, T + + vsrl.vi F, \TMP3, 31 + vsll.vi \TMP3, \TMP3, 1 + vor.vv F, F, \TMP3 + + vse32.v F, (P3) + vadd.vv \E, \E, F + + PROLD_nd T, 5, F, \A + vadd.vv \E, \E, T + \MAGIC \B, \C, \D + PROLD \B, 30, T + vadd.vv \E, \E, F +.endm + + .option arch, +v + .global sha1_mb_rvv_x4 + .type sha1_mb_rvv_x4, %function + +sha1_mb_rvv_x4: + + beqz loops, .done + + BACKUP_STACKS + + addi sp, sp, -(FRAMESZ+128) + + li K1, 0x5A827999 + li K2, 0x6ED9EBA1 + li K3, 0x8F1BBCDC + li K4, 0xCA62C1D6 + + la key_table, GATHER_PATTERN + + vsetivli zero, 4, e32, m1, ta, ma + + addi P1, a0, 64*1 + addi P2, a0, 64*2 + addi P3, a0, 64*3 + addi P4, a0, 64*4 + + vle32.v A, (a0) + vle32.v B, (P1) + vle32.v C, (P2) + vle32.v D, (P3) + vle32.v E, (P4) + + ld inp0, _data_ptr + 0*8(a0) + ld inp1, _data_ptr + 1*8(a0) + ld inp2, _data_ptr + 2*8(a0) + ld inp3, _data_ptr + 3*8(a0) + + # add sp for transpose data pointer + addi transpose_ptr_inp0, sp, FRAMESZ + 0*16 + addi transpose_ptr_inp1, sp, FRAMESZ + 1*16 + addi transpose_ptr_inp2, sp, FRAMESZ + 2*16 + addi transpose_ptr_inp3, sp, FRAMESZ + 3*16 + addi transpose_ptr_out1, sp, FRAMESZ + 1*4 + addi transpose_ptr_out2, sp, FRAMESZ + 2*4 + addi transpose_ptr_out3, sp, FRAMESZ + 3*4 + li ptrdiff_t, 16 + +.block_loop: + + vsetivli zero, 16, e8, m1, ta, ma + vle8.v EE, (key_table) + +.set I, 0 +.rept 4 + + vsetivli zero, 4, e32, m1, ta, ma + + # Load each lane one at a time, store to transpose scratch area + vle32.v AA, (inp0) + addi inp0, inp0, 16 + vse32.v AA, (transpose_ptr_inp0) + + vle32.v AA, (inp1) + addi inp1, inp1, 16 + vse32.v AA, (transpose_ptr_inp1) + + vle32.v AA, (inp2) + addi inp2, inp2, 16 + vse32.v AA, (transpose_ptr_inp2) + + vle32.v AA, (inp3) + addi inp3, inp3, 16 + vse32.v AA, (transpose_ptr_inp3) + + # Strided loads for 4x4 transpose + vlse32.v AA, (transpose_ptr_inp0), ptrdiff_t + vlse32.v BB, (transpose_ptr_out1), ptrdiff_t + vlse32.v CC, (transpose_ptr_out2), ptrdiff_t + vlse32.v DD, (transpose_ptr_out3), ptrdiff_t + + # Byte-swap (big-endian to little-endian) + vsetivli zero, 16, e8, m1, ta, ma + + vrgather.vv TT0, AA, EE + vrgather.vv TT1, BB, EE + vrgather.vv TT2, CC, EE + vrgather.vv TT3, DD, EE + + addi P0, sp, (I*4+0)*16 + addi P1, sp, (I*4+1)*16 + addi P2, sp, (I*4+2)*16 + addi P3, sp, (I*4+3)*16 + + vse8.v TT0, (P0) + vse8.v TT1, (P1) + vse8.v TT2, (P2) + vse8.v TT3, (P3) + +.set I, I+1 +.endr + vsetivli zero, 4, e32, m1, ta, ma + +.segs_loop: + + vmv.v.v AA, A + vmv.v.v BB, B + vmv.v.v CC, C + vmv.v.v DD, D + vmv.v.v EE, E + + SHA1_STEP_00_15 A, B, C, D, E, 0, K1, MAGIC_F0 + SHA1_STEP_00_15 E, A, B, C, D, 1, K1, MAGIC_F0 + SHA1_STEP_00_15 D, E, A, B, C, 2, K1, MAGIC_F0 + SHA1_STEP_00_15 C, D, E, A, B, 3, K1, MAGIC_F0 + SHA1_STEP_00_15 B, C, D, E, A, 4, K1, MAGIC_F0 + SHA1_STEP_00_15 A, B, C, D, E, 5, K1, MAGIC_F0 + SHA1_STEP_00_15 E, A, B, C, D, 6, K1, MAGIC_F0 + SHA1_STEP_00_15 D, E, A, B, C, 7, K1, MAGIC_F0 + SHA1_STEP_00_15 C, D, E, A, B, 8, K1, MAGIC_F0 + SHA1_STEP_00_15 B, C, D, E, A, 9, K1, MAGIC_F0 + SHA1_STEP_00_15 A, B, C, D, E, 10, K1, MAGIC_F0 + SHA1_STEP_00_15 E, A, B, C, D, 11, K1, MAGIC_F0 + SHA1_STEP_00_15 D, E, A, B, C, 12, K1, MAGIC_F0 + SHA1_STEP_00_15 C, D, E, A, B, 13, K1, MAGIC_F0 + SHA1_STEP_00_15 B, C, D, E, A, 14, K1, MAGIC_F0 + SHA1_STEP_00_15 A, B, C, D, E, 15, K1, MAGIC_F0 + + addi P0, sp, ((16 - 16) & 15) * 16 + addi P1, sp, ((16 - 15) & 15) * 16 + vle32.v TT3, (P0) + vle32.v TT2, (P1) + + SHA1_STEP_16_79 E, A, B, C, D, 16, K1, MAGIC_F0, TT1, TT2, TT3 + SHA1_STEP_16_79 D, E, A, B, C, 17, K1, MAGIC_F0, TT3, TT1, TT2 + SHA1_STEP_16_79 C, D, E, A, B, 18, K1, MAGIC_F0, TT2, TT3, TT1 + SHA1_STEP_16_79 B, C, D, E, A, 19, K1, MAGIC_F0, TT1, TT2, TT3 + + SHA1_STEP_16_79 A, B, C, D, E, 20, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 E, A, B, C, D, 21, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 D, E, A, B, C, 22, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 C, D, E, A, B, 23, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 B, C, D, E, A, 24, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 A, B, C, D, E, 25, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 E, A, B, C, D, 26, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 D, E, A, B, C, 27, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 C, D, E, A, B, 28, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 B, C, D, E, A, 29, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 A, B, C, D, E, 30, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 E, A, B, C, D, 31, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 D, E, A, B, C, 32, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 C, D, E, A, B, 33, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 B, C, D, E, A, 34, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 A, B, C, D, E, 35, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 E, A, B, C, D, 36, K2, MAGIC_F1, TT2, TT3, TT1 + SHA1_STEP_16_79 D, E, A, B, C, 37, K2, MAGIC_F1, TT1, TT2, TT3 + SHA1_STEP_16_79 C, D, E, A, B, 38, K2, MAGIC_F1, TT3, TT1, TT2 + SHA1_STEP_16_79 B, C, D, E, A, 39, K2, MAGIC_F1, TT2, TT3, TT1 + + SHA1_STEP_16_79 A, B, C, D, E, 40, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 E, A, B, C, D, 41, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 D, E, A, B, C, 42, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 C, D, E, A, B, 43, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 B, C, D, E, A, 44, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 A, B, C, D, E, 45, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 E, A, B, C, D, 46, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 D, E, A, B, C, 47, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 C, D, E, A, B, 48, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 B, C, D, E, A, 49, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 A, B, C, D, E, 50, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 E, A, B, C, D, 51, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 D, E, A, B, C, 52, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 C, D, E, A, B, 53, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 B, C, D, E, A, 54, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 A, B, C, D, E, 55, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 E, A, B, C, D, 56, K3, MAGIC_F2, TT3, TT1, TT2 + SHA1_STEP_16_79 D, E, A, B, C, 57, K3, MAGIC_F2, TT2, TT3, TT1 + SHA1_STEP_16_79 C, D, E, A, B, 58, K3, MAGIC_F2, TT1, TT2, TT3 + SHA1_STEP_16_79 B, C, D, E, A, 59, K3, MAGIC_F2, TT3, TT1, TT2 + + SHA1_STEP_16_79 A, B, C, D, E, 60, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 E, A, B, C, D, 61, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 D, E, A, B, C, 62, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 C, D, E, A, B, 63, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 B, C, D, E, A, 64, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 A, B, C, D, E, 65, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 E, A, B, C, D, 66, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 D, E, A, B, C, 67, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 C, D, E, A, B, 68, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 B, C, D, E, A, 69, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 A, B, C, D, E, 70, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 E, A, B, C, D, 71, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 D, E, A, B, C, 72, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 C, D, E, A, B, 73, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 B, C, D, E, A, 74, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 A, B, C, D, E, 75, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 E, A, B, C, D, 76, K4, MAGIC_F3, TT1, TT2, TT3 + SHA1_STEP_16_79 D, E, A, B, C, 77, K4, MAGIC_F3, TT3, TT1, TT2 + SHA1_STEP_16_79 C, D, E, A, B, 78, K4, MAGIC_F3, TT2, TT3, TT1 + SHA1_STEP_16_79 B, C, D, E, A, 79, K4, MAGIC_F3, TT1, TT2, TT3 + + # add old digest + # write out digests + vadd.vv A, A, AA + vadd.vv B, B, BB + vadd.vv C, C, CC + vadd.vv D, D, DD + vadd.vv E, E, EE + + addi loops, loops, -1 + bnez loops, .block_loop + + addi P0, a0, 0*64 + addi P1, a0, 1*64 + addi P2, a0, 2*64 + vse32.v A, (P0) + vse32.v B, (P1) + vse32.v C, (P2) + + addi P1, a0, 3*64 + addi P2, a0, 4*64 + vse32.v D, (P1) + vse32.v E, (P2) + + sd inp0, _data_ptr + 0*8(a0) + sd inp1, _data_ptr + 1*8(a0) + sd inp2, _data_ptr + 2*8(a0) + sd inp3, _data_ptr + 3*8(a0) + + addi sp, sp, (FRAMESZ+128) + + RESTORE_STACKS + +.done: + ret + + .size sha1_mb_rvv_x4, .-sha1_mb_rvv_x4 + + .section .rodata + .align 4 + + +GATHER_PATTERN: + .quad 0x0405060700010203, 0x0c0d0e0f08090a0b + +#endif diff --git a/sha1_mb/riscv64/sha1_mb_mgr_rvv.c b/sha1_mb/riscv64/sha1_mb_mgr_rvv.c new file mode 100644 index 0000000..76ca9b0 --- /dev/null +++ b/sha1_mb/riscv64/sha1_mb_mgr_rvv.c @@ -0,0 +1,233 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include +#include +#include +#include "endian_helper.h" + +extern void +sha1_riscv64_x1(const uint8_t *data, int num_blocks, uint32_t digest[]); + +static inline void +sha1_job_x1(ISAL_SHA1_JOB *job, int blocks) +{ + sha1_riscv64_x1(job->buffer, blocks, job->result_digest); +} + +#ifndef min +#define min(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +#define SHA1_MB_RVV_MAX_LANES 4 +void +sha1_mb_rvv_x4(ISAL_SHA1_MB_ARGS_X16 *, int); + +#define LANE_IS_NOT_FINISHED(state, i) \ + (((state->lens[i] & (~0xf)) != 0) && state->ldata[i].job_in_lane != NULL) +#define LANE_IS_FINISHED(state, i) \ + (((state->lens[i] & (~0xf)) == 0) && state->ldata[i].job_in_lane != NULL) +#define LANE_IS_FREE(state, i) \ + (((state->lens[i] & (~0xf)) == 0) && state->ldata[i].job_in_lane == NULL) +#define LANE_IS_INVALID(state, i) \ + (((state->lens[i] & (~0xf)) != 0) && state->ldata[i].job_in_lane == NULL) + +void +sha1_mb_mgr_init_rvv(ISAL_SHA1_MB_JOB_MGR *state) +{ + unsigned int i; + + state->unused_lanes = 0xf; + state->num_lanes_inuse = 0; + for (i = 0; i < SHA1_MB_RVV_MAX_LANES; i++) { + state->unused_lanes <<= 4; + state->unused_lanes |= SHA1_MB_RVV_MAX_LANES - 1 - i; + state->lens[i] = i; + state->ldata[i].job_in_lane = 0; + } + + // lanes > SHA1_MB_RVV_MAX_LANES is invalid lane + for (; i < ISAL_SHA1_MAX_LANES; i++) { + state->lens[i] = 0xf; + state->ldata[i].job_in_lane = 0; + } +} + +static void +sha1_mb_mgr_run_rvv(ISAL_SHA1_MB_JOB_MGR *state, int lane_idx_array[], int lanes, int blocks) +{ + int j, k; + + /* Marshal: copy digest and data_ptr from individual jobs into state->args */ + for (j = 0; j < lanes; j++) { + ISAL_SHA1_JOB *job = state->ldata[lane_idx_array[j]].job_in_lane; + for (k = 0; k < ISAL_SHA1_DIGEST_NWORDS; k++) + state->args.digest[k][j] = job->result_digest[k]; + state->args.data_ptr[j] = job->buffer; + } + /* Fill remaining lanes (up to 4) with dummy data from lane 0 */ + for (; j < SHA1_MB_RVV_MAX_LANES; j++) { + for (k = 0; k < ISAL_SHA1_DIGEST_NWORDS; k++) + state->args.digest[k][j] = state->args.digest[k][0]; + state->args.data_ptr[j] = state->args.data_ptr[0]; + } + + sha1_mb_rvv_x4(&state->args, blocks); + + /* Unmarshal: copy digest back from state->args to individual jobs */ + for (j = 0; j < lanes; j++) { + ISAL_SHA1_JOB *job = state->ldata[lane_idx_array[j]].job_in_lane; + for (k = 0; k < ISAL_SHA1_DIGEST_NWORDS; k++) + job->result_digest[k] = state->args.digest[k][j]; + } +} + +static int +sha1_mb_mgr_do_jobs(ISAL_SHA1_MB_JOB_MGR *state) +{ + int lane_idx, len, i, lanes, blocks; + int lane_idx_array[ISAL_SHA1_MAX_LANES]; + + if (state->num_lanes_inuse == 0) { + return -1; + } + lanes = 0, len = 0; + for (i = 0; i < ISAL_SHA1_MAX_LANES && lanes < state->num_lanes_inuse; i++) { + if (LANE_IS_NOT_FINISHED(state, i)) { + if (lanes) + len = min(len, state->lens[i]); + else + len = state->lens[i]; + lane_idx_array[lanes] = i; + lanes++; + } + } + + if (lanes == 0) + return -1; + lane_idx = len & 0xf; + len = len & (~0xf); + blocks = len >> 4; + + if (lanes >= 3) { + sha1_mb_mgr_run_rvv(state, lane_idx_array, lanes, blocks); + } else { + sha1_job_x1(state->ldata[lane_idx_array[0]].job_in_lane, blocks); + if (lanes >= 2) { + sha1_job_x1(state->ldata[lane_idx_array[1]].job_in_lane, blocks); + } + } + + // only return the min length job + for (i = 0; i < ISAL_SHA1_MAX_LANES; i++) { + if (LANE_IS_NOT_FINISHED(state, i)) { + state->lens[i] -= len; + state->ldata[i].job_in_lane->len -= len; + state->ldata[i].job_in_lane->buffer += len << 2; + } + } + return lane_idx; +} + +static ISAL_SHA1_JOB * +sha1_mb_mgr_free_lane(ISAL_SHA1_MB_JOB_MGR *state) +{ + int i; + ISAL_SHA1_JOB *ret = NULL; + + for (i = 0; i < SHA1_MB_RVV_MAX_LANES; i++) { + if (LANE_IS_FINISHED(state, i)) { + state->unused_lanes <<= 4; + state->unused_lanes |= i; + state->num_lanes_inuse--; + ret = state->ldata[i].job_in_lane; + ret->status = ISAL_STS_COMPLETED; + state->ldata[i].job_in_lane = NULL; + break; + } + } + return ret; +} + +static void +sha1_mb_mgr_insert_job(ISAL_SHA1_MB_JOB_MGR *state, ISAL_SHA1_JOB *job) +{ + int lane_idx; + // add job into lanes + lane_idx = state->unused_lanes & 0xf; + // fatal error + assert(lane_idx < SHA1_MB_RVV_MAX_LANES); + state->lens[lane_idx] = (job->len << 4) | lane_idx; + state->ldata[lane_idx].job_in_lane = job; + state->unused_lanes >>= 4; + state->num_lanes_inuse++; +} + +ISAL_SHA1_JOB * +sha1_mb_mgr_submit_rvv(ISAL_SHA1_MB_JOB_MGR *state, ISAL_SHA1_JOB *job) +{ +#ifndef NDEBUG + int lane_idx; +#endif + ISAL_SHA1_JOB *ret; + + // add job into lanes + sha1_mb_mgr_insert_job(state, job); + + ret = sha1_mb_mgr_free_lane(state); + if (ret != NULL) { + return ret; + } + // submit will wait all lane has data + if (state->num_lanes_inuse < SHA1_MB_RVV_MAX_LANES) + return NULL; +#ifndef NDEBUG + lane_idx = sha1_mb_mgr_do_jobs(state); + assert(lane_idx != -1); +#else + sha1_mb_mgr_do_jobs(state); +#endif + + // ~ i = lane_idx; + ret = sha1_mb_mgr_free_lane(state); + return ret; +} + +ISAL_SHA1_JOB * +sha1_mb_mgr_flush_rvv(ISAL_SHA1_MB_JOB_MGR *state) +{ + ISAL_SHA1_JOB *ret; + ret = sha1_mb_mgr_free_lane(state); + if (ret) { + return ret; + } + + sha1_mb_mgr_do_jobs(state); + return sha1_mb_mgr_free_lane(state); +} diff --git a/sha1_mb/riscv64/sha1_mb_multibinary.S b/sha1_mb/riscv64/sha1_mb_multibinary.S new file mode 100644 index 0000000..f3c97c6 --- /dev/null +++ b/sha1_mb/riscv64/sha1_mb_multibinary.S @@ -0,0 +1,34 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include "riscv64_multibinary.h" + +mbin_interface _sha1_ctx_mgr_submit +mbin_interface _sha1_ctx_mgr_init +mbin_interface _sha1_ctx_mgr_flush diff --git a/sha1_mb/riscv64/sha1_mb_riscv64_dispatcher.c b/sha1_mb/riscv64/sha1_mb_riscv64_dispatcher.c new file mode 100644 index 0000000..401b75f --- /dev/null +++ b/sha1_mb/riscv64/sha1_mb_riscv64_dispatcher.c @@ -0,0 +1,60 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include + +DEFINE_INTERFACE_DISPATCHER(_sha1_ctx_mgr_submit) +{ +#if HAVE_RVV + const unsigned long hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_RV('V')) + return PROVIDER_INFO(sha1_ctx_mgr_submit_rvv); +#endif + return PROVIDER_BASIC(_sha1_ctx_mgr_submit); +} + +DEFINE_INTERFACE_DISPATCHER(_sha1_ctx_mgr_init) +{ +#if HAVE_RVV + const unsigned long hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_RV('V')) + return PROVIDER_INFO(sha1_ctx_mgr_init_rvv); +#endif + return PROVIDER_BASIC(_sha1_ctx_mgr_init); +} + +DEFINE_INTERFACE_DISPATCHER(_sha1_ctx_mgr_flush) +{ +#if HAVE_RVV + const unsigned long hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_RV('V')) + return PROVIDER_INFO(sha1_ctx_mgr_flush_rvv); +#endif + return PROVIDER_BASIC(_sha1_ctx_mgr_flush); +} diff --git a/sha1_mb/riscv64/sha1_riscv64_x1.S b/sha1_mb/riscv64/sha1_riscv64_x1.S new file mode 100644 index 0000000..c7e07e4 --- /dev/null +++ b/sha1_mb/riscv64/sha1_riscv64_x1.S @@ -0,0 +1,314 @@ +/********************************************************************** + Copyright (c) 2026 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#define WA s0 +#define WB s1 +#define WC s2 +#define WD s3 +#define WE s4 + +#define AA s5 +#define BB s6 +#define CC s7 +#define DD s8 +#define EE s9 + +#define DGST s10 + +#define WK t3 + +#define T0 t0 +#define T1 t1 +#define T2 t2 +#define T4 t4 +#define T5 t5 + +#define K1 0x5A827999 +#define K2 0x6ED9EBA1 +#define K3 0x8F1BBCDC +#define K4 0xCA62C1D6 + +#define FRAMESZ 64 +#define FRAMESAVES 64 + +.macro BACKUP_STACKS + addi sp, sp, -(FRAMESZ + 12*8) + sd ra, FRAMESAVES+ 0(sp) + sd s0, FRAMESAVES+ 8(sp) + sd s1, FRAMESAVES+16(sp) + sd s2, FRAMESAVES+24(sp) + sd s3, FRAMESAVES+32(sp) + sd s4, FRAMESAVES+40(sp) + sd s5, FRAMESAVES+48(sp) + sd s6, FRAMESAVES+56(sp) + sd s7, FRAMESAVES+64(sp) + sd s8, FRAMESAVES+72(sp) + sd s9, FRAMESAVES+80(sp) + sd s10, FRAMESAVES+88(sp) +.endm + +.macro RESTORE_STACKS + ld ra, FRAMESAVES+ 0(sp) + ld s0, FRAMESAVES+ 8(sp) + ld s1, FRAMESAVES+16(sp) + ld s2, FRAMESAVES+24(sp) + ld s3, FRAMESAVES+32(sp) + ld s4, FRAMESAVES+40(sp) + ld s5, FRAMESAVES+48(sp) + ld s6, FRAMESAVES+56(sp) + ld s7, FRAMESAVES+64(sp) + ld s8, FRAMESAVES+72(sp) + ld s9, FRAMESAVES+80(sp) + ld s10, FRAMESAVES+88(sp) + addi sp, sp, (FRAMESZ + 12*8) +.endm + +.macro BSWAP reg + rev8 \reg, \reg + srli \reg, \reg, 32 +.endm + +.macro ROL1 reg + roriw \reg, \reg, 31 +.endm + +# F0 = D ^ (B & (C ^ D)) +.macro MAGIC_F0 + xor T0, WC, WD + and T0, WB, T0 + xor T0, WD, T0 +.endm + +# F1 = B ^ C ^ D +.macro MAGIC_F1 + xor T0, WB, WC + xor T0, T0, WD +.endm + +# F2 = (B & C) | (D & (B | C)) +.macro MAGIC_F2 + and T0, WB, WC + or T1, WB, WC + and T1, WD, T1 + or T0, T0, T1 +.endm + +.macro MAGIC_F3 + MAGIC_F1 +.endm + +.macro SHA1_ROUND_BODY + # T2 = ROL5(WA) + roriw T2, WA, 27 + + # T2 = ROL5(A) + E + K + W[i] + F + addw T2, T2, WE + addw T2, T2, WK + addw T2, T2, T4 + addw T2, T2, T0 + + # T0 = ROL30(WB) = new C + roriw T0, WB, 2 + + # rotate state + mv WE, WD + mv WD, WC + mv WC, T0 + mv WB, WA + mv WA, T2 +.endm + +.macro SHA1_STEP_00_15 round, MAGIC + lwu T4, ((\round) * 4)(a0) + BSWAP T4 + sw T4, ((\round & 15) * 4)(sp) + + \MAGIC + SHA1_ROUND_BODY +.endm + +.macro SHA1_STEP_16_79 round, MAGIC + lwu T4, (((\round - 16) & 15) * 4)(sp) + lwu T5, (((\round - 14) & 15) * 4)(sp) + xor T4, T4, T5 + lwu T5, (((\round - 8) & 15) * 4)(sp) + xor T4, T4, T5 + lwu T5, (((\round - 3) & 15) * 4)(sp) + xor T4, T4, T5 + ROL1 T4 + sw T4, ((\round & 15) * 4)(sp) + + \MAGIC + SHA1_ROUND_BODY +.endm + + .text + .option arch, +zbb + .global sha1_riscv64_x1 + .type sha1_riscv64_x1, %function + +sha1_riscv64_x1: + + beqz a1, .done + + BACKUP_STACKS + + mv DGST, a2 + + lwu WA, 0(a2) + lwu WB, 4(a2) + lwu WC, 8(a2) + lwu WD, 12(a2) + lwu WE, 16(a2) + +.block_loop: + mv AA, WA + mv BB, WB + mv CC, WC + mv DD, WD + mv EE, WE + + # rounds 0-19 + li WK, K1 + SHA1_STEP_00_15 0, MAGIC_F0 + SHA1_STEP_00_15 1, MAGIC_F0 + SHA1_STEP_00_15 2, MAGIC_F0 + SHA1_STEP_00_15 3, MAGIC_F0 + SHA1_STEP_00_15 4, MAGIC_F0 + SHA1_STEP_00_15 5, MAGIC_F0 + SHA1_STEP_00_15 6, MAGIC_F0 + SHA1_STEP_00_15 7, MAGIC_F0 + SHA1_STEP_00_15 8, MAGIC_F0 + SHA1_STEP_00_15 9, MAGIC_F0 + SHA1_STEP_00_15 10, MAGIC_F0 + SHA1_STEP_00_15 11, MAGIC_F0 + SHA1_STEP_00_15 12, MAGIC_F0 + SHA1_STEP_00_15 13, MAGIC_F0 + SHA1_STEP_00_15 14, MAGIC_F0 + SHA1_STEP_00_15 15, MAGIC_F0 + + addi a0, a0, 64 + + SHA1_STEP_16_79 16, MAGIC_F0 + SHA1_STEP_16_79 17, MAGIC_F0 + SHA1_STEP_16_79 18, MAGIC_F0 + SHA1_STEP_16_79 19, MAGIC_F0 + + # rounds 20-39 + li WK, K2 + SHA1_STEP_16_79 20, MAGIC_F1 + SHA1_STEP_16_79 21, MAGIC_F1 + SHA1_STEP_16_79 22, MAGIC_F1 + SHA1_STEP_16_79 23, MAGIC_F1 + SHA1_STEP_16_79 24, MAGIC_F1 + SHA1_STEP_16_79 25, MAGIC_F1 + SHA1_STEP_16_79 26, MAGIC_F1 + SHA1_STEP_16_79 27, MAGIC_F1 + SHA1_STEP_16_79 28, MAGIC_F1 + SHA1_STEP_16_79 29, MAGIC_F1 + SHA1_STEP_16_79 30, MAGIC_F1 + SHA1_STEP_16_79 31, MAGIC_F1 + SHA1_STEP_16_79 32, MAGIC_F1 + SHA1_STEP_16_79 33, MAGIC_F1 + SHA1_STEP_16_79 34, MAGIC_F1 + SHA1_STEP_16_79 35, MAGIC_F1 + SHA1_STEP_16_79 36, MAGIC_F1 + SHA1_STEP_16_79 37, MAGIC_F1 + SHA1_STEP_16_79 38, MAGIC_F1 + SHA1_STEP_16_79 39, MAGIC_F1 + + # rounds 40-59 + li WK, K3 + SHA1_STEP_16_79 40, MAGIC_F2 + SHA1_STEP_16_79 41, MAGIC_F2 + SHA1_STEP_16_79 42, MAGIC_F2 + SHA1_STEP_16_79 43, MAGIC_F2 + SHA1_STEP_16_79 44, MAGIC_F2 + SHA1_STEP_16_79 45, MAGIC_F2 + SHA1_STEP_16_79 46, MAGIC_F2 + SHA1_STEP_16_79 47, MAGIC_F2 + SHA1_STEP_16_79 48, MAGIC_F2 + SHA1_STEP_16_79 49, MAGIC_F2 + SHA1_STEP_16_79 50, MAGIC_F2 + SHA1_STEP_16_79 51, MAGIC_F2 + SHA1_STEP_16_79 52, MAGIC_F2 + SHA1_STEP_16_79 53, MAGIC_F2 + SHA1_STEP_16_79 54, MAGIC_F2 + SHA1_STEP_16_79 55, MAGIC_F2 + SHA1_STEP_16_79 56, MAGIC_F2 + SHA1_STEP_16_79 57, MAGIC_F2 + SHA1_STEP_16_79 58, MAGIC_F2 + SHA1_STEP_16_79 59, MAGIC_F2 + + # rounds 60-79 + li WK, K4 + SHA1_STEP_16_79 60, MAGIC_F3 + SHA1_STEP_16_79 61, MAGIC_F3 + SHA1_STEP_16_79 62, MAGIC_F3 + SHA1_STEP_16_79 63, MAGIC_F3 + SHA1_STEP_16_79 64, MAGIC_F3 + SHA1_STEP_16_79 65, MAGIC_F3 + SHA1_STEP_16_79 66, MAGIC_F3 + SHA1_STEP_16_79 67, MAGIC_F3 + SHA1_STEP_16_79 68, MAGIC_F3 + SHA1_STEP_16_79 69, MAGIC_F3 + SHA1_STEP_16_79 70, MAGIC_F3 + SHA1_STEP_16_79 71, MAGIC_F3 + SHA1_STEP_16_79 72, MAGIC_F3 + SHA1_STEP_16_79 73, MAGIC_F3 + SHA1_STEP_16_79 74, MAGIC_F3 + SHA1_STEP_16_79 75, MAGIC_F3 + SHA1_STEP_16_79 76, MAGIC_F3 + SHA1_STEP_16_79 77, MAGIC_F3 + SHA1_STEP_16_79 78, MAGIC_F3 + SHA1_STEP_16_79 79, MAGIC_F3 + + # add old digest + addw WA, WA, AA + addw WB, WB, BB + addw WC, WC, CC + addw WD, WD, DD + addw WE, WE, EE + + addi a1, a1, -1 + bnez a1, .block_loop + + # write out digest + sw WA, 0(DGST) + sw WB, 4(DGST) + sw WC, 8(DGST) + sw WD, 12(DGST) + sw WE, 16(DGST) + + RESTORE_STACKS + +.done: + ret + + .size sha1_riscv64_x1, .-sha1_riscv64_x1