Skip to content

Commit

Permalink
8339910: RISC-V: crc32 intrinsic with carry-less multiplication
Browse files Browse the repository at this point in the history 
Reviewed-by: rehn, luhenry
  • Loading branch information
Hamlin Li committed Dec 7, 2024
1 parent e0d6398 commit c517ffb
Show file tree
Hide file tree
Showing 7 changed files with 407 additions and 3 deletions.
1 change: 1 addition & 0 deletions src/hotspot/cpu/riscv/globals_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Use Zihintpause instructions") \
product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \
product(bool, UseZvbb, false, EXPERIMENTAL, "Use Zvbb instructions") \
product(bool, UseZvbc, false, EXPERIMENTAL, "Use Zvbc instructions") \
product(bool, UseZvfh, false, DIAGNOSTIC, "Use Zvfh instructions") \
product(bool, UseZvkn, false, EXPERIMENTAL, \
"Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt") \
Expand Down
365 changes: 363 additions & 2 deletions src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,6 +1712,359 @@ void MacroAssembler::vector_update_crc32(Register crc, Register buf, Register le
addi(buf, buf, N*4);
}
}

void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16(VectorRegister vx, VectorRegister vt,
VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
Register buf, Register tmp, const int STEP) {
assert_different_registers(vx, vt, vtmp1, vtmp2, vtmp3, vtmp4);
vclmul_vv(vtmp1, vx, vt);
vclmulh_vv(vtmp2, vx, vt);
vle64_v(vtmp4, buf); addi(buf, buf, STEP);
// low parts
vredxor_vs(vtmp3, vtmp1, vtmp4);
// high parts
vslidedown_vi(vx, vtmp4, 1);
vredxor_vs(vtmp1, vtmp2, vx);
// merge low and high back
vslideup_vi(vx, vtmp1, 1);
vmv_x_s(tmp, vtmp3);
vmv_s_x(vx, tmp);
}

void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16_2(VectorRegister vx, VectorRegister vy, VectorRegister vt,
VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
Register tmp) {
assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
vclmul_vv(vtmp1, vx, vt);
vclmulh_vv(vtmp2, vx, vt);
// low parts
vredxor_vs(vtmp3, vtmp1, vy);
// high parts
vslidedown_vi(vtmp4, vy, 1);
vredxor_vs(vtmp1, vtmp2, vtmp4);
// merge low and high back
vslideup_vi(vx, vtmp1, 1);
vmv_x_s(tmp, vtmp3);
vmv_s_x(vx, tmp);
}

void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16_3(VectorRegister vx, VectorRegister vy, VectorRegister vt,
VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
Register tmp) {
assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
vclmul_vv(vtmp1, vx, vt);
vclmulh_vv(vtmp2, vx, vt);
// low parts
vredxor_vs(vtmp3, vtmp1, vy);
// high parts
vslidedown_vi(vtmp4, vy, 1);
vredxor_vs(vtmp1, vtmp2, vtmp4);
// merge low and high back
vslideup_vi(vy, vtmp1, 1);
vmv_x_s(tmp, vtmp3);
vmv_s_x(vy, tmp);
}

void MacroAssembler::kernel_crc32_vclmul_fold_vectorsize_16(Register crc, Register buf, Register len,
Register vclmul_table, Register tmp1, Register tmp2) {
assert_different_registers(crc, buf, len, vclmul_table, tmp1, tmp2, t1);
assert(MaxVectorSize == 16, "sanity");

const int TABLE_STEP = 16;
const int STEP = 16;
const int LOOP_STEP = 128;
const int N = 2;

Register loop_step = t1;

// ======== preparation ========

mv(loop_step, LOOP_STEP);
sub(len, len, loop_step);

vsetivli(zr, N, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);
vle64_v(v0, buf); addi(buf, buf, STEP);
vle64_v(v1, buf); addi(buf, buf, STEP);
vle64_v(v2, buf); addi(buf, buf, STEP);
vle64_v(v3, buf); addi(buf, buf, STEP);
vle64_v(v4, buf); addi(buf, buf, STEP);
vle64_v(v5, buf); addi(buf, buf, STEP);
vle64_v(v6, buf); addi(buf, buf, STEP);
vle64_v(v7, buf); addi(buf, buf, STEP);

vmv_v_x(v31, zr);
vsetivli(zr, 1, Assembler::e32, Assembler::m1, Assembler::mu, Assembler::tu);
vmv_s_x(v31, crc);
vsetivli(zr, N, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);
vxor_vv(v0, v0, v31);

// load table
vle64_v(v31, vclmul_table);

Label L_16_bytes_loop;
j(L_16_bytes_loop);


// ======== folding 128 bytes in data buffer per round ========

align(OptoLoopAlignment);
bind(L_16_bytes_loop);
{
crc32_vclmul_fold_16_bytes_vectorsize_16(v0, v31, v8, v9, v10, v11, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v1, v31, v12, v13, v14, v15, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v2, v31, v16, v17, v18, v19, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v3, v31, v20, v21, v22, v23, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v4, v31, v24, v25, v26, v27, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v5, v31, v8, v9, v10, v11, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v6, v31, v12, v13, v14, v15, buf, tmp2, STEP);
crc32_vclmul_fold_16_bytes_vectorsize_16(v7, v31, v16, v17, v18, v19, buf, tmp2, STEP);
}
sub(len, len, loop_step);
bge(len, loop_step, L_16_bytes_loop);


// ======== folding into 64 bytes from 128 bytes in register ========

// load table
addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v31, vclmul_table);

crc32_vclmul_fold_16_bytes_vectorsize_16_2(v0, v4, v31, v8, v9, v10, v11, tmp2);
crc32_vclmul_fold_16_bytes_vectorsize_16_2(v1, v5, v31, v12, v13, v14, v15, tmp2);
crc32_vclmul_fold_16_bytes_vectorsize_16_2(v2, v6, v31, v16, v17, v18, v19, tmp2);
crc32_vclmul_fold_16_bytes_vectorsize_16_2(v3, v7, v31, v20, v21, v22, v23, tmp2);


// ======== folding into 16 bytes from 64 bytes in register ========

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v31, vclmul_table);
crc32_vclmul_fold_16_bytes_vectorsize_16_3(v0, v3, v31, v8, v9, v10, v11, tmp2);

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v31, vclmul_table);
crc32_vclmul_fold_16_bytes_vectorsize_16_3(v1, v3, v31, v12, v13, v14, v15, tmp2);

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v31, vclmul_table);
crc32_vclmul_fold_16_bytes_vectorsize_16_3(v2, v3, v31, v16, v17, v18, v19, tmp2);

#undef FOLD_2_VCLMUL_3


// ======== final: move result to scalar regsiters ========

vmv_x_s(tmp1, v3);
vslidedown_vi(v1, v3, 1);
vmv_x_s(tmp2, v1);
}

void MacroAssembler::crc32_vclmul_fold_to_16_bytes_vectorsize_32(VectorRegister vx, VectorRegister vy, VectorRegister vt,
VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4) {
assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
vclmul_vv(vtmp1, vx, vt);
vclmulh_vv(vtmp2, vx, vt);
// low parts
vredxor_vs(vtmp3, vtmp1, vy);
// high parts
vslidedown_vi(vtmp4, vy, 1);
vredxor_vs(vtmp1, vtmp2, vtmp4);
// merge low and high back
vslideup_vi(vy, vtmp1, 1);
vmv_x_s(t1, vtmp3);
vmv_s_x(vy, t1);
}

void MacroAssembler::kernel_crc32_vclmul_fold_vectorsize_32(Register crc, Register buf, Register len,
Register vclmul_table, Register tmp1, Register tmp2) {
assert_different_registers(crc, buf, len, vclmul_table, tmp1, tmp2, t1);
assert(MaxVectorSize >= 32, "sanity");

// utility: load table
#define CRC32_VCLMUL_LOAD_TABLE(vt, rt, vtmp, rtmp) \
vid_v(vtmp); \
mv(rtmp, 2); \
vremu_vx(vtmp, vtmp, rtmp); \
vsll_vi(vtmp, vtmp, 3); \
vluxei64_v(vt, rt, vtmp);

const int TABLE_STEP = 16;
const int STEP = 128; // 128 bytes per round
const int N = 2 * 8; // 2: 128-bits/64-bits, 8: 8 pairs of double 64-bits

Register step = tmp2;


// ======== preparation ========

mv(step, STEP);
sub(len, len, step); // 2 rounds of folding with carry-less multiplication

vsetivli(zr, N, Assembler::e64, Assembler::m4, Assembler::mu, Assembler::tu);
// load data
vle64_v(v4, buf);
add(buf, buf, step);

// load table
CRC32_VCLMUL_LOAD_TABLE(v8, vclmul_table, v28, t1);
// load mask,
// v28 should already contains: 0, 8, 0, 8, ...
vmseq_vi(v2, v28, 0);
// now, v2 should contains: 101010...
vmnand_mm(v1, v2, v2);
// now, v1 should contains: 010101...

// initial crc
vmv_v_x(v24, zr);
vsetivli(zr, 1, Assembler::e32, Assembler::m4, Assembler::mu, Assembler::tu);
vmv_s_x(v24, crc);
vsetivli(zr, N, Assembler::e64, Assembler::m4, Assembler::mu, Assembler::tu);
vxor_vv(v4, v4, v24);

Label L_128_bytes_loop;
j(L_128_bytes_loop);


// ======== folding 128 bytes in data buffer per round ========

align(OptoLoopAlignment);
bind(L_128_bytes_loop);
{
// v4: data
// v4: buf, reused
// v8: table
// v12: lows
// v16: highs
// v20: low_slides
// v24: high_slides
vclmul_vv(v12, v4, v8);
vclmulh_vv(v16, v4, v8);
vle64_v(v4, buf);
add(buf, buf, step);
// lows
vslidedown_vi(v20, v12, 1);
vmand_mm(v0, v2, v2);
vxor_vv(v12, v12, v20, v0_t);
// with buf data
vxor_vv(v4, v4, v12, v0_t);

// highs
vslideup_vi(v24, v16, 1);
vmand_mm(v0, v1, v1);
vxor_vv(v16, v16, v24, v0_t);
// with buf data
vxor_vv(v4, v4, v16, v0_t);
}
sub(len, len, step);
bge(len, step, L_128_bytes_loop);


// ======== folding into 64 bytes from 128 bytes in register ========

// load table
addi(vclmul_table, vclmul_table, TABLE_STEP);
CRC32_VCLMUL_LOAD_TABLE(v8, vclmul_table, v28, t1);

// v4: data, first (low) part, N/2 of 64-bits
// v20: data, second (high) part, N/2 of 64-bits
// v8: table
// v10: lows
// v12: highs
// v14: low_slides
// v16: high_slides

// high part
vslidedown_vi(v20, v4, N/2);

vsetivli(zr, N/2, Assembler::e64, Assembler::m2, Assembler::mu, Assembler::tu);

vclmul_vv(v10, v4, v8);
vclmulh_vv(v12, v4, v8);

// lows
vslidedown_vi(v14, v10, 1);
vmand_mm(v0, v2, v2);
vxor_vv(v10, v10, v14, v0_t);
// with data part 2
vxor_vv(v4, v20, v10, v0_t);

// highs
vslideup_vi(v16, v12, 1);
vmand_mm(v0, v1, v1);
vxor_vv(v12, v12, v16, v0_t);
// with data part 2
vxor_vv(v4, v20, v12, v0_t);


// ======== folding into 16 bytes from 64 bytes in register ========

// v4: data, first part, 2 of 64-bits
// v16: data, second part, 2 of 64-bits
// v18: data, third part, 2 of 64-bits
// v20: data, second part, 2 of 64-bits
// v8: table

vslidedown_vi(v16, v4, 2);
vslidedown_vi(v18, v4, 4);
vslidedown_vi(v20, v4, 6);

vsetivli(zr, 2, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v8, vclmul_table);
crc32_vclmul_fold_to_16_bytes_vectorsize_32(v4, v20, v8, v28, v29, v30, v31);

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v8, vclmul_table);
crc32_vclmul_fold_to_16_bytes_vectorsize_32(v16, v20, v8, v28, v29, v30, v31);

addi(vclmul_table, vclmul_table, TABLE_STEP);
vle64_v(v8, vclmul_table);
crc32_vclmul_fold_to_16_bytes_vectorsize_32(v18, v20, v8, v28, v29, v30, v31);


// ======== final: move result to scalar regsiters ========

vmv_x_s(tmp1, v20);
vslidedown_vi(v4, v20, 1);
vmv_x_s(tmp2, v4);

#undef CRC32_VCLMUL_LOAD_TABLE
}

// For more details of the algorithm, please check the paper:
// "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction - Intel"
//
// Please also refer to the corresponding code in aarch64 or x86 ones.
//
// As the riscv carry-less multiplication is a bit different from the other platforms,
// so the implementation itself is also a bit different from others.

void MacroAssembler::kernel_crc32_vclmul_fold(Register crc, Register buf, Register len,
Register table0, Register table1, Register table2, Register table3,
Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
const int64_t single_table_size = 256;
const int64_t table_num = 8; // 4 for scalar, 4 for plain vector
const ExternalAddress table_addr = StubRoutines::crc_table_addr();
Register vclmul_table = tmp3;

la(vclmul_table, table_addr);
add(vclmul_table, vclmul_table, table_num*single_table_size*sizeof(juint), tmp1);
la(table0, table_addr);

if (MaxVectorSize == 16) {
kernel_crc32_vclmul_fold_vectorsize_16(crc, buf, len, vclmul_table, tmp1, tmp2);
} else {
kernel_crc32_vclmul_fold_vectorsize_32(crc, buf, len, vclmul_table, tmp1, tmp2);
}

mv(crc, zr);
update_word_crc32(crc, tmp1, tmp3, tmp4, tmp5, table0, table1, table2, table3, false);
update_word_crc32(crc, tmp1, tmp3, tmp4, tmp5, table0, table1, table2, table3, true);
update_word_crc32(crc, tmp2, tmp3, tmp4, tmp5, table0, table1, table2, table3, false);
update_word_crc32(crc, tmp2, tmp3, tmp4, tmp5, table0, table1, table2, table3, true);
}

#endif // COMPILER2

/**
Expand Down Expand Up @@ -1765,7 +2118,9 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,

#ifdef COMPILER2
if (UseRVV) {
const int64_t tmp_limit = MaxVectorSize >= 32 ? unroll_words*3 : unroll_words*5;
const int64_t tmp_limit =
UseZvbc ? 128 * 3 // 3 rounds of folding with carry-less multiplication
: MaxVectorSize >= 32 ? unroll_words*3 : unroll_words*5;
mv(tmp1, tmp_limit);
bge(len, tmp1, L_vector_entry);
}
Expand Down Expand Up @@ -1827,7 +2182,13 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
j(L_exit);

bind(L_vector_entry);
vector_update_crc32(crc, buf, len, tmp1, tmp2, tmp3, tmp4, tmp6, table0, table3);
if (UseZvbc) { // carry-less multiplication
kernel_crc32_vclmul_fold(crc, buf, len,
table0, table1, table2, table3,
tmp1, tmp2, tmp3, tmp4, tmp6);
} else { // plain vector instructions
vector_update_crc32(crc, buf, len, tmp1, tmp2, tmp3, tmp4, tmp6, table0, table3);
}

bgtz(len, L_by4_loop_entry);
}
Expand Down
Loading

0 comments on commit c517ffb

Please sign in to comment.