#define _GF16_XORJIT_COPY_ALIGN 32 #include "gf16_xor_common.h" #undef _GF16_XORJIT_COPY_ALIGN #include #if defined(__AVX2__) && defined(PLATFORM_AMD64) int gf16_xor_available_avx2 = 1; #else int gf16_xor_available_avx2 = 0; #endif #if defined(__AVX2__) && defined(PLATFORM_AMD64) ALIGN_TO(16, __m128i xor256_jit_clut_code1[64]); ALIGN_TO(16, uint8_t xor256_jit_clut_info_mem[64]); ALIGN_TO(16, __m64 xor256_jit_nums[128]); ALIGN_TO(16, __m64 xor256_jit_rmask[128]); static int xor256_jit_created = 0; static void gf16_xor_create_jit_lut_avx2(void) { uint_fast32_t i; int j; if(xor256_jit_created) return; xor256_jit_created = 1; memset(xor256_jit_clut_code1, 0, sizeof(xor256_jit_clut_code1)); for(i=0; i<64; i++) { int m = (i&1) | ((i&8)>>2) | ((i&2)<<1) | ((i&16)>>1) | ((i&4)<<2) | (i&32); /* interleave bits */ uint_fast8_t posM = 0; uint8_t* pC = (uint8_t*)(xor256_jit_clut_code1 + i); for(j=0; j<3; j++) { int msk = m&3; if(msk) { int reg = msk-1; /* if we ever support 32-bit, need to ensure that vpxor/load is fixed length */ pC += _jit_vpxor_m(pC, reg, reg, AX, lshift32(j-4, 5)); /* advance pointers */ posM += 5; } m >>= 2; } xor256_jit_clut_info_mem[i] = posM; } memset(xor256_jit_nums, 255, sizeof(xor256_jit_nums)); memset(xor256_jit_rmask, 0, sizeof(xor256_jit_rmask)); for(i=0; i<128; i++) { uint8_t* nums = (uint8_t*)(xor256_jit_nums + i), * rmask = (uint8_t*)(xor256_jit_rmask + i); for(j=0; j<8; j++) { if(i & (1< 2) { *jitptr += _jit_vpxor_m(*jitptr, reg, (uint_fast8_t)highest, AX, lowest*32-128); } else if(highest >= 0) { *jitptr += _jit_vmovdqa_load(*jitptr, reg, AX, highest*32-128); *jitptr += _jit_vpxor_m(*jitptr, reg, reg, AX, lowest*32-128); } else *jitptr += _jit_vmovdqa_load(*jitptr, reg, AX, lowest*32-128); } else { if(highest >= 0) { /* highest dep cannot be sourced from memory */ *jitptr += _jit_vpxor_r(*jitptr, reg, (uint_fast8_t)highest, (uint_fast8_t)lowest); } else #ifdef XORDEP_AVX_XOR_OPTIMAL { /* just change XOR at end to merge from this register */ return lowest; } #else /* just a move */ *jitptr += _jit_vmovdqa(*jitptr, reg, (uint_fast8_t)lowest); #endif } } return reg; } // table originally from http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetTable // modified for our use (items pre-multiplied by 4, only 128 entries) static const unsigned char xor256_jit_len[128] = { # define B2(n) n, n+4, n+4, n+8 # define B4(n) B2(n), B2(n+4), B2(n+4), B2(n+8) # define B6(n) B4(n), B4(n+4), B4(n+4), B4(n+8) B6(0), B6(4) #undef B2 #undef B4 #undef B6 }; static inline int xor_write_avx_main_part(void* jitptr, uint8_t dep1, uint8_t dep2, int high) { uint8_t dep = dep1 | dep2; __m128i nums = _mm_loadl_epi64((__m128i*)(xor256_jit_nums + dep)); // expand to 8x32b + shift into place __m256i srcs = _mm256_slli_epi32(_mm256_cvtepu8_epi32(_mm_add_epi8(nums, _mm_set1_epi8(high ? 10 : 3))), 11); __m128i regs = _mm_loadl_epi64((__m128i*)(xor256_jit_rmask + dep1)); __m128i regs2 = _mm_loadl_epi64((__m128i*)(xor256_jit_rmask + dep2)); regs = _mm_or_si128(regs, _mm_add_epi8(regs2, regs2)); regs = _mm_shuffle_epi8(regs, nums); __m256i inst = _mm256_add_epi8( _mm256_slli_epi32(_mm256_cvtepu8_epi32(regs), 24), _mm256_set1_epi32(0xB7EFFDC5) /* VPXOR op-code, but last byte is 0xC0 - ((1<<3)+1) to offset the fact that our registers num is +1 too much */ ); _mm256_storeu_si256((__m256i*)jitptr, _mm256_xor_si256(srcs, inst)); return xor256_jit_len[dep]; } static inline void* xor_write_jit_avx(const void *HEDLEY_RESTRICT _scratch, uint8_t *HEDLEY_RESTRICT jitptr, uint16_t val, const int mode, const int prefetch) { const struct gf16_xor_scratch *HEDLEY_RESTRICT scratch = (const struct gf16_xor_scratch*)_scratch; uint_fast32_t bit; __m256i depmask = _mm256_load_si256((__m256i*)scratch->deps + (val & 0xf)*4); depmask = _mm256_xor_si256(depmask, _mm256_load_si256((__m256i*)(scratch->deps + ((val << 3) & 0x780)) + 1) ); depmask = _mm256_xor_si256(depmask, _mm256_load_si256((__m256i*)(scratch->deps + ((val >> 1) & 0x780)) + 2) ); depmask = _mm256_xor_si256(depmask, _mm256_load_si256((__m256i*)(scratch->deps + ((val >> 5) & 0x780)) + 3) ); __m128i common_mask, tmp3, tmp4; tmp3 = _mm256_castsi256_si128(depmask); tmp4 = _mm256_extracti128_si256(depmask, 1); ALIGN_TO(16, int16_t common_highest[8]); ALIGN_TO(16, int16_t common_lowest[8]); ALIGN_TO(16, int16_t dep1_highest[8]); ALIGN_TO(16, int16_t dep1_lowest[8]); ALIGN_TO(16, int16_t dep2_highest[8]); ALIGN_TO(16, int16_t dep2_lowest[8]); /* obtain index of lowest bit set, and clear it */ common_mask = _mm_and_si128(tmp3, tmp4); __m128i lowest = ssse3_tzcnt_epi16(common_mask); _mm_store_si128((__m128i*)common_lowest, lowest); __m128i common_sub1 = _mm_add_epi16(common_mask, _mm_set1_epi16(-1)); // TODO: could re-use the VPXOR constant with _mm_sign_epi16 (and invert and/not below) __m128i common_elim = _mm_andnot_si128(common_sub1, common_mask); __m128i highest; common_mask = _mm_and_si128(common_mask, common_sub1); highest = ssse3_lzcnt_epi16(common_mask); _mm_store_si128((__m128i*)common_highest, _mm_sub_epi16(_mm_set1_epi16(15), highest)); common_elim = _mm_or_si128(common_elim, sse4_lzcnt_to_mask_epi16(highest)); /* clear highest/lowest bit from tmp3/4 */ tmp3 = _mm_xor_si128(tmp3, common_elim); tmp4 = _mm_xor_si128(tmp4, common_elim); if(mode != XORDEP_JIT_MODE_MULADD) { lowest = ssse3_tzcnt_epi16(tmp3); _mm_store_si128((__m128i*)dep1_lowest, lowest); tmp3 = _mm_and_si128(tmp3, _mm_add_epi16(tmp3, _mm_set1_epi16(-1))); lowest = ssse3_tzcnt_epi16(tmp4); _mm_store_si128((__m128i*)dep2_lowest, lowest); tmp4 = _mm_and_si128(tmp4, _mm_add_epi16(tmp4, _mm_set1_epi16(-1))); } highest = ssse3_lzcnt_epi16(tmp3); _mm_store_si128((__m128i*)dep1_highest, _mm_sub_epi16(_mm_set1_epi16(15), highest)); tmp3 = _mm_xor_si128(tmp3, sse4_lzcnt_to_mask_epi16(highest)); highest = ssse3_lzcnt_epi16(tmp4); _mm_store_si128((__m128i*)dep2_highest, _mm_sub_epi16(_mm_set1_epi16(15), highest)); tmp4 = _mm_xor_si128(tmp4, sse4_lzcnt_to_mask_epi16(highest)); ALIGN_TO(16, uint16_t memDeps[8]); _mm_store_si128((__m128i*)memDeps, _mm_or_si128( _mm_and_si128(tmp3, _mm_set1_epi16(7)), _mm_slli_epi16(_mm_and_si128(tmp4, _mm_set1_epi16(7)), 3) )); ALIGN_TO(16, uint8_t deps1[16]); ALIGN_TO(16, uint8_t deps2[16]); tmp3 = _mm_srli_epi16(tmp3, 3); tmp4 = _mm_srli_epi16(tmp4, 3); tmp3 = _mm_blendv_epi8(_mm_add_epi16(tmp3, tmp3), _mm_and_si128(tmp3, _mm_set1_epi8(0x7f)), _mm_set1_epi16(0xff)); tmp4 = _mm_blendv_epi8(_mm_add_epi16(tmp4, tmp4), _mm_and_si128(tmp4, _mm_set1_epi8(0x7f)), _mm_set1_epi16(0xff)); _mm_store_si128((__m128i*)deps1, tmp3); _mm_store_si128((__m128i*)deps2, tmp4); if(prefetch) { jitptr += _jit_add_i(jitptr, SI, 256); jitptr += _jit_prefetch_m(jitptr, prefetch, SI, -128); jitptr += _jit_prefetch_m(jitptr, prefetch, SI, -64); jitptr += _jit_prefetch_m(jitptr, prefetch, SI, 0); jitptr += _jit_prefetch_m(jitptr, prefetch, SI, 64); } // TODO: optimize these #undef _LD_DQA #define _LD_DQA(yreg, mreg, offs) \ jitptr += _jit_vmovdqa_load(jitptr, yreg, mreg, offs) #undef _ST_DQA #define _ST_DQA(mreg, offs, yreg) \ jitptr += _jit_vmovdqa_store(jitptr, mreg, offs, yreg) //_jit_pxor_r(jit, r2, r1) /* #define _PXOR_R_(r2, r1, tr) \ write32(jitptr, (0xC0EF0F66 + ((r2) <<27) + ((r1) <<24)) ^ (tr)) #define _PXOR_R(r2, r1) \ _PXOR_R_(r2, r1, 0); \ jitptr += 4 #define _C_PXOR_R(r2, r1, c) \ _PXOR_R_(r2, r1, 0); \ jitptr += (c)<<2 */ #undef _PXOR_R #define _PXOR_R(r2, r1) jitptr += _jit_vpxor_r(jitptr, r2, r2, r1) #undef _C_PXOR_R #define _C_PXOR_R(rD, r2, r1, c) jitptr += _jit_vpxor_r(jitptr, rD, r2, r1) & -(c) /* generate code */ if(mode == XORDEP_JIT_MODE_MULADD) { for(bit=0; bit<8; bit++) { int destOffs = (bit<<6)-128; int destOffs2 = destOffs+32; uint8_t common_reg; /* if there's a higest bit set, do a VPXOR-load, otherwise, regular load + VPXOR-load */ if(dep1_highest[bit] > 2) { jitptr += _jit_vpxor_m(jitptr, 0, (uint_fast8_t)dep1_highest[bit], DX, destOffs); } else { _LD_DQA(0, DX, destOffs); if(dep1_highest[bit] >= 0) jitptr += _jit_vpxor_m(jitptr, 0, 0, AX, dep1_highest[bit]*32-128); } if(dep2_highest[bit] > 2) { jitptr += _jit_vpxor_m(jitptr, 1, (uint_fast8_t)dep2_highest[bit], DX, destOffs2); } else { _LD_DQA(1, DX, destOffs2); if(dep2_highest[bit] >= 0) jitptr += _jit_vpxor_m(jitptr, 1, 1, AX, dep2_highest[bit]*32-128); } /* for common mask, if two lowest bits available, do VPXOR, else if only one, just XOR at end (consider no common mask optimization to eliminate this case) */ common_reg = xor_write_avx_load_part(&jitptr, 2, common_lowest[bit], common_highest[bit]); _mm_storeu_si128((__m128i*)jitptr, _mm_load_si128(&xor256_jit_clut_code1[memDeps[bit]])); jitptr += xor256_jit_clut_info_mem[memDeps[bit]]; jitptr += xor_write_avx_main_part(jitptr, deps1[bit*2], deps2[bit*2], 0); jitptr += xor_write_avx_main_part(jitptr, deps1[bit*2+1], deps2[bit*2+1], 1); _C_PXOR_R(0, common_reg, 0, common_lowest[bit] < 16); _C_PXOR_R(1, common_reg, 1, common_lowest[bit] < 16); _ST_DQA(DX, destOffs, 0); _ST_DQA(DX, destOffs2, 1); } } else { for(bit=0; bit<8; bit++) { int destOffs = (bit<<6)-128; int destOffs2 = destOffs+32; uint8_t common_reg, reg1, reg2; reg1 = xor_write_avx_load_part(&jitptr, 0, dep1_lowest[bit], dep1_highest[bit]); reg2 = xor_write_avx_load_part(&jitptr, 1, dep2_lowest[bit], dep2_highest[bit]); common_reg = xor_write_avx_load_part(&jitptr, 2, common_lowest[bit], common_highest[bit]); _mm_storeu_si128((__m128i*)jitptr, _mm_load_si128(&xor256_jit_clut_code1[memDeps[bit]])); jitptr += xor256_jit_clut_info_mem[memDeps[bit]]; jitptr += xor_write_avx_main_part(jitptr, deps1[bit*2], deps2[bit*2], 0); jitptr += xor_write_avx_main_part(jitptr, deps1[bit*2+1], deps2[bit*2+1], 1); if(dep1_lowest[bit] < 16) { #ifdef XORDEP_AVX_XOR_OPTIMAL if(common_lowest[bit] < 16) { jitptr += _jit_vpxor_r(jitptr, 0, reg1, common_reg); _ST_DQA(DX, destOffs, 0); } else { _ST_DQA(DX, destOffs, reg1); } #else _C_PXOR_R(0, reg1, common_reg, common_lowest[bit] < 16); _ST_DQA(DX, destOffs, 0); #endif } else { /* dep1 must be sourced from the common mask */ _ST_DQA(DX, destOffs, common_reg); } if(dep2_lowest[bit] < 16) { #ifdef XORDEP_AVX_XOR_OPTIMAL if(common_lowest[bit] < 16) { jitptr += _jit_vpxor_r(jitptr, 1, reg2, common_reg); _ST_DQA(DX, destOffs2, 1); } else { _ST_DQA(DX, destOffs2, reg2); } #else _C_PXOR_R(1, reg2, common_reg, common_lowest[bit] < 16); _ST_DQA(DX, destOffs2, 1); #endif } else { _ST_DQA(DX, destOffs2, common_reg); } } } /* cmp/jcc */ write64(jitptr, 0x800FC03948 | (DX <<16) | (CX <<19) | ((uint64_t)JL <<32)); return jitptr+5; } static HEDLEY_ALWAYS_INLINE void gf16_xor_jit_mul_avx2_base(const void *HEDLEY_RESTRICT scratch, void* dst, const void* src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch, const int mode, const int doPrefetch, const void *HEDLEY_RESTRICT prefetch) { jit_wx_pair* jit = (jit_wx_pair*)mutScratch; gf16_xorjit_write_jit(scratch, coefficient, jit, mode, doPrefetch, &xor_write_jit_avx, scratch); if(mode == XORDEP_JIT_MODE_MUL_INSITU) { ALIGN_TO(32, __m256i spill[3]); gf16_xor256_jit_stub( (intptr_t)spill + 128, (intptr_t)dst + len - 384, (intptr_t)dst - 384, (intptr_t)prefetch - 128, (uint8_t*)jit->x + XORDEP_JIT_SIZE/2 ); } else { gf16_xor256_jit_stub( (intptr_t)src - 384, (intptr_t)dst + len - 384, (intptr_t)dst - 384, (intptr_t)prefetch - 128, jit->x ); } _mm256_zeroupper(); } #endif /* defined(__AVX2__) && defined(PLATFORM_AMD64) */ #ifdef PARPAR_INVERT_SUPPORT void gf16_xor_jit_mul_avx2(const void *HEDLEY_RESTRICT scratch, void* dst, const void* src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch) { #if defined(__AVX2__) && defined(PLATFORM_AMD64) if(coefficient == 0) { memset(dst, 0, len); return; } gf16_xor_jit_mul_avx2_base(scratch, dst, src, len, coefficient, mutScratch, dst==src ? XORDEP_JIT_MODE_MUL_INSITU : XORDEP_JIT_MODE_MUL, 0, NULL); #else UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch); #endif } #endif void gf16_xor_jit_muladd_avx2(const void *HEDLEY_RESTRICT scratch, void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch) { #if defined(__AVX2__) && defined(PLATFORM_AMD64) if(coefficient == 0) return; gf16_xor_jit_mul_avx2_base(scratch, dst, src, len, coefficient, mutScratch, XORDEP_JIT_MODE_MULADD, 0, NULL); #else UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch); #endif } void gf16_xor_jit_muladd_prefetch_avx2(const void *HEDLEY_RESTRICT scratch, void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch, const void *HEDLEY_RESTRICT prefetch) { #if defined(__AVX2__) && defined(PLATFORM_AMD64) if(coefficient == 0) return; gf16_xor_jit_mul_avx2_base(scratch, dst, src, len, coefficient, mutScratch, XORDEP_JIT_MODE_MULADD, _MM_HINT_T1, prefetch); #else UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch); UNUSED(prefetch); #endif } #if defined(__AVX2__) && defined(PLATFORM_AMD64) // extract top bits; interleaving of 16-bit words needed due to byte arrangement for pmovmskb static HEDLEY_ALWAYS_INLINE __m256i gf16_xor_finish_extract_bits(__m256i src) { #define EXTRACT_BITS_NIBBLE(targVec, srcVec) { \ uint32_t mskA, mskB, mskC, mskD; \ mskD = _mm256_movemask_epi8(srcVec); \ srcVec = _mm256_add_epi8(srcVec, srcVec); \ mskC = _mm256_movemask_epi8(srcVec); \ srcVec = _mm256_add_epi8(srcVec, srcVec); \ mskB = _mm256_movemask_epi8(srcVec); \ srcVec = _mm256_add_epi8(srcVec, srcVec); \ mskA = _mm256_movemask_epi8(srcVec); \ targVec = _mm_cvtsi32_si128(mskA); \ targVec = _mm_insert_epi32(targVec, mskB, 1); \ targVec = _mm_insert_epi32(targVec, mskC, 2); \ targVec = _mm_insert_epi32(targVec, mskD, 3); \ } __m128i words1, words2; EXTRACT_BITS_NIBBLE(words1, src) src = _mm256_add_epi8(src, src); EXTRACT_BITS_NIBBLE(words2, src) __m256i words = _mm256_inserti128_si256(_mm256_castsi128_si256(words2), words1, 1); words = _mm256_shuffle_epi8(words, _mm256_set_epi32( 0x0f0e0b0a, 0x07060302, 0x0d0c0908, 0x05040100, 0x0f0e0b0a, 0x07060302, 0x0d0c0908, 0x05040100 )); return _mm256_permute4x64_epi64(words, _MM_SHUFFLE(3,1,2,0)); #undef EXTRACT_BITS_NIBBLE } static HEDLEY_ALWAYS_INLINE void gf16_xor_finish_extract_bits_store(uint32_t* dst, __m256i src) { __m256i srcShifted = _mm256_add_epi8(src, src); __m256i lane = _mm256_inserti128_si256(srcShifted, _mm256_castsi256_si128(src), 1); write32(dst+3, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+2, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+1, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+0, _mm256_movemask_epi8(lane)); lane = _mm256_permute2x128_si256(srcShifted, src, 0x31); write32(dst+7, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+6, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+5, _mm256_movemask_epi8(lane)); lane = _mm256_slli_epi16(lane, 2); write32(dst+4, _mm256_movemask_epi8(lane)); } #define LOAD_HALVES(a, b, upper) \ _mm256_inserti128_si256( \ _mm256_castsi128_si256(_mm_load_si128((__m128i*)(_src + 120 + upper*4 - (a)*8))), \ _mm_load_si128((__m128i*)(_src + 120 + upper*4 - (b)*8)), \ 1 \ ) #define LOAD_X4(offs, dst1, dst2, upper) { \ __m256i in1 = LOAD_HALVES(offs+0, offs+8, upper); /* 88888888 00000000 */ \ __m256i in2 = LOAD_HALVES(offs+1, offs+9, upper); /* 99999999 11111111 */ \ dst1 = _mm256_unpacklo_epi8(in1, in2); /* 98989898 10101010 */ \ dst2 = _mm256_unpackhi_epi8(in1, in2); \ } #define UNPACK_VECTS \ srcD0a = _mm256_unpacklo_epi16(srcW0, srcW2); /* ba98ba98 32103210 */ \ srcD0b = _mm256_unpackhi_epi16(srcW0, srcW2); \ srcD0c = _mm256_unpacklo_epi16(srcW1, srcW3); \ srcD0d = _mm256_unpackhi_epi16(srcW1, srcW3); \ srcD4a = _mm256_unpacklo_epi16(srcW4, srcW6); \ srcD4b = _mm256_unpackhi_epi16(srcW4, srcW6); \ srcD4c = _mm256_unpacklo_epi16(srcW5, srcW7); \ srcD4d = _mm256_unpackhi_epi16(srcW5, srcW7); \ \ srcQa = _mm256_unpacklo_epi32(srcD0a, srcD4a); /* fedcba98 76543210 */ \ srcQb = _mm256_unpackhi_epi32(srcD0a, srcD4a); \ srcQc = _mm256_unpacklo_epi32(srcD0b, srcD4b); \ srcQd = _mm256_unpackhi_epi32(srcD0b, srcD4b); \ srcQe = _mm256_unpacklo_epi32(srcD0c, srcD4c); \ srcQf = _mm256_unpackhi_epi32(srcD0c, srcD4c); \ srcQg = _mm256_unpacklo_epi32(srcD0d, srcD4d); \ srcQh = _mm256_unpackhi_epi32(srcD0d, srcD4d); \ \ srcQa = _mm256_permute4x64_epi64(srcQa, _MM_SHUFFLE(3,1,2,0)); /* fedcba9876543210 fedcba9876543210 */ \ srcQb = _mm256_permute4x64_epi64(srcQb, _MM_SHUFFLE(3,1,2,0)); \ srcQc = _mm256_permute4x64_epi64(srcQc, _MM_SHUFFLE(3,1,2,0)); \ srcQd = _mm256_permute4x64_epi64(srcQd, _MM_SHUFFLE(3,1,2,0)); \ srcQe = _mm256_permute4x64_epi64(srcQe, _MM_SHUFFLE(3,1,2,0)); \ srcQf = _mm256_permute4x64_epi64(srcQf, _MM_SHUFFLE(3,1,2,0)); \ srcQg = _mm256_permute4x64_epi64(srcQg, _MM_SHUFFLE(3,1,2,0)); \ srcQh = _mm256_permute4x64_epi64(srcQh, _MM_SHUFFLE(3,1,2,0)) void gf16_xor_finish_block_avx2(void *HEDLEY_RESTRICT dst) { uint32_t* _dst = (uint32_t*)dst; const uint32_t* _src = _dst; __m256i srcW0, srcW1, srcW2, srcW3, srcW4, srcW5, srcW6, srcW7; __m256i srcD0a, srcD0b, srcD0c, srcD0d, srcD4a, srcD4b, srcD4c, srcD4d; __m256i srcQa, srcQb, srcQc, srcQd, srcQe, srcQf, srcQg, srcQh; // load 16x 128-bit inputs LOAD_X4(0, srcW0, srcW1, 0) LOAD_X4(2, srcW2, srcW3, 0) LOAD_X4(4, srcW4, srcW5, 0) LOAD_X4(6, srcW6, srcW7, 0) // interleave bytes in all 8 vectors UNPACK_VECTS; // save extracted bits (can't write these yet as they'd overwrite the next round) __m256i dstA = gf16_xor_finish_extract_bits(srcQa); __m256i dstB = gf16_xor_finish_extract_bits(srcQb); __m256i dstC = gf16_xor_finish_extract_bits(srcQc); __m256i dstD = gf16_xor_finish_extract_bits(srcQd); __m256i dstE = gf16_xor_finish_extract_bits(srcQe); __m256i dstF = gf16_xor_finish_extract_bits(srcQf); __m256i dstG = gf16_xor_finish_extract_bits(srcQg); __m256i dstH = gf16_xor_finish_extract_bits(srcQh); // load second half & store saved data once relevant stuff read LOAD_X4(6, srcW6, srcW7, 1) _mm256_store_si256((__m256i*)(_dst + 0), dstA); _mm256_store_si256((__m256i*)(_dst + 8), dstB); LOAD_X4(4, srcW4, srcW5, 1) _mm256_store_si256((__m256i*)(_dst + 16), dstC); _mm256_store_si256((__m256i*)(_dst + 24), dstD); LOAD_X4(2, srcW2, srcW3, 1) _mm256_store_si256((__m256i*)(_dst + 32), dstE); _mm256_store_si256((__m256i*)(_dst + 40), dstF); LOAD_X4(0, srcW0, srcW1, 1) _mm256_store_si256((__m256i*)(_dst + 48), dstG); _mm256_store_si256((__m256i*)(_dst + 56), dstH); UNPACK_VECTS; gf16_xor_finish_extract_bits_store(_dst + 64 + 0, srcQa); gf16_xor_finish_extract_bits_store(_dst + 64 + 8, srcQb); gf16_xor_finish_extract_bits_store(_dst + 64 + 16, srcQc); gf16_xor_finish_extract_bits_store(_dst + 64 + 24, srcQd); gf16_xor_finish_extract_bits_store(_dst + 64 + 32, srcQe); gf16_xor_finish_extract_bits_store(_dst + 64 + 40, srcQf); gf16_xor_finish_extract_bits_store(_dst + 64 + 48, srcQg); gf16_xor_finish_extract_bits_store(_dst + 64 + 56, srcQh); } void gf16_xor_finish_copy_block_avx2(void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src) { uint32_t* _dst = (uint32_t*)dst; const uint32_t* _src = (uint32_t*)src; __m256i srcW0, srcW1, srcW2, srcW3, srcW4, srcW5, srcW6, srcW7; __m256i srcD0a, srcD0b, srcD0c, srcD0d, srcD4a, srcD4b, srcD4c, srcD4d; __m256i srcQa, srcQb, srcQc, srcQd, srcQe, srcQf, srcQg, srcQh; // load 16x 128-bit inputs LOAD_X4(0, srcW0, srcW1, 0) LOAD_X4(2, srcW2, srcW3, 0) LOAD_X4(4, srcW4, srcW5, 0) LOAD_X4(6, srcW6, srcW7, 0) // interleave bytes in all 8 vectors UNPACK_VECTS; // save extracted bits gf16_xor_finish_extract_bits_store(_dst + 0, srcQa); gf16_xor_finish_extract_bits_store(_dst + 8, srcQb); gf16_xor_finish_extract_bits_store(_dst + 16, srcQc); gf16_xor_finish_extract_bits_store(_dst + 24, srcQd); gf16_xor_finish_extract_bits_store(_dst + 32, srcQe); gf16_xor_finish_extract_bits_store(_dst + 40, srcQf); gf16_xor_finish_extract_bits_store(_dst + 48, srcQg); gf16_xor_finish_extract_bits_store(_dst + 56, srcQh); // load second half & store saved data once relevant stuff read LOAD_X4(0, srcW0, srcW1, 1) LOAD_X4(2, srcW2, srcW3, 1) LOAD_X4(4, srcW4, srcW5, 1) LOAD_X4(6, srcW6, srcW7, 1) UNPACK_VECTS; gf16_xor_finish_extract_bits_store(_dst + 64 + 0, srcQa); gf16_xor_finish_extract_bits_store(_dst + 64 + 8, srcQb); gf16_xor_finish_extract_bits_store(_dst + 64 + 16, srcQc); gf16_xor_finish_extract_bits_store(_dst + 64 + 24, srcQd); gf16_xor_finish_extract_bits_store(_dst + 64 + 32, srcQe); gf16_xor_finish_extract_bits_store(_dst + 64 + 40, srcQf); gf16_xor_finish_extract_bits_store(_dst + 64 + 48, srcQg); gf16_xor_finish_extract_bits_store(_dst + 64 + 56, srcQh); } void gf16_xor_finish_copy_blocku_avx2(void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t bytes) { uint16_t block[256]; gf16_xor_finish_copy_block_avx2(block, src); memcpy(dst, block, bytes); } #undef UNPACK_VECTS #undef LOAD_HALVES #undef LOAD_X4 #endif #define MWORD_SIZE 32 #define _mword __m256i #define _MM(f) _mm256_ ## f #define _MMI(f) _mm256_ ## f ## _si256 #define _FNSUFFIX _avx2 #define _MM_END _mm256_zeroupper(); #if defined(__AVX2__) && defined(PLATFORM_AMD64) # define _AVAILABLE # include "gf16_checksum_x86.h" #endif #include "gf16_xor_common_funcs.h" #undef _AVAILABLE #undef MWORD_SIZE #undef _mword #undef _MM #undef _MMI #undef _FNSUFFIX #undef _MM_END #if defined(__AVX2__) && defined(PLATFORM_AMD64) GF_FINISH_PACKED_FUNCS(gf16_xor, _avx2, sizeof(__m256i)*16, gf16_xor_finish_copy_block_avx2, gf16_xor_finish_copy_blocku_avx2, 1, _mm256_zeroupper(), gf16_checksum_block_avx2, gf16_checksum_blocku_avx2, gf16_checksum_exp_avx2, &gf16_xor_finish_block_avx2, sizeof(__m256i)) #else GF_FINISH_PACKED_FUNCS_STUB(gf16_xor, _avx2) #endif #if defined(__AVX2__) && defined(PLATFORM_AMD64) static void xor_write_init_jit(uint8_t *jitCodeNorm, uint8_t *jitCodeInsitu, uint_fast16_t* sizeNorm, uint_fast16_t* sizeInsitu) { uint8_t *jitCodeStart = jitCodeNorm; jitCodeNorm += _jit_add_i(jitCodeNorm, AX, 512); jitCodeNorm += _jit_add_i(jitCodeNorm, DX, 512); /* only 64-bit supported*/ for(int i=3; i<16; i++) { jitCodeNorm += _jit_vmovdqa_load(jitCodeNorm, i, AX, lshift32(i-4, 5)); } if(sizeNorm) *sizeNorm = (uint_fast16_t)(jitCodeNorm-jitCodeStart); jitCodeStart = jitCodeInsitu; jitCodeInsitu += _jit_add_i(jitCodeInsitu, DX, 512); for(int i=0; i<16; i++) { jitCodeInsitu += _jit_vmovdqa_load(jitCodeInsitu, i, DX, lshift32(i-4, 5)); } for(int i=0; i<3; i++) { jitCodeInsitu += _jit_vmovdqa_store(jitCodeInsitu, AX, lshift32(i-4, 5), i); } if(sizeInsitu) *sizeInsitu = (uint_fast16_t)(jitCodeInsitu-jitCodeStart); } # include "gf16_bitdep_init_avx2.h" #endif void* gf16_xor_jit_init_avx2(int polynomial, int jitOptStrat) { #if defined(__AVX2__) && defined(PLATFORM_AMD64) struct gf16_xor_scratch* ret; uint8_t tmpCode[XORDEP_JIT_CODE_SIZE]; ALIGN_ALLOC(ret, sizeof(struct gf16_xor_scratch), 32); gf16_bitdep_init256(ret->deps, polynomial, 0); gf16_xor_create_jit_lut_avx2(); ret->jitOptStrat = jitOptStrat; xor_write_init_jit(tmpCode, tmpCode, &(ret->codeStart), &(ret->codeStartInsitu)); return ret; #else UNUSED(polynomial); UNUSED(jitOptStrat); return NULL; #endif } void* gf16_xor_jit_init_mut_avx2() { #if defined(__AVX2__) && defined(PLATFORM_AMD64) jit_wx_pair *jitCode = jit_alloc(XORDEP_JIT_SIZE); if(!jitCode) return NULL; xor_write_init_jit(jitCode->w, (uint8_t*)jitCode->w + XORDEP_JIT_SIZE/2, NULL, NULL); return jitCode; #else return NULL; #endif }