8000 7zip/C/Blake2s.c at 26.02 · AIahmedshrf/7zip · GitHub
[go: up one dir, main page]

Skip to content

Latest commit

 

History

History
2672 lines (2318 loc) · 77.2 KB

Blake2s.c

File metadata and controls

2672 lines (2318 loc) · 77.2 KB
/* Blake2s.c -- BLAKE2sp Hash
2024-05-18 : Igor Pavlov : Public domain
2015-2019 : Samuel Neves : original code : CC0 1.0 Universal (CC0 1.0). */
#include "Precomp.h"
// #include <stdio.h>
#include <string.h>
#include "Blake2.h"
#include "RotateDefs.h"
#include "Compiler.h"
#include "CpuArch.h"
/*
if defined(__AVX512F__) && defined(__AVX512VL__)
{
we define Z7_BLAKE2S_USE_AVX512_ALWAYS,
but the compiler can use avx512 for any code.
}
else if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
{ we use avx512 only for sse* and avx* branches of code. }
*/
// #define Z7_BLAKE2S_USE_AVX512_ALWAYS // for debug
#if defined(__SSE2__)
#define Z7_BLAKE2S_USE_VECTORS
#elif defined(MY_CPU_X86_OR_AMD64)
#if defined(_MSC_VER) && _MSC_VER > 1200 \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \
|| defined(__clang__) \
|| defined(__INTEL_COMPILER)
#define Z7_BLAKE2S_USE_VECTORS
#endif
#endif
#ifdef Z7_BLAKE2S_USE_VECTORS
#define Z7_BLAKE2SP_USE_FUNCTIONS
// define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED, if CBlake2sp can be non aligned for 32-bytes.
// #define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED
// SSSE3 : for _mm_shuffle_epi8 (pshufb) that improves the performance for 5-15%.
#if defined(__SSSE3__)
#define Z7_BLAKE2S_USE_SSSE3
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000)
#define Z7_BLAKE2S_USE_SSSE3
#endif
#ifdef Z7_BLAKE2S_USE_SSSE3
/* SSE41 : for _mm_insert_epi32 (pinsrd)
it can slightly reduce code size and improves the performance in some cases.
it's used only for last 512-1024 bytes, if FAST versions (2 or 3) of vector algos are used.
it can be used for all blocks in another algos (4+).
*/
#if defined(__SSE4_1__)
#define Z7_BLAKE2S_USE_SSE41
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000)
#define Z7_BLAKE2S_USE_SSE41
#endif
#endif // SSSE3
#if defined(__GNUC__) || defined(__clang__)
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__))
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("avx512vl,avx512f")))
#else
#if defined(Z7_BLAKE2S_USE_SSE41)
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse4.1")))
#elif defined(Z7_BLAKE2S_USE_SSSE3)
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("ssse3")))
#else
#define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse2")))
#endif
#endif
#endif
#if defined(__AVX2__)
#define Z7_BLAKE2S_USE_AVX2
#else
#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \
|| defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \
|| defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100)
#define Z7_BLAKE2S_USE_AVX2
#ifdef Z7_BLAKE2S_USE_AVX2
#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__))
#define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx512vl,avx512f")))
#else
#define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx2")))
#endif
#endif
#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \
|| defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400)
#if (Z7_MSC_VER_ORIGINAL == 1900)
#pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX
#endif
#define Z7_BLAKE2S_USE_AVX2
#endif
#endif
#ifdef Z7_BLAKE2S_USE_SSE41
#include <smmintrin.h> // SSE4.1
#elif defined(Z7_BLAKE2S_USE_SSSE3)
#include <tmmintrin.h> // SSSE3
#else
#include <emmintrin.h> // SSE2
#endif
#ifdef Z7_BLAKE2S_USE_AVX2
#include <immintrin.h>
#if defined(__clang__)
#include <avxintrin.h>
#include <avx2intrin.h>
#endif
#endif // avx2
#if defined(__AVX512F__) && defined(__AVX512VL__)
// && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL > 1930)
#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS
#define Z7_BLAKE2S_USE_AVX512_ALWAYS
#endif
// #pragma message ("=== Blake2s AVX512")
#endif
#define Z7_BLAKE2S_USE_V128_FAST
// for speed optimization for small messages:
// #define Z7_BLAKE2S_USE_V128_WAY2
#ifdef Z7_BLAKE2S_USE_AVX2
// for debug:
// gather is slow
// #define Z7_BLAKE2S_USE_GATHER
#define Z7_BLAKE2S_USE_AVX2_FAST
// for speed optimization for small messages:
// #define Z7_BLAKE2S_USE_AVX2_WAY2
// #define Z7_BLAKE2S_USE_AVX2_WAY4
#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) || \
defined(Z7_BLAKE2S_USE_AVX2_WAY4)
#define Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#endif
#endif
#define Z7_BLAKE2SP_ALGO_DEFAULT 0
#define Z7_BLAKE2SP_ALGO_SCALAR 1
#ifdef Z7_BLAKE2S_USE_V128_FAST
#define Z7_BLAKE2SP_ALGO_V128_FAST 2
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#define Z7_BLAKE2SP_ALGO_V256_FAST 3
#endif
#define Z7_BLAKE2SP_ALGO_V128_WAY1 4
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#define Z7_BLAKE2SP_ALGO_V128_WAY2 5
8
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY2
#define Z7_BLAKE2SP_ALGO_V256_WAY2 6
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY4
#define Z7_BLAKE2SP_ALGO_V256_WAY4 7
#endif
#endif // Z7_BLAKE2S_USE_VECTORS
#define BLAKE2S_FINAL_FLAG (~(UInt32)0)
#define NSW Z7_BLAKE2SP_NUM_STRUCT_WORDS
#define SUPER_BLOCK_SIZE (Z7_BLAKE2S_BLOCK_SIZE * Z7_BLAKE2SP_PARALLEL_DEGREE)
#define SUPER_BLOCK_MASK (SUPER_BLOCK_SIZE - 1)
#define V_INDEX_0_0 0
#define V_INDEX_1_0 1
#define V_INDEX_2_0 2
#define V_INDEX_3_0 3
#define V_INDEX_0_1 4
#define V_INDEX_1_1 5
#define V_INDEX_2_1 6
#define V_INDEX_3_1 7
#define V_INDEX_0_2 8
#define V_INDEX_1_2 9
#define V_INDEX_2_2 10
#define V_INDEX_3_2 11
#define V_INDEX_0_3 12
#define V_INDEX_1_3 13
#define V_INDEX_2_3 14
#define V_INDEX_3_3 15
#define V_INDEX_4_0 0
#define V_INDEX_5_0 1
#define V_INDEX_6_0 2
#define V_INDEX_7_0 3
#define V_INDEX_7_1 4
#define V_INDEX_4_1 5
#define V_INDEX_5_1 6
#define V_INDEX_6_1 7
#define V_INDEX_6_2 8
#define V_INDEX_7_2 9
#define V_INDEX_4_2 10
#define V_INDEX_5_2 11
#define V_INDEX_5_3 12
#define V_INDEX_6_3 13
#define V_INDEX_7_3 14
#define V_INDEX_4_3 15
#define V(row, col) v[V_INDEX_ ## row ## _ ## col]
#define k_Blake2s_IV_0 0x6A09E667UL
#define k_Blake2s_IV_1 0xBB67AE85UL
#define k_Blake2s_IV_2 0x3C6EF372UL
#define k_Blake2s_IV_3 0xA54FF53AUL
#define k_Blake2s_IV_4 0x510E527FUL
#define k_Blake2s_IV_5 0x9B05688CUL
#define k_Blake2s_IV_6 0x1F83D9ABUL
#define k_Blake2s_IV_7 0x5BE0CD19UL
#define KIV(n) (k_Blake2s_IV_## n)
#ifdef Z7_BLAKE2S_USE_VECTORS
MY_ALIGN(16)
static const UInt32 k_Blake2s_IV[8] =
{
KIV(0), KIV(1), KIV(2), KIV(3), KIV(4), KIV(5), KIV(6), KIV(7)
};
#endif
#define STATE_T(s) ((s) + 8)
#define STATE_F(s) ((s) + 10)
#ifdef Z7_BLAKE2S_USE_VECTORS
#define LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p))
#define LOADU_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p))
#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED
// here we use unaligned load and stores
// use this branch if CBlake2sp can be unaligned for 16 bytes
#define STOREU_128(p, r) _mm_storeu_si128((__m128i *)(void *)(p), r)
#define LOAD_128_FROM_STRUCT(p) LOADU_128(p)
#define STORE_128_TO_STRUCT(p, r) STOREU_128(p, r)
#else
// here we use aligned load and stores
// use this branch if CBlake2sp is aligned for 16 bytes
#define STORE_128(p, r) _mm_store_si128((__m128i *)(void *)(p), r)
#define LOAD_128_FROM_STRUCT(p) LOAD_128(p)
#define STORE_128_TO_STRUCT(p, r) STORE_128(p, r)
#endif
#endif // Z7_BLAKE2S_USE_VECTORS
#if 0
static void PrintState(const UInt32 *s, unsigned num)
{
unsigned i;
printf("\n");
for (i = 0; i < num; i++)
printf(" %08x", (unsigned)s[i]);
}
static void PrintStates2(const UInt32 *s, unsigned x, unsigned y)
{
unsigned i;
for (i = 0; i < y; i++)
PrintState(s + i * x, x);
printf("\n");
}
#endif
#define REP8_MACRO(m) { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) }
#define BLAKE2S_NUM_ROUNDS 10
#if defined(Z7_BLAKE2S_USE_VECTORS)
#define ROUNDS_LOOP(mac) \
{ unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) mac(r) }
#endif
/*
#define ROUNDS_LOOP_2(mac) \
{ unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r += 2) { mac(r) mac(r + 1) } }
*/
#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS)
#define ROUNDS_LOOP_UNROLLED(m) \
{ m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) }
#endif
#define SIGMA_TABLE(M) \
M( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ), \
M( 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ), \
M( 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ), \
M( 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ), \
M( 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ), \
M( 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ), \
M( 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ), \
M( 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ), \
M( 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ), \
M( 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 )
#define SIGMA_TABLE_MULT(m, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
{ a0*m,a1*m,a2*m,a3*m,a4*m,a5*m,a6*m,a7*m,a8*m,a9*m,a10*m,a11*m,a12*m,a13*m,a14*m,a15*m }
#define SIGMA_TABLE_MULT_4( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(4, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
// MY_ALIGN(32)
MY_ALIGN(16)
static const Byte k_Blake2s_Sigma_4[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_4) };
#define GET_SIGMA_PTR(p, index) \
((const void *)((const Byte *)(const void *)(p) + (index)))
#define GET_STATE_TABLE_PTR_FROM_BYTE_POS(s, pos) \
((UInt32 *)(void *)((Byte *)(void *)(s) + (pos)))
#ifdef Z7_BLAKE2S_USE_VECTORS
#if 0
// use loading constants from memory
// is faster for some compilers.
#define KK4(n) KIV(n), KIV(n), KIV(n), KIV(n)
MY_ALIGN(64)
static const UInt32 k_Blake2s_IV_WAY4[]=
{
KK4(0), KK4(1), KK4(2), KK4(3), KK4(4), KK4(5), KK4(6), KK4(7)
};
#define GET_128_IV_WAY4(i) LOAD_128(k_Blake2s_IV_WAY4 + 4 * (i))
#else
// use constant generation:
#define GET_128_IV_WAY4(i) _mm_set1_epi32((Int32)KIV(i))
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#define GET_CONST_128_FROM_ARRAY32(k) \
_mm_set_epi32((Int32)(k)[3], (Int32)(k)[2], (Int32)(k)[1], (Int32)(k)[0])
#endif
#if 0
#define k_r8 _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)
#define k_r16 _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)
#define k_inc _mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)
#define k_iv0_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0)
#define k_iv4_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4)
#else
#if defined(Z7_BLAKE2S_USE_SSSE3) && \
!defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
MY_ALIGN(16) static const Byte k_r8_arr [16] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 };
MY_ALIGN(16) static const Byte k_r16_arr[16] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 };
#define k_r8 LOAD_128(k_r8_arr)
#define k_r16 LOAD_128(k_r16_arr)
#endif
MY_ALIGN(16) static const UInt32 k_inc_arr[4] = { Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0 };
#define k_inc LOAD_128(k_inc_arr)
#define k_iv0_128 LOAD_128(k_Blake2s_IV + 0)
#define k_iv4_128 LOAD_128(k_Blake2s_IV + 4)
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#ifdef Z7_BLAKE2S_USE_AVX2
#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000)
#define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1)
#else
#define MY_mm256_set_m128i _mm256_set_m128i
#endif
#define SET_FROM_128(a) MY_mm256_set_m128i(a, a)
#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS
MY_ALIGN(32) static const Byte k_r8_arr_256 [32] =
{
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12,
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12
};
MY_ALIGN(32) static const Byte k_r16_arr_256[32] =
{
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13,
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
};
#define k_r8_256 LOAD_256(k_r8_arr_256)
#define k_r16_256 LOAD_256(k_r16_arr_256)
#endif
// #define k_r8_256 SET_FROM_128(_mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1))
// #define k_r16_256 SET_FROM_128(_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2))
// #define k_inc_256 SET_FROM_128(_mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE))
// #define k_iv0_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0))
#define k_iv4_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4))
#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW
#endif
/*
IPC(TP) ports:
1 p__5 : skl- : SSE : shufps : _mm_shuffle_ps
2 p_15 : icl+
1 p__5 : nhm-bdw : SSE : xorps : _mm_xor_ps
3 p015 : skl+
3 p015 : SSE2 : pxor : _mm_xor_si128
2 p_15: snb-bdw : SSE2 : padd : _mm_add_epi32
2 p0_5: mrm-wsm :
3 p015 : skl+
2 p_15 : ivb-,icl+ : SSE2 : punpcklqdq, punpckhqdq, punpckldq, punpckhdq
2 p_15 : : SSE2 : pshufd : _mm_shuffle_epi32
2 p_15 : : SSE2 : pshuflw : _mm_shufflelo_epi16
2 p_15 : : SSE2 : psrldq :
2 p_15 : : SSE3 : pshufb : _mm_shuffle_epi8
2 p_15 : : SSE4 : pblendw : _mm_blend_epi16
1 p__5 : hsw-skl : *
1 p0 : SSE2 : pslld (i8) : _mm_slli_si128
2 p01 : skl+ :
2 p_15 : ivb- : SSE3 : palignr
1 p__5 : hsw+
2 p_15 + p23 : ivb-, icl+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, m32, i8)
1 p__5 + p23 : hsw-skl
1 p_15 + p5 : ivb-, ice+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, r32, i8)
0.5 2*p5 : hsw-skl
2 p23 : SSE2 : movd (m32)
3 p23A : adl :
1 p5: : SSE2 : movd (r32)
*/
#if 0 && defined(__XOP__)
// we must debug and test __XOP__ instruction
#include <x86intrin.h>
#include <ammintrin.h>
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) _mm_roti_epi32(r, -(c))
#define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#elif 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS)
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) _mm_ror_epi32(r, c)
#define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#else
// MSVC_1937+ uses "orps" instruction for _mm_or_si128().
// But "orps" has low throughput: TP=1 for bdw-nhm.
// So it can be better to use _mm_add_epi32()/"paddd" (TP=2 for bdw-nhm) instead of "xorps".
// But "orps" is fast for modern cpus (skl+).
// So we are default with "or" version:
#if 0 || 0 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1937
// minor optimization for some old cpus, if "xorps" is slow.
#define MM128_EPI32_OR_or_ADD _mm_add_epi32
#else
#define MM128_EPI32_OR_or_ADD _mm_or_si128
#endif
#define MM_ROR_EPI32_VIA_SHIFT(r, c)( \
MM128_EPI32_OR_or_ADD( \
_mm_srli_epi32((r), (c)), \
_mm_slli_epi32((r), 32-(c))))
#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41)
#define LOAD_ROTATE_CONSTS \
const __m128i r8 = k_r8; \
const __m128i r16 = k_r16;
#define MM_ROR_EPI32(r, c) ( \
( 8==(c)) ? _mm_shuffle_epi8(r,r8) \
: (16==(c)) ? _mm_shuffle_epi8(r,r16) \
: MM_ROR_EPI32_VIA_SHIFT(r, c))
#else
#define LOAD_ROTATE_CONSTS
#define MM_ROR_EPI32(r, c) ( \
(16==(c)) ? _mm_shufflehi_epi16(_mm_shufflelo_epi16(r, 0xb1), 0xb1) \
: MM_ROR_EPI32_VIA_SHIFT(r, c))
#endif
#endif
/*
we have 3 main ways to load 4 32-bit integers to __m128i:
1) SSE2: _mm_set_epi32()
2) SSE2: _mm_unpacklo_epi64() / _mm_unpacklo_epi32 / _mm_cvtsi32_si128()
3) SSE41: _mm_insert_epi32() and _mm_cvtsi32_si128()
good compiler for _mm_set_epi32() generates these instructions:
{
movd xmm, [m32]; vpunpckldq; vpunpckldq; vpunpcklqdq;
}
good new compiler generates one instruction
{
for _mm_insert_epi32() : { pinsrd xmm, [m32], i }
for _mm_cvtsi32_si128() : { movd xmm, [m32] }
}
but vc2010 generates slow pair of instructions:
{
for _mm_insert_epi32() : { mov r32, [m32]; pinsrd xmm, r32, i }
for _mm_cvtsi32_si128() : { mov r32, [m32]; movd xmm, r32 }
}
_mm_insert_epi32() (pinsrd) code reduces xmm register pressure
in comparison with _mm_set_epi32() (movd + vpunpckld) code.
Note that variant with "movd xmm, r32" can be more slow,
but register pressure can be more important.
So we can force to "pinsrd" always.
*/
// #if !defined(Z7_MSC_VER_ORIGINAL) || Z7_MSC_VER_ORIGINAL > 1600 || defined(MY_CPU_X86)
#ifdef Z7_BLAKE2S_USE_SSE41
/* _mm_set_epi32() can be more effective for GCC and CLANG
_mm_insert_epi32() is more effective for MSVC */
#if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL)
#define Z7_BLAKE2S_USE_INSERT_INSTRUCTION
#endif
#endif // USE_SSE41
// #endif
#ifdef Z7_BLAKE2S_USE_INSERT_INSTRUCTION
// for SSE4.1
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_insert_epi32( \
_mm_insert_epi32( \
_mm_insert_epi32( \
_mm_cvtsi32_si128( \
*(const Int32 *)p0), \
*(const Int32 *)p1, 1), \
*(const Int32 *)p2, 2), \
*(const Int32 *)p3, 3)
#elif 0 || 1 && defined(Z7_MSC_VER_ORIGINAL)
/* MSVC 1400 implements _mm_set_epi32() via slow memory write/read.
Also _mm_unpacklo_epi32 is more effective for another MSVC compilers.
But _mm_set_epi32() is more effective for GCC and CLANG.
So we use _mm_unpacklo_epi32 for MSVC only */
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_unpacklo_epi64( \
_mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p0), \
_mm_cvtsi32_si128(*(const Int32 *)p1)), \
_mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p2), \
_mm_cvtsi32_si128(*(const Int32 *)p3)))
#else
#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \
_mm_set_epi32( \
*(const Int32 *)p3, \
*(const Int32 *)p2, \
*(const Int32 *)p1, \
*(const Int32 *)p0)
#endif
#define SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) \
MM_LOAD_EPI32_FROM_4_POINTERS( \
GET_SIGMA_PTR(input, i0), \
GET_SIGMA_PTR(input, i1), \
GET_SIGMA_PTR(input, i2), \
GET_SIGMA_PTR(input, i3))
#define SET_ROW_FROM_SIGMA(input, sigma_index) \
SET_ROW_FROM_SIGMA_BASE(input, \
sigma[(sigma_index) ], \
sigma[(sigma_index) + 2 * 1], \
sigma[(sigma_index) + 2 * 2], \
sigma[(sigma_index) + 2 * 3]) \
#define ADD_128(a, b) _mm_add_epi32(a, b)
#define XOR_128(a, b) _mm_xor_si128(a, b)
#define D_ADD_128(dest, src) dest = ADD_128(dest, src)
#define D_XOR_128(dest, src) dest = XOR_128(dest, src)
#define D_ROR_128(dest, shift) dest = MM_ROR_EPI32(dest, shift)
#define D_ADD_EPI64_128(dest, src) dest = _mm_add_epi64(dest, src)
#define AXR(a, b, d, shift) \
D_ADD_128(a, b); \
D_XOR_128(d, a); \
D_ROR_128(d, shift);
#define AXR2(a, b, c, d, input, sigma_index, shift1, shift2) \
a = _mm_add_epi32 (a, SET_ROW_FROM_SIGMA(input, sigma_index)); \
AXR(a, b, d, shift1) \
AXR(c, d, b, shift2)
#define ROTATE_WORDS_TO_RIGHT(a, n) \
a = _mm_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3));
#define AXR4(a, b, c, d, input, sigma_index) \
AXR2(a, b, c, d, input, sigma_index, 16, 12) \
AXR2(a, b, c, d, input, sigma_index + 1, 8, 7) \
#define RR2(a, b, c, d, input) \
{ \
AXR4(a, b, c, d, input, 0) \
ROTATE_WORDS_TO_RIGHT(b, 1) \
ROTATE_WORDS_TO_RIGHT(c, 2) \
ROTATE_WORDS_TO_RIGHT(d, 3) \
AXR4(a, b, c, d, input, 8) \
ROTATE_WORDS_TO_RIGHT(b, 3) \
ROTATE_WORDS_TO_RIGHT(c, 2) \
ROTATE_WORDS_TO_RIGHT(d, 1) \
}
/*
Way1:
per 64 bytes block:
10 rounds * 4 iters * (7 + 2) = 360 cycles = if pslld TP=1
* (7 + 1) = 320 cycles = if pslld TP=2 (skl+)
additional operations per 7_op_iter :
4 movzx byte mem
1 movd mem
3 pinsrd mem
1.5 pshufd
*/
static
#if 0 || 0 && (defined(Z7_BLAKE2S_USE_V128_WAY2) || \
defined(Z7_BLAKE2S_USE_V256_WAY2))
Z7_NO_INLINE
#else
Z7_FORCE_INLINE
#endif
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2s_Compress_V128_Way1(UInt32 * const s, const Byte * const input)
{
__m128i a, b, c, d;
__m128i f0, f1;
LOAD_ROTATE_CONSTS
d = LOAD_128_FROM_STRUCT(STATE_T(s));
c = k_iv0_128;
a = f0 = LOAD_128_FROM_STRUCT(s);
b = f1 = LOAD_128_FROM_STRUCT(s + 4);
D_ADD_EPI64_128(d, k_inc);
STORE_128_TO_STRUCT (STATE_T(s), d);
D_XOR_128(d, k_iv4_128);
#define RR(r) { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \
RR2(a, b, c, d, input) }
ROUNDS_LOOP(RR)
#undef RR
STORE_128_TO_STRUCT(s , XOR_128(f0, XOR_128(a, c)));
STORE_128_TO_STRUCT(s + 4, XOR_128(f1, XOR_128(b, d)));
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_V128_Way1(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Blake2s_Compress_V128_Way1(s, data);
data += Z7_BLAKE2S_BLOCK_SIZE;
pos += Z7_BLAKE2S_BLOCK_SIZE;
pos &= SUPER_BLOCK_MASK;
}
while (data != end);
}
#if defined(Z7_BLAKE2S_USE_V128_WAY2) || \
defined(Z7_BLAKE2S_USE_AVX2_WAY2)
#if 1
#define Z7_BLAKE2S_CompressSingleBlock(s, data) \
Blake2sp_Compress2_V128_Way1(s, data, \
(const Byte *)(const void *)(data) + Z7_BLAKE2S_BLOCK_SIZE)
#else
#define Z7_BLAKE2S_CompressSingleBlock Blake2s_Compress_V128_Way1
#endif
#endif
#if (defined(Z7_BLAKE2S_USE_AVX2_WAY_SLOW) || \
defined(Z7_BLAKE2S_USE_V128_WAY2)) && \
!defined(Z7_BLAKE2S_USE_GATHER)
#define AXR2_LOAD_INDEXES(sigma_index) \
const unsigned i0 = sigma[(sigma_index)]; \
const unsigned i1 = sigma[(sigma_index) + 2 * 1]; \
const unsigned i2 = sigma[(sigma_index) + 2 * 2]; \
const unsigned i3 = sigma[(sigma_index) + 2 * 3]; \
#define SET_ROW_FROM_SIGMA_W(input) \
SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3)
#endif
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#if 1 || !defined(Z7_BLAKE2S_USE_SSE41)
/* we use SET_ROW_FROM_SIGMA_BASE, that uses
(SSE4) _mm_insert_epi32(), if Z7_BLAKE2S_USE_INSERT_INSTRUCTION is defined
(SSE2) _mm_set_epi32()
MSVC can be faster for this branch:
*/
#define AXR2_W(sigma_index, shift1, shift2) \
{ \
AXR2_LOAD_INDEXES(sigma_index) \
a0 = _mm_add_epi32(a0, SET_ROW_FROM_SIGMA_W(data)); \
a1 = _mm_add_epi32(a1, SET_ROW_FROM_SIGMA_W(data + Z7_BLAKE2S_BLOCK_SIZE)); \
AXR(a0, b0, d0, shift1) \
AXR(a1, b1, d1, shift1) \
AXR(c0, d0, b0, shift2) \
AXR(c1, d1, b1, shift2) \
}
#else
/* we use interleaved _mm_insert_epi32():
GCC can be faster for this branch:
*/
#define AXR2_W_PRE_INSERT(sigma_index, i) \
{ const unsigned ii = sigma[(sigma_index) + i * 2]; \
t0 = _mm_insert_epi32(t0, *(const Int32 *)GET_SIGMA_PTR(data, ii), i); \
t1 = _mm_insert_epi32(t1, *(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii), i); \
}
#define AXR2_W(sigma_index, shift1, shift2) \
{ __m128i t0, t1; \
{ const unsigned ii = sigma[sigma_index]; \
t0 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, ii)); \
t1 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii)); \
} \
AXR2_W_PRE_INSERT(sigma_index, 1) \
AXR2_W_PRE_INSERT(sigma_index, 2) \
AXR2_W_PRE_INSERT(sigma_index, 3) \
a0 = _mm_add_epi32(a0, t0); \
a1 = _mm_add_epi32(a1, t1); \
AXR(a0, b0, d0, shift1) \
AXR(a1, b1, d1, shift1) \
AXR(c0, d0, b0, shift2) \
AXR(c1, d1, b1, shift2) \
}
#endif
#define AXR4_W(sigma_index) \
AXR2_W(sigma_index, 16, 12) \
AXR2_W(sigma_index + 1, 8, 7) \
#define WW(r) \
{ const Byte * const sigma = k_Blake2s_Sigma_4[r]; \
AXR4_W(0) \
ROTATE_WORDS_TO_RIGHT(b0, 1) \
ROTATE_WORDS_TO_RIGHT(b1, 1) \
ROTATE_WORDS_TO_RIGHT(c0, 2) \
ROTATE_WORDS_TO_RIGHT(c1, 2) \
ROTATE_WORDS_TO_RIGHT(d0, 3) \
ROTATE_WORDS_TO_RIGHT(d1, 3) \
AXR4_W(8) \
ROTATE_WORDS_TO_RIGHT(b0, 3) \
ROTATE_WORDS_TO_RIGHT(b1, 3) \
ROTATE_WORDS_TO_RIGHT(c0, 2) \
ROTATE_WORDS_TO_RIGHT(c1, 2) \
ROTATE_WORDS_TO_RIGHT(d0, 1) \
ROTATE_WORDS_TO_RIGHT(d1, 1) \
}
static
Z7_NO_INLINE
#ifdef BLAKE2S_ATTRIB_128BIT
BLAKE2S_ATTRIB_128BIT
#endif
void
Z7_FASTCALL
Blake2sp_Compress2_V128_Way2(UInt32 *s_items, const Byte *data, const Byte *end)
{
size_t pos = 0;
end -= Z7_BLAKE2S_BLOCK_SIZE;
if (data != end)
{
LOAD_ROTATE_CONSTS
do
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
__m128i a0, b0, c0, d0;
__m128i a1, b1, c1, d1;
{
const __m128i inc = k_inc;
const __m128i temp = k_iv4_128;
d0 = LOAD_128_FROM_STRUCT (STATE_T(s));
d1 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW));
D_ADD_EPI64_128(d0, inc);
D_ADD_EPI64_128(d1, inc);
STORE_128_TO_STRUCT (STATE_T(s ), d0);
STORE_128_TO_STRUCT (STATE_T(s + NSW), d1);
D_XOR_128(d0, temp);
D_XOR_128(d1, temp);
}
c1 = c0 = k_iv0_128;
a0 = LOAD_128_FROM_STRUCT(s);
b0 = LOAD_128_FROM_STRUCT(s + 4);
a1 = LOAD_128_FROM_STRUCT(s + NSW);
b1 = LOAD_128_FROM_STRUCT(s + NSW + 4);
ROUNDS_LOOP (WW)
#undef WW
D_XOR_128(a0, c0);
D_XOR_128(b0, d0);
D_XOR_128(a1, c1);
D_XOR_128(b1, d1);
D_XOR_128(a0, LOAD_128_FROM_STRUCT(s));
D_XOR_128(b0, LOAD_128_FROM_STRUCT(s + 4));
D_XOR_128(a1, LOAD_128_FROM_STRUCT(s + NSW));
D_XOR_128(b1, LOAD_128_FROM_STRUCT(s + NSW + 4));
STORE_128_TO_STRUCT(s, a0);
STORE_128_TO_STRUCT(s + 4, b0);
STORE_128_TO_STRUCT(s + NSW, a1);
STORE_128_TO_STRUCT(s + NSW + 4, b1);
data += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos += Z7_BLAKE2S_BLOCK_SIZE * 2;
pos &= SUPER_BLOCK_MASK;
}
while (data < end);
if (data != end)
return;
}
{
UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos);
Z7_BLAKE2S_CompressSingleBlock(s, data);
}
}
#endif // Z7_BLAKE2S_USE_V128_WAY2
#ifdef Z7_BLAKE2S_USE_V128_WAY2
#define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way2
#else
#define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way1
#endif
#ifdef Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED
#define ROT_128_8(x) MM_ROR_EPI32(x, 8)
#define ROT_128_16(x) MM_ROR_EPI32(x, 16)
#define ROT_128_7(x) MM_ROR_EPI32(x, 7)
#define ROT_128_12(x) MM_ROR_EPI32(x, 12)
#else
#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41)
#define ROT_128_8(x) _mm_shuffle_epi8(x, r8) // k_r8
#define ROT_128_16(x) _mm_shuffle_epi8(x, r16) // k_r16
#else
#define ROT_128_8(x) MM_ROR_EPI32_VIA_SHIFT(x, 8)
#define ROT_128_16(x) MM_ROR_EPI32_VIA_SHIFT(x, 16)
#endif
#define ROT_128_7(x) MM_ROR_EPI32_VIA_SHIFT(x, 7)
#define ROT_128_12(x) MM_ROR_EPI32_VIA_SHIFT(x, 12)
#endif
#if 1
// this branch can provide similar speed on x86* in most cases,
// because [base + index*4] provides same speed as [base + index].
// but some compilers can generate different code with this branch, that can be faster sometimes.
// this branch uses additional table of 10*16=160 bytes.
#define SIGMA_TABLE_MULT_16( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(16, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
MY_ALIGN(16)
static const Byte k_Blake2s_Sigma_16[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_16) };
#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_16[r];
#define GET_SIGMA_VAL_128(n) (sigma[n])
#else
#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_4[r];
#define GET_SIGMA_VAL_128(n) (4 * (size_t)sigma[n])
#endif
#ifdef Z7_BLAKE2S_USE_AVX2_FAST
#if 1
#define SIGMA_TABLE_MULT_32( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
SIGMA_TABLE_MULT(32, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15)
MY_ALIGN(64)
static const UInt16 k_Blake2s_Sigma_32[BLAKE2S_NUM_ROUNDS][16] =
{ SIGMA_TABLE(SIGMA_TABLE_MULT_32) };
#define GET_SIGMA_PTR_256(r) const UInt16 * const sigma = k_Blake2s_Sigma_32[r];
#define GET_SIGMA_VAL_256(n) (sigma[n])
#else
#define GET_SIGMA_PTR_256(r) const Byte * const sigma = k_Blake2s_Sigma_4[r];
#define GET_SIGMA_VAL_256(n) (8 * (size_t)sigma[n])
#endif
#endif // Z7_BLAKE2S_USE_AVX2_FAST
#define D_ROT_128_7(dest) dest = ROT_128_7(dest)
#define D_ROT_128_8(dest) dest = ROT_128_8(dest)
#define D_ROT_128_12(dest) dest = ROT_128_12(dest)
#define D_ROT_128_16(dest) dest = ROT_128_16(dest)
#define OP_L(a, i) D_ADD_128 (V(a, 0), \
LOAD_128((const Byte *)(w) + GET_SIGMA_VAL_128(2*(a)+(i))));
#define OP_0(a) OP_L(a, 0)
#define OP_7(a) OP_L(a, 1)
#define OP_1(a) D_ADD_128 (V(a, 0), V(a, 1));
#define OP_2(a) D_XOR_128 (V(a, 3), V(a, 0));
#define OP_4(a) D_ADD_128 (V(a, 2), V(a, 3));
#define OP_5(a) D_XOR_128 (V(a, 1), V(a, 2));
#define OP_3(a) D_ROT_128_16 (V(a, 3));
#define OP_6(a) D_ROT_128_12 (V(a, 1));
#define OP_8(a) D_ROT_128_8 (V(a, 3));
#define OP_9(a) D_ROT_128_7 (V(a, 1));
// for 32-bit x86 : interleave mode works slower, because of register pressure.
#if 0 || 1 && (defined(MY_CPU_X86) \
|| defined(__GNUC__) && !defined(__clang__))
// non-inteleaved version:
// is fast for x86 32-bit.
// is fast for GCC x86-64.
#define V4G(a) \
OP_0 (a) \
OP_1 (a) \
OP_2 (a) \
OP_3 (a) \
OP_4 (a) \
OP_5 (a) \
OP_6 (a) \
OP_7 (a) \
OP_1 (a) \
OP_2 (a) \
OP_8 (a) \
OP_4 (a) \
OP_5 (a) \
OP_9 (a) \
#define V4R \
{ \
V4G (0) \
V4G (1) \
V4G (2) \
V4G (3) \
V4G (4) \
V4G (5) \
V4G (6) \
V4G (7) \
}
#elif 0 || 1 && defined(MY_CPU_X86)
#define OP_INTER_2(op, a,b) \
op (a) \
op (b) \
#define V4G(a,b) \
OP_INTER_2 (OP_0, a,b) \
OP_INTER_2 (OP_1, a,b) \
OP_INTER_2 (OP_2, a,b) \
OP_INTER_2 (OP_3, a,b) \
OP_INTER_2 (OP_4, a,b) \
OP_INTER_2 (OP_5, a,b) \
OP_INTER_2 (OP_6, a,b) \
OP_INTER_2 (OP_7, a,b) \
OP_INTER_2 (OP_1, a,b) \
OP_INTER_2 (OP_2, a,b) \
OP_INTER_2 (OP_8, a,b) \
OP_INTER_2 (OP_4, a,b) \
OP_INTER_2 (OP_5, a,b) \
OP_INTER_2 (OP_9, a,b) \
#define V4R \
{ \
V4G (0, 1) \
V4G (2, 3) \
V4G (4, 5) \
V4G (6, 7) \
}
#else
// iterleave-4 version is fast for x64 (MSVC/CLANG)
#define OP_INTER_4(op, a,b,c,d) \
op (a) \
0