Currently blake2s's assembly routines claim they have a zero byte frame and manually subtract upwards of 704 bytes from the stack pointer without cooperating with Go's ABI. As a result, these functions may not grow the stack when necessary, leading to memory corruption. Fix this by using the correct stack frame sizes so the generated stack growth prologue is correct, and aligning the SP up instead of down. Change-Id: Ic426338c45c94a2c01d549860c2295a0ee9200bf Reviewed-on: https://go-review.googlesource.com/31583 Reviewed-by: Adam Langley <agl@golang.org> Reviewed-by: Andreas Auernhammer <aead@mail.de> Reviewed-by: Minux Ma <minux@golang.org>
464 lines
16 KiB
ArmAsm
464 lines
16 KiB
ArmAsm
// Copyright 2016 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build amd64, !gccgo, !appengine
|
|
|
|
#include "textflag.h"
|
|
|
|
DATA iv0<>+0x00(SB)/4, $0x6a09e667
|
|
DATA iv0<>+0x04(SB)/4, $0xbb67ae85
|
|
DATA iv0<>+0x08(SB)/4, $0x3c6ef372
|
|
DATA iv0<>+0x0c(SB)/4, $0xa54ff53a
|
|
GLOBL iv0<>(SB), (NOPTR+RODATA), $16
|
|
|
|
DATA iv1<>+0x00(SB)/4, $0x510e527f
|
|
DATA iv1<>+0x04(SB)/4, $0x9b05688c
|
|
DATA iv1<>+0x08(SB)/4, $0x1f83d9ab
|
|
DATA iv1<>+0x0c(SB)/4, $0x5be0cd19
|
|
GLOBL iv1<>(SB), (NOPTR+RODATA), $16
|
|
|
|
DATA rol16<>+0x00(SB)/8, $0x0504070601000302
|
|
DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
|
|
GLOBL rol16<>(SB), (NOPTR+RODATA), $16
|
|
|
|
DATA rol8<>+0x00(SB)/8, $0x0407060500030201
|
|
DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09
|
|
GLOBL rol8<>(SB), (NOPTR+RODATA), $16
|
|
|
|
DATA counter<>+0x00(SB)/8, $0x40
|
|
DATA counter<>+0x08(SB)/8, $0x0
|
|
GLOBL counter<>(SB), (NOPTR+RODATA), $16
|
|
|
|
#define ROTL_SSE2(n, t, v) \
|
|
MOVO v, t; \
|
|
PSLLL $n, t; \
|
|
PSRLL $(32-n), v; \
|
|
PXOR t, v
|
|
|
|
#define ROTL_SSSE3(c, v) \
|
|
PSHUFB c, v
|
|
|
|
#define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \
|
|
PADDL m0, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSE2(16, t, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(20, t, v1); \
|
|
PADDL m1, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSE2(24, t, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(25, t, v1); \
|
|
PSHUFL $0x39, v1, v1; \
|
|
PSHUFL $0x4E, v2, v2; \
|
|
PSHUFL $0x93, v3, v3; \
|
|
PADDL m2, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSE2(16, t, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(20, t, v1); \
|
|
PADDL m3, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSE2(24, t, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(25, t, v1); \
|
|
PSHUFL $0x39, v3, v3; \
|
|
PSHUFL $0x4E, v2, v2; \
|
|
PSHUFL $0x93, v1, v1
|
|
|
|
#define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \
|
|
PADDL m0, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSSE3(c16, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(20, t, v1); \
|
|
PADDL m1, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSSE3(c8, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(25, t, v1); \
|
|
PSHUFL $0x39, v1, v1; \
|
|
PSHUFL $0x4E, v2, v2; \
|
|
PSHUFL $0x93, v3, v3; \
|
|
PADDL m2, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSSE3(c16, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(20, t, v1); \
|
|
PADDL m3, v0; \
|
|
PADDL v1, v0; \
|
|
PXOR v0, v3; \
|
|
ROTL_SSSE3(c8, v3); \
|
|
PADDL v3, v2; \
|
|
PXOR v2, v1; \
|
|
ROTL_SSE2(25, t, v1); \
|
|
PSHUFL $0x39, v3, v3; \
|
|
PSHUFL $0x4E, v2, v2; \
|
|
PSHUFL $0x93, v1, v1
|
|
|
|
|
|
#define LOAD_MSG_SSE4(m0, m1, m2, m3, src, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15) \
|
|
MOVL i0*4(src), m0; \
|
|
PINSRD $1, i1*4(src), m0; \
|
|
PINSRD $2, i2*4(src), m0; \
|
|
PINSRD $3, i3*4(src), m0; \
|
|
MOVL i4*4(src), m1; \
|
|
PINSRD $1, i5*4(src), m1; \
|
|
PINSRD $2, i6*4(src), m1; \
|
|
PINSRD $3, i7*4(src), m1; \
|
|
MOVL i8*4(src), m2; \
|
|
PINSRD $1, i9*4(src), m2; \
|
|
PINSRD $2, i10*4(src), m2; \
|
|
PINSRD $3, i11*4(src), m2; \
|
|
MOVL i12*4(src), m3; \
|
|
PINSRD $1, i13*4(src), m3; \
|
|
PINSRD $2, i14*4(src), m3; \
|
|
PINSRD $3, i15*4(src), m3
|
|
|
|
#define PRECOMPUTE_MSG(dst, off, src, R8, R9, R10, R11, R12, R13, R14, R15) \
|
|
MOVQ 0*4(src), R8; \
|
|
MOVQ 2*4(src), R9; \
|
|
MOVQ 4*4(src), R10; \
|
|
MOVQ 6*4(src), R11; \
|
|
MOVQ 8*4(src), R12; \
|
|
MOVQ 10*4(src), R13; \
|
|
MOVQ 12*4(src), R14; \
|
|
MOVQ 14*4(src), R15; \
|
|
\
|
|
MOVL R8, 0*4+off+0(dst); \
|
|
MOVL R8, 9*4+off+64(dst); \
|
|
MOVL R8, 5*4+off+128(dst); \
|
|
MOVL R8, 14*4+off+192(dst); \
|
|
MOVL R8, 4*4+off+256(dst); \
|
|
MOVL R8, 2*4+off+320(dst); \
|
|
MOVL R8, 8*4+off+384(dst); \
|
|
MOVL R8, 12*4+off+448(dst); \
|
|
MOVL R8, 3*4+off+512(dst); \
|
|
MOVL R8, 15*4+off+576(dst); \
|
|
SHRQ $32, R8; \
|
|
MOVL R8, 4*4+off+0(dst); \
|
|
MOVL R8, 8*4+off+64(dst); \
|
|
MOVL R8, 14*4+off+128(dst); \
|
|
MOVL R8, 5*4+off+192(dst); \
|
|
MOVL R8, 12*4+off+256(dst); \
|
|
MOVL R8, 11*4+off+320(dst); \
|
|
MOVL R8, 1*4+off+384(dst); \
|
|
MOVL R8, 6*4+off+448(dst); \
|
|
MOVL R8, 10*4+off+512(dst); \
|
|
MOVL R8, 3*4+off+576(dst); \
|
|
\
|
|
MOVL R9, 1*4+off+0(dst); \
|
|
MOVL R9, 13*4+off+64(dst); \
|
|
MOVL R9, 6*4+off+128(dst); \
|
|
MOVL R9, 8*4+off+192(dst); \
|
|
MOVL R9, 2*4+off+256(dst); \
|
|
MOVL R9, 0*4+off+320(dst); \
|
|
MOVL R9, 14*4+off+384(dst); \
|
|
MOVL R9, 11*4+off+448(dst); \
|
|
MOVL R9, 12*4+off+512(dst); \
|
|
MOVL R9, 4*4+off+576(dst); \
|
|
SHRQ $32, R9; \
|
|
MOVL R9, 5*4+off+0(dst); \
|
|
MOVL R9, 15*4+off+64(dst); \
|
|
MOVL R9, 9*4+off+128(dst); \
|
|
MOVL R9, 1*4+off+192(dst); \
|
|
MOVL R9, 11*4+off+256(dst); \
|
|
MOVL R9, 7*4+off+320(dst); \
|
|
MOVL R9, 13*4+off+384(dst); \
|
|
MOVL R9, 3*4+off+448(dst); \
|
|
MOVL R9, 6*4+off+512(dst); \
|
|
MOVL R9, 10*4+off+576(dst); \
|
|
\
|
|
MOVL R10, 2*4+off+0(dst); \
|
|
MOVL R10, 1*4+off+64(dst); \
|
|
MOVL R10, 15*4+off+128(dst); \
|
|
MOVL R10, 10*4+off+192(dst); \
|
|
MOVL R10, 6*4+off+256(dst); \
|
|
MOVL R10, 8*4+off+320(dst); \
|
|
MOVL R10, 3*4+off+384(dst); \
|
|
MOVL R10, 13*4+off+448(dst); \
|
|
MOVL R10, 14*4+off+512(dst); \
|
|
MOVL R10, 5*4+off+576(dst); \
|
|
SHRQ $32, R10; \
|
|
MOVL R10, 6*4+off+0(dst); \
|
|
MOVL R10, 11*4+off+64(dst); \
|
|
MOVL R10, 2*4+off+128(dst); \
|
|
MOVL R10, 9*4+off+192(dst); \
|
|
MOVL R10, 1*4+off+256(dst); \
|
|
MOVL R10, 13*4+off+320(dst); \
|
|
MOVL R10, 4*4+off+384(dst); \
|
|
MOVL R10, 8*4+off+448(dst); \
|
|
MOVL R10, 15*4+off+512(dst); \
|
|
MOVL R10, 7*4+off+576(dst); \
|
|
\
|
|
MOVL R11, 3*4+off+0(dst); \
|
|
MOVL R11, 7*4+off+64(dst); \
|
|
MOVL R11, 13*4+off+128(dst); \
|
|
MOVL R11, 12*4+off+192(dst); \
|
|
MOVL R11, 10*4+off+256(dst); \
|
|
MOVL R11, 1*4+off+320(dst); \
|
|
MOVL R11, 9*4+off+384(dst); \
|
|
MOVL R11, 14*4+off+448(dst); \
|
|
MOVL R11, 0*4+off+512(dst); \
|
|
MOVL R11, 6*4+off+576(dst); \
|
|
SHRQ $32, R11; \
|
|
MOVL R11, 7*4+off+0(dst); \
|
|
MOVL R11, 14*4+off+64(dst); \
|
|
MOVL R11, 10*4+off+128(dst); \
|
|
MOVL R11, 0*4+off+192(dst); \
|
|
MOVL R11, 5*4+off+256(dst); \
|
|
MOVL R11, 9*4+off+320(dst); \
|
|
MOVL R11, 12*4+off+384(dst); \
|
|
MOVL R11, 1*4+off+448(dst); \
|
|
MOVL R11, 13*4+off+512(dst); \
|
|
MOVL R11, 2*4+off+576(dst); \
|
|
\
|
|
MOVL R12, 8*4+off+0(dst); \
|
|
MOVL R12, 5*4+off+64(dst); \
|
|
MOVL R12, 4*4+off+128(dst); \
|
|
MOVL R12, 15*4+off+192(dst); \
|
|
MOVL R12, 14*4+off+256(dst); \
|
|
MOVL R12, 3*4+off+320(dst); \
|
|
MOVL R12, 11*4+off+384(dst); \
|
|
MOVL R12, 10*4+off+448(dst); \
|
|
MOVL R12, 7*4+off+512(dst); \
|
|
MOVL R12, 1*4+off+576(dst); \
|
|
SHRQ $32, R12; \
|
|
MOVL R12, 12*4+off+0(dst); \
|
|
MOVL R12, 2*4+off+64(dst); \
|
|
MOVL R12, 11*4+off+128(dst); \
|
|
MOVL R12, 4*4+off+192(dst); \
|
|
MOVL R12, 0*4+off+256(dst); \
|
|
MOVL R12, 15*4+off+320(dst); \
|
|
MOVL R12, 10*4+off+384(dst); \
|
|
MOVL R12, 7*4+off+448(dst); \
|
|
MOVL R12, 5*4+off+512(dst); \
|
|
MOVL R12, 9*4+off+576(dst); \
|
|
\
|
|
MOVL R13, 9*4+off+0(dst); \
|
|
MOVL R13, 4*4+off+64(dst); \
|
|
MOVL R13, 8*4+off+128(dst); \
|
|
MOVL R13, 13*4+off+192(dst); \
|
|
MOVL R13, 3*4+off+256(dst); \
|
|
MOVL R13, 5*4+off+320(dst); \
|
|
MOVL R13, 7*4+off+384(dst); \
|
|
MOVL R13, 15*4+off+448(dst); \
|
|
MOVL R13, 11*4+off+512(dst); \
|
|
MOVL R13, 0*4+off+576(dst); \
|
|
SHRQ $32, R13; \
|
|
MOVL R13, 13*4+off+0(dst); \
|
|
MOVL R13, 10*4+off+64(dst); \
|
|
MOVL R13, 0*4+off+128(dst); \
|
|
MOVL R13, 3*4+off+192(dst); \
|
|
MOVL R13, 9*4+off+256(dst); \
|
|
MOVL R13, 6*4+off+320(dst); \
|
|
MOVL R13, 15*4+off+384(dst); \
|
|
MOVL R13, 4*4+off+448(dst); \
|
|
MOVL R13, 2*4+off+512(dst); \
|
|
MOVL R13, 12*4+off+576(dst); \
|
|
\
|
|
MOVL R14, 10*4+off+0(dst); \
|
|
MOVL R14, 12*4+off+64(dst); \
|
|
MOVL R14, 1*4+off+128(dst); \
|
|
MOVL R14, 6*4+off+192(dst); \
|
|
MOVL R14, 13*4+off+256(dst); \
|
|
MOVL R14, 4*4+off+320(dst); \
|
|
MOVL R14, 0*4+off+384(dst); \
|
|
MOVL R14, 2*4+off+448(dst); \
|
|
MOVL R14, 8*4+off+512(dst); \
|
|
MOVL R14, 14*4+off+576(dst); \
|
|
SHRQ $32, R14; \
|
|
MOVL R14, 14*4+off+0(dst); \
|
|
MOVL R14, 3*4+off+64(dst); \
|
|
MOVL R14, 7*4+off+128(dst); \
|
|
MOVL R14, 2*4+off+192(dst); \
|
|
MOVL R14, 15*4+off+256(dst); \
|
|
MOVL R14, 12*4+off+320(dst); \
|
|
MOVL R14, 6*4+off+384(dst); \
|
|
MOVL R14, 0*4+off+448(dst); \
|
|
MOVL R14, 9*4+off+512(dst); \
|
|
MOVL R14, 11*4+off+576(dst); \
|
|
\
|
|
MOVL R15, 11*4+off+0(dst); \
|
|
MOVL R15, 0*4+off+64(dst); \
|
|
MOVL R15, 12*4+off+128(dst); \
|
|
MOVL R15, 7*4+off+192(dst); \
|
|
MOVL R15, 8*4+off+256(dst); \
|
|
MOVL R15, 14*4+off+320(dst); \
|
|
MOVL R15, 2*4+off+384(dst); \
|
|
MOVL R15, 5*4+off+448(dst); \
|
|
MOVL R15, 1*4+off+512(dst); \
|
|
MOVL R15, 13*4+off+576(dst); \
|
|
SHRQ $32, R15; \
|
|
MOVL R15, 15*4+off+0(dst); \
|
|
MOVL R15, 6*4+off+64(dst); \
|
|
MOVL R15, 3*4+off+128(dst); \
|
|
MOVL R15, 11*4+off+192(dst); \
|
|
MOVL R15, 7*4+off+256(dst); \
|
|
MOVL R15, 10*4+off+320(dst); \
|
|
MOVL R15, 5*4+off+384(dst); \
|
|
MOVL R15, 9*4+off+448(dst); \
|
|
MOVL R15, 4*4+off+512(dst); \
|
|
MOVL R15, 8*4+off+576(dst)
|
|
|
|
#define BLAKE2s_SSE2() \
|
|
PRECOMPUTE_MSG(SP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X8); \
|
|
ROUND_SSE2(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X8)
|
|
|
|
#define BLAKE2s_SSSE3() \
|
|
PRECOMPUTE_MSG(SP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X8, X13, X14); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X8, X13, X14)
|
|
|
|
#define BLAKE2s_SSE4() \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14); \
|
|
LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0); \
|
|
ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
|
|
|
|
#define HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_FUNC) \
|
|
MOVQ h, AX; \
|
|
MOVQ c, BX; \
|
|
MOVL flag, CX; \
|
|
MOVQ blocks_base, SI; \
|
|
MOVQ blocks_len, DX; \
|
|
\
|
|
MOVQ SP, BP; \
|
|
MOVQ SP, R9; \
|
|
ADDQ $15, R9; \
|
|
ANDQ $~15, R9; \
|
|
MOVQ R9, SP; \
|
|
\
|
|
MOVQ 0(BX), R9; \
|
|
MOVQ R9, 0(SP); \
|
|
XORQ R9, R9; \
|
|
MOVQ R9, 8(SP); \
|
|
MOVL CX, 8(SP); \
|
|
\
|
|
MOVOU 0(AX), X0; \
|
|
MOVOU 16(AX), X1; \
|
|
MOVOU iv0<>(SB), X2; \
|
|
MOVOU iv1<>(SB), X3 \
|
|
\
|
|
MOVOU counter<>(SB), X12; \
|
|
MOVOU rol16<>(SB), X13; \
|
|
MOVOU rol8<>(SB), X14; \
|
|
MOVO 0(SP), X15; \
|
|
\
|
|
loop: \
|
|
MOVO X0, X4; \
|
|
MOVO X1, X5; \
|
|
MOVO X2, X6; \
|
|
MOVO X3, X7; \
|
|
\
|
|
PADDQ X12, X15; \
|
|
PXOR X15, X7; \
|
|
\
|
|
BLAKE2s_FUNC(); \
|
|
\
|
|
PXOR X4, X0; \
|
|
PXOR X5, X1; \
|
|
PXOR X6, X0; \
|
|
PXOR X7, X1; \
|
|
\
|
|
LEAQ 64(SI), SI; \
|
|
SUBQ $64, DX; \
|
|
JNE loop; \
|
|
\
|
|
MOVO X15, 0(SP); \
|
|
MOVQ 0(SP), R9; \
|
|
MOVQ R9, 0(BX); \
|
|
\
|
|
MOVOU X0, 0(AX); \
|
|
MOVOU X1, 16(AX); \
|
|
\
|
|
MOVQ BP, SP
|
|
|
|
// func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
|
|
TEXT ·hashBlocksSSE2(SB), 4, $672-48 // frame = 656 + 16 byte alignment
|
|
HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE2)
|
|
RET
|
|
|
|
// func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
|
|
TEXT ·hashBlocksSSSE3(SB), 4, $672-48 // frame = 656 + 16 byte alignment
|
|
HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSSE3)
|
|
RET
|
|
|
|
// func hashBlocksSSE4(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
|
|
TEXT ·hashBlocksSSE4(SB), 4, $16-48 // frame = 0 + 16 byte alignment
|
|
HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE4)
|
|
RET
|
|
|
|
// func supportSSE4() bool
|
|
TEXT ·supportSSE4(SB), 4, $0-1
|
|
MOVL $1, AX
|
|
CPUID
|
|
SHRL $19, CX // Bit 19 indicates SSE4.1.
|
|
ANDL $1, CX
|
|
MOVB CX, ret+0(FP)
|
|
RET
|
|
|
|
// func supportSSSE3() bool
|
|
TEXT ·supportSSSE3(SB), 4, $0-1
|
|
MOVL $1, AX
|
|
CPUID
|
|
MOVL CX, BX
|
|
ANDL $0x1, BX // Bit zero indicates SSE3 support.
|
|
JZ FALSE
|
|
ANDL $0x200, CX // Bit nine indicates SSSE3 support.
|
|
JZ FALSE
|
|
MOVB $1, ret+0(FP)
|
|
RET
|
|
|
|
FALSE:
|
|
MOVB $0, ret+0(FP)
|
|
RET
|