Files
crypto/blake2s/blake2s_386.s
T
Austin Clements 5953a478da blake2s: use proper Go frame sizes
Currently blake2s's assembly routines claim they have a zero byte
frame and manually subtract upwards of 704 bytes from the stack
pointer without cooperating with Go's ABI. As a result, these
functions may not grow the stack when necessary, leading to memory
corruption.

Fix this by using the correct stack frame sizes so the generated stack
growth prologue is correct, and aligning the SP up instead of down.

Change-Id: Ic426338c45c94a2c01d549860c2295a0ee9200bf
Reviewed-on: https://go-review.googlesource.com/31583
Reviewed-by: Adam Langley <agl@golang.org>
Reviewed-by: Andreas Auernhammer <aead@mail.de>
Reviewed-by: Minux Ma <minux@golang.org>
2016-10-21 19:51:11 +00:00

461 lines
12 KiB
ArmAsm

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build 386, !gccgo, !appengine
#include "textflag.h"
DATA iv0<>+0x00(SB)/4, $0x6a09e667
DATA iv0<>+0x04(SB)/4, $0xbb67ae85
DATA iv0<>+0x08(SB)/4, $0x3c6ef372
DATA iv0<>+0x0c(SB)/4, $0xa54ff53a
GLOBL iv0<>(SB), (NOPTR+RODATA), $16
DATA iv1<>+0x00(SB)/4, $0x510e527f
DATA iv1<>+0x04(SB)/4, $0x9b05688c
DATA iv1<>+0x08(SB)/4, $0x1f83d9ab
DATA iv1<>+0x0c(SB)/4, $0x5be0cd19
GLOBL iv1<>(SB), (NOPTR+RODATA), $16
DATA rol16<>+0x00(SB)/8, $0x0504070601000302
DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
GLOBL rol16<>(SB), (NOPTR+RODATA), $16
DATA rol8<>+0x00(SB)/8, $0x0407060500030201
DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09
GLOBL rol8<>(SB), (NOPTR+RODATA), $16
DATA counter<>+0x00(SB)/8, $0x40
DATA counter<>+0x08(SB)/8, $0x0
GLOBL counter<>(SB), (NOPTR+RODATA), $16
#define ROTL_SSE2(n, t, v) \
MOVO v, t; \
PSLLL $n, t; \
PSRLL $(32-n), v; \
PXOR t, v
#define ROTL_SSSE3(c, v) \
PSHUFB c, v
#define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \
PADDL m0, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSE2(16, t, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(20, t, v1); \
PADDL m1, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSE2(24, t, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(25, t, v1); \
PSHUFL $0x39, v1, v1; \
PSHUFL $0x4E, v2, v2; \
PSHUFL $0x93, v3, v3; \
PADDL m2, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSE2(16, t, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(20, t, v1); \
PADDL m3, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSE2(24, t, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(25, t, v1); \
PSHUFL $0x39, v3, v3; \
PSHUFL $0x4E, v2, v2; \
PSHUFL $0x93, v1, v1
#define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \
PADDL m0, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSSE3(c16, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(20, t, v1); \
PADDL m1, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSSE3(c8, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(25, t, v1); \
PSHUFL $0x39, v1, v1; \
PSHUFL $0x4E, v2, v2; \
PSHUFL $0x93, v3, v3; \
PADDL m2, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSSE3(c16, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(20, t, v1); \
PADDL m3, v0; \
PADDL v1, v0; \
PXOR v0, v3; \
ROTL_SSSE3(c8, v3); \
PADDL v3, v2; \
PXOR v2, v1; \
ROTL_SSE2(25, t, v1); \
PSHUFL $0x39, v3, v3; \
PSHUFL $0x4E, v2, v2; \
PSHUFL $0x93, v1, v1
#define PRECOMPUTE(dst, off, src, t) \
MOVL 0*4(src), t; \
MOVL t, 0*4+off+0(dst); \
MOVL t, 9*4+off+64(dst); \
MOVL t, 5*4+off+128(dst); \
MOVL t, 14*4+off+192(dst); \
MOVL t, 4*4+off+256(dst); \
MOVL t, 2*4+off+320(dst); \
MOVL t, 8*4+off+384(dst); \
MOVL t, 12*4+off+448(dst); \
MOVL t, 3*4+off+512(dst); \
MOVL t, 15*4+off+576(dst); \
MOVL 1*4(src), t; \
MOVL t, 4*4+off+0(dst); \
MOVL t, 8*4+off+64(dst); \
MOVL t, 14*4+off+128(dst); \
MOVL t, 5*4+off+192(dst); \
MOVL t, 12*4+off+256(dst); \
MOVL t, 11*4+off+320(dst); \
MOVL t, 1*4+off+384(dst); \
MOVL t, 6*4+off+448(dst); \
MOVL t, 10*4+off+512(dst); \
MOVL t, 3*4+off+576(dst); \
MOVL 2*4(src), t; \
MOVL t, 1*4+off+0(dst); \
MOVL t, 13*4+off+64(dst); \
MOVL t, 6*4+off+128(dst); \
MOVL t, 8*4+off+192(dst); \
MOVL t, 2*4+off+256(dst); \
MOVL t, 0*4+off+320(dst); \
MOVL t, 14*4+off+384(dst); \
MOVL t, 11*4+off+448(dst); \
MOVL t, 12*4+off+512(dst); \
MOVL t, 4*4+off+576(dst); \
MOVL 3*4(src), t; \
MOVL t, 5*4+off+0(dst); \
MOVL t, 15*4+off+64(dst); \
MOVL t, 9*4+off+128(dst); \
MOVL t, 1*4+off+192(dst); \
MOVL t, 11*4+off+256(dst); \
MOVL t, 7*4+off+320(dst); \
MOVL t, 13*4+off+384(dst); \
MOVL t, 3*4+off+448(dst); \
MOVL t, 6*4+off+512(dst); \
MOVL t, 10*4+off+576(dst); \
MOVL 4*4(src), t; \
MOVL t, 2*4+off+0(dst); \
MOVL t, 1*4+off+64(dst); \
MOVL t, 15*4+off+128(dst); \
MOVL t, 10*4+off+192(dst); \
MOVL t, 6*4+off+256(dst); \
MOVL t, 8*4+off+320(dst); \
MOVL t, 3*4+off+384(dst); \
MOVL t, 13*4+off+448(dst); \
MOVL t, 14*4+off+512(dst); \
MOVL t, 5*4+off+576(dst); \
MOVL 5*4(src), t; \
MOVL t, 6*4+off+0(dst); \
MOVL t, 11*4+off+64(dst); \
MOVL t, 2*4+off+128(dst); \
MOVL t, 9*4+off+192(dst); \
MOVL t, 1*4+off+256(dst); \
MOVL t, 13*4+off+320(dst); \
MOVL t, 4*4+off+384(dst); \
MOVL t, 8*4+off+448(dst); \
MOVL t, 15*4+off+512(dst); \
MOVL t, 7*4+off+576(dst); \
MOVL 6*4(src), t; \
MOVL t, 3*4+off+0(dst); \
MOVL t, 7*4+off+64(dst); \
MOVL t, 13*4+off+128(dst); \
MOVL t, 12*4+off+192(dst); \
MOVL t, 10*4+off+256(dst); \
MOVL t, 1*4+off+320(dst); \
MOVL t, 9*4+off+384(dst); \
MOVL t, 14*4+off+448(dst); \
MOVL t, 0*4+off+512(dst); \
MOVL t, 6*4+off+576(dst); \
MOVL 7*4(src), t; \
MOVL t, 7*4+off+0(dst); \
MOVL t, 14*4+off+64(dst); \
MOVL t, 10*4+off+128(dst); \
MOVL t, 0*4+off+192(dst); \
MOVL t, 5*4+off+256(dst); \
MOVL t, 9*4+off+320(dst); \
MOVL t, 12*4+off+384(dst); \
MOVL t, 1*4+off+448(dst); \
MOVL t, 13*4+off+512(dst); \
MOVL t, 2*4+off+576(dst); \
MOVL 8*4(src), t; \
MOVL t, 8*4+off+0(dst); \
MOVL t, 5*4+off+64(dst); \
MOVL t, 4*4+off+128(dst); \
MOVL t, 15*4+off+192(dst); \
MOVL t, 14*4+off+256(dst); \
MOVL t, 3*4+off+320(dst); \
MOVL t, 11*4+off+384(dst); \
MOVL t, 10*4+off+448(dst); \
MOVL t, 7*4+off+512(dst); \
MOVL t, 1*4+off+576(dst); \
MOVL 9*4(src), t; \
MOVL t, 12*4+off+0(dst); \
MOVL t, 2*4+off+64(dst); \
MOVL t, 11*4+off+128(dst); \
MOVL t, 4*4+off+192(dst); \
MOVL t, 0*4+off+256(dst); \
MOVL t, 15*4+off+320(dst); \
MOVL t, 10*4+off+384(dst); \
MOVL t, 7*4+off+448(dst); \
MOVL t, 5*4+off+512(dst); \
MOVL t, 9*4+off+576(dst); \
MOVL 10*4(src), t; \
MOVL t, 9*4+off+0(dst); \
MOVL t, 4*4+off+64(dst); \
MOVL t, 8*4+off+128(dst); \
MOVL t, 13*4+off+192(dst); \
MOVL t, 3*4+off+256(dst); \
MOVL t, 5*4+off+320(dst); \
MOVL t, 7*4+off+384(dst); \
MOVL t, 15*4+off+448(dst); \
MOVL t, 11*4+off+512(dst); \
MOVL t, 0*4+off+576(dst); \
MOVL 11*4(src), t; \
MOVL t, 13*4+off+0(dst); \
MOVL t, 10*4+off+64(dst); \
MOVL t, 0*4+off+128(dst); \
MOVL t, 3*4+off+192(dst); \
MOVL t, 9*4+off+256(dst); \
MOVL t, 6*4+off+320(dst); \
MOVL t, 15*4+off+384(dst); \
MOVL t, 4*4+off+448(dst); \
MOVL t, 2*4+off+512(dst); \
MOVL t, 12*4+off+576(dst); \
MOVL 12*4(src), t; \
MOVL t, 10*4+off+0(dst); \
MOVL t, 12*4+off+64(dst); \
MOVL t, 1*4+off+128(dst); \
MOVL t, 6*4+off+192(dst); \
MOVL t, 13*4+off+256(dst); \
MOVL t, 4*4+off+320(dst); \
MOVL t, 0*4+off+384(dst); \
MOVL t, 2*4+off+448(dst); \
MOVL t, 8*4+off+512(dst); \
MOVL t, 14*4+off+576(dst); \
MOVL 13*4(src), t; \
MOVL t, 14*4+off+0(dst); \
MOVL t, 3*4+off+64(dst); \
MOVL t, 7*4+off+128(dst); \
MOVL t, 2*4+off+192(dst); \
MOVL t, 15*4+off+256(dst); \
MOVL t, 12*4+off+320(dst); \
MOVL t, 6*4+off+384(dst); \
MOVL t, 0*4+off+448(dst); \
MOVL t, 9*4+off+512(dst); \
MOVL t, 11*4+off+576(dst); \
MOVL 14*4(src), t; \
MOVL t, 11*4+off+0(dst); \
MOVL t, 0*4+off+64(dst); \
MOVL t, 12*4+off+128(dst); \
MOVL t, 7*4+off+192(dst); \
MOVL t, 8*4+off+256(dst); \
MOVL t, 14*4+off+320(dst); \
MOVL t, 2*4+off+384(dst); \
MOVL t, 5*4+off+448(dst); \
MOVL t, 1*4+off+512(dst); \
MOVL t, 13*4+off+576(dst); \
MOVL 15*4(src), t; \
MOVL t, 15*4+off+0(dst); \
MOVL t, 6*4+off+64(dst); \
MOVL t, 3*4+off+128(dst); \
MOVL t, 11*4+off+192(dst); \
MOVL t, 7*4+off+256(dst); \
MOVL t, 10*4+off+320(dst); \
MOVL t, 5*4+off+384(dst); \
MOVL t, 9*4+off+448(dst); \
MOVL t, 4*4+off+512(dst); \
MOVL t, 8*4+off+576(dst)
// func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
TEXT ·hashBlocksSSE2(SB), 4, $672-24 // frame = 656 + 16 byte alignment
MOVL h+0(FP), AX
MOVL c+4(FP), BX
MOVL flag+8(FP), CX
MOVL blocks_base+12(FP), SI
MOVL blocks_len+16(FP), DX
MOVL SP, BP
MOVL SP, DI
ADDL $15, DI
ANDL $~15, DI
MOVL DI, SP
MOVL CX, 8(SP)
MOVL 0(BX), CX
MOVL CX, 0(SP)
MOVL 4(BX), CX
MOVL CX, 4(SP)
XORL CX, CX
MOVL CX, 12(SP)
MOVOU 0(AX), X0
MOVOU 16(AX), X1
MOVOU counter<>(SB), X2
loop:
MOVO X0, X4
MOVO X1, X5
MOVOU iv0<>(SB), X6
MOVOU iv1<>(SB), X7
MOVO 0(SP), X3
PADDQ X2, X3
PXOR X3, X7
MOVO X3, 0(SP)
PRECOMPUTE(SP, 16, SI, CX)
ROUND_SSE2(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3)
ROUND_SSE2(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3)
PXOR X4, X0
PXOR X5, X1
PXOR X6, X0
PXOR X7, X1
LEAL 64(SI), SI
SUBL $64, DX
JNE loop
MOVL 0(SP), CX
MOVL CX, 0(BX)
MOVL 4(SP), CX
MOVL CX, 4(BX)
MOVOU X0, 0(AX)
MOVOU X1, 16(AX)
MOVL BP, SP
RET
// func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
TEXT ·hashBlocksSSSE3(SB), 4, $704-24 // frame = 688 + 16 byte alignment
MOVL h+0(FP), AX
MOVL c+4(FP), BX
MOVL flag+8(FP), CX
MOVL blocks_base+12(FP), SI
MOVL blocks_len+16(FP), DX
MOVL SP, BP
MOVL SP, DI
ADDL $15, DI
ANDL $~15, DI
MOVL DI, SP
MOVL CX, 8(SP)
MOVL 0(BX), CX
MOVL CX, 0(SP)
MOVL 4(BX), CX
MOVL CX, 4(SP)
XORL CX, CX
MOVL CX, 12(SP)
MOVOU 0(AX), X0
MOVOU 16(AX), X1
MOVOU counter<>(SB), X2
loop:
MOVO X0, 656(SP)
MOVO X1, 672(SP)
MOVO X0, X4
MOVO X1, X5
MOVOU iv0<>(SB), X6
MOVOU iv1<>(SB), X7
MOVO 0(SP), X3
PADDQ X2, X3
PXOR X3, X7
MOVO X3, 0(SP)
MOVOU rol16<>(SB), X0
MOVOU rol8<>(SB), X1
PRECOMPUTE(SP, 16, SI, CX)
ROUND_SSSE3(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3, X0, X1)
ROUND_SSSE3(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3, X0, X1)
MOVO 656(SP), X0
MOVO 672(SP), X1
PXOR X4, X0
PXOR X5, X1
PXOR X6, X0
PXOR X7, X1
LEAL 64(SI), SI
SUBL $64, DX
JNE loop
MOVL 0(SP), CX
MOVL CX, 0(BX)
MOVL 4(SP), CX
MOVL CX, 4(BX)
MOVOU X0, 0(AX)
MOVOU X1, 16(AX)
MOVL BP, SP
RET
// func supportSSSE3() bool
TEXT ·supportSSSE3(SB), 4, $0-1
MOVL $1, AX
CPUID
MOVL CX, BX
ANDL $0x1, BX // supports SSE3
JZ FALSE
ANDL $0x200, CX // supports SSSE3
JZ FALSE
MOVB $1, ret+0(FP)
RET
FALSE:
MOVB $0, ret+0(FP)
RET
// func supportSSE2() bool
TEXT ·supportSSE2(SB), 4, $0-1
MOVL $1, AX
CPUID
SHRL $26, DX
ANDL $1, DX // DX != 0 if support SSE2
MOVB DX, ret+0(FP)
RET