eiac/crypto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ebc8e463153182e44695bfc4e4ae8cfdc00aa86d
crypto/blake2b/_asm/AVX2/blake2bAVX2_amd64_asm.go
T
Garrett Bodley 0484c26df7 blake2b: port blake2bAVX2_amd64.s to Avo 
This implementation utilizes the same registers found in the reference
implementation, aiming to produce a minimal semantic diff between the
Avo-generated output and the original hand-written assembly.

To verify the Avo implementation, the reference and Avo-generated
assembly files are fed to `go tool asm`, capturing the debug output into
corresponding temp files. The debug output contains supplementary
metadata (line numbers, instruction offsets, and source file references)
that must be removed in order to obtain a semantic diff of the two
files. This is accomplished via a small utility script written in awk.

Commands used to verify Avo output:

GOROOT=$(go env GOROOT)
ASM_PATH="blake2b/blake2bAVX2_amd64.s"
REFERENCE="b2d3a6a4b4d36521cd7f653879cf6981e7c5c340"

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  <(git cat-file -p "$REFERENCE:$ASM_PATH") \
  > /tmp/reference.s

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  "$ASM_PATH" \
  > /tmp/avo.s

normalize(){
  awk '{
    $1=$2=$3="";
    print substr($0,4)
  }'
}

diff <(normalize < /tmp/reference.s) <(normalize < /tmp/avo.s)

Change-Id: Ia2af1b82c871e26b89bd9a2d9fb187cc49e18341
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/600836
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Reviewed-by: Roland Shoemaker <roland@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
2024-09-04 20:34:34 +00:00

1229 lines
27 KiB
Go
Raw Blame History

// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
. "github.com/mmcloughlin/avo/build"
"github.com/mmcloughlin/avo/ir"
. "github.com/mmcloughlin/avo/operand"
. "github.com/mmcloughlin/avo/reg"
_ "golang.org/x/crypto/blake2b"
)
//go:generate go run . -out ../../blake2bAVX2_amd64.s -pkg blake2b
const ThatPeskyUnicodeDot = "\u00b7"
func main() {
Package("golang.org/x/crypto/blake2b")
ConstraintExpr("amd64,gc,!purego")
hashBlocksAVX2()
hashBlocksAVX()
Generate()
}
// Utility function to emit a BYTE instruction
func BYTE(imm Op) {
Instruction(&ir.Instruction{Opcode: "BYTE", Operands: []Op{imm}})
}
func VPERMQ_0x39_Y1_Y1() {
BYTE(U8(0xc4))
BYTE(U8(0xe3))
BYTE(U8(0xfd))
BYTE(U8(0x00))
BYTE(U8(0xc9))
BYTE(U8(0x39))
}
func VPERMQ_0x93_Y1_Y1() {
BYTE(U8(0xc4))
BYTE(U8(0xe3))
BYTE(U8(0xfd))
BYTE(U8(0x00))
BYTE(U8(0xc9))
BYTE(U8(0x93))
}
func VPERMQ_0x4E_Y2_Y2() {
BYTE(U8(0xc4))
BYTE(U8(0xe3))
BYTE(U8(0xfd))
BYTE(U8(0x00))
BYTE(U8(0xd2))
BYTE(U8(0x4e))
}
func VPERMQ_0x93_Y3_Y3() {
BYTE(U8(0xc4))
BYTE(U8(0xe3))
BYTE(U8(0xfd))
BYTE(U8(0x00))
BYTE(U8(0xdb))
BYTE(U8(0x93))
}
func VPERMQ_0x39_Y3_Y3() {
BYTE(U8(0xc4))
BYTE(U8(0xe3))
BYTE(U8(0xfd))
BYTE(U8(0x00))
BYTE(U8(0xdb))
BYTE(U8(0x39))
}
func ROUND_AVX2(m0, m1, m2, m3 Op, t, c40, c48 VecPhysical) {
VPADDQ(m0, Y0, Y0)
VPADDQ(Y1, Y0, Y0)
VPXOR(Y0, Y3, Y3)
VPSHUFD(I8(-79), Y3, Y3)
VPADDQ(Y3, Y2, Y2)
VPXOR(Y2, Y1, Y1)
VPSHUFB(c40, Y1, Y1)
VPADDQ(m1, Y0, Y0)
VPADDQ(Y1, Y0, Y0)
VPXOR(Y0, Y3, Y3)
VPSHUFB(c48, Y3, Y3)
VPADDQ(Y3, Y2, Y2)
VPXOR(Y2, Y1, Y1)
VPADDQ(Y1, Y1, t)
VPSRLQ(Imm(63), Y1, Y1)
VPXOR(t, Y1, Y1)
VPERMQ_0x39_Y1_Y1()
VPERMQ_0x4E_Y2_Y2()
VPERMQ_0x93_Y3_Y3()
VPADDQ(m2, Y0, Y0)
VPADDQ(Y1, Y0, Y0)
VPXOR(Y0, Y3, Y3)
VPSHUFD(I8(-79), Y3, Y3)
VPADDQ(Y3, Y2, Y2)
VPXOR(Y2, Y1, Y1)
VPSHUFB(c40, Y1, Y1)
VPADDQ(m3, Y0, Y0)
VPADDQ(Y1, Y0, Y0)
VPXOR(Y0, Y3, Y3)
VPSHUFB(c48, Y3, Y3)
VPADDQ(Y3, Y2, Y2)
VPXOR(Y2, Y1, Y1)
VPADDQ(Y1, Y1, t)
VPSRLQ(Imm(63), Y1, Y1)
VPXOR(t, Y1, Y1)
VPERMQ_0x39_Y3_Y3()
VPERMQ_0x4E_Y2_Y2()
VPERMQ_0x93_Y1_Y1()
}
func VMOVQ_SI_X11_0() {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x1E))
}
func VMOVQ_SI_X12_0() {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x26))
}
func VMOVQ_SI_X13_0() {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x2E))
}
func VMOVQ_SI_X14_0() {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x36))
}
func VMOVQ_SI_X15_0() {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x3E))
}
func VMOVQ_SI_X11(n uint8) {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x5E))
BYTE(U8(n))
}
func VMOVQ_SI_X12(n uint8) {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x66))
BYTE(U8(n))
}
func VMOVQ_SI_X13(n uint8) {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x6E))
BYTE(U8(n))
}
func VMOVQ_SI_X14(n uint8) {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x76))
BYTE(U8(n))
}
func VMOVQ_SI_X15(n uint8) {
BYTE(U8(0xC5))
BYTE(U8(0x7A))
BYTE(U8(0x7E))
BYTE(U8(0x7E))
BYTE(U8(n))
}
func VPINSRQ_1_SI_X11_0() {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0xA1))
BYTE(U8(0x22))
BYTE(U8(0x1E))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X12_0() {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x99))
BYTE(U8(0x22))
BYTE(U8(0x26))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X13_0() {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x91))
BYTE(U8(0x22))
BYTE(U8(0x2E))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X14_0() {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x89))
BYTE(U8(0x22))
BYTE(U8(0x36))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X15_0() {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x81))
BYTE(U8(0x22))
BYTE(U8(0x3E))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X11(n uint8) {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0xA1))
BYTE(U8(0x22))
BYTE(U8(0x5E))
BYTE(U8(n))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X12(n uint8) {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x99))
BYTE(U8(0x22))
BYTE(U8(0x66))
BYTE(U8(n))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X13(n uint8) {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x91))
BYTE(U8(0x22))
BYTE(U8(0x6E))
BYTE(U8(n))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X14(n uint8) {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x89))
BYTE(U8(0x22))
BYTE(U8(0x76))
BYTE(U8(n))
BYTE(U8(0x01))
}
func VPINSRQ_1_SI_X15(n uint8) {
BYTE(U8(0xC4))
BYTE(U8(0x63))
BYTE(U8(0x81))
BYTE(U8(0x22))
BYTE(U8(0x7E))
BYTE(U8(n))
BYTE(U8(0x01))
}
func VMOVQ_R8_X15() {
BYTE(U8(0xC4))
BYTE(U8(0x41))
BYTE(U8(0xF9))
BYTE(U8(0x6E))
BYTE(U8(0xF8))
}
func VPINSRQ_1_R9_X15() {
BYTE(U8(0xC4))
BYTE(U8(0x43))
BYTE(U8(0x81))
BYTE(U8(0x22))
BYTE(U8(0xF9))
BYTE(U8(0x01))
}
// load msg:
//
// Y12 = (i0, i1, i2, i3)
//
// i0, i1, i2, i3 must not be 0
func LOAD_MSG_AVX2_Y12(i0, i1, i2, i3 uint8) {
VMOVQ_SI_X12(i0 * 8)
VMOVQ_SI_X11(i2 * 8)
VPINSRQ_1_SI_X12(i1 * 8)
VPINSRQ_1_SI_X11(i3 * 8)
VINSERTI128(Imm(1), X11, Y12, Y12)
}
// load msg:
//
// Y13 = (i0, i1, i2, i3)
//
// i0, i1, i2, i3 must not be 0
func LOAD_MSG_AVX2_Y13(i0, i1, i2, i3 uint8) {
VMOVQ_SI_X13(i0 * 8)
VMOVQ_SI_X11(i2 * 8)
VPINSRQ_1_SI_X13(i1 * 8)
VPINSRQ_1_SI_X11(i3 * 8)
VINSERTI128(Imm(1), X11, Y13, Y13)
}
// load msg:
//
// Y14 = (i0, i1, i2, i3)
//
// i0, i1, i2, i3 must not be 0
func LOAD_MSG_AVX2_Y14(i0, i1, i2, i3 uint8) {
VMOVQ_SI_X14(i0 * 8)
VMOVQ_SI_X11(i2 * 8)
VPINSRQ_1_SI_X14(i1 * 8)
VPINSRQ_1_SI_X11(i3 * 8)
VINSERTI128(Imm(1), X11, Y14, Y14)
}
// load msg:
//
// Y15 = (i0, i1, i2, i3)
//
// i0, i1, i2, i3 must not be 0
func LOAD_MSG_AVX2_Y15(i0, i1, i2, i3 uint8) {
VMOVQ_SI_X15(i0 * 8)
VMOVQ_SI_X11(i2 * 8)
VPINSRQ_1_SI_X15(i1 * 8)
VPINSRQ_1_SI_X11(i3 * 8)
VINSERTI128(Imm(1), X11, Y15, Y15)
}
func LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15() {
VMOVQ_SI_X12_0()
VMOVQ_SI_X11(4 * 8)
VPINSRQ_1_SI_X12(2 * 8)
VPINSRQ_1_SI_X11(6 * 8)
VINSERTI128(Imm(1), X11, Y12, Y12)
LOAD_MSG_AVX2_Y13(1, 3, 5, 7)
LOAD_MSG_AVX2_Y14(8, 10, 12, 14)
LOAD_MSG_AVX2_Y15(9, 11, 13, 15)
}
func LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3() {
LOAD_MSG_AVX2_Y12(14, 4, 9, 13)
LOAD_MSG_AVX2_Y13(10, 8, 15, 6)
VMOVQ_SI_X11(11 * 8)
VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(0*8), X14)
VPINSRQ_1_SI_X11(5 * 8)
VINSERTI128(Imm(1), X11, Y14, Y14)
LOAD_MSG_AVX2_Y15(12, 2, 7, 3)
}
func LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4() {
VMOVQ_SI_X11(5 * 8)
VMOVDQU(Mem{Base: SI}.Offset(11*8), X12)
VPINSRQ_1_SI_X11(15 * 8)
VINSERTI128(Imm(1), X11, Y12, Y12)
VMOVQ_SI_X13(8 * 8)
VMOVQ_SI_X11(2 * 8)
VPINSRQ_1_SI_X13_0()
VPINSRQ_1_SI_X11(13 * 8)
VINSERTI128(Imm(1), X11, Y13, Y13)
LOAD_MSG_AVX2_Y14(10, 3, 7, 9)
LOAD_MSG_AVX2_Y15(14, 6, 1, 4)
}
func LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8() {
LOAD_MSG_AVX2_Y12(7, 3, 13, 11)
LOAD_MSG_AVX2_Y13(9, 1, 12, 14)
LOAD_MSG_AVX2_Y14(2, 5, 4, 15)
VMOVQ_SI_X15(6 * 8)
VMOVQ_SI_X11_0()
VPINSRQ_1_SI_X15(10 * 8)
VPINSRQ_1_SI_X11(8 * 8)
VINSERTI128(Imm(1), X11, Y15, Y15)
}
func LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13() {
LOAD_MSG_AVX2_Y12(9, 5, 2, 10)
VMOVQ_SI_X13_0()
VMOVQ_SI_X11(4 * 8)
VPINSRQ_1_SI_X13(7 * 8)
VPINSRQ_1_SI_X11(15 * 8)
VINSERTI128(Imm(1), X11, Y13, Y13)
LOAD_MSG_AVX2_Y14(14, 11, 6, 3)
LOAD_MSG_AVX2_Y15(1, 12, 8, 13)
}
func LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9() {
VMOVQ_SI_X12(2 * 8)
VMOVQ_SI_X11_0()
VPINSRQ_1_SI_X12(6 * 8)
VPINSRQ_1_SI_X11(8 * 8)
VINSERTI128(Imm(1), X11, Y12, Y12)
LOAD_MSG_AVX2_Y13(12, 10, 11, 3)
LOAD_MSG_AVX2_Y14(4, 7, 15, 1)
LOAD_MSG_AVX2_Y15(13, 5, 14, 9)
}
func LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11() {
LOAD_MSG_AVX2_Y12(12, 1, 14, 4)
LOAD_MSG_AVX2_Y13(5, 15, 13, 10)
VMOVQ_SI_X14_0()
VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(8*8), X11)
VPINSRQ_1_SI_X14(6 * 8)
VINSERTI128(Imm(1), X11, Y14, Y14)
LOAD_MSG_AVX2_Y15(7, 3, 2, 11)
}
func LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10() {
LOAD_MSG_AVX2_Y12(13, 7, 12, 3)
LOAD_MSG_AVX2_Y13(11, 14, 1, 9)
LOAD_MSG_AVX2_Y14(5, 15, 8, 2)
VMOVQ_SI_X15_0()
VMOVQ_SI_X11(6 * 8)
VPINSRQ_1_SI_X15(4 * 8)
VPINSRQ_1_SI_X11(10 * 8)
VINSERTI128(Imm(1), X11, Y15, Y15)
}
func LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5() {
VMOVQ_SI_X12(6 * 8)
VMOVQ_SI_X11(11 * 8)
VPINSRQ_1_SI_X12(14 * 8)
VPINSRQ_1_SI_X11_0()
VINSERTI128(Imm(1), X11, Y12, Y12)
LOAD_MSG_AVX2_Y13(15, 9, 3, 8)
VMOVQ_SI_X11(1 * 8)
VMOVDQU(Mem{Base: SI}.Offset(12*8), X14)
VPINSRQ_1_SI_X11(10 * 8)
VINSERTI128(Imm(1), X11, Y14, Y14)
VMOVQ_SI_X15(2 * 8)
VMOVDQU(Mem{Base: SI}.Offset(4*8), X11)
VPINSRQ_1_SI_X15(7 * 8)
VINSERTI128(Imm(1), X11, Y15, Y15)
}
func LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0() {
LOAD_MSG_AVX2_Y12(10, 8, 7, 1)
VMOVQ_SI_X13(2 * 8)
VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(5*8), X11)
VPINSRQ_1_SI_X13(4 * 8)
VINSERTI128(Imm(1), X11, Y13, Y13)
LOAD_MSG_AVX2_Y14(15, 9, 3, 13)
VMOVQ_SI_X15(11 * 8)
VMOVQ_SI_X11(12 * 8)
VPINSRQ_1_SI_X15(14 * 8)
VPINSRQ_1_SI_X11_0()
VINSERTI128(Imm(1), X11, Y15, Y15)
}
func hashBlocksAVX2() {
Implement("hashBlocksAVX2")
Attributes(4)
AllocLocal(320) // frame size = 288 + 32 byte alignment
Load(Param("h"), RAX)
Load(Param("c"), RBX)
Load(Param("flag"), RCX)
Load(Param("blocks").Base(), RSI)
Load(Param("blocks").Len(), RDI)
MOVQ(RSP, RDX)
ADDQ(I32(31), RDX)
ANDQ(I32(^31), RDX)
MOVQ(RCX, Mem{Base: DX}.Offset(16))
XORQ(RCX, RCX)
MOVQ(RCX, Mem{Base: DX}.Offset(24))
AVX2_c40 := AVX2_c40_DATA()
AVX2_c48 := AVX2_c48_DATA()
VMOVDQU(AVX2_c40, Y4)
VMOVDQU(AVX2_c48, Y5)
VMOVDQU(Mem{Base: AX}.Offset(0), Y8)
VMOVDQU(Mem{Base: AX}.Offset(32), Y9)
AVX2_iv0 := AVX2_iv0_DATA()
AVX2_iv1 := AVX2_iv1_DATA()
VMOVDQU(AVX2_iv0, Y6)
VMOVDQU(AVX2_iv1, Y7)
MOVQ(Mem{Base: BX}.Offset(0), R8)
MOVQ(Mem{Base: BX}.Offset(8), R9)
MOVQ(R9, Mem{Base: DX}.Offset(8))
loop_AVX2()
noinc_AVX2()
}
func loop_AVX2() {
Label("loop")
ADDQ(Imm(128), R8)
MOVQ(R8, Mem{Base: DX}.Offset(0))
CMPQ(R8, Imm(128))
JGE(LabelRef("noinc"))
INCQ(R9)
MOVQ(R9, Mem{Base: DX}.Offset(8))
}
// line 312
func noinc_AVX2() {
Label("noinc")
VMOVDQA(Y8, Y0)
VMOVDQA(Y9, Y1)
VMOVDQA(Y6, Y2)
VPXOR(Mem{Base: DX}.Offset(0), Y7, Y3)
LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15()
VMOVDQA(Y12, Mem{Base: DX}.Offset(32))
VMOVDQA(Y13, Mem{Base: DX}.Offset(64))
VMOVDQA(Y14, Mem{Base: DX}.Offset(96))
VMOVDQA(Y15, Mem{Base: DX}.Offset(128))
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3()
VMOVDQA(Y12, Mem{Base: DX}.Offset(160))
VMOVDQA(Y13, Mem{Base: DX}.Offset(192))
VMOVDQA(Y14, Mem{Base: DX}.Offset(224))
VMOVDQA(Y15, Mem{Base: DX}.Offset(256))
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0()
ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
ROUND_AVX2(Mem{Base: DX}.Offset(32), Mem{Base: DX}.Offset(64), Mem{Base: DX}.Offset(96), Mem{Base: DX}.Offset(128), Y10, Y4, Y5)
ROUND_AVX2(Mem{Base: DX}.Offset(160), Mem{Base: DX}.Offset(192), Mem{Base: DX}.Offset(224), Mem{Base: DX}.Offset(256), Y10, Y4, Y5)
VPXOR(Y0, Y8, Y8)
VPXOR(Y1, Y9, Y9)
VPXOR(Y2, Y8, Y8)
VPXOR(Y3, Y9, Y9)
LEAQ(Mem{Base: SI}.Offset(128), RSI)
SUBQ(Imm(128), RDI)
JNE(LabelRef("loop"))
MOVQ(R8, Mem{Base: BX}.Offset(0))
MOVQ(R9, Mem{Base: BX}.Offset(8))
VMOVDQU(Y8, Mem{Base: AX}.Offset(0))
VMOVDQU(Y9, Mem{Base: AX}.Offset(32))
VZEROUPPER()
RET()
}
func VPUNPCKLQDQ_X2_X2_X15() {
BYTE(U8(0xC5))
BYTE(U8(0x69))
BYTE(U8(0x6C))
BYTE(U8(0xFA))
}
func VPUNPCKLQDQ_X3_X3_X15() {
BYTE(U8(0xC5))
BYTE(U8(0x61))
BYTE(U8(0x6C))
BYTE(U8(0xFB))
}
func VPUNPCKLQDQ_X7_X7_X15() {
BYTE(U8(0xC5))
BYTE(U8(0x41))
BYTE(U8(0x6C))
BYTE(U8(0xFF))
}
func VPUNPCKLQDQ_X13_X13_X15() {
BYTE(U8(0xC4))
BYTE(U8(0x41))
BYTE(U8(0x11))
BYTE(U8(0x6C))
BYTE(U8(0xFD))
}
func VPUNPCKLQDQ_X14_X14_X15() {
BYTE(U8(0xC4))
BYTE(U8(0x41))
BYTE(U8(0x09))
BYTE(U8(0x6C))
BYTE(U8(0xFE))
}
func VPUNPCKHQDQ_X15_X2_X2() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x69))
BYTE(U8(0x6D))
BYTE(U8(0xD7))
}
func VPUNPCKHQDQ_X15_X3_X3() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x61))
BYTE(U8(0x6D))
BYTE(U8(0xDF))
}
func VPUNPCKHQDQ_X15_X6_X6() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x49))
BYTE(U8(0x6D))
BYTE(U8(0xF7))
}
func VPUNPCKHQDQ_X15_X7_X7() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x41))
BYTE(U8(0x6D))
BYTE(U8(0xFF))
}
func VPUNPCKHQDQ_X15_X3_X2() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x61))
BYTE(U8(0x6D))
BYTE(U8(0xD7))
}
func VPUNPCKHQDQ_X15_X7_X6() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x41))
BYTE(U8(0x6D))
BYTE(U8(0xF7))
}
func VPUNPCKHQDQ_X15_X13_X3() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x11))
BYTE(U8(0x6D))
BYTE(U8(0xDF))
}
func VPUNPCKHQDQ_X15_X13_X7() {
BYTE(U8(0xC4))
BYTE(U8(0xC1))
BYTE(U8(0x11))
BYTE(U8(0x6D))
BYTE(U8(0xFF))
}
func SHUFFLE_AVX() {
VMOVDQA(X6, X13)
VMOVDQA(X2, X14)
VMOVDQA(X4, X6)
VPUNPCKLQDQ_X13_X13_X15()
VMOVDQA(X5, X4)
VMOVDQA(X6, X5)
VPUNPCKHQDQ_X15_X7_X6()
VPUNPCKLQDQ_X7_X7_X15()
VPUNPCKHQDQ_X15_X13_X7()
VPUNPCKLQDQ_X3_X3_X15()
VPUNPCKHQDQ_X15_X2_X2()
VPUNPCKLQDQ_X14_X14_X15()
VPUNPCKHQDQ_X15_X3_X3()
}
func SHUFFLE_AVX_INV() {
VMOVDQA(X2, X13)
VMOVDQA(X4, X14)
VPUNPCKLQDQ_X2_X2_X15()
VMOVDQA(X5, X4)
VPUNPCKHQDQ_X15_X3_X2()
VMOVDQA(X14, X5)
VPUNPCKLQDQ_X3_X3_X15()
VMOVDQA(X6, X14)
VPUNPCKHQDQ_X15_X13_X3()
VPUNPCKLQDQ_X7_X7_X15()
VPUNPCKHQDQ_X15_X6_X6()
VPUNPCKLQDQ_X14_X14_X15()
VPUNPCKHQDQ_X15_X7_X7()
}
func HALF_ROUND_AVX(v0, v1, v2, v3, v4, v5, v6, v7 VecPhysical, m0, m1, m2, m3 Op, t0, c40, c48 VecPhysical) {
VPADDQ(m0, v0, v0)
VPADDQ(v2, v0, v0)
VPADDQ(m1, v1, v1)
VPADDQ(v3, v1, v1)
VPXOR(v0, v6, v6)
VPXOR(v1, v7, v7)
VPSHUFD(I8(-79), v6, v6)
VPSHUFD(I8(-79), v7, v7)
VPADDQ(v6, v4, v4)
VPADDQ(v7, v5, v5)
VPXOR(v4, v2, v2)
VPXOR(v5, v3, v3)
VPSHUFB(c40, v2, v2)
VPSHUFB(c40, v3, v3)
VPADDQ(m2, v0, v0)
VPADDQ(v2, v0, v0)
VPADDQ(m3, v1, v1)
VPADDQ(v3, v1, v1)
VPXOR(v0, v6, v6)
VPXOR(v1, v7, v7)
VPSHUFB(c48, v6, v6)
VPSHUFB(c48, v7, v7)
VPADDQ(v6, v4, v4)
VPADDQ(v7, v5, v5)
VPXOR(v4, v2, v2)
VPXOR(v5, v3, v3)
VPADDQ(v2, v2, t0)
VPSRLQ(Imm(63), v2, v2)
VPXOR(t0, v2, v2)
VPADDQ(v3, v3, t0)
VPSRLQ(Imm(63), v3, v3)
VPXOR(t0, v3, v3)
}
// load msg:
//
// X12 = (i0, i1), X13 = (i2, i3), X14 = (i4, i5), X15 = (i6, i7)
//
// i0, i1, i2, i3, i4, i5, i6, i7 must not be 0
func LOAD_MSG_AVX(i0, i1, i2, i3, i4, i5, i6, i7 uint8) {
VMOVQ_SI_X12(i0 * 8)
VMOVQ_SI_X13(i2 * 8)
VMOVQ_SI_X14(i4 * 8)
VMOVQ_SI_X15(i6 * 8)
VPINSRQ_1_SI_X12(i1 * 8)
VPINSRQ_1_SI_X13(i3 * 8)
VPINSRQ_1_SI_X14(i5 * 8)
VPINSRQ_1_SI_X15(i7 * 8)
}
// load msg:
//
// X12 = (0, 2), X13 = (4, 6), X14 = (1, 3), X15 = (5, 7)
func LOAD_MSG_AVX_0_2_4_6_1_3_5_7() {
VMOVQ_SI_X12_0()
VMOVQ_SI_X13(4 * 8)
VMOVQ_SI_X14(1 * 8)
VMOVQ_SI_X15(5 * 8)
VPINSRQ_1_SI_X12(2 * 8)
VPINSRQ_1_SI_X13(6 * 8)
VPINSRQ_1_SI_X14(3 * 8)
VPINSRQ_1_SI_X15(7 * 8)
}
// load msg:
//
// X12 = (1, 0), X13 = (11, 5), X14 = (12, 2), X15 = (7, 3)
func LOAD_MSG_AVX_1_0_11_5_12_2_7_3() {
VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(0*8), X12)
VMOVQ_SI_X13(11 * 8)
VMOVQ_SI_X14(12 * 8)
VMOVQ_SI_X15(7 * 8)
VPINSRQ_1_SI_X13(5 * 8)
VPINSRQ_1_SI_X14(2 * 8)
VPINSRQ_1_SI_X15(3 * 8)
}
// load msg:
//
// X12 = (11, 12), X13 = (5, 15), X14 = (8, 0), X15 = (2, 13)
func LOAD_MSG_AVX_11_12_5_15_8_0_2_13() {
VMOVDQU(Mem{Base: SI}.Offset(11*8), X12)
VMOVQ_SI_X13(5 * 8)
VMOVQ_SI_X14(8 * 8)
VMOVQ_SI_X15(2 * 8)
VPINSRQ_1_SI_X13(15 * 8)
VPINSRQ_1_SI_X14_0()
VPINSRQ_1_SI_X15(13 * 8)
}
// load msg:
//
// X12 = (2, 5), X13 = (4, 15), X14 = (6, 10), X15 = (0, 8)
func LOAD_MSG_AVX_2_5_4_15_6_10_0_8() {
VMOVQ_SI_X12(2 * 8)
VMOVQ_SI_X13(4 * 8)
VMOVQ_SI_X14(6 * 8)
VMOVQ_SI_X15_0()
VPINSRQ_1_SI_X12(5 * 8)
VPINSRQ_1_SI_X13(15 * 8)
VPINSRQ_1_SI_X14(10 * 8)
VPINSRQ_1_SI_X15(8 * 8)
}
// load msg:
//
// X12 = (9, 5), X13 = (2, 10), X14 = (0, 7), X15 = (4, 15)
func LOAD_MSG_AVX_9_5_2_10_0_7_4_15() {
VMOVQ_SI_X12(9 * 8)
VMOVQ_SI_X13(2 * 8)
VMOVQ_SI_X14_0()
VMOVQ_SI_X15(4 * 8)
VPINSRQ_1_SI_X12(5 * 8)
VPINSRQ_1_SI_X13(10 * 8)
VPINSRQ_1_SI_X14(7 * 8)
VPINSRQ_1_SI_X15(15 * 8)
}
// load msg:
//
// X12 = (2, 6), X13 = (0, 8), X14 = (12, 10), X15 = (11, 3)
func LOAD_MSG_AVX_2_6_0_8_12_10_11_3() {
VMOVQ_SI_X12(2 * 8)
VMOVQ_SI_X13_0()
VMOVQ_SI_X14(12 * 8)
VMOVQ_SI_X15(11 * 8)
VPINSRQ_1_SI_X12(6 * 8)
VPINSRQ_1_SI_X13(8 * 8)
VPINSRQ_1_SI_X14(10 * 8)
VPINSRQ_1_SI_X15(3 * 8)
}
// load msg:
//
// X12 = (0, 6), X13 = (9, 8), X14 = (7, 3), X15 = (2, 11)
func LOAD_MSG_AVX_0_6_9_8_7_3_2_11() {
MOVQ(Mem{Base: SI}.Offset(0*8), X12)
VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(8*8), X13)
MOVQ(Mem{Base: SI}.Offset(7*8), X14)
MOVQ(Mem{Base: SI}.Offset(2*8), X15)
VPINSRQ_1_SI_X12(6 * 8)
VPINSRQ_1_SI_X14(3 * 8)
VPINSRQ_1_SI_X15(11 * 8)
}
// load msg:
//
// X12 = (6, 14), X13 = (11, 0), X14 = (15, 9), X15 = (3, 8)
func LOAD_MSG_AVX_6_14_11_0_15_9_3_8() {
MOVQ(Mem{Base: SI}.Offset(6*8), X12)
MOVQ(Mem{Base: SI}.Offset(11*8), X13)
MOVQ(Mem{Base: SI}.Offset(15*8), X14)
MOVQ(Mem{Base: SI}.Offset(3*8), X15)
VPINSRQ_1_SI_X12(14 * 8)
VPINSRQ_1_SI_X13_0()
VPINSRQ_1_SI_X14(9 * 8)
VPINSRQ_1_SI_X15(8 * 8)
}
// load msg:
//
// X12 = (5, 15), X13 = (8, 2), X14 = (0, 4), X15 = (6, 10)
func LOAD_MSG_AVX_5_15_8_2_0_4_6_10() {
MOVQ(Mem{Base: SI}.Offset(5*8), X12)
MOVQ(Mem{Base: SI}.Offset(8*8), X13)
MOVQ(Mem{Base: SI}.Offset(0*8), X14)
MOVQ(Mem{Base: SI}.Offset(6*8), X15)
VPINSRQ_1_SI_X12(15 * 8)
VPINSRQ_1_SI_X13(2 * 8)
VPINSRQ_1_SI_X14(4 * 8)
VPINSRQ_1_SI_X15(10 * 8)
}
// load msg:
//
// X12 = (12, 13), X13 = (1, 10), X14 = (2, 7), X15 = (4, 5)
func LOAD_MSG_AVX_12_13_1_10_2_7_4_5() {
VMOVDQU(Mem{Base: SI}.Offset(12*8), X12)
MOVQ(Mem{Base: SI}.Offset(1*8), X13)
MOVQ(Mem{Base: SI}.Offset(2*8), X14)
VPINSRQ_1_SI_X13(10 * 8)
VPINSRQ_1_SI_X14(7 * 8)
VMOVDQU(Mem{Base: SI}.Offset(4*8), X15)
}
// load msg:
//
// X12 = (15, 9), X13 = (3, 13), X14 = (11, 14), X15 = (12, 0)
func LOAD_MSG_AVX_15_9_3_13_11_14_12_0() {
MOVQ(Mem{Base: SI}.Offset(15*8), X12)
MOVQ(Mem{Base: SI}.Offset(3*8), X13)
MOVQ(Mem{Base: SI}.Offset(11*8), X14)
MOVQ(Mem{Base: SI}.Offset(12*8), X15)
VPINSRQ_1_SI_X12(9 * 8)
VPINSRQ_1_SI_X13(13 * 8)
VPINSRQ_1_SI_X14(14 * 8)
VPINSRQ_1_SI_X15_0()
}
func hashBlocksAVX() {
Implement("hashBlocksAVX")
Attributes(4)
AllocLocal(288) // frame size = 272 + 16 byte alignment
Load(Param("h"), RAX)
Load(Param("c"), RBX)
Load(Param("flag"), RCX)
Load(Param("blocks").Base(), RSI)
Load(Param("blocks").Len(), RDI)
MOVQ(RSP, R10)
ADDQ(Imm(15), R10)
ANDQ(I32(^15), R10)
AVX_c40 := AVX_c40_DATA()
AVX_c48 := AVX_c48_DATA()
VMOVDQU(AVX_c40, X0)
VMOVDQU(AVX_c48, X1)
VMOVDQA(X0, X8)
VMOVDQA(X1, X9)
AVX_iv3 := AVX_iv3_DATA()
VMOVDQU(AVX_iv3, X0)
VMOVDQA(X0, Mem{Base: R10}.Offset(0))
XORQ(RCX, Mem{Base: R10}.Offset(0)) // 0(R10) = ·AVX_iv3 ^ (CX || 0)
VMOVDQU(Mem{Base: AX}.Offset(0), X10)
VMOVDQU(Mem{Base: AX}.Offset(16), X11)
VMOVDQU(Mem{Base: AX}.Offset(32), X2)
VMOVDQU(Mem{Base: AX}.Offset(48), X3)
MOVQ(Mem{Base: BX}.Offset(0), R8)
MOVQ(Mem{Base: BX}.Offset(8), R9)
loop_AVX()
noinc_AVX()
}
func loop_AVX() {
Label("loop")
ADDQ(Imm(128), R8)
CMPQ(R8, Imm(128))
JGE(LabelRef("noinc"))
INCQ(R9)
}
func noinc_AVX() {
Label("noinc")
VMOVQ_R8_X15()
VPINSRQ_1_R9_X15()
AVX_iv0 := AVX_iv0_DATA()
AVX_iv1 := AVX_iv1_DATA()
AVX_iv2 := AVX_iv2_DATA()
VMOVDQA(X10, X0)
VMOVDQA(X11, X1)
VMOVDQU(AVX_iv0, X4)
VMOVDQU(AVX_iv1, X5)
VMOVDQU(AVX_iv2, X6)
VPXOR(X15, X6, X6)
VMOVDQA(Mem{Base: R10}.Offset(0), X7)
LOAD_MSG_AVX_0_2_4_6_1_3_5_7()
VMOVDQA(X12, Mem{Base: R10}.Offset(16))
VMOVDQA(X13, Mem{Base: R10}.Offset(32))
VMOVDQA(X14, Mem{Base: R10}.Offset(48))
VMOVDQA(X15, Mem{Base: R10}.Offset(64))
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX(8, 10, 12, 14, 9, 11, 13, 15)
VMOVDQA(X12, Mem{Base: R10}.Offset(80))
VMOVDQA(X13, Mem{Base: R10}.Offset(96))
VMOVDQA(X14, Mem{Base: R10}.Offset(112))
VMOVDQA(X15, Mem{Base: R10}.Offset(128))
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX(14, 4, 9, 13, 10, 8, 15, 6)
VMOVDQA(X12, Mem{Base: R10}.Offset(144))
VMOVDQA(X13, Mem{Base: R10}.Offset(160))
VMOVDQA(X14, Mem{Base: R10}.Offset(176))
VMOVDQA(X15, Mem{Base: R10}.Offset(192))
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_1_0_11_5_12_2_7_3()
VMOVDQA(X12, Mem{Base: R10}.Offset(208))
VMOVDQA(X13, Mem{Base: R10}.Offset(224))
VMOVDQA(X14, Mem{Base: R10}.Offset(240))
VMOVDQA(X15, Mem{Base: R10}.Offset(256))
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX_11_12_5_15_8_0_2_13()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX(10, 3, 7, 9, 14, 6, 1, 4)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX(7, 3, 13, 11, 9, 1, 12, 14)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_2_5_4_15_6_10_0_8()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX_9_5_2_10_0_7_4_15()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX(14, 11, 6, 3, 1, 12, 8, 13)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX_2_6_0_8_12_10_11_3()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX(4, 7, 15, 1, 13, 5, 14, 9)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX(12, 1, 14, 4, 5, 15, 13, 10)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_0_6_9_8_7_3_2_11()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX(13, 7, 12, 3, 11, 14, 1, 9)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_5_15_8_2_0_4_6_10()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX_6_14_11_0_15_9_3_8()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_12_13_1_10_2_7_4_5()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
LOAD_MSG_AVX(10, 8, 7, 1, 2, 4, 6, 5)
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX()
LOAD_MSG_AVX_15_9_3_13_11_14_12_0()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
SHUFFLE_AVX_INV()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(16), Mem{Base: R10}.Offset(32), Mem{Base: R10}.Offset(48), Mem{Base: R10}.Offset(64), X15, X8, X9)
SHUFFLE_AVX()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(80), Mem{Base: R10}.Offset(96), Mem{Base: R10}.Offset(112), Mem{Base: R10}.Offset(128), X15, X8, X9)
SHUFFLE_AVX_INV()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(144), Mem{Base: R10}.Offset(160), Mem{Base: R10}.Offset(176), Mem{Base: R10}.Offset(192), X15, X8, X9)
SHUFFLE_AVX()
HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(208), Mem{Base: R10}.Offset(224), Mem{Base: R10}.Offset(240), Mem{Base: R10}.Offset(256), X15, X8, X9)
SHUFFLE_AVX_INV()
VMOVDQU(Mem{Base: AX}.Offset(32), X14)
VMOVDQU(Mem{Base: AX}.Offset(48), X15)
VPXOR(X0, X10, X10)
VPXOR(X1, X11, X11)
VPXOR(X2, X14, X14)
VPXOR(X3, X15, X15)
VPXOR(X4, X10, X10)
VPXOR(X5, X11, X11)
VPXOR(X6, X14, X2)
VPXOR(X7, X15, X3)
VMOVDQU(X2, Mem{Base: AX}.Offset(32))
VMOVDQU(X3, Mem{Base: AX}.Offset(48))
LEAQ(Mem{Base: SI}.Offset(128), RSI)
SUBQ(Imm(128), RDI)
JNE(LabelRef("loop"))
VMOVDQU(X10, Mem{Base: AX}.Offset(0))
VMOVDQU(X11, Mem{Base: AX}.Offset(16))
MOVQ(R8, Mem{Base: BX}.Offset(0))
MOVQ(R9, Mem{Base: BX}.Offset(8))
VZEROUPPER()
RET()
}
// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
var (
AVX2_iv0_ptr,
AVX2_iv1_ptr,
AVX2_c40_ptr,
AVX2_c48_ptr,
AVX_iv0_ptr,
AVX_iv1_ptr,
AVX_iv2_ptr,
AVX_iv3_ptr,
AVX_c40_ptr,
AVX_c48_ptr *Mem
)
func AVX2_iv0_DATA() Mem {
if AVX2_iv0_ptr != nil {
return *AVX2_iv0_ptr
}
AVX2_iv0 := GLOBL(ThatPeskyUnicodeDot+"AVX2_iv0", NOPTR|RODATA)
DATA(0x00, U64(0x6a09e667f3bcc908))
DATA(0x08, U64(0xbb67ae8584caa73b))
DATA(0x10, U64(0x3c6ef372fe94f82b))
DATA(0x18, U64(0xa54ff53a5f1d36f1))
return AVX2_iv0
}
func AVX2_iv1_DATA() Mem {
if AVX2_iv1_ptr != nil {
return *AVX2_iv1_ptr
}
AVX2_iv1 := GLOBL(ThatPeskyUnicodeDot+"AVX2_iv1", NOPTR|RODATA)
DATA(0x00, U64(0x510e527fade682d1))
DATA(0x08, U64(0x9b05688c2b3e6c1f))
DATA(0x10, U64(0x1f83d9abfb41bd6b))
DATA(0x18, U64(0x5be0cd19137e2179))
return AVX2_iv1
}
func AVX2_c40_DATA() Mem {
if AVX2_c40_ptr != nil {
return *AVX2_c40_ptr
}
AVX2_c40 := GLOBL(ThatPeskyUnicodeDot+"AVX2_c40", NOPTR|RODATA)
DATA(0x00, U64(0x0201000706050403))
DATA(0x08, U64(0x0a09080f0e0d0c0b))
DATA(0x10, U64(0x0201000706050403))
DATA(0x18, U64(0x0a09080f0e0d0c0b))
return AVX2_c40
}
func AVX2_c48_DATA() Mem {
if AVX2_c48_ptr != nil {
return *AVX2_c48_ptr
}
AVX2_c48 := GLOBL(ThatPeskyUnicodeDot+"AVX2_c48", NOPTR|RODATA)
DATA(0x00, U64(0x0100070605040302))
DATA(0x08, U64(0x09080f0e0d0c0b0a))
DATA(0x10, U64(0x0100070605040302))
DATA(0x18, U64(0x09080f0e0d0c0b0a))
return AVX2_c48
}
func AVX_iv0_DATA() Mem {
if AVX_iv0_ptr != nil {
return *AVX_iv0_ptr
}
AVX_iv0 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv0", NOPTR|RODATA)
DATA(0x00, U64(0x6a09e667f3bcc908))
DATA(0x08, U64(0xbb67ae8584caa73b))
return AVX_iv0
}
func AVX_iv1_DATA() Mem {
if AVX_iv1_ptr != nil {
return *AVX_iv1_ptr
}
AVX_iv1 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv1", NOPTR|RODATA)
DATA(0x00, U64(0x3c6ef372fe94f82b))
DATA(0x08, U64(0xa54ff53a5f1d36f1))
return AVX_iv1
}
func AVX_iv2_DATA() Mem {
if AVX_iv2_ptr != nil {
return *AVX_iv2_ptr
}
AVX_iv2 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv2", NOPTR|RODATA)
DATA(0x00, U64(0x510e527fade682d1))
DATA(0x08, U64(0x9b05688c2b3e6c1f))
return AVX_iv2
}
func AVX_iv3_DATA() Mem {
if AVX_iv3_ptr != nil {
return *AVX_iv3_ptr
}
AVX_iv3 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv3", NOPTR|RODATA)
DATA(0x00, U64(0x1f83d9abfb41bd6b))
DATA(0x08, U64(0x5be0cd19137e2179))
return AVX_iv3
}
func AVX_c40_DATA() Mem {
if AVX_c40_ptr != nil {
return *AVX_c40_ptr
}
AVX_c40 := GLOBL(ThatPeskyUnicodeDot+"AVX_c40", NOPTR|RODATA)
DATA(0x00, U64(0x0201000706050403))
DATA(0x08, U64(0x0a09080f0e0d0c0b))
return AVX_c40
}
func AVX_c48_DATA() Mem {
if AVX_c48_ptr != nil {
return *AVX_c48_ptr
}
AVX_c48 := GLOBL(ThatPeskyUnicodeDot+"AVX_c48", NOPTR|RODATA)
DATA(0x00, U64(0x0100070605040302))
DATA(0x08, U64(0x09080f0e0d0c0b0a))
return AVX_c48
}
Reference in New Issue View Git Blame Copy Permalink