From 7ee59707ff302869e7667032223ed3f10711c3b3 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Wed, 14 May 2025 13:20:15 -0700 Subject: [PATCH] chacha20poly1305: drop pre-AVX assembly impl Updates golang/go#69587 Change-Id: Ic158558f879b8b8ad23155bf887e083169096d19 Reviewed-on: https://go-review.googlesource.com/c/crypto/+/672837 Auto-Submit: Roland Shoemaker Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: Daniel McCarney --- .../_asm/chacha20poly1305_amd64_asm.go | 2411 +------- chacha20poly1305/chacha20poly1305_amd64.go | 6 +- chacha20poly1305/chacha20poly1305_amd64.s | 5132 +---------------- 3 files changed, 517 insertions(+), 7032 deletions(-) diff --git a/chacha20poly1305/_asm/chacha20poly1305_amd64_asm.go b/chacha20poly1305/_asm/chacha20poly1305_amd64_asm.go index 66af868..c25199d 100644 --- a/chacha20poly1305/_asm/chacha20poly1305_amd64_asm.go +++ b/chacha20poly1305/_asm/chacha20poly1305_amd64_asm.go @@ -7,7 +7,6 @@ package main import ( - "fmt" "os" "strings" @@ -38,35 +37,11 @@ var ( t3 = R8 // Register and stack allocation for the SSE code - rStore Mem = Mem{Base: BP}.Offset(0 * 16) - sStore = Mem{Base: BP}.Offset(1 * 16) - state1Store = Mem{Base: BP}.Offset(2 * 16) - state2Store = Mem{Base: BP}.Offset(3 * 16) - tmpStore = Mem{Base: BP}.Offset(4 * 16) - ctr0Store = Mem{Base: BP}.Offset(5 * 16) - ctr1Store = Mem{Base: BP}.Offset(6 * 16) - ctr2Store = Mem{Base: BP}.Offset(7 * 16) - ctr3Store = Mem{Base: BP}.Offset(8 * 16) - A0 VecPhysical = X0 - A1 = X1 - A2 = X2 - B0 = X3 - B1 = X4 - B2 = X5 - C0 = X6 - C1 = X7 - C2 = X8 - D0 = X9 - D1 = X10 - D2 = X11 - T0 = X12 - T1 = X13 - T2 = X14 - T3 = X15 - A3 = T0 - B3 = T1 - C3 = T2 - D3 = T3 + sStore = Mem{Base: BP}.Offset(1 * 16) + tmpStore = Mem{Base: BP}.Offset(4 * 16) + A0 VecPhysical = X0 + A1 = X1 + T0 = X12 // Register and stack allocation for the AVX2 code rsStoreAVX2 Mem = Mem{Base: BP}.Offset(0 * 32) @@ -118,319 +93,6 @@ func BYTE(u8 U8) { Instruction(&ir.Instruction{Opcode: "BYTE", Operands: []Op{u8}}) } -// PALIGNR $4, X3, X3 -func shiftB0Left() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xdb)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X4, X4 -func shiftB1Left() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xe4)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X5, X5 -func shiftB2Left() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xed)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X13, X13 -func shiftB3Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xed)) - BYTE(U8(0x04)) -} - -// PALIGNR $8, X6, X6 -func shiftC0Left() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xf6)) - BYTE(U8(0x08)) -} - -// PALIGNR $8, X7, X7 -func shiftC1Left() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xff)) - BYTE(U8(0x08)) -} - -// PALIGNR $8, X8, X8 -func shiftC2Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xc0)) - BYTE(U8(0x08)) -} - -// PALIGNR $8, X14, X14 -func shiftC3Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xf6)) - BYTE(U8(0x08)) -} - -// PALIGNR $12, X9, X9 -func shiftD0Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xc9)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X10, X10 -func shiftD1Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xd2)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X11, X11 -func shiftD2Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xdb)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X15, X15 -func shiftD3Left() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xff)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X3, X3 -func shiftB0Right() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xdb)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X4, X4 -func shiftB1Right() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xe4)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X5, X5 -func shiftB2Right() { - BYTE(U8(0x66)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xed)) - BYTE(U8(0x0c)) -} - -// PALIGNR $12, X13, X13 -func shiftB3Right() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xed)) - BYTE(U8(0x0c)) -} - -func shiftC0Right() { - shiftC0Left() -} - -func shiftC1Right() { - shiftC1Left() -} - -func shiftC2Right() { - shiftC2Left() -} - -func shiftC3Right() { - shiftC3Left() -} - -// PALIGNR $4, X9, X9 -func shiftD0Right() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xc9)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X10, X10 -func shiftD1Right() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xd2)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X11, X11 -func shiftD2Right() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xdb)) - BYTE(U8(0x04)) -} - -// PALIGNR $4, X15, X15 -func shiftD3Right() { - BYTE(U8(0x66)) - BYTE(U8(0x45)) - BYTE(U8(0x0f)) - BYTE(U8(0x3a)) - BYTE(U8(0x0f)) - BYTE(U8(0xff)) - BYTE(U8(0x04)) -} - -// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~SOME MACROS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~## - -// Hack: ROL must be a #define macro as it is referenced by other macros -func defineROL() { - definition := - `#define ROL(N, R, T) \ - MOVO R, T; \ - PSLLL $(N), T; \ - PSRLL $(32-(N)), R; \ - PXOR T, R` - Comment("ROL rotates the uint32s in register R left by N bits, using temporary T.") - Instruction(&ir.Instruction{Opcode: definition}) -} - -// ROL rotates the uint32s in register R left by N bits, using temporary T. -func ROL(N uint64, R, T VecPhysical) { - // Hack: ROL must be a #define macro as it is referenced by other macros - Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL(%s, %s, %s)", I8(N).Asm(), R.Asm(), T.Asm())}) -} - -// Hack to get Avo to generate an #ifdef -// -// ROL16(R, T) definition depends on a compiler flag that specifies amd64 architectural level. -func defineROL16() { - definition := - `#ifdef GOAMD64_v2 - #define ROL16(R, T) PSHUFB ·rol16<>(SB), R - #else - #define ROL16(R, T) ROL(16, R, T) - #endif` - - Comment("ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.") - Instruction(&ir.Instruction{Opcode: definition}) -} - -// Hack to emit macro call -// -// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. -func ROL16(R, T VecPhysical) { - Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL16(%s, %s)", R.Asm(), T.Asm())}) -} - -// Hack to get Avo to generate an #ifdef -// -// ROL8(R, T) definition depends on a compiler flag that specifies amd64 architectural level. -func defineROL8() { - definition := - `#ifdef GOAMD64_v2 - #define ROL8(R, T) PSHUFB ·rol8<>(SB), R - #else - #define ROL8(R, T) ROL(8, R, T) - #endif` - - Comment("ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.") - Instruction(&ir.Instruction{Opcode: definition}) -} - -// Hack to emit macro call -// -// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. -func ROL8(R, T VecPhysical) { - Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL8(%s, %s)", R.Asm(), T.Asm())}) -} - -func chachaQR(A, B, C, D, T VecPhysical) { - PADDD(B, A) - PXOR(A, D) - ROL16(D, T) - PADDD(D, C) - PXOR(C, B) - MOVO(B, T) - PSLLL(Imm(12), T) - PSRLL(Imm(20), B) - PXOR(T, B) - PADDD(B, A) - PXOR(A, D) - ROL8(D, T) - PADDD(D, C) - PXOR(C, B) - MOVO(B, T) - PSLLL(Imm(7), T) - PSRLL(Imm(25), B) - PXOR(T, B) -} - func chachaQR_AVX2(A, B, C, D, T VecPhysical) { VPADDD(B, A, A) VPXOR(A, D, D) @@ -555,11 +217,6 @@ func polyHashADInternal() { Attributes(NOSPLIT) AllocLocal(0) - Comment("Hack: Must declare #define macros inside of a function due to Avo constraints") - defineROL() - defineROL8() - defineROL16() - // adp points to beginning of additional data // itr2 holds ad length XORQ(acc0, acc0) @@ -638,8 +295,115 @@ func hashADDone() { RET() } +func openSSEFinalize() { + Label("openSSEFinalize") + Comment("Hash in the PT, AAD lengths") + ADDQ(NewParamAddr("ad_len", 80), acc0) + ADCQ(NewParamAddr("src_len", 56), acc1) + ADCQ(Imm(1), acc2) + polyMul() + + Comment("Final reduce") + MOVQ(acc0, t0) + MOVQ(acc1, t1) + MOVQ(acc2, t2) + SUBQ(I8(-5), acc0) + SBBQ(I8(-1), acc1) + SBBQ(Imm(3), acc2) + CMOVQCS(t0, acc0) + CMOVQCS(t1, acc1) + CMOVQCS(t2, acc2) + + Comment("Add in the \"s\" part of the key") + ADDQ(sStore.Offset(0), acc0) + ADCQ(sStore.Offset(8), acc1) + + Comment("Finally, constant time compare to the tag at the end of the message") + XORQ(RAX, RAX) + MOVQ(U32(1), RDX) + XORQ(Mem{Base: inp}.Offset(0*8), acc0) + XORQ(Mem{Base: inp}.Offset(1*8), acc1) + ORQ(acc1, acc0) + CMOVQEQ(RDX, RAX) + + Comment("Return true iff tags are equal") + // Hack to get Avo to emit: + // MOVB AX, ret+96(FP) + Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{AX, NewParamAddr("ret", 96)}}) + RET() +} + // ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- +// Special optimization for buffers smaller than 129 bytes + +func openSSETail16() { + Label("openSSETail16") + TESTQ(inl, inl) + JE(LabelRef("openSSEFinalize")) + + Comment("We can safely load the CT from the end, because it is padded with the MAC") + MOVQ(inl, itr2) + SHLQ(Imm(4), itr2) + andMask := andMask_DATA() + LEAQ(andMask, t0) + MOVOU(Mem{Base: inp}, T0) + ADDQ(inl, inp) + PAND(Mem{Base: t0, Index: itr2, Scale: 1}.Offset(-16), T0) + MOVO(T0, tmpStore.Offset(0)) + MOVQ(T0, t0) + MOVQ(tmpStore.Offset(8), t1) + PXOR(A1, T0) +} + +func openSSETail16Store() { + Comment("We can only store one byte at a time, since plaintext can be shorter than 16 bytes") + Label("openSSETail16Store") + MOVQ(T0, t3) + // Hack to get Avo to emit: + // MOVB t3, (oup) + Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{t3, Mem{Base: oup}}}) + PSRLDQ(Imm(1), T0) + INCQ(oup) + DECQ(inl) + JNE(LabelRef("openSSETail16Store")) + ADDQ(t0, acc0) + ADCQ(t1, acc1) + ADCQ(Imm(1), acc2) + polyMul() + JMP(LabelRef("openSSEFinalize")) +} + +// Functions to emit AVX instructions via BYTE directive + +// broadcasti128 16(r8), ymm14 +func VBROADCASTI128_16_R8_YMM14() { + BYTE(U8(0xc4)) + BYTE(U8(0x42)) + BYTE(U8(0x7d)) + BYTE(U8(0x5a)) + BYTE(U8(0x70)) + BYTE(U8(0x10)) +} + +// broadcasti128 32(r8), ymm12 +func VBROADCASTI128_32_R8_YMM12() { + BYTE(U8(0xc4)) + BYTE(U8(0x42)) + BYTE(U8(0x7d)) + BYTE(U8(0x5a)) + BYTE(U8(0x60)) + BYTE(U8(0x20)) +} + +// broadcasti128 48(r8), ymm4 +func VBROADCASTI128_48_R8_YMM4() { + BYTE(U8(0xc4)) + BYTE(U8(0xc2)) + BYTE(U8(0x7d)) + BYTE(U8(0x5a)) + BYTE(U8(0x60)) + BYTE(U8(0x30)) +} // Implements the following function signature: // @@ -660,78 +424,40 @@ func chacha20Poly1305Open() { Load(Param("src").Len(), inl) Load(Param("ad").Base(), adp) - Comment("Check for AVX2 support") - CMPB(Mem{Symbol: Symbol{Name: ThatPeskyUnicodeDot + "useAVX2"}, Base: StaticBase}, Imm(1)) - JE(LabelRef("chacha20Poly1305Open_AVX2")) + VZEROUPPER() + chacha20Constants := chacha20Constants_DATA() + VMOVDQU(chacha20Constants, AA0) + VBROADCASTI128_16_R8_YMM14() + VBROADCASTI128_32_R8_YMM12() + VBROADCASTI128_48_R8_YMM4() + avx2InitMask := avx2InitMask_DATA() + VPADDD(avx2InitMask, DD0, DD0) Comment("Special optimization, for very short buffers") - CMPQ(inl, Imm(128)) - JBE(LabelRef("openSSE128")) // About 16% faster + CMPQ(inl, Imm(192)) + JBE(LabelRef("openAVX2192")) + CMPQ(inl, U32(320)) + JBE(LabelRef("openAVX2320")) - Comment("For long buffers, prepare the poly key first") - chacha20Constants := chacha20Constants_DATA() - MOVOU(chacha20Constants, A0) - MOVOU(Mem{Base: keyp}.Offset(1*16), B0) - MOVOU(Mem{Base: keyp}.Offset(2*16), C0) - MOVOU(Mem{Base: keyp}.Offset(3*16), D0) - MOVO(D0, T1) - - Comment("Store state on stack for future use") - MOVO(B0, state1Store) - MOVO(C0, state2Store) - MOVO(D0, ctr3Store) + Comment("For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream") + VMOVDQA(BB0, state1StoreAVX2) + VMOVDQA(CC0, state2StoreAVX2) + VMOVDQA(DD0, ctr3StoreAVX2) MOVQ(U32(10), itr2) - openSSEPreparePolyKey() - openSSEMainLoop() - openSSEInternalLoop() - openSSEMainLoopDone() - openSSEFinalize() - - // ---------------------------------------------------------------------------- - // Special optimization for buffers smaller than 129 bytes - openSSE128() - openSSE128InnerCipherLoop() - openSSE128Open() - openSSETail16() - openSSETail16Store() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 64 bytes of ciphertext - openSSETail64() - openSSETail64LoopA() - openSSETail64LoopB() - openSSETail64DecLoop() - openSSETail64DecLoopDone() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 128 bytes of ciphertext - openSSETail128() - openSSETail128LoopA() - openSSETail128LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 192 bytes of ciphertext - openSSETail192() - openSSLTail192LoopA() - openSSLTail192LoopB() - openSSLTail192Store() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 256 bytes of ciphertext - openSSETail256() - openSSETail256Loop() - openSSETail256HashLoop() - - // ---------------------------------------------------------------------------- - // ------------------------- AVX2 Code ---------------------------------------- - chacha20Poly1305Open_AVX2() openAVX2PreparePolyKey() openAVX2InitialHash64() openAVX2MainLoop() openAVX2InternalLoop() openAVX2MainLoopDone() + openSSEFinalize() + + // ---------------------------------------------------------------------------- + // Special optimization for buffers smaller than 129 bytes + openSSETail16() + openSSETail16Store() + // ---------------------------------------------------------------------------- // Special optimization for buffers smaller than 193 bytes openAVX2192() @@ -780,940 +506,6 @@ func chacha20Poly1305Open() { openAVX2Tail512HashEnd() } -func openSSEPreparePolyKey() { - Label("openSSEPreparePolyKey") - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - DECQ(itr2) - JNE(LabelRef("openSSEPreparePolyKey")) - - Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded") - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(state1Store, B0) - - Comment("Clamp and store the key") - polyClampMask := polyClampMask_DATA() - PAND(polyClampMask, A0) - MOVO(A0, rStore) - MOVO(B0, sStore) - - Comment("Hash AAD") - Load(Param("ad").Len(), itr2) - CALL(LabelRef("polyHashADInternal<>(SB)")) -} - -func openSSEMainLoop() { - Label("openSSEMainLoop") - CMPQ(inl, U32(256)) - JB(LabelRef("openSSEMainLoopDone")) - - chacha20Constants := chacha20Constants_DATA() - sseIncMask := sseIncMask_DATA() - - Comment("Load state, increment counter blocks") - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - PADDL(sseIncMask, D0) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(A2, A3) - MOVO(B2, B3) - MOVO(C2, C3) - MOVO(D2, D3) - PADDL(sseIncMask, D3) - - Comment("Store counters") - MOVO(D0, ctr0Store) - MOVO(D1, ctr1Store) - MOVO(D2, ctr2Store) - MOVO(D3, ctr3Store) - - Comment("There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash") - Comment("2 blocks, and for the remaining 4 only 1 block - for a total of 16") - MOVQ(U32(4), itr1) - MOVQ(inp, itr2) -} - -func openSSEInternalLoop() { - Label("openSSEInternalLoop") - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - polyAdd(Mem{Base: itr2}.Offset(0)) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftB3Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftC3Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - shiftD3Left() - polyMulStage1() - polyMulStage2() - LEAQ(Mem{Base: itr2}.Offset(2*8), itr2) - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - polyMulStage3() - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - polyMulReduceStage() - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftB3Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftC3Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - shiftD3Right() - DECQ(itr1) - JGE(LabelRef("openSSEInternalLoop")) - - polyAdd(Mem{Base: itr2}.Offset(0)) - polyMul() - LEAQ(Mem{Base: itr2}.Offset(2*8), itr2) - - CMPQ(itr1, I8(-6)) - JG(LabelRef("openSSEInternalLoop")) - - chacha20Constants := chacha20Constants_DATA() - Comment("Add in the state") - PADDD(chacha20Constants, A0) - PADDD(chacha20Constants, A1) - PADDD(chacha20Constants, A2) - PADDD(chacha20Constants, A3) - PADDD(state1Store, B0) - PADDD(state1Store, B1) - PADDD(state1Store, B2) - PADDD(state1Store, B3) - PADDD(state2Store, C0) - PADDD(state2Store, C1) - PADDD(state2Store, C2) - PADDD(state2Store, C3) - PADDD(ctr0Store, D0) - PADDD(ctr1Store, D1) - PADDD(ctr2Store, D2) - PADDD(ctr3Store, D3) - - Comment("Load - xor - store") - MOVO(D3, tmpStore) - MOVOU(Mem{Base: inp}.Offset(0*16), D3) - PXOR(D3, A0) - MOVOU(A0, Mem{Base: oup}.Offset(0*16)) - MOVOU(Mem{Base: inp}.Offset(1*16), D3) - PXOR(D3, B0) - MOVOU(B0, Mem{Base: oup}.Offset(1*16)) - MOVOU(Mem{Base: inp}.Offset(2*16), D3) - PXOR(D3, C0) - MOVOU(C0, Mem{Base: oup}.Offset(2*16)) - MOVOU(Mem{Base: inp}.Offset(3*16), D3) - PXOR(D3, D0) - MOVOU(D0, Mem{Base: oup}.Offset(3*16)) - MOVOU(Mem{Base: inp}.Offset(4*16), D0) - PXOR(D0, A1) - MOVOU(A1, Mem{Base: oup}.Offset(4*16)) - MOVOU(Mem{Base: inp}.Offset(5*16), D0) - PXOR(D0, B1) - MOVOU(B1, Mem{Base: oup}.Offset(5*16)) - MOVOU(Mem{Base: inp}.Offset(6*16), D0) - PXOR(D0, C1) - MOVOU(C1, Mem{Base: oup}.Offset(6*16)) - MOVOU(Mem{Base: inp}.Offset(7*16), D0) - PXOR(D0, D1) - MOVOU(D1, Mem{Base: oup}.Offset(7*16)) - MOVOU(Mem{Base: inp}.Offset(8*16), D0) - PXOR(D0, A2) - MOVOU(A2, Mem{Base: oup}.Offset(8*16)) - MOVOU(Mem{Base: inp}.Offset(9*16), D0) - PXOR(D0, B2) - MOVOU(B2, Mem{Base: oup}.Offset(9*16)) - MOVOU(Mem{Base: inp}.Offset(10*16), D0) - PXOR(D0, C2) - MOVOU(C2, Mem{Base: oup}.Offset(10*16)) - MOVOU(Mem{Base: inp}.Offset(11*16), D0) - PXOR(D0, D2) - MOVOU(D2, Mem{Base: oup}.Offset(11*16)) - MOVOU(Mem{Base: inp}.Offset(12*16), D0) - PXOR(D0, A3) - MOVOU(A3, Mem{Base: oup}.Offset(12*16)) - MOVOU(Mem{Base: inp}.Offset(13*16), D0) - PXOR(D0, B3) - MOVOU(B3, Mem{Base: oup}.Offset(13*16)) - MOVOU(Mem{Base: inp}.Offset(14*16), D0) - PXOR(D0, C3) - MOVOU(C3, Mem{Base: oup}.Offset(14*16)) - MOVOU(Mem{Base: inp}.Offset(15*16), D0) - PXOR(tmpStore, D0) - MOVOU(D0, Mem{Base: oup}.Offset(15*16)) - LEAQ(Mem{Base: inp}.Offset(256), inp) - LEAQ(Mem{Base: oup}.Offset(256), oup) - SUBQ(U32(256), inl) - JMP(LabelRef("openSSEMainLoop")) -} - -func openSSEMainLoopDone() { - Label("openSSEMainLoopDone") - Comment("Handle the various tail sizes efficiently") - TESTQ(inl, inl) - JE(LabelRef("openSSEFinalize")) - CMPQ(inl, Imm(64)) - JBE(LabelRef("openSSETail64")) - CMPQ(inl, Imm(128)) - JBE(LabelRef("openSSETail128")) - CMPQ(inl, Imm(192)) - JBE(LabelRef("openSSETail192")) - JMP(LabelRef("openSSETail256")) -} - -func openSSEFinalize() { - Label("openSSEFinalize") - Comment("Hash in the PT, AAD lengths") - ADDQ(NewParamAddr("ad_len", 80), acc0) - ADCQ(NewParamAddr("src_len", 56), acc1) - ADCQ(Imm(1), acc2) - polyMul() - - Comment("Final reduce") - MOVQ(acc0, t0) - MOVQ(acc1, t1) - MOVQ(acc2, t2) - SUBQ(I8(-5), acc0) - SBBQ(I8(-1), acc1) - SBBQ(Imm(3), acc2) - CMOVQCS(t0, acc0) - CMOVQCS(t1, acc1) - CMOVQCS(t2, acc2) - - Comment("Add in the \"s\" part of the key") - ADDQ(sStore.Offset(0), acc0) - ADCQ(sStore.Offset(8), acc1) - - Comment("Finally, constant time compare to the tag at the end of the message") - XORQ(RAX, RAX) - MOVQ(U32(1), RDX) - XORQ(Mem{Base: inp}.Offset(0*8), acc0) - XORQ(Mem{Base: inp}.Offset(1*8), acc1) - ORQ(acc1, acc0) - CMOVQEQ(RDX, RAX) - - Comment("Return true iff tags are equal") - // Hack to get Avo to emit: - // MOVB AX, ret+96(FP) - Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{AX, NewParamAddr("ret", 96)}}) - RET() -} - -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 129 bytes - -// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks -func openSSE128() { - Label("openSSE128") - - chacha20Constants := chacha20Constants_DATA() - sseIncMask := sseIncMask_DATA() - - MOVOU(chacha20Constants, A0) - MOVOU(Mem{Base: keyp}.Offset(1*16), B0) - MOVOU(Mem{Base: keyp}.Offset(2*16), C0) - MOVOU(Mem{Base: keyp}.Offset(3*16), D0) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(B0, T1) - MOVO(C0, T2) - MOVO(D1, T3) - MOVQ(U32(10), itr2) -} - -func openSSE128InnerCipherLoop() { - Label("openSSE128InnerCipherLoop") - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - DECQ(itr2) - JNE(LabelRef("openSSE128InnerCipherLoop")) - - Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded") - - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(chacha20Constants, A2) - PADDL(T1, B0) - PADDL(T1, B1) - PADDL(T1, B2) - PADDL(T2, C1) - PADDL(T2, C2) - PADDL(T3, D1) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, T3) - PADDL(T3, D2) - - Comment("Clamp and store the key") - polyClampMask := polyClampMask_DATA() - PAND(polyClampMask, A0) - MOVOU(A0, rStore) - MOVOU(B0, sStore) - - Comment("Hash") - Load(Param("ad").Len(), itr2) - CALL(LabelRef("polyHashADInternal<>(SB)")) -} - -func openSSE128Open() { - Label("openSSE128Open") - CMPQ(inl, Imm(16)) - JB(LabelRef("openSSETail16")) - SUBQ(Imm(16), inl) - - Comment("Load for hashing") - polyAdd(Mem{Base: inp}.Offset(0)) - - Comment("Load for decryption") - MOVOU(Mem{Base: inp}, T0) - PXOR(T0, A1) - MOVOU(A1, Mem{Base: oup}) - LEAQ(Mem{Base: inp}.Offset(1*16), inp) - LEAQ(Mem{Base: oup}.Offset(1*16), oup) - polyMul() - - Comment("Shift the stream \"left\"") - MOVO(B1, A1) - MOVO(C1, B1) - MOVO(D1, C1) - MOVO(A2, D1) - MOVO(B2, A2) - MOVO(C2, B2) - MOVO(D2, C2) - JMP(LabelRef("openSSE128Open")) -} - -func openSSETail16() { - Label("openSSETail16") - TESTQ(inl, inl) - JE(LabelRef("openSSEFinalize")) - - Comment("We can safely load the CT from the end, because it is padded with the MAC") - MOVQ(inl, itr2) - SHLQ(Imm(4), itr2) - andMask := andMask_DATA() - LEAQ(andMask, t0) - MOVOU(Mem{Base: inp}, T0) - ADDQ(inl, inp) - PAND(Mem{Base: t0, Index: itr2, Scale: 1}.Offset(-16), T0) - MOVO(T0, tmpStore.Offset(0)) - MOVQ(T0, t0) - MOVQ(tmpStore.Offset(8), t1) - PXOR(A1, T0) -} - -func openSSETail16Store() { - Comment("We can only store one byte at a time, since plaintext can be shorter than 16 bytes") - Label("openSSETail16Store") - MOVQ(T0, t3) - // Hack to get Avo to emit: - // MOVB t3, (oup) - Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{t3, Mem{Base: oup}}}) - PSRLDQ(Imm(1), T0) - INCQ(oup) - DECQ(inl) - JNE(LabelRef("openSSETail16Store")) - ADDQ(t0, acc0) - ADCQ(t1, acc1) - ADCQ(Imm(1), acc2) - polyMul() - JMP(LabelRef("openSSEFinalize")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of ciphertext - -// Need to decrypt up to 64 bytes - prepare single block -func openSSETail64() { - Label("openSSETail64") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D0) - MOVO(D0, ctr0Store) - XORQ(itr2, itr2) - MOVQ(inl, itr1) - CMPQ(itr1, Imm(16)) - JB(LabelRef("openSSETail64LoopB")) -} - -// Perform ChaCha rounds, while hashing the remaining input -func openSSETail64LoopA() { - Label("openSSETail64LoopA") - polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0)) - polyMul() - SUBQ(Imm(16), itr1) -} - -func openSSETail64LoopB() { - Label("openSSETail64LoopB") - ADDQ(Imm(16), itr2) - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - - CMPQ(itr1, Imm(16)) - JAE(LabelRef("openSSETail64LoopA")) - - CMPQ(itr2, Imm(160)) - JNE(LabelRef("openSSETail64LoopB")) - - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(state1Store, B0) - PADDL(state2Store, C0) - PADDL(ctr0Store, D0) -} - -func openSSETail64DecLoop() { - Label("openSSETail64DecLoop") - CMPQ(inl, Imm(16)) - JB(LabelRef("openSSETail64DecLoopDone")) - SUBQ(Imm(16), inl) - MOVOU(Mem{Base: inp}, T0) - PXOR(T0, A0) - MOVOU(A0, Mem{Base: oup}) - LEAQ(Mem{Base: inp}.Offset(16), inp) - LEAQ(Mem{Base: oup}.Offset(16), oup) - MOVO(B0, A0) - MOVO(C0, B0) - MOVO(D0, C0) - JMP(LabelRef("openSSETail64DecLoop")) -} - -func openSSETail64DecLoopDone() { - Label("openSSETail64DecLoopDone") - MOVO(A0, A1) - JMP(LabelRef("openSSETail16")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext - -// Need to decrypt up to 128 bytes - prepare two blocks -func openSSETail128() { - Label("openSSETail128") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A1) - MOVO(state1Store, B1) - MOVO(state2Store, C1) - MOVO(ctr3Store, D1) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D1) - MOVO(D1, ctr0Store) - MOVO(A1, A0) - MOVO(B1, B0) - MOVO(C1, C0) - MOVO(D1, D0) - PADDL(sseIncMask, D0) - MOVO(D0, ctr1Store) - XORQ(itr2, itr2) - MOVQ(inl, itr1) - ANDQ(I8(-16), itr1) -} - -// Perform ChaCha rounds, while hashing the remaining input -func openSSETail128LoopA() { - Label("openSSETail128LoopA") - polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0)) - polyMul() -} - -func openSSETail128LoopB() { - Label("openSSETail128LoopB") - ADDQ(Imm(16), itr2) - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - shiftB1Left() - shiftC1Left() - shiftD1Left() - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - shiftB1Right() - shiftC1Right() - shiftD1Right() - - CMPQ(itr2, itr1) - JB(LabelRef("openSSETail128LoopA")) - - CMPQ(itr2, Imm(160)) - JNE(LabelRef("openSSETail128LoopB")) - - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(state1Store, B0) - PADDL(state1Store, B1) - PADDL(state2Store, C0) - PADDL(state2Store, C1) - PADDL(ctr1Store, D0) - PADDL(ctr0Store, D1) - - MOVOU(Mem{Base: inp}.Offset(0*16), T0) - MOVOU(Mem{Base: inp}.Offset(1*16), T1) - MOVOU(Mem{Base: inp}.Offset(2*16), T2) - MOVOU(Mem{Base: inp}.Offset(3*16), T3) - PXOR(T0, A1) - PXOR(T1, B1) - PXOR(T2, C1) - PXOR(T3, D1) - MOVOU(A1, Mem{Base: oup}.Offset(0*16)) - MOVOU(B1, Mem{Base: oup}.Offset(1*16)) - MOVOU(C1, Mem{Base: oup}.Offset(2*16)) - MOVOU(D1, Mem{Base: oup}.Offset(3*16)) - - SUBQ(Imm(64), inl) - LEAQ(Mem{Base: inp}.Offset(64), inp) - LEAQ(Mem{Base: oup}.Offset(64), oup) - JMP(LabelRef("openSSETail64DecLoop")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of ciphertext - -// Need to decrypt up to 192 bytes - prepare three blocks -func openSSETail192() { - Label("openSSETail192") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A2) - MOVO(state1Store, B2) - MOVO(state2Store, C2) - MOVO(ctr3Store, D2) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D2) - MOVO(D2, ctr0Store) - MOVO(A2, A1) - MOVO(B2, B1) - MOVO(C2, C1) - MOVO(D2, D1) - PADDL(sseIncMask, D1) - MOVO(D1, ctr1Store) - MOVO(A1, A0) - MOVO(B1, B0) - MOVO(C1, C0) - MOVO(D1, D0) - PADDL(sseIncMask, D0) - MOVO(D0, ctr2Store) - - MOVQ(inl, itr1) - MOVQ(U32(160), itr2) - CMPQ(itr1, Imm(160)) - CMOVQGT(itr2, itr1) - ANDQ(I8(-16), itr1) - XORQ(itr2, itr2) -} - -// Perform ChaCha rounds, while hashing the remaining input -func openSSLTail192LoopA() { - Label("openSSLTail192LoopA") - polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0)) - polyMul() -} - -func openSSLTail192LoopB() { - Label("openSSLTail192LoopB") - ADDQ(Imm(16), itr2) - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - shiftB1Left() - shiftC1Left() - shiftD1Left() - shiftB2Left() - shiftC2Left() - shiftD2Left() - - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - shiftB1Right() - shiftC1Right() - shiftD1Right() - shiftB2Right() - shiftC2Right() - shiftD2Right() - - CMPQ(itr2, itr1) - JB(LabelRef("openSSLTail192LoopA")) - - CMPQ(itr2, Imm(160)) - JNE(LabelRef("openSSLTail192LoopB")) - - CMPQ(inl, Imm(176)) - JB(LabelRef("openSSLTail192Store")) - - polyAdd(Mem{Base: inp}.Offset(160)) - polyMul() - - CMPQ(inl, Imm(192)) - JB(LabelRef("openSSLTail192Store")) - - polyAdd(Mem{Base: inp}.Offset(176)) - polyMul() -} - -func openSSLTail192Store() { - Label("openSSLTail192Store") - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(chacha20Constants, A2) - PADDL(state1Store, B0) - PADDL(state1Store, B1) - PADDL(state1Store, B2) - PADDL(state2Store, C0) - PADDL(state2Store, C1) - PADDL(state2Store, C2) - PADDL(ctr2Store, D0) - PADDL(ctr1Store, D1) - PADDL(ctr0Store, D2) - - MOVOU(Mem{Base: inp}.Offset(0*16), T0) - MOVOU(Mem{Base: inp}.Offset(1*16), T1) - MOVOU(Mem{Base: inp}.Offset(2*16), T2) - MOVOU(Mem{Base: inp}.Offset(3*16), T3) - PXOR(T0, A2) - PXOR(T1, B2) - PXOR(T2, C2) - PXOR(T3, D2) - MOVOU(A2, Mem{Base: oup}.Offset(0*16)) - MOVOU(B2, Mem{Base: oup}.Offset(1*16)) - MOVOU(C2, Mem{Base: oup}.Offset(2*16)) - MOVOU(D2, Mem{Base: oup}.Offset(3*16)) - - MOVOU(Mem{Base: inp}.Offset(4*16), T0) - MOVOU(Mem{Base: inp}.Offset(5*16), T1) - MOVOU(Mem{Base: inp}.Offset(6*16), T2) - MOVOU(Mem{Base: inp}.Offset(7*16), T3) - PXOR(T0, A1) - PXOR(T1, B1) - PXOR(T2, C1) - PXOR(T3, D1) - MOVOU(A1, Mem{Base: oup}.Offset(4*16)) - MOVOU(B1, Mem{Base: oup}.Offset(5*16)) - MOVOU(C1, Mem{Base: oup}.Offset(6*16)) - MOVOU(D1, Mem{Base: oup}.Offset(7*16)) - - SUBQ(Imm(128), inl) - LEAQ(Mem{Base: inp}.Offset(128), inp) - LEAQ(Mem{Base: oup}.Offset(128), oup) - JMP(LabelRef("openSSETail64DecLoop")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext - -// Need to decrypt up to 256 bytes - prepare four blocks -func openSSETail256() { - Label("openSSETail256") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D0) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(A2, A3) - MOVO(B2, B3) - MOVO(C2, C3) - MOVO(D2, D3) - PADDL(sseIncMask, D3) - - Comment("Store counters") - MOVO(D0, ctr0Store) - MOVO(D1, ctr1Store) - MOVO(D2, ctr2Store) - MOVO(D3, ctr3Store) - XORQ(itr2, itr2) -} - -// This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication -func openSSETail256Loop() { - Label("openSSETail256Loop") - polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0)) - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftB3Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftC3Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - shiftD3Left() - polyMulStage1() - polyMulStage2() - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - polyMulStage3() - polyMulReduceStage() - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftB3Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftC3Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - shiftD3Right() - ADDQ(Imm(2*8), itr2) - CMPQ(itr2, Imm(160)) - JB(LabelRef("openSSETail256Loop")) - MOVQ(inl, itr1) - ANDQ(I8(-16), itr1) -} - -func openSSETail256HashLoop() { - Label("openSSETail256HashLoop") - polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0)) - polyMul() - ADDQ(Imm(2*8), itr2) - CMPQ(itr2, itr1) - JB(LabelRef("openSSETail256HashLoop")) - - Comment("Add in the state") - chacha20Constants := chacha20Constants_DATA() - PADDD(chacha20Constants, A0) - PADDD(chacha20Constants, A1) - PADDD(chacha20Constants, A2) - PADDD(chacha20Constants, A3) - PADDD(state1Store, B0) - PADDD(state1Store, B1) - PADDD(state1Store, B2) - PADDD(state1Store, B3) - PADDD(state2Store, C0) - PADDD(state2Store, C1) - PADDD(state2Store, C2) - PADDD(state2Store, C3) - PADDD(ctr0Store, D0) - PADDD(ctr1Store, D1) - PADDD(ctr2Store, D2) - PADDD(ctr3Store, D3) - MOVO(D3, tmpStore) - - Comment("Load - xor - store") - MOVOU(Mem{Base: inp}.Offset(0*16), D3) - PXOR(D3, A0) - MOVOU(Mem{Base: inp}.Offset(1*16), D3) - PXOR(D3, B0) - MOVOU(Mem{Base: inp}.Offset(2*16), D3) - PXOR(D3, C0) - MOVOU(Mem{Base: inp}.Offset(3*16), D3) - PXOR(D3, D0) - MOVOU(A0, Mem{Base: oup}.Offset(0*16)) - MOVOU(B0, Mem{Base: oup}.Offset(1*16)) - MOVOU(C0, Mem{Base: oup}.Offset(2*16)) - MOVOU(D0, Mem{Base: oup}.Offset(3*16)) - MOVOU(Mem{Base: inp}.Offset(4*16), A0) - MOVOU(Mem{Base: inp}.Offset(5*16), B0) - MOVOU(Mem{Base: inp}.Offset(6*16), C0) - MOVOU(Mem{Base: inp}.Offset(7*16), D0) - PXOR(A0, A1) - PXOR(B0, B1) - PXOR(C0, C1) - PXOR(D0, D1) - MOVOU(A1, Mem{Base: oup}.Offset(4*16)) - MOVOU(B1, Mem{Base: oup}.Offset(5*16)) - MOVOU(C1, Mem{Base: oup}.Offset(6*16)) - MOVOU(D1, Mem{Base: oup}.Offset(7*16)) - MOVOU(Mem{Base: inp}.Offset(8*16), A0) - MOVOU(Mem{Base: inp}.Offset(9*16), B0) - MOVOU(Mem{Base: inp}.Offset(10*16), C0) - MOVOU(Mem{Base: inp}.Offset(11*16), D0) - PXOR(A0, A2) - PXOR(B0, B2) - PXOR(C0, C2) - PXOR(D0, D2) - MOVOU(A2, Mem{Base: oup}.Offset(8*16)) - MOVOU(B2, Mem{Base: oup}.Offset(9*16)) - MOVOU(C2, Mem{Base: oup}.Offset(10*16)) - MOVOU(D2, Mem{Base: oup}.Offset(11*16)) - LEAQ(Mem{Base: inp}.Offset(192), inp) - LEAQ(Mem{Base: oup}.Offset(192), oup) - SUBQ(Imm(192), inl) - MOVO(A3, A0) - MOVO(B3, B0) - MOVO(C3, C0) - MOVO(tmpStore, D0) - - JMP(LabelRef("openSSETail64DecLoop")) -} - -// Functions to emit AVX instructions via BYTE directive - -// broadcasti128 16(r8), ymm14 -func VBROADCASTI128_16_R8_YMM14() { - BYTE(U8(0xc4)) - BYTE(U8(0x42)) - BYTE(U8(0x7d)) - BYTE(U8(0x5a)) - BYTE(U8(0x70)) - BYTE(U8(0x10)) -} - -// broadcasti128 32(r8), ymm12 -func VBROADCASTI128_32_R8_YMM12() { - BYTE(U8(0xc4)) - BYTE(U8(0x42)) - BYTE(U8(0x7d)) - BYTE(U8(0x5a)) - BYTE(U8(0x60)) - BYTE(U8(0x20)) -} - -// broadcasti128 48(r8), ymm4 -func VBROADCASTI128_48_R8_YMM4() { - BYTE(U8(0xc4)) - BYTE(U8(0xc2)) - BYTE(U8(0x7d)) - BYTE(U8(0x5a)) - BYTE(U8(0x60)) - BYTE(U8(0x30)) -} - -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- - -func chacha20Poly1305Open_AVX2() { - Label("chacha20Poly1305Open_AVX2") - VZEROUPPER() - chacha20Constants := chacha20Constants_DATA() - VMOVDQU(chacha20Constants, AA0) - VBROADCASTI128_16_R8_YMM14() - VBROADCASTI128_32_R8_YMM12() - VBROADCASTI128_48_R8_YMM4() - avx2InitMask := avx2InitMask_DATA() - VPADDD(avx2InitMask, DD0, DD0) - - Comment("Special optimization, for very short buffers") - CMPQ(inl, Imm(192)) - JBE(LabelRef("openAVX2192")) - CMPQ(inl, U32(320)) - JBE(LabelRef("openAVX2320")) - - Comment("For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream") - VMOVDQA(BB0, state1StoreAVX2) - VMOVDQA(CC0, state2StoreAVX2) - VMOVDQA(DD0, ctr3StoreAVX2) - MOVQ(U32(10), itr2) -} - func openAVX2PreparePolyKey() { Label("openAVX2PreparePolyKey") chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) @@ -2967,855 +1759,9 @@ func openAVX2Tail512HashEnd() { // ---------------------------------------------------------------------------- // ---------------------------------------------------------------------------- -// Implements the following function signature: -// -// func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte) -func chacha20Poly1305Seal() { - Implement("chacha20Poly1305Seal") - Attributes(0) - AllocLocal(288) - - MOVQ(RSP, RBP) - ADDQ(Imm(32), RBP) - ANDQ(I32(-32), RBP) - Load(Param("dst").Base(), oup) - Load(Param("key").Base(), keyp) - Load(Param("src").Base(), inp) - Load(Param("src").Len(), inl) - Load(Param("ad").Base(), adp) - - CMPB(Mem{Symbol: Symbol{Name: ThatPeskyUnicodeDot + "useAVX2"}, Base: StaticBase}, Imm(1)) - JE(LabelRef("chacha20Poly1305Seal_AVX2")) - - Comment("Special optimization, for very short buffers") - CMPQ(inl, Imm(128)) - JBE(LabelRef("sealSSE128")) - - Comment("In the seal case - prepare the poly key + 3 blocks of stream in the first iteration") - chacha20Constants := chacha20Constants_DATA() - MOVOU(chacha20Constants, A0) - MOVOU(Mem{Base: keyp}.Offset(1*16), B0) - MOVOU(Mem{Base: keyp}.Offset(2*16), C0) - MOVOU(Mem{Base: keyp}.Offset(3*16), D0) - - Comment("Store state on stack for future use") - MOVO(B0, state1Store) - MOVO(C0, state2Store) - - Comment("Load state, increment counter blocks") - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(A2, A3) - MOVO(B2, B3) - MOVO(C2, C3) - MOVO(D2, D3) - PADDL(sseIncMask, D3) - - Comment("Store counters") - MOVO(D0, ctr0Store) - MOVO(D1, ctr1Store) - MOVO(D2, ctr2Store) - MOVO(D3, ctr3Store) - MOVQ(U32(10), itr2) - - sealSSEIntroLoop() - sealSSEMainLoop() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 64 bytes of plaintext - sealSSETail64() - sealSSETail64LoopA() - sealSSETail64LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 128 bytes of plaintext - sealSSETail128() - sealSSETail128LoopA() - sealSSETail128LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 192 bytes of plaintext - sealSSETail192() - sealSSETail192LoopA() - sealSSETail192LoopB() - - // ---------------------------------------------------------------------------- - // Special seal optimization for buffers smaller than 129 bytes - sealSSE128() - sealSSE128SealHash() - sealSSE128Seal() - sealSSETail() - sealSSETailLoadLoop() - sealSSEFinalize() - - // ---------------------------------------------------------------------------- - // ------------------------- AVX2 Code ---------------------------------------- - chacha20Poly1305Seal_AVX2() - sealAVX2IntroLoop() - sealAVX2MainLoop() - sealAVX2InternalLoop() - sealAVX2InternalLoopStart() - - // ---------------------------------------------------------------------------- - // Special optimization for buffers smaller than 193 bytes - seal192AVX2() - sealAVX2192InnerCipherLoop() - sealAVX2ShortSeal() - sealAVX2SealHash() - sealAVX2ShortSealLoop() - sealAVX2ShortTail32() - sealAVX2ShortDone() - - // ---------------------------------------------------------------------------- - // Special optimization for buffers smaller than 321 bytes - seal320AVX2() - sealAVX2320InnerCipherLoop() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 128 bytes of ciphertext - sealAVX2Tail128() - sealAVX2Tail128LoopA() - sealAVX2Tail128LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 256 bytes of ciphertext - sealAVX2Tail256() - sealAVX2Tail256LoopA() - sealAVX2Tail256LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 384 bytes of ciphertext - sealAVX2Tail384() - sealAVX2Tail384LoopA() - sealAVX2Tail384LoopB() - - // ---------------------------------------------------------------------------- - // Special optimization for the last 512 bytes of ciphertext - sealAVX2Tail512() - sealAVX2Tail512LoopA() - sealAVX2Tail512LoopB() -} - -func sealSSEIntroLoop() { - Label("sealSSEIntroLoop") - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftB3Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftC3Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - shiftD3Left() - - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftB3Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftC3Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - shiftD3Right() - DECQ(itr2) - JNE(LabelRef("sealSSEIntroLoop")) - - Comment("Add in the state") - chacha20Constants := chacha20Constants_DATA() - PADDD(chacha20Constants, A0) - PADDD(chacha20Constants, A1) - PADDD(chacha20Constants, A2) - PADDD(chacha20Constants, A3) - PADDD(state1Store, B0) - PADDD(state1Store, B1) - PADDD(state1Store, B2) - PADDD(state1Store, B3) - PADDD(state2Store, C1) - PADDD(state2Store, C2) - PADDD(state2Store, C3) - PADDD(ctr1Store, D1) - PADDD(ctr2Store, D2) - PADDD(ctr3Store, D3) - - Comment("Clamp and store the key") - polyClampMask := polyClampMask_DATA() - PAND(polyClampMask, A0) - MOVO(A0, rStore) - MOVO(B0, sStore) - - Comment("Hash AAD") - MOVQ(NewParamAddr("ad_len", 80), itr2) - CALL(LabelRef("polyHashADInternal<>(SB)")) - - MOVOU(Mem{Base: inp}.Offset(0*16), A0) - MOVOU(Mem{Base: inp}.Offset(1*16), B0) - MOVOU(Mem{Base: inp}.Offset(2*16), C0) - MOVOU(Mem{Base: inp}.Offset(3*16), D0) - PXOR(A0, A1) - PXOR(B0, B1) - PXOR(C0, C1) - PXOR(D0, D1) - MOVOU(A1, Mem{Base: oup}.Offset(0*16)) - MOVOU(B1, Mem{Base: oup}.Offset(1*16)) - MOVOU(C1, Mem{Base: oup}.Offset(2*16)) - MOVOU(D1, Mem{Base: oup}.Offset(3*16)) - MOVOU(Mem{Base: inp}.Offset(4*16), A0) - MOVOU(Mem{Base: inp}.Offset(5*16), B0) - MOVOU(Mem{Base: inp}.Offset(6*16), C0) - MOVOU(Mem{Base: inp}.Offset(7*16), D0) - PXOR(A0, A2) - PXOR(B0, B2) - PXOR(C0, C2) - PXOR(D0, D2) - MOVOU(A2, Mem{Base: oup}.Offset(4*16)) - MOVOU(B2, Mem{Base: oup}.Offset(5*16)) - MOVOU(C2, Mem{Base: oup}.Offset(6*16)) - MOVOU(D2, Mem{Base: oup}.Offset(7*16)) - - MOVQ(U32(128), itr1) - SUBQ(Imm(128), inl) - LEAQ(Mem{Base: inp}.Offset(128), inp) - - MOVO(A3, A1) - MOVO(B3, B1) - MOVO(C3, C1) - MOVO(D3, D1) - - CMPQ(inl, Imm(64)) - JBE(LabelRef("sealSSE128SealHash")) - - MOVOU(Mem{Base: inp}.Offset(0*16), A0) - MOVOU(Mem{Base: inp}.Offset(1*16), B0) - MOVOU(Mem{Base: inp}.Offset(2*16), C0) - MOVOU(Mem{Base: inp}.Offset(3*16), D0) - PXOR(A0, A3) - PXOR(B0, B3) - PXOR(C0, C3) - PXOR(D0, D3) - MOVOU(A3, Mem{Base: oup}.Offset(8*16)) - MOVOU(B3, Mem{Base: oup}.Offset(9*16)) - MOVOU(C3, Mem{Base: oup}.Offset(10*16)) - MOVOU(D3, Mem{Base: oup}.Offset(11*16)) - - ADDQ(Imm(64), itr1) - SUBQ(Imm(64), inl) - LEAQ(Mem{Base: inp}.Offset(64), inp) - - MOVQ(U32(2), itr1) - MOVQ(U32(8), itr2) - - CMPQ(inl, Imm(64)) - JBE(LabelRef("sealSSETail64")) - CMPQ(inl, Imm(128)) - JBE(LabelRef("sealSSETail128")) - CMPQ(inl, Imm(192)) - JBE(LabelRef("sealSSETail192")) -} - -func sealSSEMainLoop() { - Label("sealSSEMainLoop") - Comment("Load state, increment counter blocks") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D0) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(A2, A3) - MOVO(B2, B3) - MOVO(C2, C3) - MOVO(D2, D3) - PADDL(sseIncMask, D3) - - Comment("Store counters") - MOVO(D0, ctr0Store) - MOVO(D1, ctr1Store) - MOVO(D2, ctr2Store) - MOVO(D3, ctr3Store) - - Label("sealSSEInnerLoop") - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - polyAdd(Mem{Base: oup}.Offset(0)) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftB3Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftC3Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - shiftD3Left() - polyMulStage1() - polyMulStage2() - LEAQ(Mem{Base: oup}.Offset(2*8), oup) - MOVO(C3, tmpStore) - chachaQR(A0, B0, C0, D0, C3) - chachaQR(A1, B1, C1, D1, C3) - chachaQR(A2, B2, C2, D2, C3) - MOVO(tmpStore, C3) - MOVO(C1, tmpStore) - polyMulStage3() - chachaQR(A3, B3, C3, D3, C1) - MOVO(tmpStore, C1) - polyMulReduceStage() - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftB3Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftC3Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - shiftD3Right() - DECQ(itr2) - JGE(LabelRef("sealSSEInnerLoop")) - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(2*8), oup) - DECQ(itr1) - JG(LabelRef("sealSSEInnerLoop")) - - Comment("Add in the state") - PADDD(chacha20Constants, A0) - PADDD(chacha20Constants, A1) - PADDD(chacha20Constants, A2) - PADDD(chacha20Constants, A3) - PADDD(state1Store, B0) - PADDD(state1Store, B1) - PADDD(state1Store, B2) - PADDD(state1Store, B3) - PADDD(state2Store, C0) - PADDD(state2Store, C1) - PADDD(state2Store, C2) - PADDD(state2Store, C3) - PADDD(ctr0Store, D0) - PADDD(ctr1Store, D1) - PADDD(ctr2Store, D2) - PADDD(ctr3Store, D3) - MOVO(D3, tmpStore) - - Comment("Load - xor - store") - MOVOU(Mem{Base: inp}.Offset(0*16), D3) - PXOR(D3, A0) - MOVOU(Mem{Base: inp}.Offset(1*16), D3) - PXOR(D3, B0) - MOVOU(Mem{Base: inp}.Offset(2*16), D3) - PXOR(D3, C0) - MOVOU(Mem{Base: inp}.Offset(3*16), D3) - PXOR(D3, D0) - MOVOU(A0, Mem{Base: oup}.Offset(0*16)) - MOVOU(B0, Mem{Base: oup}.Offset(1*16)) - MOVOU(C0, Mem{Base: oup}.Offset(2*16)) - MOVOU(D0, Mem{Base: oup}.Offset(3*16)) - MOVO(tmpStore, D3) - - MOVOU(Mem{Base: inp}.Offset(4*16), A0) - MOVOU(Mem{Base: inp}.Offset(5*16), B0) - MOVOU(Mem{Base: inp}.Offset(6*16), C0) - MOVOU(Mem{Base: inp}.Offset(7*16), D0) - PXOR(A0, A1) - PXOR(B0, B1) - PXOR(C0, C1) - PXOR(D0, D1) - MOVOU(A1, Mem{Base: oup}.Offset(4*16)) - MOVOU(B1, Mem{Base: oup}.Offset(5*16)) - MOVOU(C1, Mem{Base: oup}.Offset(6*16)) - MOVOU(D1, Mem{Base: oup}.Offset(7*16)) - MOVOU(Mem{Base: inp}.Offset(8*16), A0) - MOVOU(Mem{Base: inp}.Offset(9*16), B0) - MOVOU(Mem{Base: inp}.Offset(10*16), C0) - MOVOU(Mem{Base: inp}.Offset(11*16), D0) - PXOR(A0, A2) - PXOR(B0, B2) - PXOR(C0, C2) - PXOR(D0, D2) - MOVOU(A2, Mem{Base: oup}.Offset(8*16)) - MOVOU(B2, Mem{Base: oup}.Offset(9*16)) - MOVOU(C2, Mem{Base: oup}.Offset(10*16)) - MOVOU(D2, Mem{Base: oup}.Offset(11*16)) - ADDQ(Imm(192), inp) - MOVQ(U32(192), itr1) - SUBQ(Imm(192), inl) - MOVO(A3, A1) - MOVO(B3, B1) - MOVO(C3, C1) - MOVO(D3, D1) - CMPQ(inl, Imm(64)) - JBE(LabelRef("sealSSE128SealHash")) - MOVOU(Mem{Base: inp}.Offset(0*16), A0) - MOVOU(Mem{Base: inp}.Offset(1*16), B0) - MOVOU(Mem{Base: inp}.Offset(2*16), C0) - MOVOU(Mem{Base: inp}.Offset(3*16), D0) - PXOR(A0, A3) - PXOR(B0, B3) - PXOR(C0, C3) - PXOR(D0, D3) - MOVOU(A3, Mem{Base: oup}.Offset(12*16)) - MOVOU(B3, Mem{Base: oup}.Offset(13*16)) - MOVOU(C3, Mem{Base: oup}.Offset(14*16)) - MOVOU(D3, Mem{Base: oup}.Offset(15*16)) - LEAQ(Mem{Base: inp}.Offset(64), inp) - SUBQ(Imm(64), inl) - MOVQ(U32(6), itr1) - MOVQ(U32(4), itr2) - CMPQ(inl, Imm(192)) - JG(LabelRef("sealSSEMainLoop")) - - MOVQ(inl, itr1) - TESTQ(inl, inl) - JE(LabelRef("sealSSE128SealHash")) - MOVQ(U32(6), itr1) - CMPQ(inl, Imm(64)) - JBE(LabelRef("sealSSETail64")) - CMPQ(inl, Imm(128)) - JBE(LabelRef("sealSSETail128")) - JMP(LabelRef("sealSSETail192")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of plaintext - -// Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes -func sealSSETail64() { - Label("sealSSETail64") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A1) - MOVO(state1Store, B1) - MOVO(state2Store, C1) - MOVO(ctr3Store, D1) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D1) - MOVO(D1, ctr0Store) -} - -// Perform ChaCha rounds, while hashing the previously encrypted ciphertext -func sealSSETail64LoopA() { - Label("sealSSETail64LoopA") - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) -} - -func sealSSETail64LoopB() { - Label("sealSSETail64LoopB") - chachaQR(A1, B1, C1, D1, T1) - shiftB1Left() - shiftC1Left() - shiftD1Left() - chachaQR(A1, B1, C1, D1, T1) - shiftB1Right() - shiftC1Right() - shiftD1Right() - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) - - DECQ(itr1) - JG(LabelRef("sealSSETail64LoopA")) - - DECQ(itr2) - JGE(LabelRef("sealSSETail64LoopB")) - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A1) - PADDL(state1Store, B1) - PADDL(state2Store, C1) - PADDL(ctr0Store, D1) - - JMP(LabelRef("sealSSE128Seal")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of plaintext - -// Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes -func sealSSETail128() { - Label("sealSSETail128") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D0) - MOVO(D0, ctr0Store) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(D1, ctr1Store) -} - -// Perform ChaCha rounds, while hashing the previously encrypted ciphertext -func sealSSETail128LoopA() { - Label("sealSSETail128LoopA") - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) -} - -func sealSSETail128LoopB() { - Label("sealSSETail128LoopB") - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - shiftB1Left() - shiftC1Left() - shiftD1Left() - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - shiftB1Right() - shiftC1Right() - shiftD1Right() - - DECQ(itr1) - JG(LabelRef("sealSSETail128LoopA")) - - DECQ(itr2) - JGE(LabelRef("sealSSETail128LoopB")) - - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(state1Store, B0) - PADDL(state1Store, B1) - PADDL(state2Store, C0) - PADDL(state2Store, C1) - PADDL(ctr0Store, D0) - PADDL(ctr1Store, D1) - - MOVOU(Mem{Base: inp}.Offset(0*16), T0) - MOVOU(Mem{Base: inp}.Offset(1*16), T1) - MOVOU(Mem{Base: inp}.Offset(2*16), T2) - MOVOU(Mem{Base: inp}.Offset(3*16), T3) - PXOR(T0, A0) - PXOR(T1, B0) - PXOR(T2, C0) - PXOR(T3, D0) - MOVOU(A0, Mem{Base: oup}.Offset(0*16)) - MOVOU(B0, Mem{Base: oup}.Offset(1*16)) - MOVOU(C0, Mem{Base: oup}.Offset(2*16)) - MOVOU(D0, Mem{Base: oup}.Offset(3*16)) - - MOVQ(U32(64), itr1) - LEAQ(Mem{Base: inp}.Offset(64), inp) - SUBQ(Imm(64), inl) - - JMP(LabelRef("sealSSE128SealHash")) -} - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of plaintext - -// Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes -func sealSSETail192() { - Label("sealSSETail192") - chacha20Constants := chacha20Constants_DATA() - MOVO(chacha20Constants, A0) - MOVO(state1Store, B0) - MOVO(state2Store, C0) - MOVO(ctr3Store, D0) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D0) - MOVO(D0, ctr0Store) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - PADDL(sseIncMask, D1) - MOVO(D1, ctr1Store) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(D2, ctr2Store) -} - -// Perform ChaCha rounds, while hashing the previously encrypted ciphertext -func sealSSETail192LoopA() { - Label("sealSSETail192LoopA") - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) -} - -func sealSSETail192LoopB() { - Label("sealSSETail192LoopB") - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Left() - shiftC0Left() - shiftD0Left() - shiftB1Left() - shiftC1Left() - shiftD1Left() - shiftB2Left() - shiftC2Left() - shiftD2Left() - - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - LEAQ(Mem{Base: oup}.Offset(16), oup) - - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Right() - shiftC0Right() - shiftD0Right() - shiftB1Right() - shiftC1Right() - shiftD1Right() - shiftB2Right() - shiftC2Right() - shiftD2Right() - - DECQ(itr1) - JG(LabelRef("sealSSETail192LoopA")) - - DECQ(itr2) - JGE(LabelRef("sealSSETail192LoopB")) - - chacha20Constants := chacha20Constants_DATA() - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(chacha20Constants, A2) - PADDL(state1Store, B0) - PADDL(state1Store, B1) - PADDL(state1Store, B2) - PADDL(state2Store, C0) - PADDL(state2Store, C1) - PADDL(state2Store, C2) - PADDL(ctr0Store, D0) - PADDL(ctr1Store, D1) - PADDL(ctr2Store, D2) - - MOVOU(Mem{Base: inp}.Offset(0*16), T0) - MOVOU(Mem{Base: inp}.Offset(1*16), T1) - MOVOU(Mem{Base: inp}.Offset(2*16), T2) - MOVOU(Mem{Base: inp}.Offset(3*16), T3) - PXOR(T0, A0) - PXOR(T1, B0) - PXOR(T2, C0) - PXOR(T3, D0) - MOVOU(A0, Mem{Base: oup}.Offset(0*16)) - MOVOU(B0, Mem{Base: oup}.Offset(1*16)) - MOVOU(C0, Mem{Base: oup}.Offset(2*16)) - MOVOU(D0, Mem{Base: oup}.Offset(3*16)) - MOVOU(Mem{Base: inp}.Offset(4*16), T0) - MOVOU(Mem{Base: inp}.Offset(5*16), T1) - MOVOU(Mem{Base: inp}.Offset(6*16), T2) - MOVOU(Mem{Base: inp}.Offset(7*16), T3) - PXOR(T0, A1) - PXOR(T1, B1) - PXOR(T2, C1) - PXOR(T3, D1) - MOVOU(A1, Mem{Base: oup}.Offset(4*16)) - MOVOU(B1, Mem{Base: oup}.Offset(5*16)) - MOVOU(C1, Mem{Base: oup}.Offset(6*16)) - MOVOU(D1, Mem{Base: oup}.Offset(7*16)) - - MOVO(A2, A1) - MOVO(B2, B1) - MOVO(C2, C1) - MOVO(D2, D1) - MOVQ(U32(128), itr1) - LEAQ(Mem{Base: inp}.Offset(128), inp) - SUBQ(Imm(128), inl) - - JMP(LabelRef("sealSSE128SealHash")) -} - // ---------------------------------------------------------------------------- // Special seal optimization for buffers smaller than 129 bytes -// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks -func sealSSE128() { - Label("sealSSE128") - chacha20Constants := chacha20Constants_DATA() - MOVOU(chacha20Constants, A0) - MOVOU(Mem{Base: keyp}.Offset(1*16), B0) - MOVOU(Mem{Base: keyp}.Offset(2*16), C0) - MOVOU(Mem{Base: keyp}.Offset(3*16), D0) - MOVO(A0, A1) - MOVO(B0, B1) - MOVO(C0, C1) - MOVO(D0, D1) - sseIncMask := sseIncMask_DATA() - PADDL(sseIncMask, D1) - MOVO(A1, A2) - MOVO(B1, B2) - MOVO(C1, C2) - MOVO(D1, D2) - PADDL(sseIncMask, D2) - MOVO(B0, T1) - MOVO(C0, T2) - MOVO(D1, T3) - MOVQ(U32(10), itr2) - - Label("sealSSE128InnerCipherLoop") - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Left() - shiftB1Left() - shiftB2Left() - shiftC0Left() - shiftC1Left() - shiftC2Left() - shiftD0Left() - shiftD1Left() - shiftD2Left() - chachaQR(A0, B0, C0, D0, T0) - chachaQR(A1, B1, C1, D1, T0) - chachaQR(A2, B2, C2, D2, T0) - shiftB0Right() - shiftB1Right() - shiftB2Right() - shiftC0Right() - shiftC1Right() - shiftC2Right() - shiftD0Right() - shiftD1Right() - shiftD2Right() - DECQ(itr2) - JNE(LabelRef("sealSSE128InnerCipherLoop")) - - Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded") - PADDL(chacha20Constants, A0) - PADDL(chacha20Constants, A1) - PADDL(chacha20Constants, A2) - PADDL(T1, B0) - PADDL(T1, B1) - PADDL(T1, B2) - PADDL(T2, C1) - PADDL(T2, C2) - PADDL(T3, D1) - PADDL(sseIncMask, T3) - PADDL(T3, D2) - polyClampMask := polyClampMask_DATA() - PAND(polyClampMask, A0) - MOVOU(A0, rStore) - MOVOU(B0, sStore) - - Comment("Hash") - MOVQ(NewParamAddr("ad_len", 80), itr2) - CALL(LabelRef("polyHashADInternal<>(SB)")) - XORQ(itr1, itr1) -} - -// itr1 holds the number of bytes encrypted but not yet hashed -func sealSSE128SealHash() { - Label("sealSSE128SealHash") - CMPQ(itr1, Imm(16)) - JB(LabelRef("sealSSE128Seal")) - polyAdd(Mem{Base: oup}.Offset(0)) - polyMul() - - SUBQ(Imm(16), itr1) - ADDQ(Imm(16), oup) - - JMP(LabelRef("sealSSE128SealHash")) -} - -func sealSSE128Seal() { - Label("sealSSE128Seal") - CMPQ(inl, Imm(16)) - JB(LabelRef("sealSSETail")) - SUBQ(Imm(16), inl) - - Comment("Load for decryption") - MOVOU(Mem{Base: inp}, T0) - PXOR(T0, A1) - MOVOU(A1, Mem{Base: oup}) - LEAQ(Mem{Base: inp}.Offset(1*16), inp) - LEAQ(Mem{Base: oup}.Offset(1*16), oup) - - Comment("Extract for hashing") - MOVQ(A1, t0) - PSRLDQ(Imm(8), A1) - MOVQ(A1, t1) - ADDQ(t0, acc0) - ADCQ(t1, acc1) - ADCQ(Imm(1), acc2) - polyMul() - - Comment("Shift the stream \"left\"") - MOVO(B1, A1) - MOVO(C1, B1) - MOVO(D1, C1) - MOVO(A2, D1) - MOVO(B2, A2) - MOVO(C2, B2) - MOVO(D2, C2) - JMP(LabelRef("sealSSE128Seal")) -} - func sealSSETail() { Label("sealSSETail") TESTQ(inl, inl) @@ -3890,11 +1836,20 @@ func sealSSEFinalize() { RET() } -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- +func chacha20Poly1305Seal() { + Implement("chacha20Poly1305Seal") + Attributes(0) + AllocLocal(288) + + MOVQ(RSP, RBP) + ADDQ(Imm(32), RBP) + ANDQ(I32(-32), RBP) + Load(Param("dst").Base(), oup) + Load(Param("key").Base(), keyp) + Load(Param("src").Base(), inp) + Load(Param("src").Len(), inl) + Load(Param("ad").Base(), adp) -func chacha20Poly1305Seal_AVX2() { - Label("chacha20Poly1305Seal_AVX2") VZEROUPPER() chacha20Constants := chacha20Constants_DATA() VMOVDQU(chacha20Constants, AA0) @@ -3931,6 +1886,56 @@ func chacha20Poly1305Seal_AVX2() { VMOVDQA(DD2, ctr2StoreAVX2) VMOVDQA(DD3, ctr3StoreAVX2) MOVQ(U32(10), itr2) + + sealAVX2IntroLoop() + sealAVX2MainLoop() + sealAVX2InternalLoop() + sealAVX2InternalLoopStart() + + // ---------------------------------------------------------------------------- + // Special optimization for buffers smaller than 129 bytes + sealSSETail() + sealSSETailLoadLoop() + sealSSEFinalize() + + // ---------------------------------------------------------------------------- + // Special optimization for buffers smaller than 193 bytes + seal192AVX2() + sealAVX2192InnerCipherLoop() + sealAVX2ShortSeal() + sealAVX2SealHash() + sealAVX2ShortSealLoop() + sealAVX2ShortTail32() + sealAVX2ShortDone() + + // ---------------------------------------------------------------------------- + // Special optimization for buffers smaller than 321 bytes + seal320AVX2() + sealAVX2320InnerCipherLoop() + + // ---------------------------------------------------------------------------- + // Special optimization for the last 128 bytes of ciphertext + sealAVX2Tail128() + sealAVX2Tail128LoopA() + sealAVX2Tail128LoopB() + + // ---------------------------------------------------------------------------- + // Special optimization for the last 256 bytes of ciphertext + sealAVX2Tail256() + sealAVX2Tail256LoopA() + sealAVX2Tail256LoopB() + + // ---------------------------------------------------------------------------- + // Special optimization for the last 384 bytes of ciphertext + sealAVX2Tail384() + sealAVX2Tail384LoopA() + sealAVX2Tail384LoopB() + + // ---------------------------------------------------------------------------- + // Special optimization for the last 512 bytes of ciphertext + sealAVX2Tail512() + sealAVX2Tail512LoopA() + sealAVX2Tail512LoopB() } func sealAVX2IntroLoop() { @@ -5420,24 +3425,6 @@ func polyClampMask_DATA() Mem { return polyClampMask } -var sseIncMaskConsts = [2]uint64{ - 0x1, - 0x0, -} - -func sseIncMask_DATA() Mem { - if sseIncMask_DATA_ptr != nil { - return *sseIncMask_DATA_ptr - } - - sseIncMask := GLOBL(ThatPeskyUnicodeDot+"sseIncMask", NOPTR|RODATA) - sseIncMask_DATA_ptr = &sseIncMask - for i, v := range sseIncMaskConsts { - DATA(i*8, U64(v)) - } - return sseIncMask -} - var andMaskConsts = [30]uint64{ 0x00000000000000ff, 0x0000000000000000, diff --git a/chacha20poly1305/chacha20poly1305_amd64.go b/chacha20poly1305/chacha20poly1305_amd64.go index b850e77..bfe546b 100644 --- a/chacha20poly1305/chacha20poly1305_amd64.go +++ b/chacha20poly1305/chacha20poly1305_amd64.go @@ -20,7 +20,7 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte) var ( - useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2 + useAVX2 = cpu.X86.HasSSSE3 && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 ) // setupState writes a ChaCha20 input matrix to state. See @@ -47,7 +47,7 @@ func setupState(state *[16]uint32, key *[32]byte, nonce []byte) { } func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte { - if !cpu.X86.HasSSSE3 { + if !useAVX2 { return c.sealGeneric(dst, nonce, plaintext, additionalData) } @@ -66,7 +66,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) [] } func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { - if !cpu.X86.HasSSSE3 { + if !useAVX2 { return c.openGeneric(dst, nonce, ciphertext, additionalData) } diff --git a/chacha20poly1305/chacha20poly1305_amd64.s b/chacha20poly1305/chacha20poly1305_amd64.s index fd5ee84..7c0dcd1 100644 --- a/chacha20poly1305/chacha20poly1305_amd64.s +++ b/chacha20poly1305/chacha20poly1305_amd64.s @@ -6,27 +6,6 @@ // func polyHashADInternal<>() TEXT polyHashADInternal<>(SB), NOSPLIT, $0 - // Hack: Must declare #define macros inside of a function due to Avo constraints - // ROL rotates the uint32s in register R left by N bits, using temporary T. - #define ROL(N, R, T) \ - MOVO R, T; \ - PSLLL $(N), T; \ - PSRLL $(32-(N)), R; \ - PXOR T, R - - // ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. - #ifdef GOAMD64_v2 - #define ROL8(R, T) PSHUFB ·rol8<>(SB), R - #else - #define ROL8(R, T) ROL(8, R, T) - #endif - - // ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. - #ifdef GOAMD64_v2 - #define ROL16(R, T) PSHUFB ·rol16<>(SB), R - #else - #define ROL16(R, T) ROL(16, R, T) - #endif XORQ R10, R10 XORQ R11, R11 XORQ R12, R12 @@ -192,2447 +171,14 @@ hashADDone: // Requires: AVX, AVX2, BMI2, CMOV, SSE2 TEXT ·chacha20Poly1305Open(SB), $288-97 // For aligned stack access - MOVQ SP, BP - ADDQ $0x20, BP - ANDQ $-32, BP - MOVQ dst_base+0(FP), DI - MOVQ key_base+24(FP), R8 - MOVQ src_base+48(FP), SI - MOVQ src_len+56(FP), BX - MOVQ ad_base+72(FP), CX - - // Check for AVX2 support - CMPB ·useAVX2+0(SB), $0x01 - JE chacha20Poly1305Open_AVX2 - - // Special optimization, for very short buffers - CMPQ BX, $0x80 - JBE openSSE128 - - // For long buffers, prepare the poly key first - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X9, X13 - - // Store state on stack for future use - MOVO X3, 32(BP) - MOVO X6, 48(BP) - MOVO X9, 128(BP) - MOVQ $0x0000000a, R9 - -openSSEPreparePolyKey: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - DECQ R9 - JNE openSSEPreparePolyKey - - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL 32(BP), X3 - - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVO X0, (BP) - MOVO X3, 16(BP) - - // Hash AAD - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - -openSSEMainLoop: - CMPQ BX, $0x00000100 - JB openSSEMainLoopDone - - // Load state, increment counter blocks - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - - // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash - // 2 blocks, and for the remaining 4 only 1 block - for a total of 16 - MOVQ $0x00000004, CX - MOVQ SI, R9 - -openSSEInternalLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - LEAQ 16(R9), R9 - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ CX - JGE openSSEInternalLoop - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(R9), R9 - CMPQ CX, $-6 - JG openSSEInternalLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - - // Load - xor - store - MOVO X15, 64(BP) - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU X0, (DI) - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU X3, 16(DI) - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU X6, 32(DI) - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X9, 48(DI) - MOVOU 64(SI), X9 - PXOR X9, X1 - MOVOU X1, 64(DI) - MOVOU 80(SI), X9 - PXOR X9, X4 - MOVOU X4, 80(DI) - MOVOU 96(SI), X9 - PXOR X9, X7 - MOVOU X7, 96(DI) - MOVOU 112(SI), X9 - PXOR X9, X10 - MOVOU X10, 112(DI) - MOVOU 128(SI), X9 - PXOR X9, X2 - MOVOU X2, 128(DI) - MOVOU 144(SI), X9 - PXOR X9, X5 - MOVOU X5, 144(DI) - MOVOU 160(SI), X9 - PXOR X9, X8 - MOVOU X8, 160(DI) - MOVOU 176(SI), X9 - PXOR X9, X11 - MOVOU X11, 176(DI) - MOVOU 192(SI), X9 - PXOR X9, X12 - MOVOU X12, 192(DI) - MOVOU 208(SI), X9 - PXOR X9, X13 - MOVOU X13, 208(DI) - MOVOU 224(SI), X9 - PXOR X9, X14 - MOVOU X14, 224(DI) - MOVOU 240(SI), X9 - PXOR 64(BP), X9 - MOVOU X9, 240(DI) - LEAQ 256(SI), SI - LEAQ 256(DI), DI - SUBQ $0x00000100, BX - JMP openSSEMainLoop - -openSSEMainLoopDone: - // Handle the various tail sizes efficiently - TESTQ BX, BX - JE openSSEFinalize - CMPQ BX, $0x40 - JBE openSSETail64 - CMPQ BX, $0x80 - JBE openSSETail128 - CMPQ BX, $0xc0 - JBE openSSETail192 - JMP openSSETail256 - -openSSEFinalize: - // Hash in the PT, AAD lengths - ADDQ ad_len+80(FP), R10 - ADCQ src_len+56(FP), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Final reduce - MOVQ R10, R13 - MOVQ R11, R14 - MOVQ R12, R15 - SUBQ $-5, R10 - SBBQ $-1, R11 - SBBQ $0x03, R12 - CMOVQCS R13, R10 - CMOVQCS R14, R11 - CMOVQCS R15, R12 - - // Add in the "s" part of the key - ADDQ 16(BP), R10 - ADCQ 24(BP), R11 - - // Finally, constant time compare to the tag at the end of the message - XORQ AX, AX - MOVQ $0x00000001, DX - XORQ (SI), R10 - XORQ 8(SI), R11 - ORQ R11, R10 - CMOVQEQ DX, AX - - // Return true iff tags are equal - MOVB AX, ret+96(FP) - RET - -openSSE128: - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X3, X13 - MOVO X6, X14 - MOVO X10, X15 - MOVQ $0x0000000a, R9 - -openSSE128InnerCipherLoop: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ R9 - JNE openSSE128InnerCipherLoop - - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL X13, X3 - PADDL X13, X4 - PADDL X13, X5 - PADDL X14, X7 - PADDL X14, X8 - PADDL X15, X10 - PADDL ·sseIncMask<>+0(SB), X15 - PADDL X15, X11 - - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVOU X0, (BP) - MOVOU X3, 16(BP) - - // Hash - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - -openSSE128Open: - CMPQ BX, $0x10 - JB openSSETail16 - SUBQ $0x10, BX - - // Load for hashing - ADDQ (SI), R10 - ADCQ 8(SI), R11 - ADCQ $0x01, R12 - - // Load for decryption - MOVOU (SI), X12 - PXOR X12, X1 - MOVOU X1, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Shift the stream "left" - MOVO X4, X1 - MOVO X7, X4 - MOVO X10, X7 - MOVO X2, X10 - MOVO X5, X2 - MOVO X8, X5 - MOVO X11, X8 - JMP openSSE128Open - -openSSETail16: - TESTQ BX, BX - JE openSSEFinalize - - // We can safely load the CT from the end, because it is padded with the MAC - MOVQ BX, R9 - SHLQ $0x04, R9 - LEAQ ·andMask<>+0(SB), R13 - MOVOU (SI), X12 - ADDQ BX, SI - PAND -16(R13)(R9*1), X12 - MOVO X12, 64(BP) - MOVQ X12, R13 - MOVQ 72(BP), R14 - PXOR X1, X12 - - // We can only store one byte at a time, since plaintext can be shorter than 16 bytes -openSSETail16Store: - MOVQ X12, R8 - MOVB R8, (DI) - PSRLDQ $0x01, X12 - INCQ DI - DECQ BX - JNE openSSETail16Store - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - JMP openSSEFinalize - -openSSETail64: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - XORQ R9, R9 - MOVQ BX, CX - CMPQ CX, $0x10 - JB openSSETail64LoopB - -openSSETail64LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - SUBQ $0x10, CX - -openSSETail64LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - CMPQ CX, $0x10 - JAE openSSETail64LoopA - CMPQ R9, $0xa0 - JNE openSSETail64LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL 32(BP), X3 - PADDL 48(BP), X6 - PADDL 80(BP), X9 - -openSSETail64DecLoop: - CMPQ BX, $0x10 - JB openSSETail64DecLoopDone - SUBQ $0x10, BX - MOVOU (SI), X12 - PXOR X12, X0 - MOVOU X0, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - MOVO X3, X0 - MOVO X6, X3 - MOVO X9, X6 - JMP openSSETail64DecLoop - -openSSETail64DecLoopDone: - MOVO X0, X1 - JMP openSSETail16 - -openSSETail128: - MOVO ·chacha20Constants<>+0(SB), X1 - MOVO 32(BP), X4 - MOVO 48(BP), X7 - MOVO 128(BP), X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 80(BP) - MOVO X1, X0 - MOVO X4, X3 - MOVO X7, X6 - MOVO X10, X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 96(BP) - XORQ R9, R9 - MOVQ BX, CX - ANDQ $-16, CX - -openSSETail128LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSETail128LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - CMPQ R9, CX - JB openSSETail128LoopA - CMPQ R9, $0xa0 - JNE openSSETail128LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 96(BP), X9 - PADDL 80(BP), X10 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, (DI) - MOVOU X4, 16(DI) - MOVOU X7, 32(DI) - MOVOU X10, 48(DI) - SUBQ $0x40, BX - LEAQ 64(SI), SI - LEAQ 64(DI), DI - JMP openSSETail64DecLoop - -openSSETail192: - MOVO ·chacha20Constants<>+0(SB), X2 - MOVO 32(BP), X5 - MOVO 48(BP), X8 - MOVO 128(BP), X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X11, 80(BP) - MOVO X2, X1 - MOVO X5, X4 - MOVO X8, X7 - MOVO X11, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - MOVO X1, X0 - MOVO X4, X3 - MOVO X7, X6 - MOVO X10, X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 112(BP) - MOVQ BX, CX - MOVQ $0x000000a0, R9 - CMPQ CX, $0xa0 - CMOVQGT R9, CX - ANDQ $-16, CX - XORQ R9, R9 - -openSSLTail192LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSLTail192LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - CMPQ R9, CX - JB openSSLTail192LoopA - CMPQ R9, $0xa0 - JNE openSSLTail192LoopB - CMPQ BX, $0xb0 - JB openSSLTail192Store - ADDQ 160(SI), R10 - ADCQ 168(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - CMPQ BX, $0xc0 - JB openSSLTail192Store - ADDQ 176(SI), R10 - ADCQ 184(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSLTail192Store: - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 32(BP), X5 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 48(BP), X8 - PADDL 112(BP), X9 - PADDL 96(BP), X10 - PADDL 80(BP), X11 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X2 - PXOR X13, X5 - PXOR X14, X8 - PXOR X15, X11 - MOVOU X2, (DI) - MOVOU X5, 16(DI) - MOVOU X8, 32(DI) - MOVOU X11, 48(DI) - MOVOU 64(SI), X12 - MOVOU 80(SI), X13 - MOVOU 96(SI), X14 - MOVOU 112(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - SUBQ $0x80, BX - LEAQ 128(SI), SI - LEAQ 128(DI), DI - JMP openSSETail64DecLoop - -openSSETail256: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - XORQ R9, R9 - -openSSETail256Loop: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - ADDQ $0x10, R9 - CMPQ R9, $0xa0 - JB openSSETail256Loop - MOVQ BX, CX - ANDQ $-16, CX - -openSSETail256HashLoop: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ $0x10, R9 - CMPQ R9, CX - JB openSSETail256HashLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - MOVO X15, 64(BP) - - // Load - xor - store - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVOU 128(SI), X0 - MOVOU 144(SI), X3 - MOVOU 160(SI), X6 - MOVOU 176(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 128(DI) - MOVOU X5, 144(DI) - MOVOU X8, 160(DI) - MOVOU X11, 176(DI) - LEAQ 192(SI), SI - LEAQ 192(DI), DI - SUBQ $0xc0, BX - MOVO X12, X0 - MOVO X13, X3 - MOVO X14, X6 - MOVO 64(BP), X9 - JMP openSSETail64DecLoop - -chacha20Poly1305Open_AVX2: + MOVQ SP, BP + ADDQ $0x20, BP + ANDQ $-32, BP + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX VZEROUPPER VMOVDQU ·chacha20Constants<>+0(SB), Y0 BYTE $0xc4 @@ -3223,6 +769,144 @@ openAVX2MainLoopDone: JBE openAVX2Tail384 JMP openAVX2Tail512 +openSSEFinalize: + // Hash in the PT, AAD lengths + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + + // Final reduce + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 + + // Add in the "s" part of the key + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 + + // Finally, constant time compare to the tag at the end of the message + XORQ AX, AX + MOVQ $0x00000001, DX + XORQ (SI), R10 + XORQ 8(SI), R11 + ORQ R11, R10 + CMOVQEQ DX, AX + + // Return true iff tags are equal + MOVB AX, ret+96(FP) + RET + +openSSETail16: + TESTQ BX, BX + JE openSSEFinalize + + // We can safely load the CT from the end, because it is padded with the MAC + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVOU (SI), X12 + ADDQ BX, SI + PAND -16(R13)(R9*1), X12 + MOVO X12, 64(BP) + MOVQ X12, R13 + MOVQ 72(BP), R14 + PXOR X1, X12 + + // We can only store one byte at a time, since plaintext can be shorter than 16 bytes +openSSETail16Store: + MOVQ X12, R8 + MOVB R8, (DI) + PSRLDQ $0x01, X12 + INCQ DI + DECQ BX + JNE openSSETail16Store + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + JMP openSSEFinalize + openAVX2192: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 @@ -4739,15 +2423,35 @@ DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 +DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 +DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 + +DATA ·rol16<>+0(SB)/8, $0x0504070601000302 +DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a +DATA ·rol16<>+16(SB)/8, $0x0504070601000302 +DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a +GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 + +DATA ·rol8<>+0(SB)/8, $0x0605040702010003 +DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b +DATA ·rol8<>+16(SB)/8, $0x0605040702010003 +DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b +GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 + DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 -DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001 -DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000 -GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16 +DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 DATA ·andMask<>+0(SB)/8, $0x00000000000000ff DATA ·andMask<>+8(SB)/8, $0x0000000000000000 @@ -4781,2361 +2485,17 @@ DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 -DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 -DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 -DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 -DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 -GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 - -DATA ·rol16<>+0(SB)/8, $0x0504070601000302 -DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a -DATA ·rol16<>+16(SB)/8, $0x0504070601000302 -DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a -GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 - -DATA ·rol8<>+0(SB)/8, $0x0605040702010003 -DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b -DATA ·rol8<>+16(SB)/8, $0x0605040702010003 -DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b -GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 - -DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 -DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 -DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 -DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 -GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 - // func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) // Requires: AVX, AVX2, BMI2, CMOV, SSE2 TEXT ·chacha20Poly1305Seal(SB), $288-96 - MOVQ SP, BP - ADDQ $0x20, BP - ANDQ $-32, BP - MOVQ dst_base+0(FP), DI - MOVQ key_base+24(FP), R8 - MOVQ src_base+48(FP), SI - MOVQ src_len+56(FP), BX - MOVQ ad_base+72(FP), CX - CMPB ·useAVX2+0(SB), $0x01 - JE chacha20Poly1305Seal_AVX2 - - // Special optimization, for very short buffers - CMPQ BX, $0x80 - JBE sealSSE128 - - // In the seal case - prepare the poly key + 3 blocks of stream in the first iteration - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - - // Store state on stack for future use - MOVO X3, 32(BP) - MOVO X6, 48(BP) - - // Load state, increment counter blocks - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - MOVQ $0x0000000a, R9 - -sealSSEIntroLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ R9 - JNE sealSSEIntroLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVO X0, (BP) - MOVO X3, 16(BP) - - // Hash AAD - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, (DI) - MOVOU X4, 16(DI) - MOVOU X7, 32(DI) - MOVOU X10, 48(DI) - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 64(DI) - MOVOU X5, 80(DI) - MOVOU X8, 96(DI) - MOVOU X11, 112(DI) - MOVQ $0x00000080, CX - SUBQ $0x80, BX - LEAQ 128(SI), SI - MOVO X12, X1 - MOVO X13, X4 - MOVO X14, X7 - MOVO X15, X10 - CMPQ BX, $0x40 - JBE sealSSE128SealHash - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X12 - PXOR X3, X13 - PXOR X6, X14 - PXOR X9, X15 - MOVOU X12, 128(DI) - MOVOU X13, 144(DI) - MOVOU X14, 160(DI) - MOVOU X15, 176(DI) - ADDQ $0x40, CX - SUBQ $0x40, BX - LEAQ 64(SI), SI - MOVQ $0x00000002, CX - MOVQ $0x00000008, R9 - CMPQ BX, $0x40 - JBE sealSSETail64 - CMPQ BX, $0x80 - JBE sealSSETail128 - CMPQ BX, $0xc0 - JBE sealSSETail192 - -sealSSEMainLoop: - // Load state, increment counter blocks - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - -sealSSEInnerLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - LEAQ 16(DI), DI - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ R9 - JGE sealSSEInnerLoop - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - DECQ CX - JG sealSSEInnerLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - MOVO X15, 64(BP) - - // Load - xor - store - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVO 64(BP), X15 - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVOU 128(SI), X0 - MOVOU 144(SI), X3 - MOVOU 160(SI), X6 - MOVOU 176(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 128(DI) - MOVOU X5, 144(DI) - MOVOU X8, 160(DI) - MOVOU X11, 176(DI) - ADDQ $0xc0, SI - MOVQ $0x000000c0, CX - SUBQ $0xc0, BX - MOVO X12, X1 - MOVO X13, X4 - MOVO X14, X7 - MOVO X15, X10 - CMPQ BX, $0x40 - JBE sealSSE128SealHash - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X12 - PXOR X3, X13 - PXOR X6, X14 - PXOR X9, X15 - MOVOU X12, 192(DI) - MOVOU X13, 208(DI) - MOVOU X14, 224(DI) - MOVOU X15, 240(DI) - LEAQ 64(SI), SI - SUBQ $0x40, BX - MOVQ $0x00000006, CX - MOVQ $0x00000004, R9 - CMPQ BX, $0xc0 - JG sealSSEMainLoop - MOVQ BX, CX - TESTQ BX, BX - JE sealSSE128SealHash - MOVQ $0x00000006, CX - CMPQ BX, $0x40 - JBE sealSSETail64 - CMPQ BX, $0x80 - JBE sealSSETail128 - JMP sealSSETail192 - -sealSSETail64: - MOVO ·chacha20Constants<>+0(SB), X1 - MOVO 32(BP), X4 - MOVO 48(BP), X7 - MOVO 128(BP), X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 80(BP) - -sealSSETail64LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - -sealSSETail64LoopB: - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x0c, X13 - PSRLL $0x14, X4 - PXOR X13, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x07, X13 - PSRLL $0x19, X4 - PXOR X13, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x0c, X13 - PSRLL $0x14, X4 - PXOR X13, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x07, X13 - PSRLL $0x19, X4 - PXOR X13, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - DECQ CX - JG sealSSETail64LoopA - DECQ R9 - JGE sealSSETail64LoopB - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X4 - PADDL 48(BP), X7 - PADDL 80(BP), X10 - JMP sealSSE128Seal - -sealSSETail128: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - -sealSSETail128LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - -sealSSETail128LoopB: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - DECQ CX - JG sealSSETail128LoopA - DECQ R9 - JGE sealSSETail128LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 80(BP), X9 - PADDL 96(BP), X10 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X0 - PXOR X13, X3 - PXOR X14, X6 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVQ $0x00000040, CX - LEAQ 64(SI), SI - SUBQ $0x40, BX - JMP sealSSE128SealHash - -sealSSETail192: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X11, 112(BP) - -sealSSETail192LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - -sealSSETail192LoopB: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ CX - JG sealSSETail192LoopA - DECQ R9 - JGE sealSSETail192LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 32(BP), X5 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 48(BP), X8 - PADDL 80(BP), X9 - PADDL 96(BP), X10 - PADDL 112(BP), X11 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X0 - PXOR X13, X3 - PXOR X14, X6 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVOU 64(SI), X12 - MOVOU 80(SI), X13 - MOVOU 96(SI), X14 - MOVOU 112(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVO X2, X1 - MOVO X5, X4 - MOVO X8, X7 - MOVO X11, X10 - MOVQ $0x00000080, CX - LEAQ 128(SI), SI - SUBQ $0x80, BX - JMP sealSSE128SealHash - -sealSSE128: - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X3, X13 - MOVO X6, X14 - MOVO X10, X15 - MOVQ $0x0000000a, R9 - -sealSSE128InnerCipherLoop: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ R9 - JNE sealSSE128InnerCipherLoop - - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL X13, X3 - PADDL X13, X4 - PADDL X13, X5 - PADDL X14, X7 - PADDL X14, X8 - PADDL X15, X10 - PADDL ·sseIncMask<>+0(SB), X15 - PADDL X15, X11 - PAND ·polyClampMask<>+0(SB), X0 - MOVOU X0, (BP) - MOVOU X3, 16(BP) - - // Hash - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - XORQ CX, CX - -sealSSE128SealHash: - CMPQ CX, $0x10 - JB sealSSE128Seal - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - SUBQ $0x10, CX - ADDQ $0x10, DI - JMP sealSSE128SealHash - -sealSSE128Seal: - CMPQ BX, $0x10 - JB sealSSETail - SUBQ $0x10, BX - - // Load for decryption - MOVOU (SI), X12 - PXOR X12, X1 - MOVOU X1, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - - // Extract for hashing - MOVQ X1, R13 - PSRLDQ $0x08, X1 - MOVQ X1, R14 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Shift the stream "left" - MOVO X4, X1 - MOVO X7, X4 - MOVO X10, X7 - MOVO X2, X10 - MOVO X5, X2 - MOVO X8, X5 - MOVO X11, X8 - JMP sealSSE128Seal - -sealSSETail: - TESTQ BX, BX - JE sealSSEFinalize - - // We can only load the PT one byte at a time to avoid read after end of buffer - MOVQ BX, R9 - SHLQ $0x04, R9 - LEAQ ·andMask<>+0(SB), R13 - MOVQ BX, CX - LEAQ -1(SI)(BX*1), SI - XORQ R15, R15 - XORQ R8, R8 - XORQ AX, AX - -sealSSETailLoadLoop: - SHLQ $0x08, R15, R8 - SHLQ $0x08, R15 - MOVB (SI), AX - XORQ AX, R15 - LEAQ -1(SI), SI - DECQ CX - JNE sealSSETailLoadLoop - MOVQ R15, 64(BP) - MOVQ R8, 72(BP) - PXOR 64(BP), X1 - MOVOU X1, (DI) - MOVOU -16(R13)(R9*1), X12 - PAND X12, X1 - MOVQ X1, R13 - PSRLDQ $0x08, X1 - MOVQ X1, R14 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ BX, DI - -sealSSEFinalize: - // Hash in the buffer lengths - ADDQ ad_len+80(FP), R10 - ADCQ src_len+56(FP), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Final reduce - MOVQ R10, R13 - MOVQ R11, R14 - MOVQ R12, R15 - SUBQ $-5, R10 - SBBQ $-1, R11 - SBBQ $0x03, R12 - CMOVQCS R13, R10 - CMOVQCS R14, R11 - CMOVQCS R15, R12 - - // Add in the "s" part of the key - ADDQ 16(BP), R10 - ADCQ 24(BP), R11 - - // Finally store the tag at the end of the message - MOVQ R10, (DI) - MOVQ R11, 8(DI) - RET - -chacha20Poly1305Seal_AVX2: + MOVQ SP, BP + ADDQ $0x20, BP + ANDQ $-32, BP + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX VZEROUPPER VMOVDQU ·chacha20Constants<>+0(SB), Y0 BYTE $0xc4 @@ -8170,6 +3530,144 @@ sealAVX2InternalLoopStart: JBE sealAVX2Tail384 JMP sealAVX2Tail512 +sealSSETail: + TESTQ BX, BX + JE sealSSEFinalize + + // We can only load the PT one byte at a time to avoid read after end of buffer + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVQ BX, CX + LEAQ -1(SI)(BX*1), SI + XORQ R15, R15 + XORQ R8, R8 + XORQ AX, AX + +sealSSETailLoadLoop: + SHLQ $0x08, R15, R8 + SHLQ $0x08, R15 + MOVB (SI), AX + XORQ AX, R15 + LEAQ -1(SI), SI + DECQ CX + JNE sealSSETailLoadLoop + MOVQ R15, 64(BP) + MOVQ R8, 72(BP) + PXOR 64(BP), X1 + MOVOU X1, (DI) + MOVOU -16(R13)(R9*1), X12 + PAND X12, X1 + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ BX, DI + +sealSSEFinalize: + // Hash in the buffer lengths + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + + // Final reduce + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 + + // Add in the "s" part of the key + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 + + // Finally store the tag at the end of the message + MOVQ R10, (DI) + MOVQ R11, 8(DI) + RET + seal192AVX2: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9