diff --git a/poly1305/poly1305_arm.s b/poly1305/poly1305_arm.s index c9ceaeb..c153867 100644 --- a/poly1305/poly1305_arm.s +++ b/poly1305/poly1305_arm.s @@ -47,6 +47,16 @@ TEXT poly1305_init_ext_armv6<>(SB),4,$-4 MOVM.IA.W (R13), [R4-R11] RET +#define MOVW_UNALIGNED(Rsrc, Rdst, Rtmp, offset) \ + MOVBU (offset+0)(Rsrc), Rtmp; \ + MOVBU Rtmp, (offset+0)(Rdst); \ + MOVBU (offset+1)(Rsrc), Rtmp; \ + MOVBU Rtmp, (offset+1)(Rdst); \ + MOVBU (offset+2)(Rsrc), Rtmp; \ + MOVBU Rtmp, (offset+2)(Rdst); \ + MOVBU (offset+3)(Rsrc), Rtmp; \ + MOVBU Rtmp, (offset+3)(Rdst) + TEXT poly1305_blocks_armv6<>(SB),4,$-4 MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13) SUB $128, R13 @@ -66,7 +76,19 @@ TEXT poly1305_blocks_armv6<>(SB),4,$-4 CMP $16, R12 BLO poly1305_blocks_armv6_done poly1305_blocks_armv6_mainloop: + WORD $0xe31e0003 // TST R14, #3 not working see issue 5921 + BEQ poly1305_blocks_armv6_mainloop_aligned + ADD $48, R13, g + MOVW_UNALIGNED(R14, g, R0, 0) + MOVW_UNALIGNED(R14, g, R0, 4) + MOVW_UNALIGNED(R14, g, R0, 8) + MOVW_UNALIGNED(R14, g, R0, 12) + MOVM.IA (g), [R0-R3] + ADD $16, R14 + B poly1305_blocks_armv6_mainloop_loaded +poly1305_blocks_armv6_mainloop_aligned: MOVM.IA.W (R14), [R0-R3] +poly1305_blocks_armv6_mainloop_loaded: MOVW R0>>26, g MOVW R1>>20, R11 MOVW R2>>14, R12 @@ -174,6 +196,16 @@ poly1305_blocks_armv6_done: MOVM.IA.W (R13), [R4, R5, R6, R7, R8, R9, g, R11, R14] RET +#define MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) \ + MOVBU.P 1(Rsrc), Rtmp; \ + MOVBU.P Rtmp, 1(Rdst); \ + MOVBU.P 1(Rsrc), Rtmp; \ + MOVBU.P Rtmp, 1(Rdst) + +#define MOVWP_UNALIGNED(Rsrc, Rdst, Rtmp) \ + MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp); \ + MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) + TEXT poly1305_finish_ext_armv6<>(SB),4,$-4 MOVM.DB.W [R4, R5, R6, R7, R8, R9, g, R11, R14], (R13) SUB $16, R13, R13 @@ -189,16 +221,32 @@ TEXT poly1305_finish_ext_armv6<>(SB),4,$-4 MOVW R0, 4(R13) MOVW R0, 8(R13) MOVW R0, 12(R13) + WORD $0xe3110003 // TST R1, #3 not working see issue 5921 + BEQ poly1305_finish_ext_armv6_aligned WORD $0xe3120008 // TST R2, #8 not working see issue 5921 BEQ poly1305_finish_ext_armv6_skip8 - MOVM.IA.W (R1), [g-R11] - MOVM.IA.W [g-R11], (R9) + MOVWP_UNALIGNED(R1, R9, g) + MOVWP_UNALIGNED(R1, R9, g) poly1305_finish_ext_armv6_skip8: WORD $0xe3120004 // TST $4, R2 not working see issue 5921 BEQ poly1305_finish_ext_armv6_skip4 + MOVWP_UNALIGNED(R1, R9, g) +poly1305_finish_ext_armv6_skip4: + WORD $0xe3120002 // TST $2, R2 not working see issue 5921 + BEQ poly1305_finish_ext_armv6_skip2 + MOVHUP_UNALIGNED(R1, R9, g) + B poly1305_finish_ext_armv6_skip2 +poly1305_finish_ext_armv6_aligned: + WORD $0xe3120008 // TST R2, #8 not working see issue 5921 + BEQ poly1305_finish_ext_armv6_skip8_aligned + MOVM.IA.W (R1), [g-R11] + MOVM.IA.W [g-R11], (R9) +poly1305_finish_ext_armv6_skip8_aligned: + WORD $0xe3120004 // TST $4, R2 not working see issue 5921 + BEQ poly1305_finish_ext_armv6_skip4_aligned MOVW.P 4(R1), g MOVW.P g, 4(R9) -poly1305_finish_ext_armv6_skip4: +poly1305_finish_ext_armv6_skip4_aligned: WORD $0xe3120002 // TST $2, R2 not working see issue 5921 BEQ poly1305_finish_ext_armv6_skip2 MOVHU.P 2(R1), g diff --git a/poly1305/poly1305_test.go b/poly1305/poly1305_test.go index 2c6d1bc..b3e9231 100644 --- a/poly1305/poly1305_test.go +++ b/poly1305/poly1305_test.go @@ -7,6 +7,7 @@ package poly1305 import ( "bytes" "testing" + "unsafe" ) var testData = []struct { @@ -34,41 +35,52 @@ var testData = []struct { }, } -func TestSum(t *testing.T) { +func testSum(t *testing.T, unaligned bool) { var out [16]byte var key [32]byte for i, v := range testData { + in := v.in + if unaligned { + in = unalignBytes(in) + } copy(key[:], v.k) - Sum(&out, v.in, &key) + Sum(&out, in, &key) if !bytes.Equal(out[:], v.correct) { t.Errorf("%d: expected %x, got %x", i, v.correct, out[:]) } } } -func Benchmark1K(b *testing.B) { - b.StopTimer() +func TestSum(t *testing.T) { testSum(t, false) } +func TestSumUnaligned(t *testing.T) { testSum(t, true) } + +func benchmark(b *testing.B, size int, unaligned bool) { var out [16]byte var key [32]byte - in := make([]byte, 1024) + in := make([]byte, size) + if unaligned { + in = unalignBytes(in) + } b.SetBytes(int64(len(in))) - b.StartTimer() - + b.ResetTimer() for i := 0; i < b.N; i++ { Sum(&out, in, &key) } } -func Benchmark64(b *testing.B) { - b.StopTimer() - var out [16]byte - var key [32]byte - in := make([]byte, 64) - b.SetBytes(int64(len(in))) - b.StartTimer() +func Benchmark64(b *testing.B) { benchmark(b, 64, false) } +func Benchmark1K(b *testing.B) { benchmark(b, 1024, false) } +func Benchmark64Unaligned(b *testing.B) { benchmark(b, 64, true) } +func Benchmark1KUnaligned(b *testing.B) { benchmark(b, 1024, true) } - for i := 0; i < b.N; i++ { - Sum(&out, in, &key) +func unalignBytes(in []byte) []byte { + out := make([]byte, len(in)+1) + if uintptr(unsafe.Pointer(&out[0]))&(unsafe.Alignof(uint32(0))-1) == 0 { + out = out[1:] + } else { + out = out[:len(in)] } + copy(out, in) + return out }