mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-08-30 00:14:51 +02:00
909 lines
24 KiB
ArmAsm
909 lines
24 KiB
ArmAsm
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build s390x,go1.11,!gccgo,!appengine
|
|
|
|
#include "textflag.h"
|
|
|
|
// Implementation of Poly1305 using the vector facility (vx) and the VMSL instruction.
|
|
|
|
// constants
|
|
#define EX0 V1
|
|
#define EX1 V2
|
|
#define EX2 V3
|
|
|
|
// temporaries
|
|
#define T_0 V4
|
|
#define T_1 V5
|
|
#define T_2 V6
|
|
#define T_3 V7
|
|
#define T_4 V8
|
|
#define T_5 V9
|
|
#define T_6 V10
|
|
#define T_7 V11
|
|
#define T_8 V12
|
|
#define T_9 V13
|
|
#define T_10 V14
|
|
|
|
// r**2 & r**4
|
|
#define R_0 V15
|
|
#define R_1 V16
|
|
#define R_2 V17
|
|
#define R5_1 V18
|
|
#define R5_2 V19
|
|
// key (r)
|
|
#define RSAVE_0 R7
|
|
#define RSAVE_1 R8
|
|
#define RSAVE_2 R9
|
|
#define R5SAVE_1 R10
|
|
#define R5SAVE_2 R11
|
|
|
|
// message block
|
|
#define M0 V20
|
|
#define M1 V21
|
|
#define M2 V22
|
|
#define M3 V23
|
|
#define M4 V24
|
|
#define M5 V25
|
|
|
|
// accumulator
|
|
#define H0_0 V26
|
|
#define H1_0 V27
|
|
#define H2_0 V28
|
|
#define H0_1 V29
|
|
#define H1_1 V30
|
|
#define H2_1 V31
|
|
|
|
GLOBL ·keyMask<>(SB), RODATA, $16
|
|
DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f
|
|
DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f
|
|
|
|
GLOBL ·bswapMask<>(SB), RODATA, $16
|
|
DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908
|
|
DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100
|
|
|
|
GLOBL ·constants<>(SB), RODATA, $48
|
|
// EX0
|
|
DATA ·constants<>+0(SB)/8, $0x18191a1b1c1d1e1f
|
|
DATA ·constants<>+8(SB)/8, $0x0000050403020100
|
|
// EX1
|
|
DATA ·constants<>+16(SB)/8, $0x18191a1b1c1d1e1f
|
|
DATA ·constants<>+24(SB)/8, $0x00000a0908070605
|
|
// EX2
|
|
DATA ·constants<>+32(SB)/8, $0x18191a1b1c1d1e1f
|
|
DATA ·constants<>+40(SB)/8, $0x0000000f0e0d0c0b
|
|
|
|
GLOBL ·c<>(SB), RODATA, $48
|
|
// EX0
|
|
DATA ·c<>+0(SB)/8, $0x0000050403020100
|
|
DATA ·c<>+8(SB)/8, $0x0000151413121110
|
|
// EX1
|
|
DATA ·c<>+16(SB)/8, $0x00000a0908070605
|
|
DATA ·c<>+24(SB)/8, $0x00001a1918171615
|
|
// EX2
|
|
DATA ·c<>+32(SB)/8, $0x0000000f0e0d0c0b
|
|
DATA ·c<>+40(SB)/8, $0x0000001f1e1d1c1b
|
|
|
|
GLOBL ·reduce<>(SB), RODATA, $32
|
|
// 44 bit
|
|
DATA ·reduce<>+0(SB)/8, $0x0
|
|
DATA ·reduce<>+8(SB)/8, $0xfffffffffff
|
|
// 42 bit
|
|
DATA ·reduce<>+16(SB)/8, $0x0
|
|
DATA ·reduce<>+24(SB)/8, $0x3ffffffffff
|
|
|
|
// h = (f*g) % (2**130-5) [partial reduction]
|
|
// uses T_0...T_9 temporary registers
|
|
// input: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2
|
|
// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9
|
|
// output: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2
|
|
#define MULTIPLY(m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) \
|
|
\ // Eliminate the dependency for the last 2 VMSLs
|
|
VMSLG m02_0, r_2, m4_2, m4_2 \
|
|
VMSLG m13_0, r_2, m5_2, m5_2 \ // 8 VMSLs pipelined
|
|
VMSLG m02_0, r_0, m4_0, m4_0 \
|
|
VMSLG m02_1, r5_2, V0, T_0 \
|
|
VMSLG m02_0, r_1, m4_1, m4_1 \
|
|
VMSLG m02_1, r_0, V0, T_1 \
|
|
VMSLG m02_1, r_1, V0, T_2 \
|
|
VMSLG m02_2, r5_1, V0, T_3 \
|
|
VMSLG m02_2, r5_2, V0, T_4 \
|
|
VMSLG m13_0, r_0, m5_0, m5_0 \
|
|
VMSLG m13_1, r5_2, V0, T_5 \
|
|
VMSLG m13_0, r_1, m5_1, m5_1 \
|
|
VMSLG m13_1, r_0, V0, T_6 \
|
|
VMSLG m13_1, r_1, V0, T_7 \
|
|
VMSLG m13_2, r5_1, V0, T_8 \
|
|
VMSLG m13_2, r5_2, V0, T_9 \
|
|
VMSLG m02_2, r_0, m4_2, m4_2 \
|
|
VMSLG m13_2, r_0, m5_2, m5_2 \
|
|
VAQ m4_0, T_0, m02_0 \
|
|
VAQ m4_1, T_1, m02_1 \
|
|
VAQ m5_0, T_5, m13_0 \
|
|
VAQ m5_1, T_6, m13_1 \
|
|
VAQ m02_0, T_3, m02_0 \
|
|
VAQ m02_1, T_4, m02_1 \
|
|
VAQ m13_0, T_8, m13_0 \
|
|
VAQ m13_1, T_9, m13_1 \
|
|
VAQ m4_2, T_2, m02_2 \
|
|
VAQ m5_2, T_7, m13_2 \
|
|
|
|
// SQUARE uses three limbs of r and r_2*5 to output square of r
|
|
// uses T_1, T_5 and T_7 temporary registers
|
|
// input: r_0, r_1, r_2, r5_2
|
|
// temp: TEMP0, TEMP1, TEMP2
|
|
// output: p0, p1, p2
|
|
#define SQUARE(r_0, r_1, r_2, r5_2, p0, p1, p2, TEMP0, TEMP1, TEMP2) \
|
|
VMSLG r_0, r_0, p0, p0 \
|
|
VMSLG r_1, r5_2, V0, TEMP0 \
|
|
VMSLG r_2, r5_2, p1, p1 \
|
|
VMSLG r_0, r_1, V0, TEMP1 \
|
|
VMSLG r_1, r_1, p2, p2 \
|
|
VMSLG r_0, r_2, V0, TEMP2 \
|
|
VAQ TEMP0, p0, p0 \
|
|
VAQ TEMP1, p1, p1 \
|
|
VAQ TEMP2, p2, p2 \
|
|
VAQ TEMP0, p0, p0 \
|
|
VAQ TEMP1, p1, p1 \
|
|
VAQ TEMP2, p2, p2 \
|
|
|
|
// carry h0->h1->h2->h0 || h3->h4->h5->h3
|
|
// uses T_2, T_4, T_5, T_7, T_8, T_9
|
|
// t6, t7, t8, t9, t10, t11
|
|
// input: h0, h1, h2, h3, h4, h5
|
|
// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11
|
|
// output: h0, h1, h2, h3, h4, h5
|
|
#define REDUCE(h0, h1, h2, h3, h4, h5, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) \
|
|
VLM (R12), t6, t7 \ // 44 and 42 bit clear mask
|
|
VLEIB $7, $0x28, t10 \ // 5 byte shift mask
|
|
VREPIB $4, t8 \ // 4 bit shift mask
|
|
VREPIB $2, t11 \ // 2 bit shift mask
|
|
VSRLB t10, h0, t0 \ // h0 byte shift
|
|
VSRLB t10, h1, t1 \ // h1 byte shift
|
|
VSRLB t10, h2, t2 \ // h2 byte shift
|
|
VSRLB t10, h3, t3 \ // h3 byte shift
|
|
VSRLB t10, h4, t4 \ // h4 byte shift
|
|
VSRLB t10, h5, t5 \ // h5 byte shift
|
|
VSRL t8, t0, t0 \ // h0 bit shift
|
|
VSRL t8, t1, t1 \ // h2 bit shift
|
|
VSRL t11, t2, t2 \ // h2 bit shift
|
|
VSRL t8, t3, t3 \ // h3 bit shift
|
|
VSRL t8, t4, t4 \ // h4 bit shift
|
|
VESLG $2, t2, t9 \ // h2 carry x5
|
|
VSRL t11, t5, t5 \ // h5 bit shift
|
|
VN t6, h0, h0 \ // h0 clear carry
|
|
VAQ t2, t9, t2 \ // h2 carry x5
|
|
VESLG $2, t5, t9 \ // h5 carry x5
|
|
VN t6, h1, h1 \ // h1 clear carry
|
|
VN t7, h2, h2 \ // h2 clear carry
|
|
VAQ t5, t9, t5 \ // h5 carry x5
|
|
VN t6, h3, h3 \ // h3 clear carry
|
|
VN t6, h4, h4 \ // h4 clear carry
|
|
VN t7, h5, h5 \ // h5 clear carry
|
|
VAQ t0, h1, h1 \ // h0->h1
|
|
VAQ t3, h4, h4 \ // h3->h4
|
|
VAQ t1, h2, h2 \ // h1->h2
|
|
VAQ t4, h5, h5 \ // h4->h5
|
|
VAQ t2, h0, h0 \ // h2->h0
|
|
VAQ t5, h3, h3 \ // h5->h3
|
|
VREPG $1, t6, t6 \ // 44 and 42 bit masks across both halves
|
|
VREPG $1, t7, t7 \
|
|
VSLDB $8, h0, h0, h0 \ // set up [h0/1/2, h3/4/5]
|
|
VSLDB $8, h1, h1, h1 \
|
|
VSLDB $8, h2, h2, h2 \
|
|
VO h0, h3, h3 \
|
|
VO h1, h4, h4 \
|
|
VO h2, h5, h5 \
|
|
VESRLG $44, h3, t0 \ // 44 bit shift right
|
|
VESRLG $44, h4, t1 \
|
|
VESRLG $42, h5, t2 \
|
|
VN t6, h3, h3 \ // clear carry bits
|
|
VN t6, h4, h4 \
|
|
VN t7, h5, h5 \
|
|
VESLG $2, t2, t9 \ // multiply carry by 5
|
|
VAQ t9, t2, t2 \
|
|
VAQ t0, h4, h4 \
|
|
VAQ t1, h5, h5 \
|
|
VAQ t2, h3, h3 \
|
|
|
|
// carry h0->h1->h2->h0
|
|
// input: h0, h1, h2
|
|
// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8
|
|
// output: h0, h1, h2
|
|
#define REDUCE2(h0, h1, h2, t0, t1, t2, t3, t4, t5, t6, t7, t8) \
|
|
VLEIB $7, $0x28, t3 \ // 5 byte shift mask
|
|
VREPIB $4, t4 \ // 4 bit shift mask
|
|
VREPIB $2, t7 \ // 2 bit shift mask
|
|
VGBM $0x003F, t5 \ // mask to clear carry bits
|
|
VSRLB t3, h0, t0 \
|
|
VSRLB t3, h1, t1 \
|
|
VSRLB t3, h2, t2 \
|
|
VESRLG $4, t5, t5 \ // 44 bit clear mask
|
|
VSRL t4, t0, t0 \
|
|
VSRL t4, t1, t1 \
|
|
VSRL t7, t2, t2 \
|
|
VESRLG $2, t5, t6 \ // 42 bit clear mask
|
|
VESLG $2, t2, t8 \
|
|
VAQ t8, t2, t2 \
|
|
VN t5, h0, h0 \
|
|
VN t5, h1, h1 \
|
|
VN t6, h2, h2 \
|
|
VAQ t0, h1, h1 \
|
|
VAQ t1, h2, h2 \
|
|
VAQ t2, h0, h0 \
|
|
VSRLB t3, h0, t0 \
|
|
VSRLB t3, h1, t1 \
|
|
VSRLB t3, h2, t2 \
|
|
VSRL t4, t0, t0 \
|
|
VSRL t4, t1, t1 \
|
|
VSRL t7, t2, t2 \
|
|
VN t5, h0, h0 \
|
|
VN t5, h1, h1 \
|
|
VESLG $2, t2, t8 \
|
|
VN t6, h2, h2 \
|
|
VAQ t0, h1, h1 \
|
|
VAQ t8, t2, t2 \
|
|
VAQ t1, h2, h2 \
|
|
VAQ t2, h0, h0 \
|
|
|
|
// expands two message blocks into the lower halfs of the d registers
|
|
// moves the contents of the d registers into upper halfs
|
|
// input: in1, in2, d0, d1, d2, d3, d4, d5
|
|
// temp: TEMP0, TEMP1, TEMP2, TEMP3
|
|
// output: d0, d1, d2, d3, d4, d5
|
|
#define EXPACC(in1, in2, d0, d1, d2, d3, d4, d5, TEMP0, TEMP1, TEMP2, TEMP3) \
|
|
VGBM $0xff3f, TEMP0 \
|
|
VGBM $0xff1f, TEMP1 \
|
|
VESLG $4, d1, TEMP2 \
|
|
VESLG $4, d4, TEMP3 \
|
|
VESRLG $4, TEMP0, TEMP0 \
|
|
VPERM in1, d0, EX0, d0 \
|
|
VPERM in2, d3, EX0, d3 \
|
|
VPERM in1, d2, EX2, d2 \
|
|
VPERM in2, d5, EX2, d5 \
|
|
VPERM in1, TEMP2, EX1, d1 \
|
|
VPERM in2, TEMP3, EX1, d4 \
|
|
VN TEMP0, d0, d0 \
|
|
VN TEMP0, d3, d3 \
|
|
VESRLG $4, d1, d1 \
|
|
VESRLG $4, d4, d4 \
|
|
VN TEMP1, d2, d2 \
|
|
VN TEMP1, d5, d5 \
|
|
VN TEMP0, d1, d1 \
|
|
VN TEMP0, d4, d4 \
|
|
|
|
// expands one message block into the lower halfs of the d registers
|
|
// moves the contents of the d registers into upper halfs
|
|
// input: in, d0, d1, d2
|
|
// temp: TEMP0, TEMP1, TEMP2
|
|
// output: d0, d1, d2
|
|
#define EXPACC2(in, d0, d1, d2, TEMP0, TEMP1, TEMP2) \
|
|
VGBM $0xff3f, TEMP0 \
|
|
VESLG $4, d1, TEMP2 \
|
|
VGBM $0xff1f, TEMP1 \
|
|
VPERM in, d0, EX0, d0 \
|
|
VESRLG $4, TEMP0, TEMP0 \
|
|
VPERM in, d2, EX2, d2 \
|
|
VPERM in, TEMP2, EX1, d1 \
|
|
VN TEMP0, d0, d0 \
|
|
VN TEMP1, d2, d2 \
|
|
VESRLG $4, d1, d1 \
|
|
VN TEMP0, d1, d1 \
|
|
|
|
// pack h2:h0 into h1:h0 (no carry)
|
|
// input: h0, h1, h2
|
|
// output: h0, h1, h2
|
|
#define PACK(h0, h1, h2) \
|
|
VMRLG h1, h2, h2 \ // copy h1 to upper half h2
|
|
VESLG $44, h1, h1 \ // shift limb 1 44 bits, leaving 20
|
|
VO h0, h1, h0 \ // combine h0 with 20 bits from limb 1
|
|
VESRLG $20, h2, h1 \ // put top 24 bits of limb 1 into h1
|
|
VLEIG $1, $0, h1 \ // clear h2 stuff from lower half of h1
|
|
VO h0, h1, h0 \ // h0 now has 88 bits (limb 0 and 1)
|
|
VLEIG $0, $0, h2 \ // clear upper half of h2
|
|
VESRLG $40, h2, h1 \ // h1 now has upper two bits of result
|
|
VLEIB $7, $88, h1 \ // for byte shift (11 bytes)
|
|
VSLB h1, h2, h2 \ // shift h2 11 bytes to the left
|
|
VO h0, h2, h0 \ // combine h0 with 20 bits from limb 1
|
|
VLEIG $0, $0, h1 \ // clear upper half of h1
|
|
|
|
// if h > 2**130-5 then h -= 2**130-5
|
|
// input: h0, h1
|
|
// temp: t0, t1, t2
|
|
// output: h0
|
|
#define MOD(h0, h1, t0, t1, t2) \
|
|
VZERO t0 \
|
|
VLEIG $1, $5, t0 \
|
|
VACCQ h0, t0, t1 \
|
|
VAQ h0, t0, t0 \
|
|
VONE t2 \
|
|
VLEIG $1, $-4, t2 \
|
|
VAQ t2, t1, t1 \
|
|
VACCQ h1, t1, t1 \
|
|
VONE t2 \
|
|
VAQ t2, t1, t1 \
|
|
VN h0, t1, t2 \
|
|
VNC t0, t1, t1 \
|
|
VO t1, t2, h0 \
|
|
|
|
// func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]key)
|
|
TEXT ·poly1305vmsl(SB), $0-32
|
|
// This code processes 6 + up to 4 blocks (32 bytes) per iteration
|
|
// using the algorithm described in:
|
|
// NEON crypto, Daniel J. Bernstein & Peter Schwabe
|
|
// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
|
|
// And as moddified for VMSL as described in
|
|
// Accelerating Poly1305 Cryptographic Message Authentication on the z14
|
|
// O'Farrell et al, CASCON 2017, p48-55
|
|
// https://ibm.ent.box.com/s/jf9gedj0e9d2vjctfyh186shaztavnht
|
|
|
|
LMG out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key
|
|
VZERO V0 // c
|
|
|
|
// load EX0, EX1 and EX2
|
|
MOVD $·constants<>(SB), R5
|
|
VLM (R5), EX0, EX2 // c
|
|
|
|
// setup r
|
|
VL (R4), T_0
|
|
MOVD $·keyMask<>(SB), R6
|
|
VL (R6), T_1
|
|
VN T_0, T_1, T_0
|
|
VZERO T_2 // limbs for r
|
|
VZERO T_3
|
|
VZERO T_4
|
|
EXPACC2(T_0, T_2, T_3, T_4, T_1, T_5, T_7)
|
|
|
|
// T_2, T_3, T_4: [0, r]
|
|
|
|
// setup r*20
|
|
VLEIG $0, $0, T_0
|
|
VLEIG $1, $20, T_0 // T_0: [0, 20]
|
|
VZERO T_5
|
|
VZERO T_6
|
|
VMSLG T_0, T_3, T_5, T_5
|
|
VMSLG T_0, T_4, T_6, T_6
|
|
|
|
// store r for final block in GR
|
|
VLGVG $1, T_2, RSAVE_0 // c
|
|
VLGVG $1, T_3, RSAVE_1 // c
|
|
VLGVG $1, T_4, RSAVE_2 // c
|
|
VLGVG $1, T_5, R5SAVE_1 // c
|
|
VLGVG $1, T_6, R5SAVE_2 // c
|
|
|
|
// initialize h
|
|
VZERO H0_0
|
|
VZERO H1_0
|
|
VZERO H2_0
|
|
VZERO H0_1
|
|
VZERO H1_1
|
|
VZERO H2_1
|
|
|
|
// initialize pointer for reduce constants
|
|
MOVD $·reduce<>(SB), R12
|
|
|
|
// calculate r**2 and 20*(r**2)
|
|
VZERO R_0
|
|
VZERO R_1
|
|
VZERO R_2
|
|
SQUARE(T_2, T_3, T_4, T_6, R_0, R_1, R_2, T_1, T_5, T_7)
|
|
REDUCE2(R_0, R_1, R_2, M0, M1, M2, M3, M4, R5_1, R5_2, M5, T_1)
|
|
VZERO R5_1
|
|
VZERO R5_2
|
|
VMSLG T_0, R_1, R5_1, R5_1
|
|
VMSLG T_0, R_2, R5_2, R5_2
|
|
|
|
// skip r**4 calculation if 3 blocks or less
|
|
CMPBLE R3, $48, b4
|
|
|
|
// calculate r**4 and 20*(r**4)
|
|
VZERO T_8
|
|
VZERO T_9
|
|
VZERO T_10
|
|
SQUARE(R_0, R_1, R_2, R5_2, T_8, T_9, T_10, T_1, T_5, T_7)
|
|
REDUCE2(T_8, T_9, T_10, M0, M1, M2, M3, M4, T_2, T_3, M5, T_1)
|
|
VZERO T_2
|
|
VZERO T_3
|
|
VMSLG T_0, T_9, T_2, T_2
|
|
VMSLG T_0, T_10, T_3, T_3
|
|
|
|
// put r**2 to the right and r**4 to the left of R_0, R_1, R_2
|
|
VSLDB $8, T_8, T_8, T_8
|
|
VSLDB $8, T_9, T_9, T_9
|
|
VSLDB $8, T_10, T_10, T_10
|
|
VSLDB $8, T_2, T_2, T_2
|
|
VSLDB $8, T_3, T_3, T_3
|
|
|
|
VO T_8, R_0, R_0
|
|
VO T_9, R_1, R_1
|
|
VO T_10, R_2, R_2
|
|
VO T_2, R5_1, R5_1
|
|
VO T_3, R5_2, R5_2
|
|
|
|
CMPBLE R3, $80, load // less than or equal to 5 blocks in message
|
|
|
|
// 6(or 5+1) blocks
|
|
SUB $81, R3
|
|
VLM (R2), M0, M4
|
|
VLL R3, 80(R2), M5
|
|
ADD $1, R3
|
|
MOVBZ $1, R0
|
|
CMPBGE R3, $16, 2(PC)
|
|
VLVGB R3, R0, M5
|
|
MOVD $96(R2), R2
|
|
EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
|
|
EXPACC(M2, M3, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
|
|
VLEIB $2, $1, H2_0
|
|
VLEIB $2, $1, H2_1
|
|
VLEIB $10, $1, H2_0
|
|
VLEIB $10, $1, H2_1
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO T_4
|
|
VZERO T_10
|
|
EXPACC(M4, M5, M0, M1, M2, M3, T_4, T_10, T_0, T_1, T_2, T_3)
|
|
VLR T_4, M4
|
|
VLEIB $10, $1, M2
|
|
CMPBLT R3, $16, 2(PC)
|
|
VLEIB $10, $1, T_10
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
VMRHG V0, H0_1, H0_0
|
|
VMRHG V0, H1_1, H1_0
|
|
VMRHG V0, H2_1, H2_0
|
|
VMRLG V0, H0_1, H0_1
|
|
VMRLG V0, H1_1, H1_1
|
|
VMRLG V0, H2_1, H2_1
|
|
|
|
SUB $16, R3
|
|
CMPBLE R3, $0, square
|
|
|
|
load:
|
|
// load EX0, EX1 and EX2
|
|
MOVD $·c<>(SB), R5
|
|
VLM (R5), EX0, EX2
|
|
|
|
loop:
|
|
CMPBLE R3, $64, add // b4 // last 4 or less blocks left
|
|
|
|
// next 4 full blocks
|
|
VLM (R2), M2, M5
|
|
SUB $64, R3
|
|
MOVD $64(R2), R2
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, T_0, T_1, T_3, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
|
|
// expacc in-lined to create [m2, m3] limbs
|
|
VGBM $0x3f3f, T_0 // 44 bit clear mask
|
|
VGBM $0x1f1f, T_1 // 40 bit clear mask
|
|
VPERM M2, M3, EX0, T_3
|
|
VESRLG $4, T_0, T_0 // 44 bit clear mask ready
|
|
VPERM M2, M3, EX1, T_4
|
|
VPERM M2, M3, EX2, T_5
|
|
VN T_0, T_3, T_3
|
|
VESRLG $4, T_4, T_4
|
|
VN T_1, T_5, T_5
|
|
VN T_0, T_4, T_4
|
|
VMRHG H0_1, T_3, H0_0
|
|
VMRHG H1_1, T_4, H1_0
|
|
VMRHG H2_1, T_5, H2_0
|
|
VMRLG H0_1, T_3, H0_1
|
|
VMRLG H1_1, T_4, H1_1
|
|
VMRLG H2_1, T_5, H2_1
|
|
VLEIB $10, $1, H2_0
|
|
VLEIB $10, $1, H2_1
|
|
VPERM M4, M5, EX0, T_3
|
|
VPERM M4, M5, EX1, T_4
|
|
VPERM M4, M5, EX2, T_5
|
|
VN T_0, T_3, T_3
|
|
VESRLG $4, T_4, T_4
|
|
VN T_1, T_5, T_5
|
|
VN T_0, T_4, T_4
|
|
VMRHG V0, T_3, M0
|
|
VMRHG V0, T_4, M1
|
|
VMRHG V0, T_5, M2
|
|
VMRLG V0, T_3, M3
|
|
VMRLG V0, T_4, M4
|
|
VMRLG V0, T_5, M5
|
|
VLEIB $10, $1, M2
|
|
VLEIB $10, $1, M5
|
|
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
CMPBNE R3, $0, loop
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
VMRHG V0, H0_1, H0_0
|
|
VMRHG V0, H1_1, H1_0
|
|
VMRHG V0, H2_1, H2_0
|
|
VMRLG V0, H0_1, H0_1
|
|
VMRLG V0, H1_1, H1_1
|
|
VMRLG V0, H2_1, H2_1
|
|
|
|
// load EX0, EX1, EX2
|
|
MOVD $·constants<>(SB), R5
|
|
VLM (R5), EX0, EX2
|
|
|
|
// sum vectors
|
|
VAQ H0_0, H0_1, H0_0
|
|
VAQ H1_0, H1_1, H1_0
|
|
VAQ H2_0, H2_1, H2_0
|
|
|
|
// h may be >= 2*(2**130-5) so we need to reduce it again
|
|
// M0...M4 are used as temps here
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
|
|
|
|
next: // carry h1->h2
|
|
VLEIB $7, $0x28, T_1
|
|
VREPIB $4, T_2
|
|
VGBM $0x003F, T_3
|
|
VESRLG $4, T_3
|
|
|
|
// byte shift
|
|
VSRLB T_1, H1_0, T_4
|
|
|
|
// bit shift
|
|
VSRL T_2, T_4, T_4
|
|
|
|
// clear h1 carry bits
|
|
VN T_3, H1_0, H1_0
|
|
|
|
// add carry
|
|
VAQ T_4, H2_0, H2_0
|
|
|
|
// h is now < 2*(2**130-5)
|
|
// pack h into h1 (hi) and h0 (lo)
|
|
PACK(H0_0, H1_0, H2_0)
|
|
|
|
// if h > 2**130-5 then h -= 2**130-5
|
|
MOD(H0_0, H1_0, T_0, T_1, T_2)
|
|
|
|
// h += s
|
|
MOVD $·bswapMask<>(SB), R5
|
|
VL (R5), T_1
|
|
VL 16(R4), T_0
|
|
VPERM T_0, T_0, T_1, T_0 // reverse bytes (to big)
|
|
VAQ T_0, H0_0, H0_0
|
|
VPERM H0_0, H0_0, T_1, H0_0 // reverse bytes (to little)
|
|
VST H0_0, (R1)
|
|
RET
|
|
|
|
add:
|
|
// load EX0, EX1, EX2
|
|
MOVD $·constants<>(SB), R5
|
|
VLM (R5), EX0, EX2
|
|
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
VMRHG V0, H0_1, H0_0
|
|
VMRHG V0, H1_1, H1_0
|
|
VMRHG V0, H2_1, H2_0
|
|
VMRLG V0, H0_1, H0_1
|
|
VMRLG V0, H1_1, H1_1
|
|
VMRLG V0, H2_1, H2_1
|
|
CMPBLE R3, $64, b4
|
|
|
|
b4:
|
|
CMPBLE R3, $48, b3 // 3 blocks or less
|
|
|
|
// 4(3+1) blocks remaining
|
|
SUB $49, R3
|
|
VLM (R2), M0, M2
|
|
VLL R3, 48(R2), M3
|
|
ADD $1, R3
|
|
MOVBZ $1, R0
|
|
CMPBEQ R3, $16, 2(PC)
|
|
VLVGB R3, R0, M3
|
|
MOVD $64(R2), R2
|
|
EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
|
|
VLEIB $10, $1, H2_0
|
|
VLEIB $10, $1, H2_1
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M4
|
|
VZERO M5
|
|
VZERO T_4
|
|
VZERO T_10
|
|
EXPACC(M2, M3, M0, M1, M4, M5, T_4, T_10, T_0, T_1, T_2, T_3)
|
|
VLR T_4, M2
|
|
VLEIB $10, $1, M4
|
|
CMPBNE R3, $16, 2(PC)
|
|
VLEIB $10, $1, T_10
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M4, M5, M2, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
VMRHG V0, H0_1, H0_0
|
|
VMRHG V0, H1_1, H1_0
|
|
VMRHG V0, H2_1, H2_0
|
|
VMRLG V0, H0_1, H0_1
|
|
VMRLG V0, H1_1, H1_1
|
|
VMRLG V0, H2_1, H2_1
|
|
SUB $16, R3
|
|
CMPBLE R3, $0, square // this condition must always hold true!
|
|
|
|
b3:
|
|
CMPBLE R3, $32, b2
|
|
|
|
// 3 blocks remaining
|
|
|
|
// setup [r²,r]
|
|
VSLDB $8, R_0, R_0, R_0
|
|
VSLDB $8, R_1, R_1, R_1
|
|
VSLDB $8, R_2, R_2, R_2
|
|
VSLDB $8, R5_1, R5_1, R5_1
|
|
VSLDB $8, R5_2, R5_2, R5_2
|
|
|
|
VLVGG $1, RSAVE_0, R_0
|
|
VLVGG $1, RSAVE_1, R_1
|
|
VLVGG $1, RSAVE_2, R_2
|
|
VLVGG $1, R5SAVE_1, R5_1
|
|
VLVGG $1, R5SAVE_2, R5_2
|
|
|
|
// setup [h0, h1]
|
|
VSLDB $8, H0_0, H0_0, H0_0
|
|
VSLDB $8, H1_0, H1_0, H1_0
|
|
VSLDB $8, H2_0, H2_0, H2_0
|
|
VO H0_1, H0_0, H0_0
|
|
VO H1_1, H1_0, H1_0
|
|
VO H2_1, H2_0, H2_0
|
|
VZERO H0_1
|
|
VZERO H1_1
|
|
VZERO H2_1
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
|
|
// H*[r**2, r]
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, T_10, M5)
|
|
|
|
SUB $33, R3
|
|
VLM (R2), M0, M1
|
|
VLL R3, 32(R2), M2
|
|
ADD $1, R3
|
|
MOVBZ $1, R0
|
|
CMPBEQ R3, $16, 2(PC)
|
|
VLVGB R3, R0, M2
|
|
|
|
// H += m0
|
|
VZERO T_1
|
|
VZERO T_2
|
|
VZERO T_3
|
|
EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)
|
|
VLEIB $10, $1, T_3
|
|
VAG H0_0, T_1, H0_0
|
|
VAG H1_0, T_2, H1_0
|
|
VAG H2_0, T_3, H2_0
|
|
|
|
VZERO M0
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
VZERO T_10
|
|
|
|
// (H+m0)*r
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M3, M4, M5, V0, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_10, H0_1, H1_1, H2_1, T_9)
|
|
|
|
// H += m1
|
|
VZERO V0
|
|
VZERO T_1
|
|
VZERO T_2
|
|
VZERO T_3
|
|
EXPACC2(M1, T_1, T_2, T_3, T_4, T_5, T_6)
|
|
VLEIB $10, $1, T_3
|
|
VAQ H0_0, T_1, H0_0
|
|
VAQ H1_0, T_2, H1_0
|
|
VAQ H2_0, T_3, H2_0
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
|
|
|
|
// [H, m2] * [r**2, r]
|
|
EXPACC2(M2, H0_0, H1_0, H2_0, T_1, T_2, T_3)
|
|
CMPBNE R3, $16, 2(PC)
|
|
VLEIB $10, $1, H2_0
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, M5, T_10)
|
|
SUB $16, R3
|
|
CMPBLE R3, $0, next // this condition must always hold true!
|
|
|
|
b2:
|
|
CMPBLE R3, $16, b1
|
|
|
|
// 2 blocks remaining
|
|
|
|
// setup [r²,r]
|
|
VSLDB $8, R_0, R_0, R_0
|
|
VSLDB $8, R_1, R_1, R_1
|
|
VSLDB $8, R_2, R_2, R_2
|
|
VSLDB $8, R5_1, R5_1, R5_1
|
|
VSLDB $8, R5_2, R5_2, R5_2
|
|
|
|
VLVGG $1, RSAVE_0, R_0
|
|
VLVGG $1, RSAVE_1, R_1
|
|
VLVGG $1, RSAVE_2, R_2
|
|
VLVGG $1, R5SAVE_1, R5_1
|
|
VLVGG $1, R5SAVE_2, R5_2
|
|
|
|
// setup [h0, h1]
|
|
VSLDB $8, H0_0, H0_0, H0_0
|
|
VSLDB $8, H1_0, H1_0, H1_0
|
|
VSLDB $8, H2_0, H2_0, H2_0
|
|
VO H0_1, H0_0, H0_0
|
|
VO H1_1, H1_0, H1_0
|
|
VO H2_1, H2_0, H2_0
|
|
VZERO H0_1
|
|
VZERO H1_1
|
|
VZERO H2_1
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
|
|
// H*[r**2, r]
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
|
|
VMRHG V0, H0_1, H0_0
|
|
VMRHG V0, H1_1, H1_0
|
|
VMRHG V0, H2_1, H2_0
|
|
VMRLG V0, H0_1, H0_1
|
|
VMRLG V0, H1_1, H1_1
|
|
VMRLG V0, H2_1, H2_1
|
|
|
|
// move h to the left and 0s at the right
|
|
VSLDB $8, H0_0, H0_0, H0_0
|
|
VSLDB $8, H1_0, H1_0, H1_0
|
|
VSLDB $8, H2_0, H2_0, H2_0
|
|
|
|
// get message blocks and append 1 to start
|
|
SUB $17, R3
|
|
VL (R2), M0
|
|
VLL R3, 16(R2), M1
|
|
ADD $1, R3
|
|
MOVBZ $1, R0
|
|
CMPBEQ R3, $16, 2(PC)
|
|
VLVGB R3, R0, M1
|
|
VZERO T_6
|
|
VZERO T_7
|
|
VZERO T_8
|
|
EXPACC2(M0, T_6, T_7, T_8, T_1, T_2, T_3)
|
|
EXPACC2(M1, T_6, T_7, T_8, T_1, T_2, T_3)
|
|
VLEIB $2, $1, T_8
|
|
CMPBNE R3, $16, 2(PC)
|
|
VLEIB $10, $1, T_8
|
|
|
|
// add [m0, m1] to h
|
|
VAG H0_0, T_6, H0_0
|
|
VAG H1_0, T_7, H1_0
|
|
VAG H2_0, T_8, H2_0
|
|
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
VZERO T_10
|
|
VZERO M0
|
|
|
|
// at this point R_0 .. R5_2 look like [r**2, r]
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M2, M3, M4, M5, T_10, M0, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M2, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
|
|
SUB $16, R3, R3
|
|
CMPBLE R3, $0, next
|
|
|
|
b1:
|
|
CMPBLE R3, $0, next
|
|
|
|
// 1 block remaining
|
|
|
|
// setup [r²,r]
|
|
VSLDB $8, R_0, R_0, R_0
|
|
VSLDB $8, R_1, R_1, R_1
|
|
VSLDB $8, R_2, R_2, R_2
|
|
VSLDB $8, R5_1, R5_1, R5_1
|
|
VSLDB $8, R5_2, R5_2, R5_2
|
|
|
|
VLVGG $1, RSAVE_0, R_0
|
|
VLVGG $1, RSAVE_1, R_1
|
|
VLVGG $1, RSAVE_2, R_2
|
|
VLVGG $1, R5SAVE_1, R5_1
|
|
VLVGG $1, R5SAVE_2, R5_2
|
|
|
|
// setup [h0, h1]
|
|
VSLDB $8, H0_0, H0_0, H0_0
|
|
VSLDB $8, H1_0, H1_0, H1_0
|
|
VSLDB $8, H2_0, H2_0, H2_0
|
|
VO H0_1, H0_0, H0_0
|
|
VO H1_1, H1_0, H1_0
|
|
VO H2_1, H2_0, H2_0
|
|
VZERO H0_1
|
|
VZERO H1_1
|
|
VZERO H2_1
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
|
|
// H*[r**2, r]
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
|
|
|
|
// set up [0, m0] limbs
|
|
SUB $1, R3
|
|
VLL R3, (R2), M0
|
|
ADD $1, R3
|
|
MOVBZ $1, R0
|
|
CMPBEQ R3, $16, 2(PC)
|
|
VLVGB R3, R0, M0
|
|
VZERO T_1
|
|
VZERO T_2
|
|
VZERO T_3
|
|
EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)// limbs: [0, m]
|
|
CMPBNE R3, $16, 2(PC)
|
|
VLEIB $10, $1, T_3
|
|
|
|
// h+m0
|
|
VAQ H0_0, T_1, H0_0
|
|
VAQ H1_0, T_2, H1_0
|
|
VAQ H2_0, T_3, H2_0
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
|
|
|
|
BR next
|
|
|
|
square:
|
|
// setup [r²,r]
|
|
VSLDB $8, R_0, R_0, R_0
|
|
VSLDB $8, R_1, R_1, R_1
|
|
VSLDB $8, R_2, R_2, R_2
|
|
VSLDB $8, R5_1, R5_1, R5_1
|
|
VSLDB $8, R5_2, R5_2, R5_2
|
|
|
|
VLVGG $1, RSAVE_0, R_0
|
|
VLVGG $1, RSAVE_1, R_1
|
|
VLVGG $1, RSAVE_2, R_2
|
|
VLVGG $1, R5SAVE_1, R5_1
|
|
VLVGG $1, R5SAVE_2, R5_2
|
|
|
|
// setup [h0, h1]
|
|
VSLDB $8, H0_0, H0_0, H0_0
|
|
VSLDB $8, H1_0, H1_0, H1_0
|
|
VSLDB $8, H2_0, H2_0, H2_0
|
|
VO H0_1, H0_0, H0_0
|
|
VO H1_1, H1_0, H1_0
|
|
VO H2_1, H2_0, H2_0
|
|
VZERO H0_1
|
|
VZERO H1_1
|
|
VZERO H2_1
|
|
|
|
VZERO M0
|
|
VZERO M1
|
|
VZERO M2
|
|
VZERO M3
|
|
VZERO M4
|
|
VZERO M5
|
|
|
|
// (h0*r**2) + (h1*r)
|
|
MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
|
|
REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
|
|
BR next
|