CURRENT PATH:
/
opt
/
golang
/
1.22.0
/
src
/
crypto
/
aes
/
KEMBALI
|
HOME
Upload File Local:
Upload
Upload via URL:
Download
Dir Baru
File Baru
Editing:
gcm_arm64.s
// Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "textflag.h" #define B0 V0 #define B1 V1 #define B2 V2 #define B3 V3 #define B4 V4 #define B5 V5 #define B6 V6 #define B7 V7 #define ACC0 V8 #define ACC1 V9 #define ACCM V10 #define T0 V11 #define T1 V12 #define T2 V13 #define T3 V14 #define POLY V15 #define ZERO V16 #define INC V17 #define CTR V18 #define K0 V19 #define K1 V20 #define K2 V21 #define K3 V22 #define K4 V23 #define K5 V24 #define K6 V25 #define K7 V26 #define K8 V27 #define K9 V28 #define K10 V29 #define K11 V30 #define KLAST V31 #define reduce() \ VEOR ACC0.B16, ACCM.B16, ACCM.B16 \ VEOR ACC1.B16, ACCM.B16, ACCM.B16 \ VEXT $8, ZERO.B16, ACCM.B16, T0.B16 \ VEXT $8, ACCM.B16, ZERO.B16, ACCM.B16 \ VEOR ACCM.B16, ACC0.B16, ACC0.B16 \ VEOR T0.B16, ACC1.B16, ACC1.B16 \ VPMULL POLY.D1, ACC0.D1, T0.Q1 \ VEXT $8, ACC0.B16, ACC0.B16, ACC0.B16 \ VEOR T0.B16, ACC0.B16, ACC0.B16 \ VPMULL POLY.D1, ACC0.D1, T0.Q1 \ VEOR T0.B16, ACC1.B16, ACC1.B16 \ VEXT $8, ACC1.B16, ACC1.B16, ACC1.B16 \ VEOR ACC1.B16, ACC0.B16, ACC0.B16 \ // func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64) TEXT ·gcmAesFinish(SB),NOSPLIT,$0 #define pTbl R0 #define tMsk R1 #define tPtr R2 #define plen R3 #define dlen R4 MOVD $0xC2, R1 LSL $56, R1 MOVD $1, R0 VMOV R1, POLY.D[0] VMOV R0, POLY.D[1] VEOR ZERO.B16, ZERO.B16, ZERO.B16 MOVD productTable+0(FP), pTbl MOVD tagMask+8(FP), tMsk MOVD T+16(FP), tPtr MOVD pLen+24(FP), plen MOVD dLen+32(FP), dlen VLD1 (tPtr), [ACC0.B16] VLD1 (tMsk), [B1.B16] LSL $3, plen LSL $3, dlen VMOV dlen, B0.D[0] VMOV plen, B0.D[1] ADD $14*16, pTbl VLD1.P (pTbl), [T1.B16, T2.B16] VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 reduce() VREV64 ACC0.B16, ACC0.B16 VEOR B1.B16, ACC0.B16, ACC0.B16 VST1 [ACC0.B16], (tPtr) RET #undef pTbl #undef tMsk #undef tPtr #undef plen #undef dlen // func gcmAesInit(productTable *[256]byte, ks []uint32) TEXT ·gcmAesInit(SB),NOSPLIT,$0 #define pTbl R0 #define KS R1 #define NR R2 #define I R3 MOVD productTable+0(FP), pTbl MOVD ks_base+8(FP), KS MOVD ks_len+16(FP), NR MOVD $0xC2, I LSL $56, I VMOV I, POLY.D[0] MOVD $1, I VMOV I, POLY.D[1] VEOR ZERO.B16, ZERO.B16, ZERO.B16 // Encrypt block 0 with the AES key to generate the hash key H VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16] VEOR B0.B16, B0.B16, B0.B16 AESE T0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T3.B16, B0.B16 AESMC B0.B16, B0.B16 VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16] AESE T0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T3.B16, B0.B16 AESMC B0.B16, B0.B16 TBZ $4, NR, initEncFinish VLD1.P 32(KS), [T0.B16, T1.B16] AESE T0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T1.B16, B0.B16 AESMC B0.B16, B0.B16 TBZ $3, NR, initEncFinish VLD1.P 32(KS), [T0.B16, T1.B16] AESE T0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T1.B16, B0.B16 AESMC B0.B16, B0.B16 initEncFinish: VLD1 (KS), [T0.B16, T1.B16, T2.B16] AESE T0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE T1.B16, B0.B16 VEOR T2.B16, B0.B16, B0.B16 VREV64 B0.B16, B0.B16 // Multiply by 2 modulo P VMOV B0.D[0], I ASR $63, I VMOV I, T1.D[0] VMOV I, T1.D[1] VAND POLY.B16, T1.B16, T1.B16 VUSHR $63, B0.D2, T2.D2 VEXT $8, ZERO.B16, T2.B16, T2.B16 VSHL $1, B0.D2, B0.D2 VEOR T1.B16, B0.B16, B0.B16 VEOR T2.B16, B0.B16, B0.B16 // Can avoid this when VSLI is available // Karatsuba pre-computation VEXT $8, B0.B16, B0.B16, B1.B16 VEOR B0.B16, B1.B16, B1.B16 ADD $14*16, pTbl VST1 [B0.B16, B1.B16], (pTbl) SUB $2*16, pTbl VMOV B0.B16, B2.B16 VMOV B1.B16, B3.B16 MOVD $7, I initLoop: // Compute powers of H SUBS $1, I VPMULL B0.D1, B2.D1, T1.Q1 VPMULL2 B0.D2, B2.D2, T0.Q1 VPMULL B1.D1, B3.D1, T2.Q1 VEOR T0.B16, T2.B16, T2.B16 VEOR T1.B16, T2.B16, T2.B16 VEXT $8, ZERO.B16, T2.B16, T3.B16 VEXT $8, T2.B16, ZERO.B16, T2.B16 VEOR T2.B16, T0.B16, T0.B16 VEOR T3.B16, T1.B16, T1.B16 VPMULL POLY.D1, T0.D1, T2.Q1 VEXT $8, T0.B16, T0.B16, T0.B16 VEOR T2.B16, T0.B16, T0.B16 VPMULL POLY.D1, T0.D1, T2.Q1 VEXT $8, T0.B16, T0.B16, T0.B16 VEOR T2.B16, T0.B16, T0.B16 VEOR T1.B16, T0.B16, B2.B16 VMOV B2.B16, B3.B16 VEXT $8, B2.B16, B2.B16, B2.B16 VEOR B2.B16, B3.B16, B3.B16 VST1 [B2.B16, B3.B16], (pTbl) SUB $2*16, pTbl BNE initLoop RET #undef I #undef NR #undef KS #undef pTbl // func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte) TEXT ·gcmAesData(SB),NOSPLIT,$0 #define pTbl R0 #define aut R1 #define tPtr R2 #define autLen R3 #define H0 R4 #define pTblSave R5 #define mulRound(X) \ VLD1.P 32(pTbl), [T1.B16, T2.B16] \ VREV64 X.B16, X.B16 \ VEXT $8, X.B16, X.B16, T0.B16 \ VEOR X.B16, T0.B16, T0.B16 \ VPMULL X.D1, T1.D1, T3.Q1 \ VEOR T3.B16, ACC1.B16, ACC1.B16 \ VPMULL2 X.D2, T1.D2, T3.Q1 \ VEOR T3.B16, ACC0.B16, ACC0.B16 \ VPMULL T0.D1, T2.D1, T3.Q1 \ VEOR T3.B16, ACCM.B16, ACCM.B16 MOVD productTable+0(FP), pTbl MOVD data_base+8(FP), aut MOVD data_len+16(FP), autLen MOVD T+32(FP), tPtr VEOR ACC0.B16, ACC0.B16, ACC0.B16 CBZ autLen, dataBail MOVD $0xC2, H0 LSL $56, H0 VMOV H0, POLY.D[0] MOVD $1, H0 VMOV H0, POLY.D[1] VEOR ZERO.B16, ZERO.B16, ZERO.B16 MOVD pTbl, pTblSave CMP $13, autLen BEQ dataTLS CMP $128, autLen BLT startSinglesLoop B octetsLoop dataTLS: ADD $14*16, pTbl VLD1.P (pTbl), [T1.B16, T2.B16] VEOR B0.B16, B0.B16, B0.B16 MOVD (aut), H0 VMOV H0, B0.D[0] MOVW 8(aut), H0 VMOV H0, B0.S[2] MOVB 12(aut), H0 VMOV H0, B0.B[12] MOVD $0, autLen B dataMul octetsLoop: CMP $128, autLen BLT startSinglesLoop SUB $128, autLen VLD1.P 32(aut), [B0.B16, B1.B16] VLD1.P 32(pTbl), [T1.B16, T2.B16] VREV64 B0.B16, B0.B16 VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 mulRound(B1) VLD1.P 32(aut), [B2.B16, B3.B16] mulRound(B2) mulRound(B3) VLD1.P 32(aut), [B4.B16, B5.B16] mulRound(B4) mulRound(B5) VLD1.P 32(aut), [B6.B16, B7.B16] mulRound(B6) mulRound(B7) MOVD pTblSave, pTbl reduce() B octetsLoop startSinglesLoop: ADD $14*16, pTbl VLD1.P (pTbl), [T1.B16, T2.B16] singlesLoop: CMP $16, autLen BLT dataEnd SUB $16, autLen VLD1.P 16(aut), [B0.B16] dataMul: VREV64 B0.B16, B0.B16 VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 reduce() B singlesLoop dataEnd: CBZ autLen, dataBail VEOR B0.B16, B0.B16, B0.B16 ADD autLen, aut dataLoadLoop: MOVB.W -1(aut), H0 VEXT $15, B0.B16, ZERO.B16, B0.B16 VMOV H0, B0.B[0] SUBS $1, autLen BNE dataLoadLoop B dataMul dataBail: VST1 [ACC0.B16], (tPtr) RET #undef pTbl #undef aut #undef tPtr #undef autLen #undef H0 #undef pTblSave // func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) TEXT ·gcmAesEnc(SB),NOSPLIT,$0 #define pTbl R0 #define dstPtr R1 #define ctrPtr R2 #define srcPtr R3 #define ks R4 #define tPtr R5 #define srcPtrLen R6 #define aluCTR R7 #define aluTMP R8 #define aluK R9 #define NR R10 #define H0 R11 #define H1 R12 #define curK R13 #define pTblSave R14 #define aesrndx8(K) \ AESE K.B16, B0.B16 \ AESMC B0.B16, B0.B16 \ AESE K.B16, B1.B16 \ AESMC B1.B16, B1.B16 \ AESE K.B16, B2.B16 \ AESMC B2.B16, B2.B16 \ AESE K.B16, B3.B16 \ AESMC B3.B16, B3.B16 \ AESE K.B16, B4.B16 \ AESMC B4.B16, B4.B16 \ AESE K.B16, B5.B16 \ AESMC B5.B16, B5.B16 \ AESE K.B16, B6.B16 \ AESMC B6.B16, B6.B16 \ AESE K.B16, B7.B16 \ AESMC B7.B16, B7.B16 #define aesrndlastx8(K) \ AESE K.B16, B0.B16 \ AESE K.B16, B1.B16 \ AESE K.B16, B2.B16 \ AESE K.B16, B3.B16 \ AESE K.B16, B4.B16 \ AESE K.B16, B5.B16 \ AESE K.B16, B6.B16 \ AESE K.B16, B7.B16 MOVD productTable+0(FP), pTbl MOVD dst+8(FP), dstPtr MOVD src_base+32(FP), srcPtr MOVD src_len+40(FP), srcPtrLen MOVD ctr+56(FP), ctrPtr MOVD T+64(FP), tPtr MOVD ks_base+72(FP), ks MOVD ks_len+80(FP), NR MOVD $0xC2, H1 LSL $56, H1 MOVD $1, H0 VMOV H1, POLY.D[0] VMOV H0, POLY.D[1] VEOR ZERO.B16, ZERO.B16, ZERO.B16 // Compute NR from len(ks) MOVD pTbl, pTblSave // Current tag, after AAD VLD1 (tPtr), [ACC0.B16] VEOR ACC1.B16, ACC1.B16, ACC1.B16 VEOR ACCM.B16, ACCM.B16, ACCM.B16 // Prepare initial counter, and the increment vector VLD1 (ctrPtr), [CTR.B16] VEOR INC.B16, INC.B16, INC.B16 MOVD $1, H0 VMOV H0, INC.S[3] VREV32 CTR.B16, CTR.B16 VADD CTR.S4, INC.S4, CTR.S4 // Skip to <8 blocks loop CMP $128, srcPtrLen MOVD ks, H0 // For AES-128 round keys are stored in: K0 .. K10, KLAST VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16] VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16] VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16] VMOV K10.B16, KLAST.B16 BLT startSingles // There are at least 8 blocks to encrypt TBZ $4, NR, octetsLoop // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST VMOV K8.B16, K10.B16 VMOV K9.B16, K11.B16 VMOV KLAST.B16, K8.B16 VLD1.P 16(H0), [K9.B16] VLD1.P 16(H0), [KLAST.B16] TBZ $3, NR, octetsLoop // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST VMOV KLAST.B16, K8.B16 VLD1.P 16(H0), [K9.B16] VLD1.P 16(H0), [KLAST.B16] ADD $10*16, ks, H0 MOVD H0, curK octetsLoop: SUB $128, srcPtrLen VMOV CTR.B16, B0.B16 VADD B0.S4, INC.S4, B1.S4 VREV32 B0.B16, B0.B16 VADD B1.S4, INC.S4, B2.S4 VREV32 B1.B16, B1.B16 VADD B2.S4, INC.S4, B3.S4 VREV32 B2.B16, B2.B16 VADD B3.S4, INC.S4, B4.S4 VREV32 B3.B16, B3.B16 VADD B4.S4, INC.S4, B5.S4 VREV32 B4.B16, B4.B16 VADD B5.S4, INC.S4, B6.S4 VREV32 B5.B16, B5.B16 VADD B6.S4, INC.S4, B7.S4 VREV32 B6.B16, B6.B16 VADD B7.S4, INC.S4, CTR.S4 VREV32 B7.B16, B7.B16 aesrndx8(K0) aesrndx8(K1) aesrndx8(K2) aesrndx8(K3) aesrndx8(K4) aesrndx8(K5) aesrndx8(K6) aesrndx8(K7) TBZ $4, NR, octetsFinish aesrndx8(K10) aesrndx8(K11) TBZ $3, NR, octetsFinish VLD1.P 32(curK), [T1.B16, T2.B16] aesrndx8(T1) aesrndx8(T2) MOVD H0, curK octetsFinish: aesrndx8(K8) aesrndlastx8(K9) VEOR KLAST.B16, B0.B16, B0.B16 VEOR KLAST.B16, B1.B16, B1.B16 VEOR KLAST.B16, B2.B16, B2.B16 VEOR KLAST.B16, B3.B16, B3.B16 VEOR KLAST.B16, B4.B16, B4.B16 VEOR KLAST.B16, B5.B16, B5.B16 VEOR KLAST.B16, B6.B16, B6.B16 VEOR KLAST.B16, B7.B16, B7.B16 VLD1.P 32(srcPtr), [T1.B16, T2.B16] VEOR B0.B16, T1.B16, B0.B16 VEOR B1.B16, T2.B16, B1.B16 VST1.P [B0.B16, B1.B16], 32(dstPtr) VLD1.P 32(srcPtr), [T1.B16, T2.B16] VEOR B2.B16, T1.B16, B2.B16 VEOR B3.B16, T2.B16, B3.B16 VST1.P [B2.B16, B3.B16], 32(dstPtr) VLD1.P 32(srcPtr), [T1.B16, T2.B16] VEOR B4.B16, T1.B16, B4.B16 VEOR B5.B16, T2.B16, B5.B16 VST1.P [B4.B16, B5.B16], 32(dstPtr) VLD1.P 32(srcPtr), [T1.B16, T2.B16] VEOR B6.B16, T1.B16, B6.B16 VEOR B7.B16, T2.B16, B7.B16 VST1.P [B6.B16, B7.B16], 32(dstPtr) VLD1.P 32(pTbl), [T1.B16, T2.B16] VREV64 B0.B16, B0.B16 VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 mulRound(B1) mulRound(B2) mulRound(B3) mulRound(B4) mulRound(B5) mulRound(B6) mulRound(B7) MOVD pTblSave, pTbl reduce() CMP $128, srcPtrLen BGE octetsLoop startSingles: CBZ srcPtrLen, done ADD $14*16, pTbl // Preload H and its Karatsuba precomp VLD1.P (pTbl), [T1.B16, T2.B16] // Preload AES round keys ADD $128, ks VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16] VMOV K10.B16, KLAST.B16 TBZ $4, NR, singlesLoop VLD1.P 32(ks), [B1.B16, B2.B16] VMOV B2.B16, KLAST.B16 TBZ $3, NR, singlesLoop VLD1.P 32(ks), [B3.B16, B4.B16] VMOV B4.B16, KLAST.B16 singlesLoop: CMP $16, srcPtrLen BLT tail SUB $16, srcPtrLen VLD1.P 16(srcPtr), [T0.B16] VEOR KLAST.B16, T0.B16, T0.B16 VREV32 CTR.B16, B0.B16 VADD CTR.S4, INC.S4, CTR.S4 AESE K0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K3.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K4.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K5.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K6.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K7.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K8.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K9.B16, B0.B16 TBZ $4, NR, singlesLast AESMC B0.B16, B0.B16 AESE K10.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B1.B16, B0.B16 TBZ $3, NR, singlesLast AESMC B0.B16, B0.B16 AESE B2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B3.B16, B0.B16 singlesLast: VEOR T0.B16, B0.B16, B0.B16 encReduce: VST1.P [B0.B16], 16(dstPtr) VREV64 B0.B16, B0.B16 VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 reduce() B singlesLoop tail: CBZ srcPtrLen, done VEOR T0.B16, T0.B16, T0.B16 VEOR T3.B16, T3.B16, T3.B16 MOVD $0, H1 SUB $1, H1 ADD srcPtrLen, srcPtr TBZ $3, srcPtrLen, ld4 MOVD.W -8(srcPtr), H0 VMOV H0, T0.D[0] VMOV H1, T3.D[0] ld4: TBZ $2, srcPtrLen, ld2 MOVW.W -4(srcPtr), H0 VEXT $12, T0.B16, ZERO.B16, T0.B16 VEXT $12, T3.B16, ZERO.B16, T3.B16 VMOV H0, T0.S[0] VMOV H1, T3.S[0] ld2: TBZ $1, srcPtrLen, ld1 MOVH.W -2(srcPtr), H0 VEXT $14, T0.B16, ZERO.B16, T0.B16 VEXT $14, T3.B16, ZERO.B16, T3.B16 VMOV H0, T0.H[0] VMOV H1, T3.H[0] ld1: TBZ $0, srcPtrLen, ld0 MOVB.W -1(srcPtr), H0 VEXT $15, T0.B16, ZERO.B16, T0.B16 VEXT $15, T3.B16, ZERO.B16, T3.B16 VMOV H0, T0.B[0] VMOV H1, T3.B[0] ld0: MOVD ZR, srcPtrLen VEOR KLAST.B16, T0.B16, T0.B16 VREV32 CTR.B16, B0.B16 AESE K0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K3.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K4.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K5.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K6.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K7.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K8.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K9.B16, B0.B16 TBZ $4, NR, tailLast AESMC B0.B16, B0.B16 AESE K10.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B1.B16, B0.B16 TBZ $3, NR, tailLast AESMC B0.B16, B0.B16 AESE B2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B3.B16, B0.B16 tailLast: VEOR T0.B16, B0.B16, B0.B16 VAND T3.B16, B0.B16, B0.B16 B encReduce done: VST1 [ACC0.B16], (tPtr) RET // func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) TEXT ·gcmAesDec(SB),NOSPLIT,$0 MOVD productTable+0(FP), pTbl MOVD dst+8(FP), dstPtr MOVD src_base+32(FP), srcPtr MOVD src_len+40(FP), srcPtrLen MOVD ctr+56(FP), ctrPtr MOVD T+64(FP), tPtr MOVD ks_base+72(FP), ks MOVD ks_len+80(FP), NR MOVD $0xC2, H1 LSL $56, H1 MOVD $1, H0 VMOV H1, POLY.D[0] VMOV H0, POLY.D[1] VEOR ZERO.B16, ZERO.B16, ZERO.B16 // Compute NR from len(ks) MOVD pTbl, pTblSave // Current tag, after AAD VLD1 (tPtr), [ACC0.B16] VEOR ACC1.B16, ACC1.B16, ACC1.B16 VEOR ACCM.B16, ACCM.B16, ACCM.B16 // Prepare initial counter, and the increment vector VLD1 (ctrPtr), [CTR.B16] VEOR INC.B16, INC.B16, INC.B16 MOVD $1, H0 VMOV H0, INC.S[3] VREV32 CTR.B16, CTR.B16 VADD CTR.S4, INC.S4, CTR.S4 MOVD ks, H0 // For AES-128 round keys are stored in: K0 .. K10, KLAST VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16] VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16] VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16] VMOV K10.B16, KLAST.B16 // Skip to <8 blocks loop CMP $128, srcPtrLen BLT startSingles // There are at least 8 blocks to encrypt TBZ $4, NR, octetsLoop // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST VMOV K8.B16, K10.B16 VMOV K9.B16, K11.B16 VMOV KLAST.B16, K8.B16 VLD1.P 16(H0), [K9.B16] VLD1.P 16(H0), [KLAST.B16] TBZ $3, NR, octetsLoop // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST VMOV KLAST.B16, K8.B16 VLD1.P 16(H0), [K9.B16] VLD1.P 16(H0), [KLAST.B16] ADD $10*16, ks, H0 MOVD H0, curK octetsLoop: SUB $128, srcPtrLen VMOV CTR.B16, B0.B16 VADD B0.S4, INC.S4, B1.S4 VREV32 B0.B16, B0.B16 VADD B1.S4, INC.S4, B2.S4 VREV32 B1.B16, B1.B16 VADD B2.S4, INC.S4, B3.S4 VREV32 B2.B16, B2.B16 VADD B3.S4, INC.S4, B4.S4 VREV32 B3.B16, B3.B16 VADD B4.S4, INC.S4, B5.S4 VREV32 B4.B16, B4.B16 VADD B5.S4, INC.S4, B6.S4 VREV32 B5.B16, B5.B16 VADD B6.S4, INC.S4, B7.S4 VREV32 B6.B16, B6.B16 VADD B7.S4, INC.S4, CTR.S4 VREV32 B7.B16, B7.B16 aesrndx8(K0) aesrndx8(K1) aesrndx8(K2) aesrndx8(K3) aesrndx8(K4) aesrndx8(K5) aesrndx8(K6) aesrndx8(K7) TBZ $4, NR, octetsFinish aesrndx8(K10) aesrndx8(K11) TBZ $3, NR, octetsFinish VLD1.P 32(curK), [T1.B16, T2.B16] aesrndx8(T1) aesrndx8(T2) MOVD H0, curK octetsFinish: aesrndx8(K8) aesrndlastx8(K9) VEOR KLAST.B16, B0.B16, T1.B16 VEOR KLAST.B16, B1.B16, T2.B16 VEOR KLAST.B16, B2.B16, B2.B16 VEOR KLAST.B16, B3.B16, B3.B16 VEOR KLAST.B16, B4.B16, B4.B16 VEOR KLAST.B16, B5.B16, B5.B16 VEOR KLAST.B16, B6.B16, B6.B16 VEOR KLAST.B16, B7.B16, B7.B16 VLD1.P 32(srcPtr), [B0.B16, B1.B16] VEOR B0.B16, T1.B16, T1.B16 VEOR B1.B16, T2.B16, T2.B16 VST1.P [T1.B16, T2.B16], 32(dstPtr) VLD1.P 32(pTbl), [T1.B16, T2.B16] VREV64 B0.B16, B0.B16 VEOR ACC0.B16, B0.B16, B0.B16 VEXT $8, B0.B16, B0.B16, T0.B16 VEOR B0.B16, T0.B16, T0.B16 VPMULL B0.D1, T1.D1, ACC1.Q1 VPMULL2 B0.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 mulRound(B1) VLD1.P 32(srcPtr), [B0.B16, B1.B16] VEOR B2.B16, B0.B16, T1.B16 VEOR B3.B16, B1.B16, T2.B16 VST1.P [T1.B16, T2.B16], 32(dstPtr) mulRound(B0) mulRound(B1) VLD1.P 32(srcPtr), [B0.B16, B1.B16] VEOR B4.B16, B0.B16, T1.B16 VEOR B5.B16, B1.B16, T2.B16 VST1.P [T1.B16, T2.B16], 32(dstPtr) mulRound(B0) mulRound(B1) VLD1.P 32(srcPtr), [B0.B16, B1.B16] VEOR B6.B16, B0.B16, T1.B16 VEOR B7.B16, B1.B16, T2.B16 VST1.P [T1.B16, T2.B16], 32(dstPtr) mulRound(B0) mulRound(B1) MOVD pTblSave, pTbl reduce() CMP $128, srcPtrLen BGE octetsLoop startSingles: CBZ srcPtrLen, done ADD $14*16, pTbl // Preload H and its Karatsuba precomp VLD1.P (pTbl), [T1.B16, T2.B16] // Preload AES round keys ADD $128, ks VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16] VMOV K10.B16, KLAST.B16 TBZ $4, NR, singlesLoop VLD1.P 32(ks), [B1.B16, B2.B16] VMOV B2.B16, KLAST.B16 TBZ $3, NR, singlesLoop VLD1.P 32(ks), [B3.B16, B4.B16] VMOV B4.B16, KLAST.B16 singlesLoop: CMP $16, srcPtrLen BLT tail SUB $16, srcPtrLen VLD1.P 16(srcPtr), [T0.B16] VREV64 T0.B16, B5.B16 VEOR KLAST.B16, T0.B16, T0.B16 VREV32 CTR.B16, B0.B16 VADD CTR.S4, INC.S4, CTR.S4 AESE K0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K3.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K4.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K5.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K6.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K7.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K8.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K9.B16, B0.B16 TBZ $4, NR, singlesLast AESMC B0.B16, B0.B16 AESE K10.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B1.B16, B0.B16 TBZ $3, NR, singlesLast AESMC B0.B16, B0.B16 AESE B2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B3.B16, B0.B16 singlesLast: VEOR T0.B16, B0.B16, B0.B16 VST1.P [B0.B16], 16(dstPtr) VEOR ACC0.B16, B5.B16, B5.B16 VEXT $8, B5.B16, B5.B16, T0.B16 VEOR B5.B16, T0.B16, T0.B16 VPMULL B5.D1, T1.D1, ACC1.Q1 VPMULL2 B5.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 reduce() B singlesLoop tail: CBZ srcPtrLen, done VREV32 CTR.B16, B0.B16 VADD CTR.S4, INC.S4, CTR.S4 AESE K0.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K1.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K3.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K4.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K5.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K6.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K7.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K8.B16, B0.B16 AESMC B0.B16, B0.B16 AESE K9.B16, B0.B16 TBZ $4, NR, tailLast AESMC B0.B16, B0.B16 AESE K10.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B1.B16, B0.B16 TBZ $3, NR, tailLast AESMC B0.B16, B0.B16 AESE B2.B16, B0.B16 AESMC B0.B16, B0.B16 AESE B3.B16, B0.B16 tailLast: VEOR KLAST.B16, B0.B16, B0.B16 // Assuming it is safe to load past dstPtr due to the presence of the tag VLD1 (srcPtr), [B5.B16] VEOR B5.B16, B0.B16, B0.B16 VEOR T3.B16, T3.B16, T3.B16 MOVD $0, H1 SUB $1, H1 TBZ $3, srcPtrLen, ld4 VMOV B0.D[0], H0 MOVD.P H0, 8(dstPtr) VMOV H1, T3.D[0] VEXT $8, ZERO.B16, B0.B16, B0.B16 ld4: TBZ $2, srcPtrLen, ld2 VMOV B0.S[0], H0 MOVW.P H0, 4(dstPtr) VEXT $12, T3.B16, ZERO.B16, T3.B16 VMOV H1, T3.S[0] VEXT $4, ZERO.B16, B0.B16, B0.B16 ld2: TBZ $1, srcPtrLen, ld1 VMOV B0.H[0], H0 MOVH.P H0, 2(dstPtr) VEXT $14, T3.B16, ZERO.B16, T3.B16 VMOV H1, T3.H[0] VEXT $2, ZERO.B16, B0.B16, B0.B16 ld1: TBZ $0, srcPtrLen, ld0 VMOV B0.B[0], H0 MOVB.P H0, 1(dstPtr) VEXT $15, T3.B16, ZERO.B16, T3.B16 VMOV H1, T3.B[0] ld0: VAND T3.B16, B5.B16, B5.B16 VREV64 B5.B16, B5.B16 VEOR ACC0.B16, B5.B16, B5.B16 VEXT $8, B5.B16, B5.B16, T0.B16 VEOR B5.B16, T0.B16, T0.B16 VPMULL B5.D1, T1.D1, ACC1.Q1 VPMULL2 B5.D2, T1.D2, ACC0.Q1 VPMULL T0.D1, T2.D1, ACCM.Q1 reduce() done: VST1 [ACC0.B16], (tPtr) RET
SIMPAN
BATAL