CURRENT PATH:
/
opt
/
golang
/
1.22.0
/
src
/
crypto
/
aes
/
KEMBALI
|
HOME
Upload File Local:
Upload
Upload via URL:
Download
Dir Baru
File Baru
Editing:
gcm_ppc64x.s
// Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le // Portions based on CRYPTOGAMS code with the following comment: // # ==================================================================== // # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL // # project. The module is, however, dual licensed under OpenSSL and // # CRYPTOGAMS licenses depending on where you obtain it. For further // # details see http://www.openssl.org/~appro/cryptogams/. // # ==================================================================== // The implementations for gcmHash, gcmInit and gcmMul are based on the generated asm // from the script https://github.com/dot-asm/cryptogams/blob/master/ppc/ghashp8-ppc.pl // from commit d47afb3c. // Changes were made due to differences in the ABI and some register usage. // Some arguments were changed due to the way the Go code passes them. // Portions that use the stitched AES-GCM approach in counterCryptASM // are based on code found in // https://github.com/IBM/ipcri/blob/main/aes/p10_aes_gcm.s #include "textflag.h" #define XIP R3 #define HTBL R4 #define INP R5 #define LEN R6 #define XL V0 #define XM V1 #define XH V2 #define IN V3 #define ZERO V4 #define T0 V5 #define T1 V6 #define T2 V7 #define XC2 V8 #define H V9 #define HH V10 #define HL V11 #define LEMASK V12 #define XL1 V13 #define XM1 V14 #define XH1 V15 #define IN1 V16 #define H2 V17 #define H2H V18 #define H2L V19 #define XL3 V20 #define XM2 V21 #define IN2 V22 #define H3L V23 #define H3 V24 #define H3H V25 #define XH3 V26 #define XM3 V27 #define IN3 V28 #define H4L V29 #define H4 V30 #define H4H V31 #define IN0 IN #define H21L HL #define H21H HH #define LOPERM H2L #define HIPERM H2H #define VXL VS32 #define VIN VS35 #define VXC2 VS40 #define VH VS41 #define VHH VS42 #define VHL VS43 #define VIN1 VS48 #define VH2 VS49 #define VH2H VS50 #define VH2L VS51 #define VIN2 VS54 #define VH3L VS55 #define VH3 VS56 #define VH3H VS57 #define VIN3 VS60 #define VH4L VS61 #define VH4 VS62 #define VH4H VS63 #define VIN0 VIN #define ESPERM V10 #define TMP2 V11 // The following macros provide appropriate // implementations for endianness as well as // ISA specific for power8 and power9. #ifdef GOARCH_ppc64le # ifdef GOPPC64_power9 #define P8_LXVB16X(RA,RB,VT) LXVB16X (RA)(RB), VT #define P8_STXVB16X(VS,RA,RB) STXVB16X VS, (RA)(RB) # else #define NEEDS_ESPERM #define P8_LXVB16X(RA,RB,VT) \ LXVD2X (RA+RB), VT \ VPERM VT, VT, ESPERM, VT #define P8_STXVB16X(VS,RA,RB) \ VPERM VS, VS, ESPERM, TMP2; \ STXVD2X TMP2, (RA+RB) # endif #else #define P8_LXVB16X(RA,RB,VT) \ LXVD2X (RA+RB), VT #define P8_STXVB16X(VS,RA,RB) \ STXVD2X VS, (RA+RB) #endif #define MASK_PTR R8 #define MASKV V0 #define INV V1 // The following macros are used for // the stitched implementation within // counterCryptASM. // Load the initial GCM counter value // in V30 and set up the counter increment // in V31 #define SETUP_COUNTER \ P8_LXVB16X(COUNTER, R0, V30); \ VSPLTISB $1, V28; \ VXOR V31, V31, V31; \ VSLDOI $1, V31, V28, V31 // These macros set up the initial value // for a single encryption, or 4 or 8 // stitched encryptions implemented // with interleaving vciphers. // // The input value for each encryption // is generated by XORing the counter // from V30 with the first key in VS0 // and incrementing the counter. // // Single encryption in V15 #define GEN_VCIPHER_INPUT \ XXLOR VS0, VS0, V29 \ VXOR V30, V29, V15; \ VADDUWM V30, V31, V30 // 4 encryptions in V15 - V18 #define GEN_VCIPHER_4_INPUTS \ XXLOR VS0, VS0, V29; \ VXOR V30, V29, V15; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V16; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V17; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V18; \ VADDUWM V30, V31, V30 // 8 encryptions in V15 - V22 #define GEN_VCIPHER_8_INPUTS \ XXLOR VS0, VS0, V29; \ VXOR V30, V29, V15; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V16; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V17; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V18; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V19; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V20; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V21; \ VADDUWM V30, V31, V30; \ VXOR V30, V29, V22; \ VADDUWM V30, V31, V30 // Load the keys to be used for // encryption based on key_len. // Keys are in VS0 - VS14 // depending on key_len. // Valid keys sizes are verified // here. CR2 is set and used // throughout to check key_len. #define LOAD_KEYS(blk_key, key_len) \ MOVD $16, R16; \ MOVD $32, R17; \ MOVD $48, R18; \ MOVD $64, R19; \ LXVD2X (blk_key)(R0), VS0; \ LXVD2X (blk_key)(R16), VS1; \ LXVD2X (blk_key)(R17), VS2; \ LXVD2X (blk_key)(R18), VS3; \ LXVD2X (blk_key)(R19), VS4; \ ADD $64, R16; \ ADD $64, R17; \ ADD $64, R18; \ ADD $64, R19; \ LXVD2X (blk_key)(R16), VS5; \ LXVD2X (blk_key)(R17), VS6; \ LXVD2X (blk_key)(R18), VS7; \ LXVD2X (blk_key)(R19), VS8; \ ADD $64, R16; \ ADD $64, R17; \ ADD $64, R18; \ ADD $64, R19; \ LXVD2X (blk_key)(R16), VS9; \ LXVD2X (blk_key)(R17), VS10; \ CMP key_len, $12, CR2; \ CMP key_len, $10; \ BEQ keysLoaded; \ LXVD2X (blk_key)(R18), VS11; \ LXVD2X (blk_key)(R19), VS12; \ BEQ CR2, keysLoaded; \ ADD $64, R16; \ ADD $64, R17; \ LXVD2X (blk_key)(R16), VS13; \ LXVD2X (blk_key)(R17), VS14; \ CMP key_len, $14; \ BEQ keysLoaded; \ MOVD R0,0(R0); \ keysLoaded: // Encrypt 1 (vin) with first 9 // keys from VS1 - VS9. #define VCIPHER_1X9_KEYS(vin) \ XXLOR VS1, VS1, V23; \ XXLOR VS2, VS2, V24; \ XXLOR VS3, VS3, V25; \ XXLOR VS4, VS4, V26; \ XXLOR VS5, VS5, V27; \ VCIPHER vin, V23, vin; \ VCIPHER vin, V24, vin; \ VCIPHER vin, V25, vin; \ VCIPHER vin, V26, vin; \ VCIPHER vin, V27, vin; \ XXLOR VS6, VS6, V23; \ XXLOR VS7, VS7, V24; \ XXLOR VS8, VS8, V25; \ XXLOR VS9, VS9, V26; \ VCIPHER vin, V23, vin; \ VCIPHER vin, V24, vin; \ VCIPHER vin, V25, vin; \ VCIPHER vin, V26, vin // Encrypt 1 value (vin) with // 2 specified keys #define VCIPHER_1X2_KEYS(vin, key1, key2) \ XXLOR key1, key1, V25; \ XXLOR key2, key2, V26; \ VCIPHER vin, V25, vin; \ VCIPHER vin, V26, vin // Encrypt 4 values in V15 - V18 // with the specified key from // VS1 - VS9. #define VCIPHER_4X1_KEY(key) \ XXLOR key, key, V23; \ VCIPHER V15, V23, V15; \ VCIPHER V16, V23, V16; \ VCIPHER V17, V23, V17; \ VCIPHER V18, V23, V18 // Encrypt 8 values in V15 - V22 // with the specified key, // assuming it is a VSreg #define VCIPHER_8X1_KEY(key) \ XXLOR key, key, V23; \ VCIPHER V15, V23, V15; \ VCIPHER V16, V23, V16; \ VCIPHER V17, V23, V17; \ VCIPHER V18, V23, V18; \ VCIPHER V19, V23, V19; \ VCIPHER V20, V23, V20; \ VCIPHER V21, V23, V21; \ VCIPHER V22, V23, V22 // Load input block into V1-V4 // in big endian order and // update blk_inp by 64. #define LOAD_INPUT_BLOCK64(blk_inp) \ MOVD $16, R16; \ MOVD $32, R17; \ MOVD $48, R18; \ P8_LXVB16X(blk_inp,R0,V1); \ P8_LXVB16X(blk_inp,R16,V2); \ P8_LXVB16X(blk_inp,R17,V3); \ P8_LXVB16X(blk_inp,R18,V4); \ ADD $64, blk_inp // Load input block into V1-V8 // in big endian order and // Update blk_inp by 128 #define LOAD_INPUT_BLOCK128(blk_inp) \ MOVD $16, R16; \ MOVD $32, R17; \ MOVD $48, R18; \ MOVD $64, R19; \ MOVD $80, R20; \ MOVD $96, R21; \ MOVD $112, R22; \ P8_LXVB16X(blk_inp,R0,V1); \ P8_LXVB16X(blk_inp,R16,V2); \ P8_LXVB16X(blk_inp,R17,V3); \ P8_LXVB16X(blk_inp,R18,V4); \ P8_LXVB16X(blk_inp,R19,V5); \ P8_LXVB16X(blk_inp,R20,V6); \ P8_LXVB16X(blk_inp,R21,V7); \ P8_LXVB16X(blk_inp,R22,V8); \ ADD $128, blk_inp // Finish encryption on 8 streams and // XOR with input block #define VCIPHERLAST8_XOR_INPUT \ VCIPHERLAST V15, V23, V15; \ VCIPHERLAST V16, V23, V16; \ VCIPHERLAST V17, V23, V17; \ VCIPHERLAST V18, V23, V18; \ VCIPHERLAST V19, V23, V19; \ VCIPHERLAST V20, V23, V20; \ VCIPHERLAST V21, V23, V21; \ VCIPHERLAST V22, V23, V22; \ XXLXOR V1, V15, V1; \ XXLXOR V2, V16, V2; \ XXLXOR V3, V17, V3; \ XXLXOR V4, V18, V4; \ XXLXOR V5, V19, V5; \ XXLXOR V6, V20, V6; \ XXLXOR V7, V21, V7; \ XXLXOR V8, V22, V8 // Finish encryption on 4 streams and // XOR with input block #define VCIPHERLAST4_XOR_INPUT \ VCIPHERLAST V15, V23, V15; \ VCIPHERLAST V16, V23, V16; \ VCIPHERLAST V17, V23, V17; \ VCIPHERLAST V18, V23, V18; \ XXLXOR V1, V15, V1; \ XXLXOR V2, V16, V2; \ XXLXOR V3, V17, V3; \ XXLXOR V4, V18, V4 // Store output block from V1-V8 // in big endian order and // Update blk_out by 128 #define STORE_OUTPUT_BLOCK128(blk_out) \ P8_STXVB16X(V1,blk_out,R0); \ P8_STXVB16X(V2,blk_out,R16); \ P8_STXVB16X(V3,blk_out,R17); \ P8_STXVB16X(V4,blk_out,R18); \ P8_STXVB16X(V5,blk_out,R19); \ P8_STXVB16X(V6,blk_out,R20); \ P8_STXVB16X(V7,blk_out,R21); \ P8_STXVB16X(V8,blk_out,R22); \ ADD $128, blk_out // Store output block from V1-V4 // in big endian order and // Update blk_out by 64 #define STORE_OUTPUT_BLOCK64(blk_out) \ P8_STXVB16X(V1,blk_out,R0); \ P8_STXVB16X(V2,blk_out,R16); \ P8_STXVB16X(V3,blk_out,R17); \ P8_STXVB16X(V4,blk_out,R18); \ ADD $64, blk_out // func gcmInit(productTable *[256]byte, h []byte) TEXT ·gcmInit(SB), NOSPLIT, $0-32 MOVD productTable+0(FP), XIP MOVD h+8(FP), HTBL MOVD $0x10, R8 MOVD $0x20, R9 MOVD $0x30, R10 LXVD2X (HTBL)(R0), VH // Load H VSPLTISB $-16, XC2 // 0xf0 VSPLTISB $1, T0 // one VADDUBM XC2, XC2, XC2 // 0xe0 VXOR ZERO, ZERO, ZERO VOR XC2, T0, XC2 // 0xe1 VSLDOI $15, XC2, ZERO, XC2 // 0xe1... VSLDOI $1, ZERO, T0, T1 // ...1 VADDUBM XC2, XC2, XC2 // 0xc2... VSPLTISB $7, T2 VOR XC2, T1, XC2 // 0xc2....01 VSPLTB $0, H, T1 // most significant byte VSL H, T0, H // H<<=1 VSRAB T1, T2, T1 // broadcast carry bit VAND T1, XC2, T1 VXOR H, T1, IN // twisted H VSLDOI $8, IN, IN, H // twist even more ... VSLDOI $8, ZERO, XC2, XC2 // 0xc2.0 VSLDOI $8, ZERO, H, HL // ... and split VSLDOI $8, H, ZERO, HH STXVD2X VXC2, (XIP+R0) // save pre-computed table STXVD2X VHL, (XIP+R8) MOVD $0x40, R8 STXVD2X VH, (XIP+R9) MOVD $0x50, R9 STXVD2X VHH, (XIP+R10) MOVD $0x60, R10 VPMSUMD IN, HL, XL // H.lo·H.lo VPMSUMD IN, H, XM // H.hi·H.lo+H.lo·H.hi VPMSUMD IN, HH, XH // H.hi·H.hi VPMSUMD XL, XC2, T2 // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD XL, XC2, XL VXOR T1, XH, T1 VXOR XL, T1, IN1 VSLDOI $8, IN1, IN1, H2 VSLDOI $8, ZERO, H2, H2L VSLDOI $8, H2, ZERO, H2H STXVD2X VH2L, (XIP+R8) // save H^2 MOVD $0x70, R8 STXVD2X VH2, (XIP+R9) MOVD $0x80, R9 STXVD2X VH2H, (XIP+R10) MOVD $0x90, R10 VPMSUMD IN, H2L, XL // H.lo·H^2.lo VPMSUMD IN1, H2L, XL1 // H^2.lo·H^2.lo VPMSUMD IN, H2, XM // H.hi·H^2.lo+H.lo·H^2.hi VPMSUMD IN1, H2, XM1 // H^2.hi·H^2.lo+H^2.lo·H^2.hi VPMSUMD IN, H2H, XH // H.hi·H^2.hi VPMSUMD IN1, H2H, XH1 // H^2.hi·H^2.hi VPMSUMD XL, XC2, T2 // 1st reduction phase VPMSUMD XL1, XC2, HH // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VSLDOI $8, XM1, ZERO, HL VSLDOI $8, ZERO, XM1, H VXOR XL, T0, XL VXOR XH, T1, XH VXOR XL1, HL, XL1 VXOR XH1, H, XH1 VSLDOI $8, XL, XL, XL VSLDOI $8, XL1, XL1, XL1 VXOR XL, T2, XL VXOR XL1, HH, XL1 VSLDOI $8, XL, XL, T1 // 2nd reduction phase VSLDOI $8, XL1, XL1, H // 2nd reduction phase VPMSUMD XL, XC2, XL VPMSUMD XL1, XC2, XL1 VXOR T1, XH, T1 VXOR H, XH1, H VXOR XL, T1, XL VXOR XL1, H, XL1 VSLDOI $8, XL, XL, H VSLDOI $8, XL1, XL1, H2 VSLDOI $8, ZERO, H, HL VSLDOI $8, H, ZERO, HH VSLDOI $8, ZERO, H2, H2L VSLDOI $8, H2, ZERO, H2H STXVD2X VHL, (XIP+R8) // save H^3 MOVD $0xa0, R8 STXVD2X VH, (XIP+R9) MOVD $0xb0, R9 STXVD2X VHH, (XIP+R10) MOVD $0xc0, R10 STXVD2X VH2L, (XIP+R8) // save H^4 STXVD2X VH2, (XIP+R9) STXVD2X VH2H, (XIP+R10) RET // func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int) TEXT ·gcmHash(SB), NOSPLIT, $0-64 MOVD output+0(FP), XIP MOVD productTable+24(FP), HTBL MOVD inp+32(FP), INP MOVD len+56(FP), LEN MOVD $0x10, R8 MOVD $0x20, R9 MOVD $0x30, R10 LXVD2X (XIP)(R0), VXL // load Xi LXVD2X (HTBL)(R8), VHL // load pre-computed table MOVD $0x40, R8 LXVD2X (HTBL)(R9), VH MOVD $0x50, R9 LXVD2X (HTBL)(R10), VHH MOVD $0x60, R10 LXVD2X (HTBL)(R0), VXC2 #ifdef GOARCH_ppc64le LVSL (R0)(R0), LEMASK VSPLTISB $0x07, T0 VXOR LEMASK, T0, LEMASK VPERM XL, XL, LEMASK, XL #endif VXOR ZERO, ZERO, ZERO CMPU LEN, $64 BGE gcm_ghash_p8_4x LXVD2X (INP)(R0), VIN ADD $16, INP, INP SUBCCC $16, LEN, LEN #ifdef GOARCH_ppc64le VPERM IN, IN, LEMASK, IN #endif VXOR IN, XL, IN BEQ short LXVD2X (HTBL)(R8), VH2L // load H^2 MOVD $16, R8 LXVD2X (HTBL)(R9), VH2 ADD LEN, INP, R9 // end of input LXVD2X (HTBL)(R10), VH2H loop_2x: LXVD2X (INP)(R0), VIN1 #ifdef GOARCH_ppc64le VPERM IN1, IN1, LEMASK, IN1 #endif SUBC $32, LEN, LEN VPMSUMD IN, H2L, XL // H^2.lo·Xi.lo VPMSUMD IN1, HL, XL1 // H.lo·Xi+1.lo SUBE R11, R11, R11 // borrow?-1:0 VPMSUMD IN, H2, XM // H^2.hi·Xi.lo+H^2.lo·Xi.hi VPMSUMD IN1, H, XM1 // H.hi·Xi+1.lo+H.lo·Xi+1.hi AND LEN, R11, R11 VPMSUMD IN, H2H, XH // H^2.hi·Xi.hi VPMSUMD IN1, HH, XH1 // H.hi·Xi+1.hi ADD R11, INP, INP VXOR XL, XL1, XL VXOR XM, XM1, XM VPMSUMD XL, XC2, T2 // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XH, XH1, XH VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL LXVD2X (INP)(R8), VIN ADD $32, INP, INP VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD XL, XC2, XL #ifdef GOARCH_ppc64le VPERM IN, IN, LEMASK, IN #endif VXOR T1, XH, T1 VXOR IN, T1, IN VXOR IN, XL, IN CMP R9, INP BGT loop_2x // done yet? CMPWU LEN, $0 BNE even short: VPMSUMD IN, HL, XL // H.lo·Xi.lo VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi VPMSUMD IN, HH, XH // H.hi·Xi.hi VPMSUMD XL, XC2, T2 // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD XL, XC2, XL VXOR T1, XH, T1 even: VXOR XL, T1, XL #ifdef GOARCH_ppc64le VPERM XL, XL, LEMASK, XL #endif STXVD2X VXL, (XIP+R0) OR R12, R12, R12 // write out Xi RET gcm_ghash_p8_4x: LVSL (R8)(R0), T0 // 0x0001..0e0f MOVD $0x70, R8 LXVD2X (HTBL)(R9), VH2 MOVD $0x80, R9 VSPLTISB $8, T1 // 0x0808..0808 MOVD $0x90, R10 LXVD2X (HTBL)(R8), VH3L // load H^3 MOVD $0xa0, R8 LXVD2X (HTBL)(R9), VH3 MOVD $0xb0, R9 LXVD2X (HTBL)(R10), VH3H MOVD $0xc0, R10 LXVD2X (HTBL)(R8), VH4L // load H^4 MOVD $0x10, R8 LXVD2X (HTBL)(R9), VH4 MOVD $0x20, R9 LXVD2X (HTBL)(R10), VH4H MOVD $0x30, R10 VSLDOI $8, ZERO, T1, T2 // 0x0000..0808 VADDUBM T0, T2, HIPERM // 0x0001..1617 VADDUBM T1, HIPERM, LOPERM // 0x0809..1e1f SRD $4, LEN, LEN // this allows to use sign bit as carry LXVD2X (INP)(R0), VIN0 // load input LXVD2X (INP)(R8), VIN1 SUBCCC $8, LEN, LEN LXVD2X (INP)(R9), VIN2 LXVD2X (INP)(R10), VIN3 ADD $0x40, INP, INP #ifdef GOARCH_ppc64le VPERM IN0, IN0, LEMASK, IN0 VPERM IN1, IN1, LEMASK, IN1 VPERM IN2, IN2, LEMASK, IN2 VPERM IN3, IN3, LEMASK, IN3 #endif VXOR IN0, XL, XH VPMSUMD IN1, H3L, XL1 VPMSUMD IN1, H3, XM1 VPMSUMD IN1, H3H, XH1 VPERM H2, H, HIPERM, H21L VPERM IN2, IN3, LOPERM, T0 VPERM H2, H, LOPERM, H21H VPERM IN2, IN3, HIPERM, T1 VPMSUMD IN2, H2, XM2 // H^2.lo·Xi+2.hi+H^2.hi·Xi+2.lo VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+2.lo+H.lo·Xi+3.lo VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+2.hi+H.hi·Xi+3.hi VXOR XM2, XM1, XM2 VXOR XL3, XL1, XL3 VXOR XM3, XM2, XM3 VXOR XH3, XH1, XH3 BLT tail_4x loop_4x: LXVD2X (INP)(R0), VIN0 LXVD2X (INP)(R8), VIN1 SUBCCC $4, LEN, LEN LXVD2X (INP)(R9), VIN2 LXVD2X (INP)(R10), VIN3 ADD $0x40, INP, INP #ifdef GOARCH_ppc64le VPERM IN1, IN1, LEMASK, IN1 VPERM IN2, IN2, LEMASK, IN2 VPERM IN3, IN3, LEMASK, IN3 VPERM IN0, IN0, LEMASK, IN0 #endif VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi VPMSUMD IN1, H3L, XL1 VPMSUMD IN1, H3, XM1 VPMSUMD IN1, H3H, XH1 VXOR XL, XL3, XL VXOR XM, XM3, XM VXOR XH, XH3, XH VPERM IN2, IN3, LOPERM, T0 VPERM IN2, IN3, HIPERM, T1 VPMSUMD XL, XC2, T2 // 1st reduction phase VPMSUMD T0, H21L, XL3 // H.lo·Xi+3.lo +H^2.lo·Xi+2.lo VPMSUMD T1, H21H, XH3 // H.hi·Xi+3.hi +H^2.hi·Xi+2.hi VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD IN2, H2, XM2 // H^2.hi·Xi+2.lo+H^2.lo·Xi+2.hi VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi VPMSUMD XL, XC2, XL VXOR XL3, XL1, XL3 VXOR XH3, XH1, XH3 VXOR XH, IN0, XH VXOR XM2, XM1, XM2 VXOR XH, T1, XH VXOR XM3, XM2, XM3 VXOR XH, XL, XH BGE loop_4x tail_4x: VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi VXOR XL, XL3, XL VXOR XM, XM3, XM VPMSUMD XL, XC2, T2 // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XH, XH3, XH VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD XL, XC2, XL VXOR T1, XH, T1 VXOR XL, T1, XL ADDCCC $4, LEN, LEN BEQ done_4x LXVD2X (INP)(R0), VIN0 CMPU LEN, $2 MOVD $-4, LEN BLT one LXVD2X (INP)(R8), VIN1 BEQ two three: LXVD2X (INP)(R9), VIN2 #ifdef GOARCH_ppc64le VPERM IN0, IN0, LEMASK, IN0 VPERM IN1, IN1, LEMASK, IN1 VPERM IN2, IN2, LEMASK, IN2 #endif VXOR IN0, XL, XH VOR H3L, H3L, H4L VOR H3, H3, H4 VOR H3H, H3H, H4H VPERM IN1, IN2, LOPERM, T0 VPERM IN1, IN2, HIPERM, T1 VPMSUMD IN1, H2, XM2 // H^2.lo·Xi+1.hi+H^2.hi·Xi+1.lo VPMSUMD IN2, H, XM3 // H.hi·Xi+2.lo +H.lo·Xi+2.hi VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+1.lo+H.lo·Xi+2.lo VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+1.hi+H.hi·Xi+2.hi VXOR XM3, XM2, XM3 JMP tail_4x two: #ifdef GOARCH_ppc64le VPERM IN0, IN0, LEMASK, IN0 VPERM IN1, IN1, LEMASK, IN1 #endif VXOR IN, XL, XH VPERM ZERO, IN1, LOPERM, T0 VPERM ZERO, IN1, HIPERM, T1 VSLDOI $8, ZERO, H2, H4L VOR H2, H2, H4 VSLDOI $8, H2, ZERO, H4H VPMSUMD T0, H21L, XL3 // H.lo·Xi+1.lo VPMSUMD IN1, H, XM3 // H.hi·Xi+1.lo+H.lo·Xi+2.hi VPMSUMD T1, H21H, XH3 // H.hi·Xi+1.hi JMP tail_4x one: #ifdef GOARCH_ppc64le VPERM IN0, IN0, LEMASK, IN0 #endif VSLDOI $8, ZERO, H, H4L VOR H, H, H4 VSLDOI $8, H, ZERO, H4H VXOR IN0, XL, XH VXOR XL3, XL3, XL3 VXOR XM3, XM3, XM3 VXOR XH3, XH3, XH3 JMP tail_4x done_4x: #ifdef GOARCH_ppc64le VPERM XL, XL, LEMASK, XL #endif STXVD2X VXL, (XIP+R0) // write out Xi RET // func gcmMul(output []byte, productTable *[256]byte) TEXT ·gcmMul(SB), NOSPLIT, $0-32 MOVD output+0(FP), XIP MOVD productTable+24(FP), HTBL MOVD $0x10, R8 MOVD $0x20, R9 MOVD $0x30, R10 LXVD2X (XIP)(R0), VIN // load Xi LXVD2X (HTBL)(R8), VHL // Load pre-computed table LXVD2X (HTBL)(R9), VH LXVD2X (HTBL)(R10), VHH LXVD2X (HTBL)(R0), VXC2 #ifdef GOARCH_ppc64le VSPLTISB $0x07, T0 VXOR LEMASK, T0, LEMASK VPERM IN, IN, LEMASK, IN #endif VXOR ZERO, ZERO, ZERO VPMSUMD IN, HL, XL // H.lo·Xi.lo VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi VPMSUMD IN, HH, XH // H.hi·Xi.hi VPMSUMD XL, XC2, T2 // 1st reduction phase VSLDOI $8, XM, ZERO, T0 VSLDOI $8, ZERO, XM, T1 VXOR XL, T0, XL VXOR XH, T1, XH VSLDOI $8, XL, XL, XL VXOR XL, T2, XL VSLDOI $8, XL, XL, T1 // 2nd reduction phase VPMSUMD XL, XC2, XL VXOR T1, XH, T1 VXOR XL, T1, XL #ifdef GOARCH_ppc64le VPERM XL, XL, LEMASK, XL #endif STXVD2X VXL, (XIP+R0) // write out Xi RET #define BLK_INP R3 #define BLK_OUT R4 #define BLK_KEY R5 #define KEY_LEN R6 #define BLK_IDX R7 #define IDX R8 #define IN_LEN R9 #define COUNTER R10 #define CONPTR R14 #define MASK V5 // Implementation of the counterCrypt function in assembler. // Original loop is unrolled to allow for multiple encryption // streams to be done in parallel, which is achieved by interleaving // vcipher instructions from each stream. This is also referred to as // stitching, and provides significant performance improvements. // Some macros are defined which enable execution for big or little // endian as well as different ISA targets. //func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte, key[gcmBlockSize]uint32) //func counterCryptASM(xr, out, in, counter, key) TEXT ·counterCryptASM(SB), NOSPLIT, $16-72 MOVD xr(FP), KEY_LEN MOVD out+8(FP), BLK_OUT MOVD out_len+16(FP), R8 MOVD in+32(FP), BLK_INP MOVD in_len+40(FP), IN_LEN MOVD counter+56(FP), COUNTER MOVD key+64(FP), BLK_KEY // Set up permute string when needed. #ifdef NEEDS_ESPERM MOVD $·rcon(SB), R14 LVX (R14), ESPERM // Permute value for P8_ macros. #endif SETUP_COUNTER // V30 Counter V31 BE {0, 0, 0, 1} LOAD_KEYS(BLK_KEY, KEY_LEN) // VS1 - VS10/12/14 based on keysize CMP IN_LEN, $128 BLT block64 block128_loop: // Do 8 encryptions in parallel by setting // input values in V15-V22 and executing // vcipher on the updated value and the keys. GEN_VCIPHER_8_INPUTS VCIPHER_8X1_KEY(VS1) VCIPHER_8X1_KEY(VS2) VCIPHER_8X1_KEY(VS3) VCIPHER_8X1_KEY(VS4) VCIPHER_8X1_KEY(VS5) VCIPHER_8X1_KEY(VS6) VCIPHER_8X1_KEY(VS7) VCIPHER_8X1_KEY(VS8) VCIPHER_8X1_KEY(VS9) // Additional encryptions are done based on // the key length, with the last key moved // to V23 for use with VCIPHERLAST. // CR2 = CMP key_len, $12 XXLOR VS10, VS10, V23 BLT CR2, block128_last // key_len = 10 VCIPHER_8X1_KEY(VS10) VCIPHER_8X1_KEY(VS11) XXLOR VS12,VS12,V23 BEQ CR2, block128_last // ken_len = 12 VCIPHER_8X1_KEY(VS12) VCIPHER_8X1_KEY(VS13) XXLOR VS14,VS14,V23 // key_len = 14 block128_last: // vcipher encryptions are in V15-V22 at this // point with vcipherlast remaining to be done. // Load input block into V1-V8, setting index offsets // in R16-R22 to use with the STORE. LOAD_INPUT_BLOCK128(BLK_INP) // Do VCIPHERLAST on the last key for each encryption // stream and XOR the result with the corresponding // value from the input block. VCIPHERLAST8_XOR_INPUT // Store the results (8*16) and update BLK_OUT by 128. STORE_OUTPUT_BLOCK128(BLK_OUT) ADD $-128, IN_LEN // input size CMP IN_LEN, $128 // check if >= blocksize BGE block128_loop // next input block CMP IN_LEN, $0 BEQ done block64: CMP IN_LEN, $64 // Check if >= 64 BLT block16_loop // Do 4 encryptions in parallel by setting // input values in V15-V18 and executing // vcipher on the updated value and the keys. GEN_VCIPHER_4_INPUTS VCIPHER_4X1_KEY(VS1) VCIPHER_4X1_KEY(VS2) VCIPHER_4X1_KEY(VS3) VCIPHER_4X1_KEY(VS4) VCIPHER_4X1_KEY(VS5) VCIPHER_4X1_KEY(VS6) VCIPHER_4X1_KEY(VS7) VCIPHER_4X1_KEY(VS8) VCIPHER_4X1_KEY(VS9) // Check key length based on CR2 // Move last key to V23 for use with later vcipherlast XXLOR VS10, VS10, V23 BLT CR2, block64_last // size = 10 VCIPHER_4X1_KEY(VS10) // Encrypt next 2 keys VCIPHER_4X1_KEY(VS11) XXLOR VS12, VS12, V23 BEQ CR2, block64_last // size = 12 VCIPHER_4X1_KEY(VS12) // Encrypt last 2 keys VCIPHER_4X1_KEY(VS13) XXLOR VS14, VS14, V23 // size = 14 block64_last: LOAD_INPUT_BLOCK64(BLK_INP) // Load 64 bytes of input // Do VCIPHERLAST on the last for each encryption // stream and XOR the result with the corresponding // value from the input block. VCIPHERLAST4_XOR_INPUT // Store the results (4*16) and update BLK_OUT by 64. STORE_OUTPUT_BLOCK64(BLK_OUT) ADD $-64, IN_LEN // decrement input block length CMP IN_LEN, $0 // check for remaining length BEQ done block16_loop: CMP IN_LEN, $16 // More input BLT final_block // If not, then handle partial block // Single encryption, no stitching GEN_VCIPHER_INPUT // Generate input value for single encryption VCIPHER_1X9_KEYS(V15) // Encrypt V15 value with 9 keys XXLOR VS10, VS10, V23 // Last key -> V23 for later vcipiherlast // Key length based on CR2. (LT=10, EQ=12, GT=14) BLT CR2, block16_last // Finish for key size 10 VCIPHER_1X2_KEYS(V15, VS10, VS11) // Encrypt V15 with 2 more keys XXLOR VS12, VS12, V23 // Last key -> V23 for later vcipherlast BEQ CR2, block16_last // Finish for key size 12 VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys XXLOR VS14, VS14, V23 // Last key -> V23 for vcipherlast with key size 14 block16_last: P8_LXVB16X(BLK_INP, R0, V1) // Load input VCIPHERLAST V15, V23, V15 // Encrypt last value in V23 XXLXOR V15, V1, V1 // XOR with input P8_STXVB16X(V1,R0,BLK_OUT) // Store final encryption value to output ADD $16, BLK_INP // Increment input pointer ADD $16, BLK_OUT // Increment output pointer ADD $-16, IN_LEN // Decrement input length BR block16_loop // Check for next final_block: CMP IN_LEN, $0 BEQ done GEN_VCIPHER_INPUT // Generate input value for partial encryption VCIPHER_1X9_KEYS(V15) // Encrypt V15 with 9 keys XXLOR VS10, VS10, V23 // Save possible last key BLT CR2, final_block_last VCIPHER_1X2_KEYS(V15, VS10, VS11) // Encrypt V15 with next 2 keys XXLOR VS12, VS12, V23 // Save possible last key BEQ CR2, final_block_last VCIPHER_1X2_KEYS(V15, VS12, VS13) // Encrypt V15 with last 2 keys XXLOR VS14, VS14, V23 // Save last key final_block_last: VCIPHERLAST V15, V23, V15 // Finish encryption #ifdef GOPPC64_power10 // set up length SLD $56, IN_LEN, R17 LXVLL BLK_INP, R17, V25 VXOR V25, V15, V25 STXVLL V25, BLK_OUT, R17 #else ADD $32, R1, MASK_PTR MOVD $0, R16 P8_STXVB16X(V15, MASK_PTR, R0) CMP IN_LEN, $8 BLT next4 MOVD 0(MASK_PTR), R14 MOVD 0(BLK_INP), R15 XOR R14, R15, R14 MOVD R14, 0(BLK_OUT) ADD $8, R16 ADD $-8, IN_LEN next4: CMP IN_LEN, $4 BLT next2 MOVWZ (BLK_INP)(R16), R15 MOVWZ (MASK_PTR)(R16), R14 XOR R14, R15, R14 MOVW R14, (R16)(BLK_OUT) ADD $4, R16 ADD $-4, IN_LEN next2: CMP IN_LEN, $2 BLT next1 MOVHZ (BLK_INP)(R16), R15 MOVHZ (MASK_PTR)(R16), R14 XOR R14, R15, R14 MOVH R14, (R16)(BLK_OUT) ADD $2, R16 ADD $-2, IN_LEN next1: CMP IN_LEN, $1 BLT done MOVBZ (MASK_PTR)(R16), R14 MOVBZ (BLK_INP)(R16), R15 XOR R14, R15, R14 MOVB R14, (R16)(BLK_OUT) #endif done: // Save the updated counter value P8_STXVB16X(V30, COUNTER, R0) // Clear the keys XXLXOR VS0, VS0, VS0 XXLXOR VS1, VS1, VS1 XXLXOR VS2, VS2, VS2 XXLXOR VS3, VS3, VS3 XXLXOR VS4, VS4, VS4 XXLXOR VS5, VS5, VS5 XXLXOR VS6, VS6, VS6 XXLXOR VS7, VS7, VS7 XXLXOR VS8, VS8, VS8 XXLXOR VS9, VS9, VS9 XXLXOR VS10, VS10, VS10 XXLXOR VS11, VS11, VS11 XXLXOR VS12, VS12, VS12 XXLXOR VS13, VS13, VS13 XXLXOR VS14, VS14, VS14 RET
SIMPAN
BATAL