Bypass Manager Pro

Editing: sha1block_amd64.s

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// AVX2 version by Intel, same algorithm as code in Linux kernel:
// https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha1_avx2_x86_64_asm.S
// Authors:
// Ilya Albrekht <ilya.albrekht@intel.com>
// Maxim Locktyukhin <maxim.locktyukhin@intel.com>
// Ronen Zohar <ronen.zohar@intel.com>
// Chandramouli Narayanan <mouli@linux.intel.com>

#include "textflag.h"

// SHA-1 block routine. See sha1block.go for Go equivalent.
//
// There are 80 rounds of 4 types:
//   - rounds 0-15 are type 1 and load data (ROUND1 macro).
//   - rounds 16-19 are type 1 and do not load data (ROUND1x macro).
//   - rounds 20-39 are type 2 and do not load data (ROUND2 macro).
//   - rounds 40-59 are type 3 and do not load data (ROUND3 macro).
//   - rounds 60-79 are type 4 and do not load data (ROUND4 macro).
//
// Each round loads or shuffles the data, then computes a per-round
// function of b, c, d, and then mixes the result into and rotates the
// five registers a, b, c, d, e holding the intermediate results.
//
// The register rotation is implemented by rotating the arguments to
// the round macros instead of by explicit move instructions.

#define LOAD(index) \
	MOVL	(index*4)(SI), R10; \
	BSWAPL	R10; \
	MOVL	R10, (index*4)(SP)

#define SHUFFLE(index) \
	MOVL	(((index)&0xf)*4)(SP), R10; \
	XORL	(((index-3)&0xf)*4)(SP), R10; \
	XORL	(((index-8)&0xf)*4)(SP), R10; \
	XORL	(((index-14)&0xf)*4)(SP), R10; \
	ROLL	$1, R10; \
	MOVL	R10, (((index)&0xf)*4)(SP)

#define FUNC1(a, b, c, d, e) \
	MOVL	d, R9; \
	XORL	c, R9; \
	ANDL	b, R9; \
	XORL	d, R9

#define FUNC2(a, b, c, d, e) \
	MOVL	b, R9; \
	XORL	c, R9; \
	XORL	d, R9

#define FUNC3(a, b, c, d, e) \
	MOVL	b, R8; \
	ORL	c, R8; \
	ANDL	d, R8; \
	MOVL	b, R9; \
	ANDL	c, R9; \
	ORL	R8, R9

#define FUNC4 FUNC2

#define MIX(a, b, c, d, e, const) \
	ROLL	$30, b; \
	ADDL	R9, e; \
	MOVL	a, R8; \
	ROLL	$5, R8; \
	LEAL	const(e)(R10*1), e; \
	ADDL	R8, e

#define ROUND1(a, b, c, d, e, index) \
	LOAD(index); \
	FUNC1(a, b, c, d, e); \
	MIX(a, b, c, d, e, 0x5A827999)

#define ROUND1x(a, b, c, d, e, index) \
	SHUFFLE(index); \
	FUNC1(a, b, c, d, e); \
	MIX(a, b, c, d, e, 0x5A827999)

#define ROUND2(a, b, c, d, e, index) \
	SHUFFLE(index); \
	FUNC2(a, b, c, d, e); \
	MIX(a, b, c, d, e, 0x6ED9EBA1)

#define ROUND3(a, b, c, d, e, index) \
	SHUFFLE(index); \
	FUNC3(a, b, c, d, e); \
	MIX(a, b, c, d, e, 0x8F1BBCDC)

#define ROUND4(a, b, c, d, e, index) \
	SHUFFLE(index); \
	FUNC4(a, b, c, d, e); \
	MIX(a, b, c, d, e, 0xCA62C1D6)

TEXT ·blockAMD64(SB),NOSPLIT,$64-32
	MOVQ	dig+0(FP),	BP
	MOVQ	p_base+8(FP),	SI
	MOVQ	p_len+16(FP),	DX
	SHRQ	$6,		DX
	SHLQ	$6,		DX

LEAQ	(SI)(DX*1),	DI
	MOVL	(0*4)(BP),	AX
	MOVL	(1*4)(BP),	BX
	MOVL	(2*4)(BP),	CX
	MOVL	(3*4)(BP),	DX
	MOVL	(4*4)(BP),	BP

CMPQ	SI,		DI
	JEQ	end

loop:
	MOVL	AX,	R11
	MOVL	BX,	R12
	MOVL	CX,	R13
	MOVL	DX,	R14
	MOVL	BP,	R15

ROUND1(AX, BX, CX, DX, BP, 0)
	ROUND1(BP, AX, BX, CX, DX, 1)
	ROUND1(DX, BP, AX, BX, CX, 2)
	ROUND1(CX, DX, BP, AX, BX, 3)
	ROUND1(BX, CX, DX, BP, AX, 4)
	ROUND1(AX, BX, CX, DX, BP, 5)
	ROUND1(BP, AX, BX, CX, DX, 6)
	ROUND1(DX, BP, AX, BX, CX, 7)
	ROUND1(CX, DX, BP, AX, BX, 8)
	ROUND1(BX, CX, DX, BP, AX, 9)
	ROUND1(AX, BX, CX, DX, BP, 10)
	ROUND1(BP, AX, BX, CX, DX, 11)
	ROUND1(DX, BP, AX, BX, CX, 12)
	ROUND1(CX, DX, BP, AX, BX, 13)
	ROUND1(BX, CX, DX, BP, AX, 14)
	ROUND1(AX, BX, CX, DX, BP, 15)

ROUND1x(BP, AX, BX, CX, DX, 16)
	ROUND1x(DX, BP, AX, BX, CX, 17)
	ROUND1x(CX, DX, BP, AX, BX, 18)
	ROUND1x(BX, CX, DX, BP, AX, 19)

ROUND2(AX, BX, CX, DX, BP, 20)
	ROUND2(BP, AX, BX, CX, DX, 21)
	ROUND2(DX, BP, AX, BX, CX, 22)
	ROUND2(CX, DX, BP, AX, BX, 23)
	ROUND2(BX, CX, DX, BP, AX, 24)
	ROUND2(AX, BX, CX, DX, BP, 25)
	ROUND2(BP, AX, BX, CX, DX, 26)
	ROUND2(DX, BP, AX, BX, CX, 27)
	ROUND2(CX, DX, BP, AX, BX, 28)
	ROUND2(BX, CX, DX, BP, AX, 29)
	ROUND2(AX, BX, CX, DX, BP, 30)
	ROUND2(BP, AX, BX, CX, DX, 31)
	ROUND2(DX, BP, AX, BX, CX, 32)
	ROUND2(CX, DX, BP, AX, BX, 33)
	ROUND2(BX, CX, DX, BP, AX, 34)
	ROUND2(AX, BX, CX, DX, BP, 35)
	ROUND2(BP, AX, BX, CX, DX, 36)
	ROUND2(DX, BP, AX, BX, CX, 37)
	ROUND2(CX, DX, BP, AX, BX, 38)
	ROUND2(BX, CX, DX, BP, AX, 39)

ROUND3(AX, BX, CX, DX, BP, 40)
	ROUND3(BP, AX, BX, CX, DX, 41)
	ROUND3(DX, BP, AX, BX, CX, 42)
	ROUND3(CX, DX, BP, AX, BX, 43)
	ROUND3(BX, CX, DX, BP, AX, 44)
	ROUND3(AX, BX, CX, DX, BP, 45)
	ROUND3(BP, AX, BX, CX, DX, 46)
	ROUND3(DX, BP, AX, BX, CX, 47)
	ROUND3(CX, DX, BP, AX, BX, 48)
	ROUND3(BX, CX, DX, BP, AX, 49)
	ROUND3(AX, BX, CX, DX, BP, 50)
	ROUND3(BP, AX, BX, CX, DX, 51)
	ROUND3(DX, BP, AX, BX, CX, 52)
	ROUND3(CX, DX, BP, AX, BX, 53)
	ROUND3(BX, CX, DX, BP, AX, 54)
	ROUND3(AX, BX, CX, DX, BP, 55)
	ROUND3(BP, AX, BX, CX, DX, 56)
	ROUND3(DX, BP, AX, BX, CX, 57)
	ROUND3(CX, DX, BP, AX, BX, 58)
	ROUND3(BX, CX, DX, BP, AX, 59)

ROUND4(AX, BX, CX, DX, BP, 60)
	ROUND4(BP, AX, BX, CX, DX, 61)
	ROUND4(DX, BP, AX, BX, CX, 62)
	ROUND4(CX, DX, BP, AX, BX, 63)
	ROUND4(BX, CX, DX, BP, AX, 64)
	ROUND4(AX, BX, CX, DX, BP, 65)
	ROUND4(BP, AX, BX, CX, DX, 66)
	ROUND4(DX, BP, AX, BX, CX, 67)
	ROUND4(CX, DX, BP, AX, BX, 68)
	ROUND4(BX, CX, DX, BP, AX, 69)
	ROUND4(AX, BX, CX, DX, BP, 70)
	ROUND4(BP, AX, BX, CX, DX, 71)
	ROUND4(DX, BP, AX, BX, CX, 72)
	ROUND4(CX, DX, BP, AX, BX, 73)
	ROUND4(BX, CX, DX, BP, AX, 74)
	ROUND4(AX, BX, CX, DX, BP, 75)
	ROUND4(BP, AX, BX, CX, DX, 76)
	ROUND4(DX, BP, AX, BX, CX, 77)
	ROUND4(CX, DX, BP, AX, BX, 78)
	ROUND4(BX, CX, DX, BP, AX, 79)

ADDL	R11, AX
	ADDL	R12, BX
	ADDL	R13, CX
	ADDL	R14, DX
	ADDL	R15, BP

ADDQ	$64, SI
	CMPQ	SI, DI
	JB	loop

end:
	MOVQ	dig+0(FP), DI
	MOVL	AX, (0*4)(DI)
	MOVL	BX, (1*4)(DI)
	MOVL	CX, (2*4)(DI)
	MOVL	DX, (3*4)(DI)
	MOVL	BP, (4*4)(DI)
	RET

// This is the implementation using AVX2, BMI1 and BMI2. It is based on:
// "SHA-1 implementation with Intel(R) AVX2 instruction set extensions"
// From http://software.intel.com/en-us/articles
// (look for improving-the-performance-of-the-secure-hash-algorithm-1)
// This implementation is 2x unrolled, and interleaves vector instructions,
// used to precompute W, with scalar computation of current round
// for optimal scheduling.

// Trivial helper macros.
#define UPDATE_HASH(A,TB,C,D,E) \
	ADDL	(R9), A \
	MOVL	A, (R9) \
	ADDL	4(R9), TB \
	MOVL	TB, 4(R9) \
	ADDL	8(R9), C \
	MOVL	C, 8(R9) \
	ADDL	12(R9), D \
	MOVL	D, 12(R9) \
	ADDL	16(R9), E \
	MOVL	E, 16(R9)

// Helper macros for PRECALC, which does precomputations
#define PRECALC_0(OFFSET) \
	VMOVDQU   OFFSET(R10),X0

#define PRECALC_1(OFFSET) \
	VINSERTI128 $1, OFFSET(R13), Y0, Y0

#define PRECALC_2(YREG) \
	VPSHUFB Y10, Y0, YREG

#define PRECALC_4(YREG,K_OFFSET) \
	VPADDD K_OFFSET(R8), YREG, Y0

#define PRECALC_7(OFFSET) \
	VMOVDQU Y0, (OFFSET*2)(R14)

// Message scheduling pre-compute for rounds 0-15
// R13 is a pointer to even 64-byte block
// R10 is a pointer to odd 64-byte block
// R14 is a pointer to temp buffer
// X0 is used as temp register
// YREG is clobbered as part of computation
// OFFSET chooses 16 byte chunk within a block
// R8 is a pointer to constants block
// K_OFFSET chooses K constants relevant to this round
// X10 holds swap mask
#define PRECALC_00_15(OFFSET,YREG) \
	PRECALC_0(OFFSET) \
	PRECALC_1(OFFSET) \
	PRECALC_2(YREG) \
	PRECALC_4(YREG,0x0) \
	PRECALC_7(OFFSET)

// Helper macros for PRECALC_16_31
#define PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \
	VPALIGNR $8, REG_SUB_16, REG_SUB_12, REG \  // w[i-14]
	VPSRLDQ $4, REG_SUB_4, Y0 // w[i-3]

#define PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \
	VPXOR  REG_SUB_8, REG, REG \
	VPXOR  REG_SUB_16, Y0, Y0

#define PRECALC_18(REG) \
	VPXOR Y0, REG, REG \
	VPSLLDQ $12, REG, Y9

#define PRECALC_19(REG) \
	VPSLLD $1, REG, Y0 \
	VPSRLD $31, REG, REG

#define PRECALC_20(REG) \
	VPOR REG, Y0, Y0 \
	VPSLLD $2, Y9,  REG

#define PRECALC_21(REG) \
	VPSRLD $30, Y9, Y9 \
	VPXOR REG, Y0, Y0

#define PRECALC_23(REG,K_OFFSET,OFFSET) \
	VPXOR Y9, Y0, REG \
	VPADDD K_OFFSET(R8), REG, Y0 \
	VMOVDQU Y0, (OFFSET)(R14)

// Message scheduling pre-compute for rounds 16-31
// calculating last 32 w[i] values in 8 XMM registers
// pre-calculate K+w[i] values and store to mem
// for later load by ALU add instruction.
// "brute force" vectorization for rounds 16-31 only
// due to w[i]->w[i-3] dependency.
// clobbers 5 input ymm registers REG_SUB*
// uses X0 and X9 as temp registers
// As always, R8 is a pointer to constants block
// and R14 is a pointer to temp buffer
#define PRECALC_16_31(REG,REG_SUB_4,REG_SUB_8,REG_SUB_12,REG_SUB_16,K_OFFSET,OFFSET) \
	PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \
	PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \
	PRECALC_18(REG) \
	PRECALC_19(REG) \
	PRECALC_20(REG) \
	PRECALC_21(REG) \
	PRECALC_23(REG,K_OFFSET,OFFSET)

// Helper macros for PRECALC_32_79
#define PRECALC_32(REG_SUB_8,REG_SUB_4) \
	VPALIGNR $8, REG_SUB_8, REG_SUB_4, Y0

#define PRECALC_33(REG_SUB_28,REG) \
	VPXOR REG_SUB_28, REG, REG

#define PRECALC_34(REG_SUB_16) \
	VPXOR REG_SUB_16, Y0, Y0

#define PRECALC_35(REG) \
	VPXOR Y0, REG, REG

#define PRECALC_36(REG) \
	VPSLLD $2, REG, Y0

#define PRECALC_37(REG) \
	VPSRLD $30, REG, REG \
	VPOR REG, Y0, REG

#define PRECALC_39(REG,K_OFFSET,OFFSET) \
	VPADDD K_OFFSET(R8), REG, Y0 \
	VMOVDQU Y0, (OFFSET)(R14)

// Message scheduling pre-compute for rounds 32-79
// In SHA-1 specification we have:
// w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
// Which is the same as:
// w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
// This allows for more efficient vectorization,
// since w[i]->w[i-3] dependency is broken
#define PRECALC_32_79(REG,REG_SUB_4,REG_SUB_8,REG_SUB_16,REG_SUB_28,K_OFFSET,OFFSET) \
	PRECALC_32(REG_SUB_8,REG_SUB_4) \
	PRECALC_33(REG_SUB_28,REG) \
	PRECALC_34(REG_SUB_16) \
	PRECALC_35(REG) \
	PRECALC_36(REG) \
	PRECALC_37(REG) \
	PRECALC_39(REG,K_OFFSET,OFFSET)

#define PRECALC \
	PRECALC_00_15(0,Y15) \
	PRECALC_00_15(0x10,Y14) \
	PRECALC_00_15(0x20,Y13) \
	PRECALC_00_15(0x30,Y12) \
	PRECALC_16_31(Y8,Y12,Y13,Y14,Y15,0,0x80) \
	PRECALC_16_31(Y7,Y8,Y12,Y13,Y14,0x20,0xa0) \
	PRECALC_16_31(Y5,Y7,Y8,Y12,Y13,0x20,0xc0) \
	PRECALC_16_31(Y3,Y5,Y7,Y8,Y12,0x20,0xe0) \
	PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x20,0x100) \
	PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x20,0x120) \
	PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x40,0x140) \
	PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x40,0x160) \
	PRECALC_32_79(Y8,Y12,Y13,Y15,Y7,0x40,0x180) \
	PRECALC_32_79(Y7,Y8,Y12,Y14,Y5,0x40,0x1a0) \
	PRECALC_32_79(Y5,Y7,Y8,Y13,Y3,0x40,0x1c0) \
	PRECALC_32_79(Y3,Y5,Y7,Y12,Y15,0x60,0x1e0) \
	PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x60,0x200) \
	PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x60,0x220) \
	PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x60,0x240) \
	PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x60,0x260)

// Macros calculating individual rounds have general form
// CALC_ROUND_PRE + PRECALC_ROUND + CALC_ROUND_POST
// CALC_ROUND_{PRE,POST} macros follow

#define CALC_F1_PRE(OFFSET,REG_A,REG_B,REG_C,REG_E) \
	ADDL OFFSET(R15),REG_E \
	ANDNL REG_C,REG_A,BP \
	LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round
	RORXL $0x1b, REG_A, R12 \
	RORXL $2, REG_A, REG_B         // for next round

// Calculate F for the next round
#define CALC_F1_POST(REG_A,REG_B,REG_E) \
	ANDL REG_B,REG_A \             // b&c
	XORL BP, REG_A \               // F1 = (b&c) ^ (~b&d)
	LEAL (REG_E)(R12*1), REG_E     // E += A >>> 5

// Registers are cyclically rotated DX -> AX -> DI -> SI -> BX -> CX
#define CALC_0 \
	MOVL SI, BX \ // Precalculating first round
	RORXL $2, SI, SI \
	ANDNL AX, BX, BP \
	ANDL DI, BX \
	XORL BP, BX \
	CALC_F1_PRE(0x0,CX,BX,DI,DX) \
	PRECALC_0(0x80) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_1 \
	CALC_F1_PRE(0x4,DX,CX,SI,AX) \
	PRECALC_1(0x80) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_2 \
	CALC_F1_PRE(0x8,AX,DX,BX,DI) \
	PRECALC_2(Y15) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_3 \
	CALC_F1_PRE(0xc,DI,AX,CX,SI) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_4 \
	CALC_F1_PRE(0x20,SI,DI,DX,BX) \
	PRECALC_4(Y15,0x0) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_5 \
	CALC_F1_PRE(0x24,BX,SI,AX,CX) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_6 \
	CALC_F1_PRE(0x28,CX,BX,DI,DX) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_7 \
	CALC_F1_PRE(0x2c,DX,CX,SI,AX) \
	PRECALC_7(0x0) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_8 \
	CALC_F1_PRE(0x40,AX,DX,BX,DI) \
	PRECALC_0(0x90) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_9 \
	CALC_F1_PRE(0x44,DI,AX,CX,SI) \
	PRECALC_1(0x90) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_10 \
	CALC_F1_PRE(0x48,SI,DI,DX,BX) \
	PRECALC_2(Y14) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_11 \
	CALC_F1_PRE(0x4c,BX,SI,AX,CX) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_12 \
	CALC_F1_PRE(0x60,CX,BX,DI,DX) \
	PRECALC_4(Y14,0x0) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_13 \
	CALC_F1_PRE(0x64,DX,CX,SI,AX) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_14 \
	CALC_F1_PRE(0x68,AX,DX,BX,DI) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_15 \
	CALC_F1_PRE(0x6c,DI,AX,CX,SI) \
	PRECALC_7(0x10) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_16 \
	CALC_F1_PRE(0x80,SI,DI,DX,BX) \
	PRECALC_0(0xa0) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_17 \
	CALC_F1_PRE(0x84,BX,SI,AX,CX) \
	PRECALC_1(0xa0) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_18 \
	CALC_F1_PRE(0x88,CX,BX,DI,DX) \
	PRECALC_2(Y13) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_F2_PRE(OFFSET,REG_A,REG_B,REG_E) \
	ADDL OFFSET(R15),REG_E \
	LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round
	RORXL $0x1b, REG_A, R12 \
	RORXL $2, REG_A, REG_B         // for next round

#define CALC_F2_POST(REG_A,REG_B,REG_C,REG_E) \
	XORL REG_B, REG_A \
	ADDL R12, REG_E \
	XORL REG_C, REG_A

#define CALC_19 \
	CALC_F2_PRE(0x8c,DX,CX,AX) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_20 \
	CALC_F2_PRE(0xa0,AX,DX,DI) \
	PRECALC_4(Y13,0x0) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_21 \
	CALC_F2_PRE(0xa4,DI,AX,SI) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_22 \
	CALC_F2_PRE(0xa8,SI,DI,BX) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_23 \
	CALC_F2_PRE(0xac,BX,SI,CX) \
	PRECALC_7(0x20) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_24 \
	CALC_F2_PRE(0xc0,CX,BX,DX) \
	PRECALC_0(0xb0) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_25 \
	CALC_F2_PRE(0xc4,DX,CX,AX) \
	PRECALC_1(0xb0) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_26 \
	CALC_F2_PRE(0xc8,AX,DX,DI) \
	PRECALC_2(Y12) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_27 \
	CALC_F2_PRE(0xcc,DI,AX,SI) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_28 \
	CALC_F2_PRE(0xe0,SI,DI,BX) \
	PRECALC_4(Y12,0x0) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_29 \
	CALC_F2_PRE(0xe4,BX,SI,CX) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_30 \
	CALC_F2_PRE(0xe8,CX,BX,DX) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_31 \
	CALC_F2_PRE(0xec,DX,CX,AX) \
	PRECALC_7(0x30) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_32 \
	CALC_F2_PRE(0x100,AX,DX,DI) \
	PRECALC_16(Y15,Y14,Y12,Y8) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_33 \
	CALC_F2_PRE(0x104,DI,AX,SI) \
	PRECALC_17(Y15,Y13,Y8) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_34 \
	CALC_F2_PRE(0x108,SI,DI,BX) \
	PRECALC_18(Y8) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_35 \
	CALC_F2_PRE(0x10c,BX,SI,CX) \
	PRECALC_19(Y8) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_36 \
	CALC_F2_PRE(0x120,CX,BX,DX) \
	PRECALC_20(Y8) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_37 \
	CALC_F2_PRE(0x124,DX,CX,AX) \
	PRECALC_21(Y8) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_38 \
	CALC_F2_PRE(0x128,AX,DX,DI) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_F3_PRE(OFFSET,REG_E) \
	ADDL OFFSET(R15),REG_E

#define CALC_F3_POST(REG_A,REG_B,REG_C,REG_E,REG_TB) \
	LEAL (REG_E)(REG_TB*1), REG_E \ // Add F from the previous round
	MOVL REG_B, BP \
	ORL  REG_A, BP \
	RORXL $0x1b, REG_A, R12 \
	RORXL $2, REG_A, REG_TB \
	ANDL REG_C, BP \		// Calculate F for the next round
	ANDL REG_B, REG_A \
	ORL  BP, REG_A \
	ADDL R12, REG_E

#define CALC_39 \
	CALC_F3_PRE(0x12c,SI) \
	PRECALC_23(Y8,0x0,0x80) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_40 \
	CALC_F3_PRE(0x140,BX) \
	PRECALC_16(Y14,Y13,Y8,Y7) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_41 \
	CALC_F3_PRE(0x144,CX) \
	PRECALC_17(Y14,Y12,Y7) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_42 \
	CALC_F3_PRE(0x148,DX) \
	PRECALC_18(Y7) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_43 \
	CALC_F3_PRE(0x14c,AX) \
	PRECALC_19(Y7) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_44 \
	CALC_F3_PRE(0x160,DI) \
	PRECALC_20(Y7) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_45 \
	CALC_F3_PRE(0x164,SI) \
	PRECALC_21(Y7) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_46 \
	CALC_F3_PRE(0x168,BX) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_47 \
	CALC_F3_PRE(0x16c,CX) \
	VPXOR Y9, Y0, Y7 \
	VPADDD 0x20(R8), Y7, Y0 \
	VMOVDQU Y0, 0xa0(R14) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_48 \
	CALC_F3_PRE(0x180,DX) \
	PRECALC_16(Y13,Y12,Y7,Y5) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_49 \
	CALC_F3_PRE(0x184,AX) \
	PRECALC_17(Y13,Y8,Y5) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_50 \
	CALC_F3_PRE(0x188,DI) \
	PRECALC_18(Y5) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_51 \
	CALC_F3_PRE(0x18c,SI) \
	PRECALC_19(Y5) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_52 \
	CALC_F3_PRE(0x1a0,BX) \
	PRECALC_20(Y5) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_53 \
	CALC_F3_PRE(0x1a4,CX) \
	PRECALC_21(Y5) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_54 \
	CALC_F3_PRE(0x1a8,DX) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_55 \
	CALC_F3_PRE(0x1ac,AX) \
	PRECALC_23(Y5,0x20,0xc0) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_56 \
	CALC_F3_PRE(0x1c0,DI) \
	PRECALC_16(Y12,Y8,Y5,Y3) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_57 \
	CALC_F3_PRE(0x1c4,SI) \
	PRECALC_17(Y12,Y7,Y3) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_58 \
	CALC_F3_PRE(0x1c8,BX) \
	PRECALC_18(Y3) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_59 \
	CALC_F2_PRE(0x1cc,BX,SI,CX) \
	PRECALC_19(Y3) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_60 \
	CALC_F2_PRE(0x1e0,CX,BX,DX) \
	PRECALC_20(Y3) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_61 \
	CALC_F2_PRE(0x1e4,DX,CX,AX) \
	PRECALC_21(Y3) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_62 \
	CALC_F2_PRE(0x1e8,AX,DX,DI) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_63 \
	CALC_F2_PRE(0x1ec,DI,AX,SI) \
	PRECALC_23(Y3,0x20,0xe0) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_64 \
	CALC_F2_PRE(0x200,SI,DI,BX) \
	PRECALC_32(Y5,Y3) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_65 \
	CALC_F2_PRE(0x204,BX,SI,CX) \
	PRECALC_33(Y14,Y15) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_66 \
	CALC_F2_PRE(0x208,CX,BX,DX) \
	PRECALC_34(Y8) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_67 \
	CALC_F2_PRE(0x20c,DX,CX,AX) \
	PRECALC_35(Y15) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_68 \
	CALC_F2_PRE(0x220,AX,DX,DI) \
	PRECALC_36(Y15) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_69 \
	CALC_F2_PRE(0x224,DI,AX,SI) \
	PRECALC_37(Y15) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_70 \
	CALC_F2_PRE(0x228,SI,DI,BX) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_71 \
	CALC_F2_PRE(0x22c,BX,SI,CX) \
	PRECALC_39(Y15,0x20,0x100) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_72 \
	CALC_F2_PRE(0x240,CX,BX,DX) \
	PRECALC_32(Y3,Y15) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_73 \
	CALC_F2_PRE(0x244,DX,CX,AX) \
	PRECALC_33(Y13,Y14) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_74 \
	CALC_F2_PRE(0x248,AX,DX,DI) \
	PRECALC_34(Y7) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_75 \
	CALC_F2_PRE(0x24c,DI,AX,SI) \
	PRECALC_35(Y14) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_76 \
	CALC_F2_PRE(0x260,SI,DI,BX) \
	PRECALC_36(Y14) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_77 \
	CALC_F2_PRE(0x264,BX,SI,CX) \
	PRECALC_37(Y14) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_78 \
	CALC_F2_PRE(0x268,CX,BX,DX) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_79 \
	ADDL 0x26c(R15), AX \
	LEAL (AX)(CX*1), AX \
	RORXL $0x1b, DX, R12 \
	PRECALC_39(Y14,0x20,0x120) \
	ADDL R12, AX

// Similar to CALC_0
#define CALC_80 \
	MOVL CX, DX \
	RORXL $2, CX, CX \
	ANDNL SI, DX, BP \
	ANDL BX, DX \
	XORL BP, DX \
	CALC_F1_PRE(0x10,AX,DX,BX,DI) \
	PRECALC_32(Y15,Y14) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_81 \
	CALC_F1_PRE(0x14,DI,AX,CX,SI) \
	PRECALC_33(Y12,Y13) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_82 \
	CALC_F1_PRE(0x18,SI,DI,DX,BX) \
	PRECALC_34(Y5) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_83 \
	CALC_F1_PRE(0x1c,BX,SI,AX,CX) \
	PRECALC_35(Y13) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_84 \
	CALC_F1_PRE(0x30,CX,BX,DI,DX) \
	PRECALC_36(Y13) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_85 \
	CALC_F1_PRE(0x34,DX,CX,SI,AX) \
	PRECALC_37(Y13) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_86 \
	CALC_F1_PRE(0x38,AX,DX,BX,DI) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_87 \
	CALC_F1_PRE(0x3c,DI,AX,CX,SI) \
	PRECALC_39(Y13,0x40,0x140) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_88 \
	CALC_F1_PRE(0x50,SI,DI,DX,BX) \
	PRECALC_32(Y14,Y13) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_89 \
	CALC_F1_PRE(0x54,BX,SI,AX,CX) \
	PRECALC_33(Y8,Y12) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_90 \
	CALC_F1_PRE(0x58,CX,BX,DI,DX) \
	PRECALC_34(Y3) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_91 \
	CALC_F1_PRE(0x5c,DX,CX,SI,AX) \
	PRECALC_35(Y12) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_92 \
	CALC_F1_PRE(0x70,AX,DX,BX,DI) \
	PRECALC_36(Y12) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_93 \
	CALC_F1_PRE(0x74,DI,AX,CX,SI) \
	PRECALC_37(Y12) \
	CALC_F1_POST(DI,DX,SI)

#define CALC_94 \
	CALC_F1_PRE(0x78,SI,DI,DX,BX) \
	CALC_F1_POST(SI,AX,BX)

#define CALC_95 \
	CALC_F1_PRE(0x7c,BX,SI,AX,CX) \
	PRECALC_39(Y12,0x40,0x160) \
	CALC_F1_POST(BX,DI,CX)

#define CALC_96 \
	CALC_F1_PRE(0x90,CX,BX,DI,DX) \
	PRECALC_32(Y13,Y12) \
	CALC_F1_POST(CX,SI,DX)

#define CALC_97 \
	CALC_F1_PRE(0x94,DX,CX,SI,AX) \
	PRECALC_33(Y7,Y8) \
	CALC_F1_POST(DX,BX,AX)

#define CALC_98 \
	CALC_F1_PRE(0x98,AX,DX,BX,DI) \
	PRECALC_34(Y15) \
	CALC_F1_POST(AX,CX,DI)

#define CALC_99 \
	CALC_F2_PRE(0x9c,DI,AX,SI) \
	PRECALC_35(Y8) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_100 \
	CALC_F2_PRE(0xb0,SI,DI,BX) \
	PRECALC_36(Y8) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_101 \
	CALC_F2_PRE(0xb4,BX,SI,CX) \
	PRECALC_37(Y8) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_102 \
	CALC_F2_PRE(0xb8,CX,BX,DX) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_103 \
	CALC_F2_PRE(0xbc,DX,CX,AX) \
	PRECALC_39(Y8,0x40,0x180) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_104 \
	CALC_F2_PRE(0xd0,AX,DX,DI) \
	PRECALC_32(Y12,Y8) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_105 \
	CALC_F2_PRE(0xd4,DI,AX,SI) \
	PRECALC_33(Y5,Y7) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_106 \
	CALC_F2_PRE(0xd8,SI,DI,BX) \
	PRECALC_34(Y14) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_107 \
	CALC_F2_PRE(0xdc,BX,SI,CX) \
	PRECALC_35(Y7) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_108 \
	CALC_F2_PRE(0xf0,CX,BX,DX) \
	PRECALC_36(Y7) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_109 \
	CALC_F2_PRE(0xf4,DX,CX,AX) \
	PRECALC_37(Y7) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_110 \
	CALC_F2_PRE(0xf8,AX,DX,DI) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_111 \
	CALC_F2_PRE(0xfc,DI,AX,SI) \
	PRECALC_39(Y7,0x40,0x1a0) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_112 \
	CALC_F2_PRE(0x110,SI,DI,BX) \
	PRECALC_32(Y8,Y7) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_113 \
	CALC_F2_PRE(0x114,BX,SI,CX) \
	PRECALC_33(Y3,Y5) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_114 \
	CALC_F2_PRE(0x118,CX,BX,DX) \
	PRECALC_34(Y13) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_115 \
	CALC_F2_PRE(0x11c,DX,CX,AX) \
	PRECALC_35(Y5) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_116 \
	CALC_F2_PRE(0x130,AX,DX,DI) \
	PRECALC_36(Y5) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_117 \
	CALC_F2_PRE(0x134,DI,AX,SI) \
	PRECALC_37(Y5) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_118 \
	CALC_F2_PRE(0x138,SI,DI,BX) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_119 \
	CALC_F3_PRE(0x13c,CX) \
	PRECALC_39(Y5,0x40,0x1c0) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_120 \
	CALC_F3_PRE(0x150,DX) \
	PRECALC_32(Y7,Y5) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_121 \
	CALC_F3_PRE(0x154,AX) \
	PRECALC_33(Y15,Y3) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_122 \
	CALC_F3_PRE(0x158,DI) \
	PRECALC_34(Y12) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_123 \
	CALC_F3_PRE(0x15c,SI) \
	PRECALC_35(Y3) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_124 \
	CALC_F3_PRE(0x170,BX) \
	PRECALC_36(Y3) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_125 \
	CALC_F3_PRE(0x174,CX) \
	PRECALC_37(Y3) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_126 \
	CALC_F3_PRE(0x178,DX) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_127 \
	CALC_F3_PRE(0x17c,AX) \
	PRECALC_39(Y3,0x60,0x1e0) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_128 \
	CALC_F3_PRE(0x190,DI) \
	PRECALC_32(Y5,Y3) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_129 \
	CALC_F3_PRE(0x194,SI) \
	PRECALC_33(Y14,Y15) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_130 \
	CALC_F3_PRE(0x198,BX) \
	PRECALC_34(Y8) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_131 \
	CALC_F3_PRE(0x19c,CX) \
	PRECALC_35(Y15) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_132 \
	CALC_F3_PRE(0x1b0,DX) \
	PRECALC_36(Y15) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_133 \
	CALC_F3_PRE(0x1b4,AX) \
	PRECALC_37(Y15) \
	CALC_F3_POST(DX,BX,SI,AX,CX)

#define CALC_134 \
	CALC_F3_PRE(0x1b8,DI) \
	CALC_F3_POST(AX,CX,BX,DI,DX)

#define CALC_135 \
	CALC_F3_PRE(0x1bc,SI) \
	PRECALC_39(Y15,0x60,0x200) \
	CALC_F3_POST(DI,DX,CX,SI,AX)

#define CALC_136 \
	CALC_F3_PRE(0x1d0,BX) \
	PRECALC_32(Y3,Y15) \
	CALC_F3_POST(SI,AX,DX,BX,DI)

#define CALC_137 \
	CALC_F3_PRE(0x1d4,CX) \
	PRECALC_33(Y13,Y14) \
	CALC_F3_POST(BX,DI,AX,CX,SI)

#define CALC_138 \
	CALC_F3_PRE(0x1d8,DX) \
	PRECALC_34(Y7) \
	CALC_F3_POST(CX,SI,DI,DX,BX)

#define CALC_139 \
	CALC_F2_PRE(0x1dc,DX,CX,AX) \
	PRECALC_35(Y14) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_140 \
	CALC_F2_PRE(0x1f0,AX,DX,DI) \
	PRECALC_36(Y14) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_141 \
	CALC_F2_PRE(0x1f4,DI,AX,SI) \
	PRECALC_37(Y14) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_142 \
	CALC_F2_PRE(0x1f8,SI,DI,BX) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_143 \
	CALC_F2_PRE(0x1fc,BX,SI,CX) \
	PRECALC_39(Y14,0x60,0x220) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_144 \
	CALC_F2_PRE(0x210,CX,BX,DX) \
	PRECALC_32(Y15,Y14) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_145 \
	CALC_F2_PRE(0x214,DX,CX,AX) \
	PRECALC_33(Y12,Y13) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_146 \
	CALC_F2_PRE(0x218,AX,DX,DI) \
	PRECALC_34(Y5) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_147 \
	CALC_F2_PRE(0x21c,DI,AX,SI) \
	PRECALC_35(Y13) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_148 \
	CALC_F2_PRE(0x230,SI,DI,BX) \
	PRECALC_36(Y13) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_149 \
	CALC_F2_PRE(0x234,BX,SI,CX) \
	PRECALC_37(Y13) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_150 \
	CALC_F2_PRE(0x238,CX,BX,DX) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_151 \
	CALC_F2_PRE(0x23c,DX,CX,AX) \
	PRECALC_39(Y13,0x60,0x240) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_152 \
	CALC_F2_PRE(0x250,AX,DX,DI) \
	PRECALC_32(Y14,Y13) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_153 \
	CALC_F2_PRE(0x254,DI,AX,SI) \
	PRECALC_33(Y8,Y12) \
	CALC_F2_POST(DI,DX,CX,SI)

#define CALC_154 \
	CALC_F2_PRE(0x258,SI,DI,BX) \
	PRECALC_34(Y3) \
	CALC_F2_POST(SI,AX,DX,BX)

#define CALC_155 \
	CALC_F2_PRE(0x25c,BX,SI,CX) \
	PRECALC_35(Y12) \
	CALC_F2_POST(BX,DI,AX,CX)

#define CALC_156 \
	CALC_F2_PRE(0x270,CX,BX,DX) \
	PRECALC_36(Y12) \
	CALC_F2_POST(CX,SI,DI,DX)

#define CALC_157 \
	CALC_F2_PRE(0x274,DX,CX,AX) \
	PRECALC_37(Y12) \
	CALC_F2_POST(DX,BX,SI,AX)

#define CALC_158 \
	CALC_F2_PRE(0x278,AX,DX,DI) \
	CALC_F2_POST(AX,CX,BX,DI)

#define CALC_159 \
	ADDL 0x27c(R15),SI \
	LEAL (SI)(AX*1), SI \
	RORXL $0x1b, DI, R12 \
	PRECALC_39(Y12,0x60,0x260) \
	ADDL R12, SI

#define CALC \
	MOVL	(R9), CX \
	MOVL	4(R9), SI \
	MOVL	8(R9), DI \
	MOVL	12(R9), AX \
	MOVL	16(R9), DX \
	MOVQ    SP, R14 \
	LEAQ    (2*4*80+32)(SP), R15 \
	PRECALC \ // Precalc WK for first 2 blocks
	XCHGQ   R15, R14 \
loop: \  // this loops is unrolled
	CMPQ    R10, R8 \ // we use R8 value (set below) as a signal of a last block
	JNE	begin \
	VZEROUPPER \
	RET \
begin: \
	CALC_0 \
	CALC_1 \
	CALC_2 \
	CALC_3 \
	CALC_4 \
	CALC_5 \
	CALC_6 \
	CALC_7 \
	CALC_8 \
	CALC_9 \
	CALC_10 \
	CALC_11 \
	CALC_12 \
	CALC_13 \
	CALC_14 \
	CALC_15 \
	CALC_16 \
	CALC_17 \
	CALC_18 \
	CALC_19 \
	CALC_20 \
	CALC_21 \
	CALC_22 \
	CALC_23 \
	CALC_24 \
	CALC_25 \
	CALC_26 \
	CALC_27 \
	CALC_28 \
	CALC_29 \
	CALC_30 \
	CALC_31 \
	CALC_32 \
	CALC_33 \
	CALC_34 \
	CALC_35 \
	CALC_36 \
	CALC_37 \
	CALC_38 \
	CALC_39 \
	CALC_40 \
	CALC_41 \
	CALC_42 \
	CALC_43 \
	CALC_44 \
	CALC_45 \
	CALC_46 \
	CALC_47 \
	CALC_48 \
	CALC_49 \
	CALC_50 \
	CALC_51 \
	CALC_52 \
	CALC_53 \
	CALC_54 \
	CALC_55 \
	CALC_56 \
	CALC_57 \
	CALC_58 \
	CALC_59 \
	ADDQ $128, R10 \ // move to next even-64-byte block
	CMPQ R10, R11 \ // is current block the last one?
	CMOVQCC R8, R10 \ // signal the last iteration smartly
	CALC_60 \
	CALC_61 \
	CALC_62 \
	CALC_63 \
	CALC_64 \
	CALC_65 \
	CALC_66 \
	CALC_67 \
	CALC_68 \
	CALC_69 \
	CALC_70 \
	CALC_71 \
	CALC_72 \
	CALC_73 \
	CALC_74 \
	CALC_75 \
	CALC_76 \
	CALC_77 \
	CALC_78 \
	CALC_79 \
	UPDATE_HASH(AX,DX,BX,SI,DI) \
	CMPQ R10, R8 \ // is current block the last one?
	JE loop\
	MOVL DX, CX \
	CALC_80 \
	CALC_81 \
	CALC_82 \
	CALC_83 \
	CALC_84 \
	CALC_85 \
	CALC_86 \
	CALC_87 \
	CALC_88 \
	CALC_89 \
	CALC_90 \
	CALC_91 \
	CALC_92 \
	CALC_93 \
	CALC_94 \
	CALC_95 \
	CALC_96 \
	CALC_97 \
	CALC_98 \
	CALC_99 \
	CALC_100 \
	CALC_101 \
	CALC_102 \
	CALC_103 \
	CALC_104 \
	CALC_105 \
	CALC_106 \
	CALC_107 \
	CALC_108 \
	CALC_109 \
	CALC_110 \
	CALC_111 \
	CALC_112 \
	CALC_113 \
	CALC_114 \
	CALC_115 \
	CALC_116 \
	CALC_117 \
	CALC_118 \
	CALC_119 \
	CALC_120 \
	CALC_121 \
	CALC_122 \
	CALC_123 \
	CALC_124 \
	CALC_125 \
	CALC_126 \
	CALC_127 \
	CALC_128 \
	CALC_129 \
	CALC_130 \
	CALC_131 \
	CALC_132 \
	CALC_133 \
	CALC_134 \
	CALC_135 \
	CALC_136 \
	CALC_137 \
	CALC_138 \
	CALC_139 \
	ADDQ $128, R13 \ //move to next even-64-byte block
	CMPQ R13, R11 \ //is current block the last one?
	CMOVQCC R8, R10 \
	CALC_140 \
	CALC_141 \
	CALC_142 \
	CALC_143 \
	CALC_144 \
	CALC_145 \
	CALC_146 \
	CALC_147 \
	CALC_148 \
	CALC_149 \
	CALC_150 \
	CALC_151 \
	CALC_152 \
	CALC_153 \
	CALC_154 \
	CALC_155 \
	CALC_156 \
	CALC_157 \
	CALC_158 \
	CALC_159 \
	UPDATE_HASH(SI,DI,DX,CX,BX) \
	MOVL	SI, R12 \ //Reset state for  AVX2 reg permutation
	MOVL	DI, SI \
	MOVL	DX, DI \
	MOVL	BX, DX \
	MOVL	CX, AX \
	MOVL	R12, CX \
	XCHGQ   R15, R14 \
	JMP     loop

TEXT ·blockAVX2(SB),$1408-32

MOVQ	dig+0(FP),	DI
	MOVQ	p_base+8(FP),	SI
	MOVQ	p_len+16(FP),	DX
	SHRQ	$6,		DX
	SHLQ	$6,		DX

MOVQ	$K_XMM_AR<>(SB), R8

MOVQ	DI, R9
	MOVQ	SI, R10
	LEAQ	64(SI), R13

ADDQ	SI, DX
	ADDQ	$64, DX
	MOVQ	DX, R11

CMPQ	R13, R11
	CMOVQCC	R8, R13

VMOVDQU	BSWAP_SHUFB_CTL<>(SB), Y10

CALC // RET is inside macros

DATA K_XMM_AR<>+0x00(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x04(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x08(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x0c(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x10(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x14(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x18(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x1c(SB)/4,$0x5a827999
DATA K_XMM_AR<>+0x20(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x24(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x28(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x2c(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x30(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x34(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x38(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x3c(SB)/4,$0x6ed9eba1
DATA K_XMM_AR<>+0x40(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x44(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x48(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x4c(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x50(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x54(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x58(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x5c(SB)/4,$0x8f1bbcdc
DATA K_XMM_AR<>+0x60(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x64(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x68(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x6c(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x70(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x74(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x78(SB)/4,$0xca62c1d6
DATA K_XMM_AR<>+0x7c(SB)/4,$0xca62c1d6
GLOBL K_XMM_AR<>(SB),RODATA,$128

DATA BSWAP_SHUFB_CTL<>+0x00(SB)/4,$0x00010203
DATA BSWAP_SHUFB_CTL<>+0x04(SB)/4,$0x04050607
DATA BSWAP_SHUFB_CTL<>+0x08(SB)/4,$0x08090a0b
DATA BSWAP_SHUFB_CTL<>+0x0c(SB)/4,$0x0c0d0e0f
DATA BSWAP_SHUFB_CTL<>+0x10(SB)/4,$0x00010203
DATA BSWAP_SHUFB_CTL<>+0x14(SB)/4,$0x04050607
DATA BSWAP_SHUFB_CTL<>+0x18(SB)/4,$0x08090a0b
DATA BSWAP_SHUFB_CTL<>+0x1c(SB)/4,$0x0c0d0e0f
GLOBL BSWAP_SHUFB_CTL<>(SB),RODATA,$32

BATAL