PSLLDQ $8, B4
		PSRLDQ $8, T2
		PXOR B4, T0
		PXOR T2, T1

		MOVOU POLY, B2
		PCLMULQDQ $0x01, T0, B2
		PSHUFD $78, T0, T0
		PXOR B2, T0
		MOVOU POLY, B2
		PCLMULQDQ $0x01, T0, B2
		PSHUFD $78, T0, T0
		PXOR T0, B2
		PXOR T1, B2

		MOVOU B2, (16*12)(dst)
		PSHUFD $78, B2, B3
		PXOR B2, B3
		MOVOU B3, (16*13)(dst)

		DECQ AX
		LEAQ (-16*2)(dst), dst
	JNE initLoop

	RET

	MOVUPS -16(BX), X0
	MOVUPS X0, 16(DX)
	RET

TEXT _expand_key_128<>(SB),NOSPLIT,$0
	PSHUFD $0xff, X1, X1
	SHUFPS $0x10, X0, X4
	PXOR X4, X0
	SHUFPS $0x8c, X0, X4
	PXOR X4, X0
	PXOR X1, X0
	MOVUPS X0, (BX)
	ADDQ $16, BX
	RET

TEXT _expand_key_192a<>(SB),NOSPLIT,$0
	PSHUFD $0x55, X1, X1
	SHUFPS $0x10, X0, X4
	PXOR X4, X0
	SHUFPS $0x8c, X0, X4
	PXOR X4, X0
	PXOR X1, X0

	ADDQ		dataPtr, numBytes		// point numBytes to end of input buffer
	VMOVDQU		(0*16)(digestPtr), state0	// load initial hash values and reorder
	VMOVDQU		(1*16)(digestPtr), state1	// DCBA, HGFE -> ABEF, CDGH
	PSHUFD		$0xb1, state0, state0		// CDAB
	PSHUFD		$0x1b, state1, state1		// EFGH
	VMOVDQA		state0, m4
	PALIGNR		$8, state1, state0		// ABEF
	PBLENDW		$0xf0, m4, state1		// CDGH
	VMOVDQA		flip_mask<>(SB), shufMask
	LEAQ		K256<>(SB), sha256Constants

	PADDD B, A; PXOR A, D; ROL8(D, T); \
	PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B

// REPLREG replicates the register R into 4 uint32s in XR.
#define REPLREG(R, XR) \
	MOVQ R, XR; \
	PSHUFD $0, XR, XR

// REPL replicates the uint32 constant val into 4 uint32s in XR. It smashes DX.
#define REPL(val, XR) \
	MOVL $val, DX; \
	REPLREG(DX, XR)

	MOVOU yout(16*0), X2
	MOVOU yout(16*1), X3
	MOVOU zout(16*0), X4
	MOVOU zout(16*1), X5

	MOVL BX, X6
	MOVL CX, X7

	PXOR X8, X8
	PCMPEQL X9, X9

	PSHUFD $0, X6, X6
	PSHUFD $0, X7, X7

	PCMPEQL X8, X6
	PCMPEQL X8, X7

	MOVOU X6, X15
	PANDN X9, X15

	MOVOU x1in(16*0), X9
	MOVOU x1in(16*1), X10
	MOVOU y1in(16*0), X11
	MOVOU y1in(16*1), X12

	"POPCNTW",
	"POPFL",
	"POPFQ",
	"POPFW",
	"POPL",
	"POPQ",
	"POPW",
	"POR",
	"PREFETCHNTA",
	"PREFETCHT0",
	"PREFETCHT1",
	"PREFETCHT2",
	"PSADBW",
	"PSHUFB",
	"PSHUFD",
	"PSHUFHW",
	"PSHUFL",
	"PSHUFLW",
	"PSHUFW",
	"PSIGNB",
	"PSIGND",
	"PSIGNW",
	"PSLLL",
	"PSLLO",
	"PSLLQ",
	"PSLLW",
	"PSRAL",
	"PSRAW",
	"PSRLL",
	"PSRLO",

	PSHUFD $7, (BX), X2                     // 660f701307
	PSHUFL $7, (BX), X2                     // 660f701307
	PSHUFD $7, (R11), X2                    // 66410f701307
	PSHUFL $7, (R11), X2                    // 66410f701307
	PSHUFD $7, X2, X2                       // 660f70d207
	PSHUFL $7, X2, X2                       // 660f70d207
	PSHUFD $7, X11, X2                      // 66410f70d307

	PREFETCHT0:      "PREFETCHT0",
	PREFETCHT1:      "PREFETCHT1",
	PREFETCHT2:      "PREFETCHT2",
	PREFETCHW:       "PREFETCHW",
	PSADBW:          "PSADBW",
	PSHUFB:          "PSHUFB",
	PSHUFD:          "PSHUFD",
	PSHUFHW:         "PSHUFHW",
	PSHUFLW:         "PSHUFLW",
	PSHUFW:          "PSHUFW",
	PSIGNB:          "PSIGNB",
	PSIGND:          "PSIGND",
	PSIGNW:          "PSIGNW",