MULHU	X5, X6, X7				// b3335302
	MULHSU	X5, X6, X7				// b3235302
	MULW	X5, X6, X7				// bb035302
	DIV	X5, X6, X7				// b3435302
	DIVU	X5, X6, X7				// b3535302
	REM	X5, X6, X7				// b3635302
	REMU	X5, X6, X7				// b3735302
	DIVW	X5, X6, X7				// bb435302
	DIVUW	X5, X6, X7				// bb535302
	REMW	X5, X6, X7				// bb635302
	REMUW	X5, X6, X7				// bb735302

	// 8.2: Load-Reserved/Store-Conditional

//     Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
#define SHA512T2(a, b, c) \
	ROR	$28, a, X6; \
	ROR	$34, a, X7; \
	XOR	X7, X6; \
	ROR	$39, a, X8; \
	XOR	X8, X6; \
	AND	a, b, X7; \
	AND	a, c, X8; \
	XOR	X8, X7; \
	AND	b, c, X9; \
	XOR	X9, X7; \
	ADD	X7, X6

// Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
// The values for e and a are stored in d and h, ready for rotation.

	AND	$7, X10, X7
	AND	$7, X12, X8
	BNE	X7, X8, check8_unaligned
	BEQZ	X7, compare32

	// Check one byte at a time until we reach 8 byte alignment.
	SUB	X7, X0, X7
	ADD	$8, X7, X7
	SUB	X7, X5, X5
align:
	SUB	$1, X7
	MOVBU	0(X10), X8
	MOVBU	0(X12), X9
	BNE	X8, X9, cmp
	ADD	$1, X10
	ADD	$1, X12
	BNEZ	X7, align

check32:
	// X6 contains $32

TEXT addMulVVWx(SB),NOFRAME|NOSPLIT,$0
	MOV	z+0(FP), X5
	MOV	x+8(FP), X7
	MOV	y+16(FP), X6
	MOV	$0, X29

	BEQZ	X30, done
loop:
	MOV	0*8(X5), X10	// z[0]
	MOV	1*8(X5), X13	// z[1]
	MOV	2*8(X5), X16	// z[2]
	MOV	3*8(X5), X19	// z[3]

	MOV	0*8(X7), X8	// x[0]
	MOV	1*8(X7), X11	// x[1]
	MOV	2*8(X7), X14	// x[2]
	MOV	3*8(X7), X17	// x[3]

	MULHU	X8, X6, X9	// z_hi[0] = x[0] * y

	VREPF $1, KEY0, X5
	VREPF $2, KEY0, X6
	VREPF $3, KEY0, X7
	VREPF $0, KEY1, X8
	VREPF $1, KEY1, X9
	VREPF $2, KEY1, X10
	VREPF $3, KEY1, X11
	VLR   CTR, X12
	VREPF $1, NONCE, X13
	VREPF $2, NONCE, X14
	VREPF $3, NONCE, X15

	MOVD $(NUM_ROUNDS/2), R1

loop:
	ROUND4(X0, X4, X12,  X8, X1, X5, X13,  X9, X2, X6, X14, X10, X3, X7, X15, X11)

	// come in on the m->g0 stack already. Or we might already
	// be on the m->gsignal stack.
	MOV	g_m(g), X6
	MOV	m_gsignal(X6), X7
	BEQ	X7, g, g0
	MOV	m_g0(X6), X7
	BEQ	X7, g, g0

	CALL	gosave_systemstack_switch<>(SB)
	MOV	X7, g
	CALL	runtime·save_g(SB)
	MOV	(g_sched+gobuf_sp)(g), X2

	// Now on a scheduling stack (a pthread-created stack).
g0:

	QR(X1, X5, X9, X13, X15)
	MOVOU (15*16)(BX), X15 // reload X15; temp now X4
	QR(X2, X6, X10, X14, X4)
	QR(X3, X7, X11, X15, X4)

	QR(X0, X5, X10, X15, X4)
	MOVOU X15, (15*16)(BX) // save X15
	QR(X1, X6, X11, X12, X4)
	MOVOU (4*16)(BX), X4  // reload X4; temp now X15
	QR(X2, X7, X8, X13, X15)
	QR(X3, X4, X9, X14, X15)

	DECL DX
	JNZ loop

	// Store interlaced blocks back to output buffer,

		x2 := byteorder.LeUint32(q[4*0x2:])
		x3 := byteorder.LeUint32(q[4*0x3:])
		x4 := byteorder.LeUint32(q[4*0x4:])
		x5 := byteorder.LeUint32(q[4*0x5:])
		x6 := byteorder.LeUint32(q[4*0x6:])
		x7 := byteorder.LeUint32(q[4*0x7:])
		x8 := byteorder.LeUint32(q[4*0x8:])
		x9 := byteorder.LeUint32(q[4*0x9:])
		xa := byteorder.LeUint32(q[4*0xa:])
		xb := byteorder.LeUint32(q[4*0xb:])
		xc := byteorder.LeUint32(q[4*0xc:])

	MOVOU yout(16*0), X2
	MOVOU yout(16*1), X3
	MOVOU zout(16*0), X4
	MOVOU zout(16*1), X5

	MOVL BX, X6
	MOVL CX, X7

	PXOR X8, X8
	PCMPEQL X9, X9

	PSHUFD $0, X6, X6
	PSHUFD $0, X7, X7

	PCMPEQL X8, X6
	PCMPEQL X8, X7

	MOVOU X6, X15
	PANDN X9, X15

	MOVOU x1in(16*0), X9
	MOVOU x1in(16*1), X10
	MOVOU y1in(16*0), X11

	MOVOU	-16(AX)(BX*1), X7

	PXOR	X0, X4
	PXOR	X1, X5
	PXOR	X2, X6
	PXOR	X3, X7

	AESENC	X4, X4
	AESENC	X5, X5
	AESENC	X6, X6
	AESENC	X7, X7

	AESENC	X4, X4
	AESENC	X5, X5
	AESENC	X6, X6
	AESENC	X7, X7

	AESENC	X4, X4
	AESENC	X5, X5
	AESENC	X6, X6
	AESENC	X7, X7

	PXOR	X6, X4
	PXOR	X7, X5
	PXOR	X5, X4
	MOVL	X4, (DX)