ADCXQ R9, CX
	ADOXQ R9, CX

	ADDQ $64, R8
	ADDQ $64, R10
	ADDQ $8, BX

	CMPQ BX, R13
	JL adx_loop
	MOVQ z+0(FP), R10
	MOVQ x+24(FP), R8
	CMPQ BX, R11
	JL adx_short
	MOVQ CX, c+56(FP)
	RET

adx_short:
	MULXQ (R8)(BX*8), SI, DI
	ADDQ CX, SI
	ADCQ $0, DI
	ADDQ SI, (R10)(BX*8)
	ADCQ $0, DI
	MOVQ DI, CX
	ADDQ $1, BX		// i++

	CMPQ BX, R11
	JL adx_short

	XORQ  R12, R14
	RORXQ $28, R9, R12
	ADDQ  8*3+frame_YFER(SP), DX
	ORQ   R11, DI

	XORQ R12, R14
	MOVQ R9, R12
	ANDQ R10, DI
	ANDQ R11, R12
	ADDQ R13, R15

	ADDQ DX, AX
	ORQ  R12, DI
	ADDQ R14, DX

	ADDQ R15, AX

	ADDQ R15, DX

	ADDQ DI, DX

	VPADDQ  1*32(BP), Y5, Y0
	VMOVDQU Y0, frame_YFER(SP)
	ADDQ    $(2*32), BP

	MOVQ  BX, R15

	PMOVMSKB X1, AX
	XORQ	$0xffff, AX	// convert EQ to NE
	JNE	diff16	// branch if at least one byte is not equal
	ADDQ	$16, SI
	ADDQ	$16, DI
	SUBQ	$16, R8
	JMP	loop

diff64:
	ADDQ	$48, SI
	ADDQ	$48, DI
	JMP	diff16
diff48:
	ADDQ	$32, SI
	ADDQ	$32, DI
	JMP	diff16
diff32:
	ADDQ	$16, SI
	ADDQ	$16, DI
	// AX = bit mask of differences
diff16:
	BSFQ	AX, BX	// index of first byte that differs
	XORQ	AX, AX

	MOVQ dst+16(FP), DX
	MOVQ src+24(FP), BX
	MOVUPS 0(AX), X1
	MOVUPS 0(BX), X0
	ADDQ $16, AX
	PXOR X1, X0
	SUBQ $12, CX
	JE Lenc192
	JB Lenc128
Lenc256:
	MOVUPS 0(AX), X1
	AESENC X1, X0
	MOVUPS 16(AX), X1
	AESENC X1, X0
	ADDQ $32, AX
Lenc192:
	MOVUPS 0(AX), X1
	AESENC X1, X0
	MOVUPS 16(AX), X1
	AESENC X1, X0
	ADDQ $32, AX
Lenc128:
	MOVUPS 0(AX), X1
	AESENC X1, X0
	MOVUPS 16(AX), X1

	ADDQ t1, acc4
	ADCQ $0, DX
	ADDQ AX, acc4
	ADCQ DX, acc5
	ADCQ $0, acc0
	// x * y[2]
	MOVQ (8*2)(y_ptr), t0

	MOVQ (8*0)(x_ptr), AX
	MULQ t0
	ADDQ AX, acc2
	ADCQ $0, DX
	MOVQ DX, t1

	MOVQ (8*1)(x_ptr), AX
	MULQ t0
	ADDQ t1, acc3
	ADCQ $0, DX
	ADDQ AX, acc3
	ADCQ $0, DX
	MOVQ DX, t1

	MOVQ (8*2)(x_ptr), AX
	MULQ t0
	ADDQ t1, acc4
	ADCQ $0, DX
	ADDQ AX, acc4

	ADDQ	$16, SI
	MOVUPS	X0, (DI)
	ADDQ	$16, DI

	MOVUPS	(SI), X0
	ADDQ	$16, SI
	MOVUPS	X0, (DI)
	ADDQ	$16, DI

	MOVUPS	(SI), X0
	ADDQ	$16, SI
	MOVUPS	X0, (DI)
	ADDQ	$16, DI

	MOVUPS	(SI), X0
	ADDQ	$16, SI
	MOVUPS	X0, (DI)
	ADDQ	$16, DI

	MOVUPS	(SI), X0
	ADDQ	$16, SI
	MOVUPS	X0, (DI)
	ADDQ	$16, DI

	MOVUPS	(SI), X0
	ADDQ	$16, SI
	MOVUPS	X0, (DI)

	ADDQ  AX, t2;                  \
	ADCQ  DX, t3;                  \
	ADDQ  h0, t2;                  \
	ADCQ  $0, t3;                  \
	                               \
	MOVQ  t0, h0;                  \
	MOVQ  t1, h1;                  \
	MOVQ  t2, h2;                  \
	ANDQ  $3, h2;                  \
	MOVQ  t2, t0;                  \
	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
	ADDQ  t0, h0;                  \

	// Iteration 0
	MOVQ (BX), AX
	MULQ SI
	ADDQ (CX), AX
	ADCQ $0x00, DX
	ADDQ DI, AX
	ADCQ $0x00, DX
	MOVQ DX, DI
	MOVQ AX, (CX)

	// Iteration 1
	MOVQ 8(BX), AX
	MULQ SI
	ADDQ 8(CX), AX
	ADCQ $0x00, DX
	ADDQ DI, AX
	ADCQ $0x00, DX
	MOVQ DX, DI
	MOVQ AX, 8(CX)

	// Iteration 2
	MOVQ 16(BX), AX
	MULQ SI
	ADDQ 16(CX), AX
	ADCQ $0x00, DX
	ADDQ DI, AX
	ADCQ $0x00, DX
	MOVQ DX, DI

	IMUL3Q $0x13, R14, R14
	ADDQ   R14, DI
	ANDQ   AX, R9
	ADDQ   SI, R9
	ANDQ   AX, R11
	ADDQ   R8, R11
	ANDQ   AX, R13
	ADDQ   R10, R13
	ANDQ   AX, R15
	ADDQ   R12, R15

	// Second reduction chain (carryPropagate)
	MOVQ   DI, SI
	SHRQ   $0x33, SI
	MOVQ   R9, R8
	SHRQ   $0x33, R8
	MOVQ   R11, R10
	SHRQ   $0x33, R10
	MOVQ   R13, R12
	SHRQ   $0x33, R12
	MOVQ   R15, R14
	SHRQ   $0x33, R14

	RET
TEXT ·a15(SB), 0, $0-0
	CMPL runtime·writeBarrier(SB), $0
	MULXQ AX, R15, BX
	ADDQ $1, R15
	RET
TEXT ·a16(SB), 0, $0-0
	CMPL runtime·writeBarrier(SB), $0
	MULXQ AX, BX, R15
	ADDQ $1, R15
	RET
TEXT ·a17(SB), 0, $0-0
	CMPL runtime·writeBarrier(SB), $0