/*
 * forward copy loop
 */
forward:
	CMPQ	BX, $2048
	JLS	move_256through2048

	// If REP MOVSB isn't fast, don't use it
	CMPB	internal∕cpu·X86+const_offsetX86HasERMS(SB), $1 // enhanced REP MOVSB/STOSB
	JNE	fwdBy8

	// Check alignment
	MOVL	SI, AX
	ORL	DI, AX
	TESTL	$7, AX
	JEQ	fwdBy8

	// Do 1 byte at a time
	MOVQ	BX, CX
	REP;	MOVSB
	RET

fwdBy8:
	// Do 8 bytes at a time

	MOVQ	m_vdsoSP(BX), DX
	MOVQ	CX, 0(SP)
	MOVQ	DX, 8(SP)

	LEAQ	ret+0(FP), DX
	MOVQ	-8(DX), CX
	MOVQ	CX, m_vdsoPC(BX)
	MOVQ	DX, m_vdsoSP(BX)

	CMPQ	R14, m_curg(BX)	// Only switch if on curg.
	JNE	noswitch

	MOVQ	m_g0(BX), DX
	MOVQ	(g_sched+gobuf_sp)(DX), SP	// Set SP to g0 stack

noswitch:
	SUBQ	$16, SP		// Space for results
	ANDQ	$~15, SP	// Align for C code

	MOVL	$1, DI // CLOCK_MONOTONIC

	instructions["JNAE"] = x86.AJCS /* alternate */
	instructions["JNB"] = x86.AJCC  /* alternate */
	instructions["JNBE"] = x86.AJHI /* alternate */
	instructions["JNC"] = x86.AJCC  /* alternate */
	instructions["JNE"] = x86.AJNE  /* not equal (ZF = 0) */
	instructions["JNG"] = x86.AJLE  /* alternate */
	instructions["JNGE"] = x86.AJLT /* alternate */
	instructions["JNL"] = x86.AJGE  /* alternate */

	/*5294*/ uint16(xSetOp), uint16(JNE),
	/*5296*/ uint16(xReadCw),
	/*5297*/ uint16(xArgRel16),
	/*5298*/ uint16(xMatch),
	/*5299*/ uint16(xSetOp), uint16(JNE),
	/*5301*/ uint16(xReadCd),
	/*5302*/ uint16(xArgRel32),
	/*5303*/ uint16(xMatch),
	/*5304*/ uint16(xCondDataSize), 5308, 5299, 5313,
	/*5308*/ uint16(xSetOp), uint16(JNE),
	/*5310*/ uint16(xReadCd),
	/*5311*/ uint16(xArgRel32),

	LEAQ    (2*4*80+32)(SP), R15 \
	PRECALC \ // Precalc WK for first 2 blocks
	XCHGQ   R15, R14 \
loop: \  // this loops is unrolled
	CMPQ    R10, R8 \ // we use R8 value (set below) as a signal of a last block
	JNE	begin \
	VZEROUPPER \
	RET \
begin: \
	CALC_0 \
	CALC_1 \
	CALC_2 \
	CALC_3 \
	CALC_4 \
	CALC_5 \
	CALC_6 \
	CALC_7 \
	CALC_8 \
	CALC_9 \
	CALC_10 \

var isCondJmp = [maxOp + 1]bool{
	JA:  true,
	JAE: true,
	JB:  true,
	JBE: true,
	JE:  true,
	JG:  true,
	JGE: true,
	JL:  true,
	JLE: true,
	JNE: true,
	JNO: true,
	JNP: true,
	JNS: true,
	JO:  true,
	JP:  true,
	JS:  true,
}

// isLoop records the loop operators.
var isLoop = [maxOp + 1]bool{
	LOOP:   true,

	// add current hash values with previously saved
	PADDD		abefSave, state0
	PADDD		cdghSave, state1

	// advance data pointer; loop until buffer empty
	ADDQ		$64, dataPtr
	CMPQ		numBytes, dataPtr
	JNE		roundLoop

	// write hash values back in the correct order
	PSHUFD		$0x1b, state0, state0		// FEBA
	PSHUFD		$0xb1, state1, state1		// DCHG
	VMOVDQA		state0, m4
	PBLENDW		$0xf0, state1, state0		// DCBA

		// TODO: have these return flags instead of bool.  The current system generates:
		//    CMPXCHGQ ...
		//    SETEQ AX
		//    CMPB  AX, $0
		//    JNE ...
		// instead of just
		//    CMPXCHGQ ...
		//    JEQ ...
		// but we can't do that because memory-using ops can't generate flags yet
		// (flagalloc wants to move flag-generating instructions around).

	JCS 2(PC)
	//TODO: LJMPQ* (BX)                     // 48ff2b
	JCS 2(PC)
	//TODO: LJMPQ* (R11)                    // 49ff2b
	//TODO: JNE .+$0x11223344               // 480f8544332211 or 0f8544332211
	//TODO: JNE .+$0x11                     // 7511
	//TODO: JNO .+$0x11223344               // 480f8144332211 or 0f8144332211
	//TODO: JNO .+$0x11                     // 7111