TESTQ	BX, BX
	JEQ	move_0
	CMPQ	BX, $2
	JBE	move_1or2
	CMPQ	BX, $4
	JB	move_3
	JBE	move_4
	CMPQ	BX, $8
	JB	move_5through7
	JE	move_8
	CMPQ	BX, $16
	JBE	move_9through16
	CMPQ	BX, $32
	JBE	move_17through32
	CMPQ	BX, $64
	JBE	move_33through64
	CMPQ	BX, $128
	JBE	move_65through128
	CMPQ	BX, $256
	JBE	move_129through256

	TESTB	$1, runtime·useAVXmemmove(SB)
	JNZ	avxUnaligned

	PMOVMSKB X1, DX
	// Count number of matching bytes
	POPCNTL DX, DX
	// Accumulate into R12
	ADDQ DX, R12
	// Advance to next block.
	ADDQ	$16, DI
sseloopentry:
	CMPQ	DI, AX
	JBE	sseloop

	// Get the number of bytes to consider in the last 16 bytes
	ANDQ $15, BX
	JZ end

	// Create mask to ignore overlap between previous 16 byte block
	// and the next.
	MOVQ $16,CX
	SUBQ BX, CX

	JB loop33to63
fail_avx2:
	VZEROUPPER
fail:
	MOVQ $-1, (R11)
	RET
success_avx2:
	VZEROUPPER
	JMP success
sse42:
#ifndef hasSSE42
	CMPB internal∕cpu·X86+const_offsetX86HasSSE42(SB), $1
	JNE no_sse42
#endif
	CMPQ AX, $12
	// PCMPESTRI is slower than normal compare,
	// so using it makes sense only if we advance 4+ bytes per compare
	// This value was determined experimentally and is the ~same

	// find out information about the processor we're on
	MOVL	$0, AX
	CPUID
	CMPL	AX, $0
	JE	nocpuinfo

	CMPL	BX, $0x756E6547  // "Genu"
	JNE	notintel
	CMPL	DX, $0x49656E69  // "ineI"
	JNE	notintel
	CMPL	CX, $0x6C65746E  // "ntel"
	JNE	notintel
	MOVB	$1, runtime·isIntel(SB)

notintel:
	// Load EAX=1 cpuid flags
	MOVL	$1, AX
	CPUID
	MOVL	AX, runtime·processorVersionInfo(SB)

	// for large sizes. The cutover is approximately 1K.
tail:
	TESTQ	BX, BX
	JEQ	move_0
	CMPQ	BX, $2
	JBE	move_1or2
	CMPQ	BX, $4
	JBE	move_3or4
	CMPQ	BX, $8
	JB	move_5through7
	JE	move_8
	CMPQ	BX, $16
	JBE	move_9through16

/*
 * check and set for backwards
 */
	CMPQ	SI, DI
	JLS	back

/*
 * forward copy loop
 */
forward:
	MOVQ	BX, CX

	// for large sizes. The cutover is approximately 1K.
tail:
	TESTL	BX, BX
	JEQ	move_0
	CMPL	BX, $2
	JBE	move_1or2
	CMPL	BX, $4
	JB	move_3
	JE	move_4
	CMPL	BX, $8
	JBE	move_5through8
	CMPL	BX, $16
	JBE	move_9through16

/*
 * check and set for backwards
 */
	CMPL	SI, DI
	JLS	back

/*
 * forward copy loop
 */
forward:
	MOVL	BX, CX

has_cpuid:
	MOVL	$0, AX
	CPUID
	MOVL	AX, SI
	CMPL	AX, $0
	JE	nocpuinfo

	CMPL	BX, $0x756E6547  // "Genu"
	JNE	notintel
	CMPL	DX, $0x49656E69  // "ineI"
	JNE	notintel
	CMPL	CX, $0x6C65746E  // "ntel"
	JNE	notintel
	MOVB	$1, runtime·isIntel(SB)
notintel:

	// Load EAX=1 cpuid flags
	MOVL	$1, AX
	CPUID
	MOVL	CX, DI // Move to global variable clobbers CX when generating PIC

	MOVL	trap+0(FP), AX	// syscall entry
	// copy args down
	LEAL	a1+4(FP), SI
	LEAL	sysargs-144(SP), DI
	CLD
	MOVSL
	MOVSL
	MOVSL
	INT	$64
	MOVL	AX, r1+16(FP)
	MOVL	$0, r2+20(FP)
	CMPL	AX, $-1
	JNE	ok3

	LEAL	errbuf-128(SP), AX
	MOVL	AX, sysargs-144(SP)
	MOVL	$128, sysargs1-140(SP)
	MOVL	$SYS_ERRSTR, AX
	INT	$64
	CALL	runtime·exitsyscall(SB)
	MOVL	sysargs-144(SP), AX
	MOVL	AX, errbuf-148(SP)

	MOVL	(0*4)(BX), AX		// fn
	CALL	AX

	MOVL	AX, (4*4)(BX)		// r1
	MOVL	DX, (5*4)(BX)		// r2

	// Standard libc functions return -1 on error and set errno.
	CMPL	AX, $-1
	JNE	ok
	CMPL	DX, $-1
	JNE	ok

	// Get error code from libc.
	CALL	libc_errno(SB)
	MOVL	(AX), AX
	MOVW	AX, (6*4)(BX)		// err

ok:
	MOVL	$0, AX			// no error (it's ignored anyway)
	MOVL	BP, SP
	POPL	BP

func ptrEqual() {
	// amd64:-"JEQ",-"JNE"
	// ppc64x:-"BEQ",-"BNE"
	// s390x:-"BEQ",-"BNE"
	copy(x[:], x[:])
}

func ptrOneOffset() {
	// amd64:-"JEQ",-"JNE"
	// ppc64x:-"BEQ",-"BNE"
	// s390x:-"BEQ",-"BNE"
	copy(x[1:], x[:])
}

func ptrBothOffset() {
	// amd64:-"JEQ",-"JNE"
	// ppc64x:-"BEQ",-"BNE"
	// s390x:-"BEQ",-"BNE"
	copy(x[1:], x[2:])
}