RET

cmp4:	// Length 4 - 7
	CMPU	R4,$4
	BLT	cmp2
	MOVD	$-4,R11
	ADD	$-4,R4,R4

	_LWBEX	(R0)(R3),R10
	_LWBEX	(R11)(R9),R11
	CMPB	R10,R5,R10
	CMPB	R11,R5,R11
	CNTLZW	R10,R10
	CNTLZW	R11,R11
	CMPU	R10,$32
	CMPU	R11,$32,CR1
	SRD	$3,R10,R3
	SRD	$3,R11,R11
	BNE	found

	ADD	R4,R11,R4
	MOVD	$-1,R3
	ISEL	CR1EQ,R3,R4,R3
	RET

cmp2:	// Length 2 - 3

#else
tail:	// Count the last 0 - 31 bytes.
	CMP	R4, $16
	BLT	tail_8
	MOVD	(R3), R12
	MOVD	8(R3), R14
	CMPB	R12, R5, R12
	CMPB	R14, R5, R14
	POPCNTD	R12, R12
	POPCNTD	R14, R14
	ADD	R12, R18, R18
	ADD	R14, R18, R18
	ADD	$16, R3, R3
	ADD	$-16, R4, R4

tail_8:	// Count the remaining 0 - 15 bytes.
	CMP	R4, $8
	BLT	tail_4
	MOVD	(R3), R12
	CMPB	R12, R5, R12
	POPCNTD	R12, R12
	ADD	R12, R18, R18

//   b in DI
//   count in BX
// Output:
//   result in AX
TEXT memeqbody<>(SB),NOSPLIT,$0-0
	CMPQ	BX, $8
	JB	small
	CMPQ	BX, $64
	JB	bigloop
#ifndef hasAVX2
	CMPB	internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1
	JE	hugeloop_avx2

	// 64 bytes at a time using xmm registers
	PCALIGN $16
hugeloop:
	CMPQ	BX, $64
	JB	bigloop
	MOVOU	(SI), X0
	MOVOU	(DI), X1
	MOVOU	16(SI), X2
	MOVOU	16(DI), X3

#include "textflag.h"

TEXT ·Count(SB),NOSPLIT,$0-40
#ifndef hasPOPCNT
	CMPB	internal∕cpu·X86+const_offsetX86HasPOPCNT(SB), $1
	JEQ	2(PC)
	JMP	·countGeneric(SB)
#endif
	MOVQ	b_base+0(FP), SI
	MOVQ	b_len+8(FP), BX
	MOVB	c+24(FP), AL
	LEAQ	ret+32(FP), R8
	JMP	countbody<>(SB)

TEXT ·CountString(SB),NOSPLIT,$0-32
#ifndef hasPOPCNT
	CMPB	internal∕cpu·X86+const_offsetX86HasPOPCNT(SB), $1
	JEQ	2(PC)

TEXT cmpbody<>(SB),NOSPLIT,$0-0
	CMPQ	SI, DI
	JEQ	allsame
	CMPQ	BX, DX
	MOVQ	DX, R8
	CMOVQLT	BX, R8 // R8 = min(alen, blen) = # of bytes to compare
	CMPQ	R8, $8
	JB	small

	CMPQ	R8, $63
	JBE	loop
#ifndef hasAVX2
	CMPB	internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1
	JEQ     big_loop_avx2
	JMP	big_loop
#else
	JMP	big_loop_avx2
#endif
loop:
	CMPQ	R8, $16
	JBE	_0through16
	MOVOU	(SI), X0

var x32 [2]uint32
var x64 [2]uint64

func compMem1() int {
	// amd64:`CMPB\tcommand-line-arguments.x\+1\(SB\), [$]0`
	if x[1] {
		return 1
	}
	// amd64:`CMPB\tcommand-line-arguments.x8\+1\(SB\), [$]7`
	if x8[1] == 7 {
		return 1
	}
	// amd64:`CMPW\tcommand-line-arguments.x16\+2\(SB\), [$]7`
	if x16[1] == 7 {
		return 1
	}
	// amd64:`CMPL\tcommand-line-arguments.x32\+4\(SB\), [$]7`
	if x32[1] == 7 {
		return 1
	}

(CMP(Q|L|W|B)loadidx1 {sym} [off] ptr idx x mem) => (CMP(Q|L|W|B) (MOV(Q|L|W|B)loadidx1 {sym} [off] ptr idx mem) x)
(CMPQloadidx8 {sym} [off] ptr idx x mem) => (CMPQ (MOVQloadidx8 {sym} [off] ptr idx mem) x)
(CMPLloadidx4 {sym} [off] ptr idx x mem) => (CMPL (MOVLloadidx4 {sym} [off] ptr idx mem) x)
(CMPWloadidx2 {sym} [off] ptr idx x mem) => (CMPW (MOVWloadidx2 {sym} [off] ptr idx mem) x)

	MOVQ	DX, (R8)
	RET

avx2:
#ifndef hasAVX2
	CMPB   internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1
	JNE sse
#endif
	MOVD AX, X0
	LEAQ -32(SI)(BX*1), R11
	VPBROADCASTB  X0, Y1

	PCALIGN $32
avx2_loop:
	VMOVDQU (DI), Y2
	VPCMPEQB Y1, Y2, Y3
	VPTEST Y3, Y3
	JNZ avx2success
	ADDQ $32, DI
	CMPQ DI, R11
	JLT avx2_loop
	MOVQ R11, DI
	VMOVDQU (DI), Y2

	MOVQ DI, 40(SP)
	MOVQ R8, 48(SP)
	MOVQ R9, 56(SP)
	MOVQ R10, 64(SP)
	MOVQ R11, 72(SP)
	MOVQ R12, 80(SP)
	MOVQ R13, 88(SP)
	MOVQ R14, 96(SP)
	MOVQ R15, 104(SP)
	#ifdef GOOS_darwin
	#ifndef hasAVX
	CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0
	JE 2(PC)
	#endif
	VZEROUPPER
	#endif
	MOVUPS X0, 112(SP)
	MOVUPS X1, 128(SP)
	MOVUPS X2, 144(SP)
	MOVUPS X3, 160(SP)
	MOVUPS X4, 176(SP)
	MOVUPS X5, 192(SP)

	addMulVVW(1536)
	addMulVVW(2048)

	Generate()
}

func addMulVVW(bits int) {
	if bits%64 != 0 {
		panic("bit size unsupported")
	}

	Implement("addMulVVW" + strconv.Itoa(bits))

	CMPB(Mem{Symbol: Symbol{Name: "·supportADX"}, Base: StaticBase}, Imm(1))
	JEQ(LabelRef("adx"))

	z := Mem{Base: Load(Param("z"), GP64())}
	x := Mem{Base: Load(Param("x"), GP64())}
	y := Load(Param("y"), GP64())