JMP	·countGeneric(SB)
#endif
	MOVQ	b_base+0(FP), SI
	MOVQ	b_len+8(FP), BX
	MOVB	c+24(FP), AL
	LEAQ	ret+32(FP), R8
	JMP	countbody<>(SB)

TEXT ·CountString(SB),NOSPLIT,$0-32
#ifndef hasPOPCNT
	CMPB	internal∕cpu·X86+const_offsetX86HasPOPCNT(SB), $1
	JEQ	2(PC)
	JMP	·countGenericString(SB)
#endif
	MOVQ	s_base+0(FP), SI
	MOVQ	s_len+8(FP), BX
	MOVB	c+16(FP), AL
	LEAQ	ret+24(FP), R8
	JMP	countbody<>(SB)

// input:

residual:
	CMPBEQ	R3, R8, notfound
	MOVBZ	0(R3), R7
	LA	1(R3), R3
	CMPBNE	R7, R5, residual

found:
	SUB	R6, R3
	SUB	$1, R3
	MOVD	R3, 0(R2)
	RET

notfound:
	MOVD	$-1, 0(R2)
	RET

large:
	MOVBZ	internal∕cpu·S390X+const_offsetS390xHasVX(SB), R1
	CMPBNE	R1, $0, vectorimpl

srstimpl:                       // no vector facility

	RET

TEXT	·Load8(SB),NOSPLIT|NOFRAME,$0-5
	MOVW	addr+0(FP), R0
	MOVB	(R0), R1

	MOVB	runtime·goarm(SB), R11
	CMP	$7, R11
	BLT	2(PC)
	DMB	MB_ISH

	MOVB	R1, ret+4(FP)
	RET

TEXT	·Store8(SB),NOSPLIT,$0-5
	MOVW	addr+0(FP), R1
	MOVB	v+4(FP), R2

	MOVB	runtime·goarm(SB), R8
	CMP	$7, R8
	BLT	2(PC)
	DMB	MB_ISH

	MOVB	R2, (R1)

	CMP	$7, R8
	BLT	2(PC)
	DMB	MB_ISH

	RET

TEXT	·Load8(SB),NOSPLIT,$0-5
	MOVW	addr+0(FP), R0
	MOVB	(R0), R1

	MOVB	runtime·goarm(SB), R11
	CMP	$7, R11
	BGE	native_barrier
	BL	memory_barrier<>(SB)
	B	end
native_barrier:
	DMB	MB_ISH
end:
	MOVB	R1, ret+4(FP)
	RET

TEXT	·Store8(SB),NOSPLIT,$0-5
	MOVW	addr+0(FP), R1
	MOVB	v+4(FP), R2

	MOVB	runtime·goarm(SB), R8
	CMP	$7, R8
	BGE	native_barrier

	MOVWU	(R1), R6
	MOVWU	-4(R4), R7
	MOVW	R6, (R0)
	MOVW	R7, -4(R5)
	RET

copy3:
	TBZ	$1, R2, copy1
	MOVHU	(R1), R6
	MOVHU	-2(R4), R7
	MOVH	R6, (R0)
	MOVH	R7, -2(R5)
	RET

copy1:
	MOVBU	(R1), R6
	MOVB	R6, (R0)

copy0:
	RET

	// Medium copies: 33..128 bytes.
copy32_128:
	ADD	R1, R2, R4          // R4 points just past the last source byte

	BNE	R3, out

	// do one byte at a time until 8-aligned
	AND	$7, R1, R3
	BEQ	R3, words
	MOVB	R0, (R1)
	ADDV	$1, R1
	JMP	-4(PC)

words:
	// do 8 bytes at a time if there is room
	ADDV	$-7, R4, R2

	SGTU	R2, R1, R3
	BEQ	R3, out
	MOVV	R0, (R1)
	ADDV	$8, R1
	JMP	-4(PC)

out:
	BEQ	R1, R4, done
	MOVB	R0, (R1)
	ADDV	$1, R1
	JMP	-3(PC)
done:

TEXT ·Store(SB), NOSPLIT, $0
	MOVD	ptr+0(FP), R2
	MOVWZ	val+8(FP), R3
	MOVW	R3, 0(R2)
	SYNC
	RET

// func Store8(ptr *uint8, val uint8)
TEXT ·Store8(SB), NOSPLIT, $0
	MOVD	ptr+0(FP), R2
	MOVB	val+8(FP), R3
	MOVB	R3, 0(R2)
	SYNC
	RET

// func Store64(ptr *uint64, val uint64)
TEXT ·Store64(SB), NOSPLIT, $0
	MOVD	ptr+0(FP), R2
	MOVD	val+8(FP), R3
	MOVD	R3, 0(R2)
	SYNC
	RET

zero8:
	MOV	ZERO, 0(X10)
	ADD	$8, X10
	SUB	$8, X11
	BEQZ	X11, done

check4:
	MOV	$4, X9
	BLT	X11, X9, loop1
zero4:
	MOVB	ZERO, 0(X10)
	MOVB	ZERO, 1(X10)
	MOVB	ZERO, 2(X10)
	MOVB	ZERO, 3(X10)
	ADD	$4, X10
	SUB	$4, X11

loop1:
	BEQZ	X11, done
	MOVB	ZERO, 0(X10)
	ADD	$1, X10
	SUB	$1, X11
	JMP	loop1

done:

	B	equal
lt_4:
	TBZ	$1, R2, lt_2
	MOVHU.P	2(R0), R4
	MOVHU.P	2(R1), R5
	CMP	R4, R5
	BNE	not_equal
lt_2:
	TBZ	$0, R2, equal
one:
	MOVBU	(R0), R4
	MOVBU	(R1), R5
	CMP	R4, R5
	BNE	not_equal
equal:
	MOVD	$1, R0
	RET
not_equal:
	MOVB	ZR, R0

	REP;	MOVSL

	CLD
	ADDL	$4, DI
	ADDL	$4, SI
	SUBL	BX, DI
	SUBL	BX, SI
	JMP	tail

move_1or2:
	MOVB	(SI), AX
	MOVB	-1(SI)(BX*1), CX
	MOVB	AX, (DI)
	MOVB	CX, -1(DI)(BX*1)
	RET
move_0:
	RET
move_3:
	MOVW	(SI), AX
	MOVB	2(SI), CX
	MOVW	AX, (DI)
	MOVB	CX, 2(DI)
	RET
move_4:
	// We need a separate case for 4 to make sure we write pointers atomically.
	MOVL	(SI), AX