// R4 = length
	// R6 = byte to count
	MTVRD	R6, V1		// move compare byte
	MOVD	R6, R5
	VSPLTB	$7, V1, V1	// replicate byte across V1
	BR	countbytebody<>(SB)

TEXT ·CountString<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-32
	// R3 = byte array pointer
	// R4 = length
	// R5 = byte to count
	MTVRD	R5, V1		// move compare byte
	VSPLTB	$7, V1, V1	// replicate byte across V1
	BR	countbytebody<>(SB)

// R3: addr of string

	"VCMPNEB",
	"VCMPNEBCC",
	"VCMPNEH",
	"VCMPNEHCC",
	"VCMPNEW",
	"VCMPNEWCC",
	"VPERM",
	"VPERMXOR",
	"VPERMR",
	"VBPERMQ",
	"VBPERMD",
	"VSEL",
	"VSPLTB",
	"VSPLTH",
	"VSPLTW",
	"VSPLTISB",
	"VSPLTISH",
	"VSPLTISW",
	"VCIPH",
	"VCIPHER",
	"VCIPHERLAST",
	"VNCIPH",
	"VNCIPHER",
	"VNCIPHERLAST",
	"VSBOX",
	"VSHASIGMA",
	"VSHASIGMAW",
	"VSHASIGMAD",

	CMPU	R4,$32

#ifndef GOPPC64_power9
	// Load VBPERMQ constant to reduce compare into an ordered bit mask.
	MOVD	$indexbytevbperm<>+00(SB),R16
	LXVD2X	(R16),V0	// Set up swap string
#endif

	MTVRD	R5,V1
	VSPLTB	$7,V1,V1	// Replicate byte across V1

	BLT	cmp16		// Jump to the small string case if it's <32 bytes.

	CMP	R4,$64,CR1
	MOVD	$16,R11
	MOVD	R3,R8
	BLT	CR1,cmp32	// Special case for length 32 - 63
	MOVD	$32,R12

	SLD    $24, R21
	MTVSRD R21, V10
	VSPLTW $1, V10, V29
	VSLDOI $2, V29, V29, V30 // Mask 0x0000ff000000ff00...
	MOVD   $0xffff, R21
	SLD    $16, R21
	MTVSRD R21, V10
	VSPLTW $1, V10, V31      // Mask 0xffff0000ffff0000...
	VSPLTW $0, V0, V1        // Splat 1st word of separator

index4loop:
	VLOADSWAP(R7, R0, V2, V2)   // Load 16 bytes @R7 into V2

next4:
	VSPLTISB $0, V10            // Clear

	CMP     R8, R4
	BGE     loopexit        // Already at end?

	// vectorize if len(z) is >=3, else jump to scalar loop
	CMP     R4, $3
	BLT     scalar
	MTVSRD  R9, VS38        // s
	VSPLTB  $7, V6, V4
	MTVSRD  R5, VS39        // ŝ
	VSPLTB  $7, V7, V2
	ADD     $-2, R4, R16
	PCALIGN $16
loopback:
	ADD     $-1, R8, R10
	SLD     $3, R10
	LXVD2X  (R6)(R10), VS32 // load x[i-1], x[i]
	SLD     $3, R8, R12

	LXVDSX   (R1)(R18), SEL1 // VLREPG idx+32(FP), SEL1
	VSPLTB   $7, SEL1, IDX    // splat byte
	VSPLTISB $1, ONE          // VREPIB $1, ONE
	VSPLTISB $1, SEL2         // VREPIB $1, SEL2
	MOVD     $17, COUNT
	MOVD     COUNT, CTR       // set up ctr

	VSPLTISB $0, X1H // VZERO  X1H
	VSPLTISB $0, X1L // VZERO  X1L
	VSPLTISB $0, Y1H // VZERO  Y1H
	VSPLTISB $0, Y1L // VZERO  Y1L
	VSPLTISB $0, Z1H // VZERO  Z1H
	VSPLTISB $0, Z1L // VZERO  Z1L

	MOVD   $0x30, R10
	LXVD2X (HTBL)(R0), VH // Load H

	VSPLTISB $-16, XC2           // 0xf0
	VSPLTISB $1, T0              // one
	VADDUBM  XC2, XC2, XC2       // 0xe0
	VXOR     ZERO, ZERO, ZERO
	VOR      XC2, T0, XC2        // 0xe1
	VSLDOI   $15, XC2, ZERO, XC2 // 0xe1...
	VSLDOI   $1, ZERO, T0, T1    // ...1
	VADDUBM  XC2, XC2, XC2       // 0xc2...
	VSPLTISB $7, T2
	VOR      XC2, T1, XC2        // 0xc2....01

	VBPERMQ V1, V2, V3              // 1061154c
	VBPERMD V1, V2, V3              // 106115cc
	VSEL V1, V2, V3, V4             // 108110ea
	VSPLTB $1, V1, V2               // 10410a0c
	VSPLTH $1, V1, V2               // 10410a4c
	VSPLTW $1, V1, V2               // 10410a8c
	VSPLTISB $1, V1                 // 1021030c
	VSPLTISW $1, V1                 // 1021038c
	VSPLTISH $1, V1                 // 1021034c

	VSEL:           "vsel",
	VSL:            "vsl",
	VSLB:           "vslb",
	VSLDOI:         "vsldoi",
	VSLH:           "vslh",
	VSLO:           "vslo",
	VSLW:           "vslw",
	VSPLTB:         "vspltb",
	VSPLTH:         "vsplth",
	VSPLTISB:       "vspltisb",
	VSPLTISH:       "vspltish",
	VSPLTISW:       "vspltisw",
	VSPLTW:         "vspltw",
	VSR:            "vsr",
	VSRAB:          "vsrab",

		case AVBPERMQ: /* vbpermq, vbpermd */
			opset(AVBPERMD, r0)

		case AVSEL: /* vsel */
			opset(AVSEL, r0)

		case AVSPLTB: /* vspltb, vsplth, vspltw */
			opset(AVSPLTH, r0)
			opset(AVSPLTW, r0)

		case AVSPLTISB: /* vspltisb, vspltish, vspltisw */
			opset(AVSPLTISH, r0)
			opset(AVSPLTISW, r0)

		case AVCIPH: /* vcipher, vcipherlast */
			opset(AVCIPHER, r0)