MOVD	R20, CTR
	MOVD	$16, R21
	XXLXOR	V4, V4, V4
	XXLXOR	V5, V5, V5

	PCALIGN	$16
cmploop:
	LXVD2X	(R0)(R3), V0	// Count 32B per loop with two vector accumulators.
	LXVD2X	(R21)(R3), V2
	VCMPEQUB V2, V1, V2
	VCMPEQUB V0, V1, V0
	VPOPCNTD V2, V2		// A match is 0xFF or 0. Count the bits into doubleword buckets.
	VPOPCNTD V0, V0
	VADDUDM	V0, V4, V4	// Accumulate the popcounts. They are 8x the count.

	"VSRAW",
	"VSRAD",
	"VSOI",
	"VSLDOI",
	"VCLZ",
	"VCLZB",
	"VCLZH",
	"VCLZW",
	"VCLZD",
	"VPOPCNT",
	"VPOPCNTB",
	"VPOPCNTH",
	"VPOPCNTW",
	"VPOPCNTD",
	"VCMPEQ",
	"VCMPEQUB",
	"VCMPEQUBCC",
	"VCMPEQUH",
	"VCMPEQUHCC",
	"VCMPEQUW",
	"VCMPEQUWCC",
	"VCMPEQUD",
	"VCMPEQUDCC",
	"VCMPGT",
	"VCMPGTUB",
	"VCMPGTUBCC",
	"VCMPGTUH",
	"VCMPGTUHCC",

index17to32loop:
	VLOADSWAP(R7, R0, V2, V2)  // Load 16 bytes @R7 into V2

next17:
	VLOADSWAP(R7, R9, V3, V3)  // Load 16 bytes @R7+R9 into V3
	VSLO       V3, V9, V3      // Shift left
	VCMPEQUB   V0, V2, V4      // Compare first 16 bytes
	VCMPEQUB   V1, V3, V5      // Compare extra over 16 bytes
	VAND       V4, V5, V6      // Check if both equal
	VCMPEQUBCC V6, V7, V8      // All equal?
	BLT        CR6, found      // Yes

	BR	notfound


cmp8:	// Length 8 - 15
#ifdef GOPPC64_power10
	// Load all the bytes into a single VSR in BE order.
	SLD	$56,R4,R5
	LXVLL	R3,R5,V2
	// Compare and count the number which don't match.
	VCMPEQUB	V2,V1,V6
	VCLZLSBB	V6,R3
	// If count is the number of bytes, or more. No matches are found.
	CMPU	R3,R4
	MOVD	$-1,R5
	// Otherwise, the count is the index of the first match.
	ISEL	CR0LT,R3,R5,R3
	RET

	VCFSX:          "vcfsx",
	VCFUX:          "vcfux",
	VCMPBFP:        "vcmpbfp",
	VCMPBFPCC:      "vcmpbfp.",
	VCMPEQFP:       "vcmpeqfp",
	VCMPEQFPCC:     "vcmpeqfp.",
	VCMPEQUB:       "vcmpequb",
	VCMPEQUBCC:     "vcmpequb.",
	VCMPEQUH:       "vcmpequh",
	VCMPEQUHCC:     "vcmpequh.",
	VCMPEQUW:       "vcmpequw",
	VCMPEQUWCC:     "vcmpequw.",
	VCMPGEFP:       "vcmpgefp",
	VCMPGEFPCC:     "vcmpgefp.",

	VPOPCNTB V1, V2                 // 10400f03
	VPOPCNTH V1, V2                 // 10400f43
	VPOPCNTW V1, V2                 // 10400f83
	VPOPCNTD V1, V2                 // 10400fc3
	VCMPEQUB V1, V2, V3             // 10611006
	VCMPEQUBCC V1, V2, V3           // 10611406
	VCMPEQUH V1, V2, V3             // 10611046
	VCMPEQUHCC V1, V2, V3           // 10611446
	VCMPEQUW V1, V2, V3             // 10611086

		return OPVX(4, 1923, 0, 0) /* vpopcntw - v2.07 */
	case AVPOPCNTD:
		return OPVX(4, 1987, 0, 0) /* vpopcntd - v2.07 */

	case AVCMPEQUB:
		return OPVC(4, 6, 0, 0) /* vcmpequb - v2.03 */
	case AVCMPEQUBCC:
		return OPVC(4, 6, 0, 1) /* vcmpequb. - v2.03 */
	case AVCMPEQUH:
		return OPVC(4, 70, 0, 0) /* vcmpequh - v2.03 */
	case AVCMPEQUHCC:
		return OPVC(4, 70, 0, 1) /* vcmpequh. - v2.03 */
	case AVCMPEQUW: