VPERM	IN1, IN1, MASK, KEY // vperm 3,2,2,5
	VSLDOI	$12, ZERO, IN0, TMP // vsldoi 6,0,1,12
	VCIPHERLAST	KEY, RCON, KEY      // vcipherlast 3,3,4

	VXOR	IN0, TMP, IN0       // vxor 1,1,6
	VSLDOI	$12, ZERO, TMP, TMP // vsldoi 6,0,6,12
	VXOR	IN0, TMP, IN0       // vxor 1,1,6
	VSLDOI	$12, ZERO, TMP, TMP // vsldoi 6,0,6,12
	VXOR	IN0, TMP, IN0       // vxor 1,1,6

	VSLDOI	$8, ZERO, IN1, STAGE  // vsldoi 7,0,2,8

	VXOR	KI, LEMASK, LEMASK
#endif

	LXVW4X	(CTX)(R_x000), V0
	LXVW4X	(CTX)(R_x010), V4

	// unpack the input values into vector registers
	VSLDOI	$4, V0, V0, V1
	VSLDOI	$8, V0, V0, V2
	VSLDOI	$12, V0, V0, V3
	VSLDOI	$4, V4, V4, V5
	VSLDOI	$8, V4, V4, V6
	VSLDOI	$12, V4, V4, V7

	MOVD	$0x020, R_x020
	MOVD	$0x030, R_x030
	MOVD	$0x040, R_x040
	MOVD	$0x050, R_x050
	MOVD	$0x060, R_x060

	MOVD     $3, R9             // Number of bytes beyond 16
	VLOADSWAP(R7, R9, V3, V3)   // Load 16 bytes @R7+3 into V3
	VSLDOI   $13, V3, V10, V3   // Shift left last 3 bytes
	VSLDOI   $1, V2, V3, V4     // V4=(V2:V3)<<1
	VSLDOI   $2, V2, V3, V9     // V9=(V2:V3)<<2
	VSLDOI   $3, V2, V3, V10    // V10=(V2:v3)<<3
	VCMPEQUW V1, V2, V5         // compare index 0, 4, ... with sep

	VSLDOI $8, XM, ZERO, T0
	VSLDOI $8, ZERO, XM, T1
	VSLDOI $8, XM1, ZERO, HL
	VSLDOI $8, ZERO, XM1, H
	VXOR   XL, T0, XL
	VXOR   XH, T1, XH
	VXOR   XL1, HL, XL1
	VXOR   XH1, H, XH1

	VSLDOI $8, XL, XL, XL
	VSLDOI $8, XL1, XL1, XL1
	VXOR   XL, T2, XL
	VXOR   XL1, HH, XL1

	VSLDOI  $8, XL, XL, T1  // 2nd reduction phase
	VSLDOI  $8, XL1, XL1, H // 2nd reduction phase

	VXOR	KI, LEMASK, LEMASK
#endif

	LXVW4X	(CTX)(R_x000), V0
	LXVW4X	(CTX)(R_x010), V4

	// unpack the input values into vector registers
	VSLDOI	$4, V0, V0, V1
	VSLDOI	$8, V0, V0, V2
	VSLDOI	$12, V0, V0, V3
	VSLDOI	$4, V4, V4, V5
	VSLDOI	$8, V4, V4, V6
	VSLDOI	$12, V4, V4, V7

	MOVD	$0x020, R_x020
	MOVD	$0x030, R_x030
	MOVD	$0x040, R_x040
	MOVD	$0x050, R_x050
	MOVD	$0x060, R_x060

	LXVD2X	(CTX)(R_x010), VS34	// v2 = vs34
	LXVD2X	(CTX)(R_x020), VS36	// v4 = vs36

	// unpack the input values into vector registers
	VSLDOI	$8, V0, V0, V1
	LXVD2X	(CTX)(R_x030), VS38	// v6 = vs38
	VSLDOI	$8, V2, V2, V3
	VSLDOI	$8, V4, V4, V5
	VSLDOI	$8, V6, V6, V7

loop:
	MOVD	TBL_STRT, TBL
	LVX	(TBL)(R_x000), KI

	LXVD2X	(INP)(R0), VS40	// load v8 (=vs40) in advance
	ADD	$16, INP

	VXOR    V4,V12,V4
	VXOR    V5,V13,V5
	VXOR    V6,V14,V6
	VXOR    V7,V15,V7

#ifdef REFLECT
	VSLDOI  $4,V0,zeroes,V0
	VSLDOI  $4,V1,zeroes,V1
	VSLDOI  $4,V2,zeroes,V2
	VSLDOI  $4,V3,zeroes,V3
	VSLDOI  $4,V4,zeroes,V4
	VSLDOI  $4,V5,zeroes,V5
	VSLDOI  $4,V6,zeroes,V6
	VSLDOI  $4,V7,zeroes,V7
#endif

	LVX	(R4),V8
	LVX	(R4+off16),V9
	LVX	(R4+off32),V10
	LVX	(R4+off48),V11

	VPERM   T1, T0, SEL1, RED2    // d1 d0 d1 d0
	VPERM   ZER, RED2, SEL2, RED1 // 0  d1 d0  0
	VSUBUQM RED2, RED1, RED2      // VSQ   RED1, RED2, RED2      // Guaranteed not to underflow

	VSLDOI $8, T1, T0, T0 // VSLDB $8, T1, T0, T0
	VSLDOI $8, T2, T1, T1 // VSLDB $8, T2, T1, T1

	VADDCUQ  T0, RED1, CAR1       // VACCQ  T0, RED1, CAR1
	VADDUQM  T0, RED1, T0         // VAQ    T0, RED1, T0

	VADDUDM	V0, V4, V4	// Accumulate the popcounts. They are 8x the count.
	VADDUDM	V2, V5, V5	// The count will be fixed up afterwards.
	ADD	$32, R3
	BDNZ	cmploop

	VADDUDM	V4, V5, V5
	MFVSRD	V5, R18
	VSLDOI	$8, V5, V5, V5
	MFVSRD	V5, R21
	ADD	R21, R18, R18
	ANDCC	$31, R4, R4
	// Skip the tail processing if no bytes remaining.
	BEQ	tail_0

#ifdef GOPPC64_power10

	MFVSRD	VS36,R10

	CMPU	R16,R10
	BEQ	lower
	SETB_CR0_NE(R3)
	RET

	PCALIGN $16
lower:
	VSLDOI	$8,V3,V3,V3	// move lower doublewords of A and B into GPR for comparison
	MFVSRD	VS35,R16
	VSLDOI	$8,V4,V4,V4
	MFVSRD	VS36,R10

	CMPU	R16,R10
	SETB_CR0_NE(R3)
	RET

	PCALIGN $16
cmp8:	// 8 - 15B (0 - 15B if GOPPC64_power10)