"VMRGEW",
	"VMRGOW",
	"VCLZLSBB",
	"VCTZLSBB",
	"LXV",
	"LXVL",
	"LXVLL",
	"LXVD2X",
	"LXVW4X",
	"LXVH8X",
	"LXVB16X",
	"LXVX",
	"LXVDSX",
	"STXV",
	"STXVL",
	"STXVLL",
	"STXVD2X",
	"STXVW4X",
	"STXVH8X",
	"STXVB16X",
	"STXVX",
	"LXSDX",
	"STXSDX",
	"LXSIWAX",
	"LXSIWZX",
	"STXSIWX",
	"MFVSRD",
	"MFFPRD",
	"MFVRD",
	"MFVSRWZ",

	LXSDX (R3)(R0), VS1             // 7c201c98
	LXSDX (R3), VS1                 // 7c201c98
	STXVD2X VS1, (R3)(R4)           // 7c241f98
	STXVD2X VS1, (R3)(R0)           // 7c201f98
	STXVD2X VS1, (R3)               // 7c201f98
	STXVW4X VS1, (R3)(R4)           // 7c241f18
	STXVW4X VS1, (R3)(R0)           // 7c201f18
	STXVW4X VS1, (R3)               // 7c201f18
	STXV VS1,16(R3)                 // f4230015

		case ALXVD2X: /* lxvd2x, lxvdsx, lxvw4x, lxvh8x, lxvb16x */
			opset(ALXVDSX, r0)
			opset(ALXVW4X, r0)
			opset(ALXVH8X, r0)
			opset(ALXVB16X, r0)

		case ALXV: /* lxv */
			opset(ALXV, r0)

		case ALXVL: /* lxvl, lxvll, lxvx */
			opset(ALXVLL, r0)
			opset(ALXVX, r0)

		case ASTXVD2X: /* stxvd2x, stxvdsx, stxvw4x, stxvh8x, stxvb16x */
			opset(ASTXVW4X, r0)

	LXVW4X (INP)(R9), VS61
	LXVW4X (INP)(R10), VS62

	VXOR V27, V0, V27
	VXOR V28, V4, V28
	VXOR V29, V8, V29
	VXOR V30, V12, V30

	STXVW4X VS59, (OUT)(R0)
	STXVW4X VS60, (OUT)(R8)
	ADD     $64, INP
	STXVW4X VS61, (OUT)(R9)
	ADD     $-64, LEN
	STXVW4X VS62, (OUT)(R10)
	ADD     $64, OUT
	BEQ     done_vsx

	VADDUWM V1, V16, V0
	VADDUWM V5, V17, V4
	VADDUWM V9, V18, V8
	VADDUWM V13, V19, V12

	VXOR	IN0, KEY, IN0                    // vxor 1,1,3
	STXVD2X	IN0, (R0+OUTENC)
	STXVD2X	IN0, (R0+OUTDEC)

	RET

l192:
	LXSDX	(INP+R0), IN1                    // Load next 8 bytes into upper half of VSR.
	XXBRD_ON_LE(IN1, IN1)                    // and convert to BE ordering on LE hosts.
	MOVD	$4, CNT                          // li 7,4
	STXVD2X	IN0, (R0+OUTENC)
	STXVD2X	IN0, (R0+OUTDEC)
	ADD	$16, OUTENC, OUTENC

	LXVD2X	(R5)(R10), VS35
	LXVD2X	(R5)(R14), VS37
	LXVD2X	(R5)(R15), VS39
	XXLXOR	VS32, VS33, VS32
	XXLXOR	VS34, VS35, VS34
	XXLXOR	VS36, VS37, VS36
	XXLXOR	VS38, VS39, VS38
	STXVD2X	VS32, (R3)(R8)
	STXVD2X	VS34, (R3)(R10)
	STXVD2X	VS36, (R3)(R14)
	STXVD2X	VS38, (R3)(R15)
	ADD	$64, R8
	ADD	$64, R10
	ADD	$64, R14
	ADD	$64, R15
	BDNZ	loop64
	BC	12,2,LR		// BEQLR
	MOVD	R9, R6
	CMP	R6, $8
	BLE	small

	ADD     $96, P1ptr
	BDNZ    loop_select

	// STXVD2X is used here so that alignment doesn't
	// need to be verified. Since values were loaded
	// using LXVD2X this is OK.
	STXVD2X X1H, (P3ptr+R0)
	STXVD2X X1L, (P3ptr+R16)
	STXVD2X Y1H, (P3ptr+R17)
	STXVD2X Y1L, (P3ptr+R18)
	STXVD2X Z1H, (P3ptr+R19)
	STXVD2X Z1L, (P3ptr+R20)
	RET

#undef P3ptr
#undef P1ptr
#undef COUNT

	VSLDOI $8, H, ZERO, HH
	VSLDOI $8, ZERO, H2, H2L
	VSLDOI $8, H2, ZERO, H2H

	STXVD2X VHL, (XIP+R8)   // save H^3
	MOVD    $0xa0, R8
	STXVD2X VH, (XIP+R9)
	MOVD    $0xb0, R9
	STXVD2X VHH, (XIP+R10)
	MOVD    $0xc0, R10
	STXVD2X VH2L, (XIP+R8)  // save H^4
	STXVD2X VH2, (XIP+R9)
	STXVD2X VH2H, (XIP+R10)

	RET

// func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)

	AVSHASIGMAD
	AVMRGEW
	AVMRGOW
	AVCLZLSBB
	AVCTZLSBB

	/* VSX */
	ALXV
	ALXVL
	ALXVLL
	ALXVD2X
	ALXVW4X
	ALXVH8X
	ALXVB16X
	ALXVX
	ALXVDSX
	ASTXV
	ASTXVL
	ASTXVLL
	ASTXVD2X
	ASTXVW4X
	ASTXVH8X
	ASTXVB16X
	ASTXVX
	ALXSDX
	ASTXSDX
	ALXSIWAX
	ALXSIWZX
	ASTXSIWX
	AMFVSRD
	AMFFPRD
	AMFVRD
	AMFVSRWZ

	VPERM	V6, V7, KI, V6
#else
	VPERM	V1, V0, KI, V0
	VPERM	V3, V2, KI, V2
	VPERM	V5, V4, KI, V4
	VPERM	V7, V6, KI, V6
#endif
	STXVD2X	VS32, (CTX+R_x000)	// v0 = vs32
	STXVD2X	VS34, (CTX+R_x010)	// v2 = vs34
	STXVD2X	VS36, (CTX+R_x020)	// v4 = vs36
	STXVD2X	VS38, (CTX+R_x030)	// v6 = vs38

end: