PSHUFD $78, T0, T0
		PXOR T0, B2
		PXOR T1, B2

		MOVOU B2, (16*12)(dst)
		PSHUFD $78, B2, B3
		PXOR B2, B3
		MOVOU B3, (16*13)(dst)

		DECQ AX
		LEAQ (-16*2)(dst), dst
	JNE initLoop

	RET
#undef NR
#undef KS
#undef dst

// func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
TEXT ·gcmAesData(SB),NOSPLIT,$0
#define pTbl DI
#define aut SI

	MOVL	m_vdsoSP(SI), DX
	MOVL	CX, 0(SP)
	MOVL	DX, 4(SP)

	LEAL	sec+0(FP), DX
	MOVL	-4(DX), CX
	MOVL	CX, m_vdsoPC(SI)
	MOVL	DX, m_vdsoSP(SI)

	CMPL	AX, m_curg(SI)	// Only switch if on curg.
	JNE	noswitch

	MOVL	m_g0(SI), DX
	MOVL	(g_sched+gobuf_sp)(DX), SP	// Set SP to g0 stack

noswitch:
	SUBL	$16, SP		// Space for results
	ANDL	$~15, SP	// Align for C code

cat >umod-amd64.s <<'EOF'
# tu_int __umodti3(tu_int x, tu_int y)
# x is rsi:rdi, y is rcx:rdx, return result is rdx:rax.
.globl __umodti3
__umodti3:
	# specialized to u128 % u64, so verify that
	test %rcx,%rcx
	jne 1f

	# save divisor
	movq %rdx, %r8

	# reduce top 64 bits mod divisor
	movq %rsi, %rax
	xorl %edx, %edx
	divq %r8

	# reduce full 128-bit mod divisor

	MOVD    sinhxinit<>+0(SB), R1
	FMOVD   F0, F4
	MOVD    R1, R3
	BLTU    L19
	FMOVD   F0, F2
L2:
	WORD    $0xED205010     //cdb %f2,.L22-.L21(%r5)
	BYTE    $0x00
	BYTE    $0x19
	BGE     L15     //jnl   .L15
	BVS     L15
	WFCEDBS V2, V2, V0
	BEQ     L20
L12:
	FMOVD   F4, F0
	FMOVD   F0, ret+8(FP)
	RET

L15:
	WFCEDBS V2, V2, V0
	BVS     L12
	MOVD    $sinhxlim1<>+0(SB), R2
	FMOVD   0(R2), F0

	// n > 0
	MOVL z+0(FP), DI
	MOVL x+12(FP), SI
	MOVL s+24(FP), CX
	MOVL (SI)(BX*4), AX	// w1 = x[n-1]
	MOVL $0, DX
	SHLL CX, AX, DX		// w1>>ŝ
	MOVL DX, c+28(FP)

	CMPL BX, $0
	JLE X8a			// i <= 0

	// i > 0
L8:	MOVL AX, DX		// w = w1
	MOVL -4(SI)(BX*4), AX	// w1 = x[i-1]
	SHLL CX, AX, DX		// w<<s | w1>>ŝ
	MOVL DX, (DI)(BX*4)	// z[i] = w<<s | w1>>ŝ
	SUBL $1, BX		// i--

	JCXZ:            "JCXZ",
	JE:              "JE",
	JECXZ:           "JECXZ",
	JG:              "JG",
	JGE:             "JGE",
	JL:              "JL",
	JLE:             "JLE",
	JMP:             "JMP",
	JNE:             "JNE",
	JNO:             "JNO",
	JNP:             "JNP",
	JNS:             "JNS",
	JO:              "JO",
	JP:              "JP",
	JRCXZ:           "JRCXZ",

	VMOVDQU Y6, Y4
	VMOVDQU Y7, Y5

	SUBQ $1, frame_SRND(SP)
	JNE  loop2

	addm(8*0(SI),AX)
	addm(8*1(SI),BX)
	addm(8*2(SI),CX)
	addm(8*3(SI),R8)
	addm(8*4(SI),DX)
	addm(8*5(SI),R9)
	addm(8*6(SI),R10)
	addm(8*7(SI),R11)

	MOVQ frame_INP(SP), DI
	ADDQ $128, DI
	CMPQ DI, frame_INPEND(SP)
	JNE  loop0

done_hash:
	VZEROUPPER

	CMOVQCS y_ptr, acc2
	CMOVQCS t1, acc3

	MOVQ acc0, (8*0)(res_ptr)
	MOVQ acc1, (8*1)(res_ptr)
	MOVQ acc2, (8*2)(res_ptr)
	MOVQ acc3, (8*3)(res_ptr)
	MOVQ res_ptr, x_ptr
	DECQ BX
	JNE  sqrLoop

	RET
/* ---------------------------------------*/
// func p256Mul(res, in1, in2 *p256Element)
TEXT ·p256Mul(SB),NOSPLIT,$0
	MOVQ res+0(FP), res_ptr
	MOVQ in1+8(FP), x_ptr
	MOVQ in2+16(FP), y_ptr

	// First, code below assumes that we are on curg, while raceGetProcCmd
	// can be executed on g0. Second, it is called frequently, so will
	// benefit from this fast path.
	CMPQ	RARG0, $0
	JNE	rest
	get_tls(RARG0)
	MOVQ	g(RARG0), RARG0
	MOVQ	g_m(RARG0), RARG0
	MOVQ	m_p(RARG0), RARG0
	MOVQ	p_raceprocctx(RARG0), RARG0
	MOVQ	RARG0, (RARG1)
	RET

rest:
	// Transition from C ABI to Go ABI.

		"njcy;":                            '\U0000045A',
		"nlArr;":                           '\U000021CD',
		"nlarr;":                           '\U0000219A',
		"nldr;":                            '\U00002025',
		"nle;":                             '\U00002270',
		"nleftarrow;":                      '\U0000219A',
		"nleftrightarrow;":                 '\U000021AE',
		"nleq;":                            '\U00002270',