U1:	// n >= 0
	// regular loop body unrolled 4x
	ADDQ CX, CX		// restore CF
	MOVQ 0(R8)(SI*8), R11
	MOVQ 8(R8)(SI*8), R12
	MOVQ 16(R8)(SI*8), R13
	MOVQ 24(R8)(SI*8), R14
	ADCQ 0(R9)(SI*8), R11
	ADCQ 8(R9)(SI*8), R12
	ADCQ 16(R9)(SI*8), R13
	ADCQ 24(R9)(SI*8), R14
	MOVQ R11, 0(R10)(SI*8)
	MOVQ R12, 8(R10)(SI*8)
	MOVQ R13, 16(R10)(SI*8)
	MOVQ R14, 24(R10)(SI*8)
	SBBQ CX, CX		// save CF

	ADDQ $4, SI		// i += 4

	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
	ADDQ  t0, h0;                  \
	ADCQ  t3, h1;                  \
	ADCQ  $0, h2;                  \
	SHRQ  $2, t3, t2;              \
	SHRQ  $2, t3;                  \
	ADDQ  t2, h0;                  \
	ADCQ  t3, h1;                  \
	ADCQ  $0, h2

// func update(state *[7]uint64, msg []byte)
TEXT ·update(SB), $0-32
	MOVQ state+0(FP), DI
	MOVQ msg_base+8(FP), SI
	MOVQ msg_len+16(FP), R15

	JMP E1

L1:	MOVL (SI)(BX*4), AX
	ADDL DX, DX		// restore CF
	ADCL (CX)(BX*4), AX
	SBBL DX, DX		// save CF
	MOVL AX, (DI)(BX*4)
	ADDL $1, BX		// i++

E1:	CMPL BX, BP		// i < n
	JL L1

	NEGL DX
	MOVL DX, c+36(FP)
	RET


// func subVV(z, x, y []Word) (c Word)
// (same as addVV except for SBBL instead of ADCL and label names)
TEXT ·subVV(SB),NOSPLIT,$0
	MOVL z+0(FP), DI
	MOVL x+12(FP), SI

// license that can be found in the LICENSE file.

//go:build !purego && (386 || amd64 || arm || arm64 || ppc64 || ppc64le || riscv64 || s390x)

package bigmod

import "internal/cpu"

// amd64 assembly uses ADCX/ADOX/MULX if ADX is available to run two carry
// chains in the flags in parallel across the whole operation, and aggressively
// unrolls loops. arm64 processes four words at a time.
//

	IMULL	$100, CX
	MOVL	CX, nsec+8(FP)

	// DI = sec/100 (still)
	// BX = sec%100
	// construct DX:AX = 64-bit sec and store for return
	MOVL	$0, DX
	MOVL	$100, AX
	MULL	DI
	ADDL	BX, AX
	ADCL	$0, DX
	MOVL	AX, sec+0(FP)
	MOVL	DX, sec+4(FP)

	},
	{
		"LOAD macro",
		lines(
			"#define LOAD(off, reg) \\",
			"\tMOVBLZX	(off*4)(R12),	reg \\",
			"\tADDB	reg,		DX",
			"",
			"LOAD(8, AX)",
		),
		"\n.\n.MOVBLZX.(.8.*.4.).(.R12.).,.AX.\n.ADDB.AX.,.DX.\n",
	},
	{
		"nested multiline macro",
		lines(
			"#define KEYROUND(xmm, load, off, r1, r2, index) \\",
			"\tMOVBLZX	(BP)(DX*4),	R8 \\",
			"\tload((off+1), r2) \\",

	skipMask := off / goarch.PtrSize / 8
	skipBytes := skipMask * goarch.PtrSize * 8
	ptrmask := addb(gcbits, skipMask)
	src = add(src, skipBytes)
	off -= skipBytes
	size += off
	var bits uint32
	for i := uintptr(0); i < size; i += goarch.PtrSize {
		if i&(goarch.PtrSize*8-1) == 0 {
			bits = uint32(*ptrmask)
			ptrmask = addb(ptrmask, 1)
		} else {
			bits >>= 1
		}
		if off > 0 {
			off -= goarch.PtrSize

	MOVL	delta_lo+4(FP), SI
	MOVL	delta_hi+8(FP), DI
	// DX:AX = *addr
	MOVL	0(BP), AX
	MOVL	4(BP), DX
addloop:
	// CX:BX = DX:AX (*addr) + DI:SI (delta)
	MOVL	AX, BX
	MOVL	DX, CX
	ADDL	SI, BX
	ADCL	DI, CX

	// if *addr == DX:AX {
	//	*addr = CX:BX
	// } else {
	//	DX:AX = *addr
	// }
	// all in one instruction
	LOCK
	CMPXCHG8B	0(BP)

	JNZ	addloop

	// success

		Load(aX, RAX)
	default:
		Comment(fmt.Sprintf("%s += %d×%s×%s", r, i, aX, bX))
		IMUL3Q(Imm(i), Load(aX, GP64()), RAX)
	}
	MULQ(mustAddr(bX)) // RDX, RAX = RAX * bX
	ADDQ(RAX, r.lo)
	ADCQ(RDX, r.hi)
}

// shiftRightBy51 returns r >> 51 and r.lo.
//
// After this function is called, the uint128 may not be used anymore.
func shiftRightBy51(r *uint128) (out, lo GPVirtual) {
	out = r.hi