STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)
	STP.P	(ZR, ZR), 16(R20)

	SUB $8, RSP, R29
	#ifdef GOOS_ios
	MOVD R30, (RSP)
	#endif
	STP (R0, R1), 8(RSP)
	STP (R2, R3), 24(RSP)
	STP (R4, R5), 40(RSP)
	STP (R6, R7), 56(RSP)
	STP (R8, R9), 72(RSP)
	STP (R10, R11), 88(RSP)
	STP (R12, R13), 104(RSP)
	STP (R14, R15), 120(RSP)
	STP (R16, R17), 136(RSP)
	STP (R19, R20), 152(RSP)
	STP (R21, R22), 168(RSP)
	STP (R23, R24), 184(RSP)
	STP (R25, R26), 200(RSP)
	MOVD NZCV, R0
	MOVD R0, 216(RSP)

tail63:
	ANDS	$48, R1, R3
	BEQ	last16
	CMPW	$32, R3
	BEQ	last48
	BLT	last32
	STP.P	(ZR, ZR), 16(R0)
last48:
	STP.P	(ZR, ZR), 16(R0)
last32:
	STP.P	(ZR, ZR), 16(R0)
	// The last store length is at most 16, so it is safe to use
	// stp to write last 16 bytes
last16:
	ANDS	$15, R1, R1
	CBZ	R1, last_end
	ADD	R1, R0, R0
	STP	(ZR, ZR), -16(R0)
last_end:
	RET

no_zva:
	SUB	$16, R0, R0
	SUB	$64, R1, R1

	instr	$0, R5, R9;		\
	instr	$0, R6, R10;		\
	instr	$0, R7, R11;		\
	STP.P	(R8, R9), 32(R3);	\
	STP	(R10, R11), -16(R3);	\
	SUB	$4, counter;

// do one iteration of copy in addVW/subVW
#define vwOneIterCopy(counter, exit)			\
	CBZ	counter, exit;				\
	LDP.P	32(R1), (R4, R5);			\
	LDP	-16(R1), (R6, R7);			\
	STP.P	(R4, R5), 32(R3);			\
	STP	(R6, R7), -16(R3);			\
	SUB	$4, counter;

	ADD R0>>51, R11, R11
	ADD R1>>51, R12, R12
	ADD R2>>51, R13, R13
	ADD R3>>51, R14, R14
	// R4>>51 * 19 + R10 -> R10
	LSR $51, R4, R21
	MOVD $19, R22
	MADD R22, R10, R21, R10

	STP (R10, R11), 0(R20)
	STP (R12, R13), 16(R20)
	MOVD R14, 32(R20)

copy64_from_start:
	LDP	48(R1), (R2, R3)
	STP	(R6, R7), -16(R5)
	LDP	32(R1), (R6, R7)
	STP	(R8, R9), -32(R5)
	LDP	16(R1), (R8, R9)
	STP	(R10, R11), -48(R5)
	LDP	(R1), (R10, R11)
	STP	(R12, R13), -64(R5)
	STP	(R2, R3), 48(R0)
	STP	(R6, R7), 32(R0)
	STP	(R8, R9), 16(R0)
	STP	(R10, R11), (R0)

EXT(crosscall1):
	.cfi_startproc
	stp x29, x30, [sp, #-96]!
	.cfi_def_cfa_offset 96
	.cfi_offset 29, -96
	.cfi_offset 30, -88
	mov x29, sp
	.cfi_def_cfa_register 29
	stp x19, x20, [sp, #80]
	.cfi_offset 19, -16
	.cfi_offset 20, -8
	stp x21, x22, [sp, #64]
	.cfi_offset 21, -32
	.cfi_offset 22, -24
	stp x23, x24, [sp, #48]
	.cfi_offset 23, -48
	.cfi_offset 24, -40
	stp x25, x26, [sp, #32]
	.cfi_offset 25, -64

//
// R29 is not saved because Go will save and restore it.

#define SAVE_R19_TO_R28(offset) \
	STP	(R19, R20), ((offset)+0*8)(RSP) \
	STP	(R21, R22), ((offset)+2*8)(RSP) \
	STP	(R23, R24), ((offset)+4*8)(RSP) \
	STP	(R25, R26), ((offset)+6*8)(RSP) \
	STP	(R27, g), ((offset)+8*8)(RSP)

#define RESTORE_R19_TO_R28(offset) \
	LDP	((offset)+0*8)(RSP), (R19, R20) \
	LDP	((offset)+2*8)(RSP), (R21, R22) \

		LDP.P	16(b_ptr), (acc2, acc3)
		CSEL	EQ, acc2, t2, t2
		CSEL	EQ, acc3, t3, t3

		CMP	$16, const1
		BNE	loop_select

	STP	(x0, x1), 0*16(res_ptr)
	STP	(x2, x3), 1*16(res_ptr)
	STP	(y0, y1), 2*16(res_ptr)
	STP	(y2, y3), 3*16(res_ptr)
	STP	(t0, t1), 4*16(res_ptr)
	STP	(t2, t3), 5*16(res_ptr)
	RET
/* ---------------------------------------*/

	// but we are not called from Go so that space is ours to use,
	// and we must to be contiguous with the stack arguments.
	MOVD	$arg0-(7*8)(SP), R14
	STP	(R0, R1), (0*8)(R14)
	STP	(R2, R3), (2*8)(R14)
	STP	(R4, R5), (4*8)(R14)
	STP	(R6, R7), (6*8)(R14)

	// Push C callee-save registers R19-R28.
	// LR, FP already saved.
	SAVE_R19_TO_R28(8*9)

	// Create a struct callbackArgs on our stack.