CMP	$8, R2
	BLT	copy7
	MOVD	(R1), R6
	MOVD	-8(R4), R7
	MOVD	R6, (R0)
	MOVD	R7, -8(R5)
	RET

copy7:
	TBZ	$2, R2, copy3
	MOVWU	(R1), R6
	MOVWU	-4(R4), R7
	MOVW	R6, (R0)
	MOVW	R7, -4(R5)
	RET

copy3:
	TBZ	$1, R2, copy1
	MOVHU	(R1), R6
	MOVHU	-2(R4), R7
	MOVH	R6, (R0)
	MOVH	R7, -2(R5)
	RET

copy1:
	MOVBU	(R1), R6
	MOVB	R6, (R0)

copy0:
	RET

chunk16_loop:
	BEQ	R0, R10, byte_loop
	MOVV	(R3), R6
	MOVV	(R4), R7
	BNE	R6, R7, byte_loop
	MOVV	8(R3), R13
	MOVV	8(R4), R14
	ADDV	$16, R3
	ADDV	$16, R4
	SUBVU	$1, R10
	BEQ	R13, R14, chunk16_loop
	SUBV	$8, R3
	SUBV	$8, R4

byte_loop:
	BEQ	R3, R8, samebytes
	MOVBU	(R3), R6
	ADDVU	$1, R3
	MOVBU	(R4), R7
	ADDVU	$1, R4
	BEQ	R6, R7, byte_loop

byte_cmp:
	SGTU	R6, R7, R8 // R8 = 1 if (R6 > R7)
	BNE	R0, R8, ret

	BEQ	R0, R14, chunk_tail_4
	MOVV	(R1), R6
	MOVV	(R2), R7
	BNE	R6, R7, not_eq
	ADDV	$8, R1
	ADDV	$8, R2

chunk_tail_4:
	AND	$4, R3, R14
	BEQ	R0, R14, chunk_tail_2
	MOVWU	(R1), R6
	MOVWU	(R2), R7
	BNE	R6, R7, not_eq
	ADDV	$4, R1
	ADDV	$4, R2

chunk_tail_2:
	AND	$2, R3, R14
	BEQ	R0, R14, chunk_tail_1
	MOVHU	(R1), R6
	MOVHU	(R2), R7
	BNE	R6, R7, not_eq
	ADDV	$2, R1
	ADDV	$2, R2

	CMP	$0, R6
	BEQ	exit	// g.m == nil

	// restore libcall
	MOVD	96(R1), R7
	MOVD	R7, (m_libcall+libcall_fn)(R6)
	MOVD	104(R1), R7
	MOVD	R7, (m_libcall+libcall_args)(R6)
	MOVD	112(R1), R7
	MOVD	R7, (m_libcall+libcall_n)(R6)
	MOVD	120(R1), R7
	MOVD	R7, (m_libcall+libcall_r1)(R6)
	MOVD	128(R1), R7
	MOVD	R7, (m_libcall+libcall_r2)(R6)

	// restore errno
	MOVD	(m_mOS+mOS_perrno)(R6), R7

	MOVD	(R1), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	SUB	$8, R2, R6	// offset of the last 8 bytes
	MOVD	(R0)(R6), R4
	MOVD	(R1)(R6), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	B	equal
lt_8:
	TBZ	$2, R2, lt_4
	MOVWU	(R0), R4
	MOVWU	(R1), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	SUB	$4, R2, R6	// offset of the last 4 bytes
	MOVWU	(R0)(R6), R4
	MOVWU	(R1)(R6), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	B	equal
lt_4:

	// alignment requirements.
	ANDCC	$PAGE_OFFSET, R8, R6	// &sX & PAGE_OFFSET
	ANDCC	$PAGE_OFFSET, R4, R9
	SUBC	R5, $8, R12		// 8-len
	SLD	$3, R12, R14		// (8-len)*8
	CMPU	R6, R12, CR1		// Enough bytes lower in the page to load lower?
	CMPU	R9, R12, CR0
	SUB	R12, R8, R6		// compute lower load address
	SUB	R12, R4, R9
	ISEL	CR1LT, R8, R6, R8	// R8 = R6 < 0 ? R8 (&s1) : R6 (&s1 - (8-len))
	ISEL	CR0LT, R4, R9, R4	// Similar for s2

	RLWNM $3, R4, $29, $31, R6      // 54861f7e
	RLWNM $0, R4, $29, $31, R6      // 5486077e
	RLWNM R0, R4, $29, $31, R6      // 5c86077e
	RLWNM R3, R4, $7, R6            // 5c861f7e
	RLWNM R3, R4, $29, $31, R6      // 5c861f7e
	RLWNMCC $3, R4, $7, R6          // 54861f7f
	RLWNMCC $3, R4, $29, $31, R6    // 54861f7f
	RLWNMCC R3, R4, $7, R6          // 5c861f7f
	RLWNMCC R3, R4, $29, $31, R6    // 5c861f7f
	RLDMI $0, R4, $7, R6            // 7886076c

TEXT ·addMulVVW1024(SB), $0-32
	MOVD	$4, R6 // R6 = z_len/4
	JMP		addMulVVWx<>(SB)

// func addMulVVW1536(z, x *uint, y uint) (c uint)
TEXT ·addMulVVW1536(SB), $0-32
	MOVD	$6, R6 // R6 = z_len/4
	JMP		addMulVVWx<>(SB)

// func addMulVVW2048(z, x *uint, y uint) (c uint)
TEXT ·addMulVVW2048(SB), $0-32
	MOVD	$8, R6 // R6 = z_len/4
	JMP		addMulVVWx<>(SB)

// a1  | R5          | R4
// a2  | R6          | R5
// a3  | R7          | R6
// a4  | R8          | R7
// a5  | R9          | R8
// a6  | R10         | R9
//
// r1  | R4          | R4
// r2  | R5          | R5
// err | R6          | part of R4
TEXT ·Syscall6<ABIInternal>(SB),NOSPLIT,$0-80
	MOVV	R4, R11  // syscall entry
	MOVV	R5, R4
	MOVV	R6, R5
	MOVV	R7, R6
	MOVV	R8, R7
	MOVV	R9, R8
	MOVV	R10, R9

	LMG  0(R8), R5, R6
	MOVD $0, 0(R9)          // R9 address of SAVSTACK_ASYNC
	LE_CALL                 // balr R7, R6 (return #1)
	NOPH
	MOVD R3, ret+32(FP)
	CMP  R3, $-1            // compare result to -1
	BNE  done

	// retrieve errno and errno2
	MOVD  zosLibVec<>(SB), R8
	ADD   $(__errno), R8
	LMG   0(R8), R5, R6
	LE_CALL                   // balr R7, R6 __errno (return #3)
	NOPH