MOVD	0(R4), R3
	MOVD	R3, 0(R6)
	MOVD	8(R4), R3
	MOVD	R3, 8(R6)
	ADD	$16, R4
	ADD	$16, R6
	ADD	$-16, R5
	CMP	R5, $16
	BGE	forwards
	BR	start

forwards_fast:
	CMP	R5, $256
	BLE	forwards_small
	MVC	$256, 0(R4), 0(R6)
	ADD	$256, R4
	ADD	$256, R6
	ADD	$-256, R5
	BR	forwards_fast

forwards_small:
	CMPBEQ	R5, $0, done
	ADD	$-1, R5
	EXRL	$memmove_exrl_mvc<>(SB), R5
	RET

	PCALIGN	$32
	BEQ	notfound	// Is tail length 0? CR0 is set before entering loop64.

	CMP	R4,$32		// Tail length >= 32, use cmp32 path.
	CMP	R4,$16,CR1
	BGE	cmp32

	ADD	R8,R4,R9
	ADD	$-16,R9
	BLE	CR1,cmp64_tail_gt0

cmp64_tail_gt16:	// Tail length 17 - 32
	LXVD2X	(R0)(R8),V2
	VCMPEQUBCC	V2,V1,V6
	BNE	CR6,foundat0

cmp64_tail_gt0:	// Tail length 1 - 16
	MOVD	R9,R8
	LXVD2X	(R0)(R9),V2

		dummy()
	}
}

// Signed 64-bit comparison with 1/-1 to comparison with 0.
func si64x0(x chan int64) {
	// riscv64:"BGTZ"
	for <-x >= 1 {
		dummy()
	}

	// riscv64:"BLEZ"
	for <-x < 1 {
		dummy()
	}

	// riscv64:"BLTZ"
	for <-x <= -1 {
		dummy()
	}

	// riscv64:"BGEZ"
	for <-x > -1 {
		dummy()
	}
}

	BR	L1
L34:
	FMOVD	8(R9), F4
L19:
	LTDBR	F6, F6
	BLEU	L47
L18:
	WFMDB	V4, V5, V1
	BR	L1
L5:
	RISBGZ	$33, $50, $63, R3, R3
	WORD	$0xC23B4000	//alfi	%r3,1073741824
	BYTE	$0x00
	BYTE	$0x00
	RLL	$24, R3, R3
	ORW	$0x45000000, R3
	BR	L2
L45:
	WFCEDBS	V0, V0, V4
	BVS	L35
	LTDBR	F0, F0
	BLEU	L48
	FMOVD	8(R9), F4
L12:
	MOVW	R2, R6
	CMPBLT	R6, $0, L19

	BNE	0(PC)                  // a7740000
	BEQ	0(PC)                  // a7840000
	BLT	0(PC)                  // a7440000
	BLE	0(PC)                  // a7c40000
	BGT	0(PC)                  // a7240000
	BGE	0(PC)                  // a7a40000
	BLTU	0(PC)                  // a7540000
	BLEU	0(PC)                  // a7d40000

	BRCT	R1, 0(PC)              // a7160000
	BRCTG	R2, 0(PC)              // a7270000

	RET

try_zva:
	// Try using the ZVA feature to zero entire cache lines
	// It is not meaningful to use ZVA if the block size is less than 64,
	// so make sure that n is greater than or equal to 64
	CMP	$63, R1
	BLE	tail63

	CMP	$128, R1
	// Ensure n is at least 128 bytes, so that there is enough to copy after
	// alignment.
	BLT	no_zva
	// Check if ZVA is allowed from user code, and if so get the block size

	BEQ	tail_0

#ifdef GOPPC64_power10
	SRD	$3, R18, R18	// Fix the vector loop count before counting the tail on P10.

tail:	// Count the last 0 - 31 bytes.
	CMP	R4, $16
	BLE	small_tail_p10
	LXV	0(R3), V0
	VCMPEQUB V0, V1, V0
	VCNTMBB	V0, $1, R14	// Sum the value of bit 0 of each byte of the compare into R14.
	SRD	$56, R14, R14	// The result of VCNTMBB is shifted. Unshift it.
	ADD	R14, R18, R18

	"ADDEVCC",
	"ADDEV",
	"ADDZE",
	"ADDZECC",
	"ADDZEVCC",
	"ADDZEV",
	"ADDEX",
	"AND",
	"ANDCC",
	"ANDN",
	"ANDNCC",
	"ANDISCC",
	"BC",
	"BCL",
	"BEQ",
	"BGE",
	"BGT",
	"BLE",
	"BLT",
	"BNE",
	"BVC",
	"BVS",
	"BDNZ",
	"BDZ",
	"CMP",
	"CMPU",
	"CMPEQB",
	"CNTLZW",
	"CNTLZWCC",
	"CRAND",
	"CRANDN",
	"CREQV",
	"CRNAND",
	"CRNOR",
	"CROR",
	"CRORN",

	WFMADB	V10, V8, V4, V8
	FMADD	F0, F8, F10
	WFSDB	V10, V1, V10
L1:
	FMOVD	F10, ret+8(FP)
	RET

L2:
	WORD	$0xC0293FEF	//iilf	%r2,1072693247
	BYTE	$0xFF
	BYTE	$0xFF
	CMPW	R1, R2
	BLE	L12
L4:
	WORD	$0xED009020	//cdb	%f0,.L34-.L13(%r9)
	BYTE	$0x00
	BYTE	$0x19
	BEQ	L8
	WORD	$0xED009018	//cdb	%f0,.L35-.L13(%r9)
	BYTE	$0x00
	BYTE	$0x19
	BEQ	L9
	WFCEDBS	V10, V10, V0
	BVS	L1

	return x*y - z
}

func FusedSub64_b(x, y, z float64) float64 {
	// arm64:"FMSUBD"
	// riscv64:"FNMSUBD\t"
	return z - x*y
}

func Cmp(f float64) bool {
	// arm64:"FCMPD","(BGT|BLE|BMI|BPL)",-"CSET\tGT",-"CBZ"
	return f > 4 || f < -4
}

func CmpZero64(f float64) bool {
	// s390x:"LTDBR",-"FCMPU"
	return f <= 0
}

func CmpZero32(f float32) bool {
	// s390x:"LTEBR",-"CEBR"