(MOVWUreg x:(MOVWUloadidx4 _ _ _)) => (MOVDreg x)

// fold double extensions
(MOVBreg  x:(MOVBreg  _)) => (MOVDreg x)
(MOVBUreg x:(MOVBUreg _)) => (MOVDreg x)
(MOVHreg  x:(MOVBreg  _)) => (MOVDreg x)
(MOVHreg  x:(MOVBUreg _)) => (MOVDreg x)
(MOVHreg  x:(MOVHreg  _)) => (MOVDreg x)
(MOVHUreg x:(MOVBUreg _)) => (MOVDreg x)
(MOVHUreg x:(MOVHUreg _)) => (MOVDreg x)
(MOVWreg  x:(MOVBreg  _)) => (MOVDreg x)
(MOVWreg  x:(MOVBUreg _)) => (MOVDreg x)

TEXT ·Cas(SB), NOSPLIT, $0-17
	MOVD	ptr+0(FP), R0
	MOVW	old+8(FP), R1
	MOVW	new+12(FP), R2
#ifndef GOARM64_LSE
	MOVBU	internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
	CBZ 	R4, load_store_loop
#endif
	MOVD	R1, R3
	CASALW	R3, (R0), R2
	CMP 	R1, R3
	CSET	EQ, R0
	MOVB	R0, ret+16(FP)
	RET
#ifndef GOARM64_LSE
load_store_loop:
	LDAXRW	(R0), R3
	CMPW	R1, R3
	BNE	ok
	STLXRW	R2, (R0), R3

TEXT runtime·memequal<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-25
	BEQ	R4, R5, eq
	ADDV	R4, R6, R7
	PCALIGN	$16
loop:
	BNE	R4, R7, test
	MOVV	$1, R4
	RET
test:
	MOVBU	(R4), R9
	ADDV	$1, R4
	MOVBU	(R5), R10
	ADDV	$1, R5
	BEQ	R9, R10, loop

	MOVB    R0, R4
	RET
eq:
	MOVV	$1, R4
	RET

// memequal_varlen(a, b unsafe.Pointer) bool
TEXT runtime·memequal_varlen<ABIInternal>(SB),NOSPLIT,$40-17
	BEQ	R4, R5, eq

// license that can be found in the LICENSE file.

//go:build !purego

#include "textflag.h"

// func block(dig *digest, p []byte)
TEXT ·block(SB), NOSPLIT|NOFRAME, $0-32
	MOVBZ  ·useAsm(SB), R4
	LMG    dig+0(FP), R1, R3            // R2 = &p[0], R3 = len(p)
	MOVBZ  $2, R0                       // SHA-256 function code
	CMPBEQ R4, $0, generic

loop:
	KIMD R0, R2      // compute intermediate message digest (KIMD)

// license that can be found in the LICENSE file.

//go:build !purego

#include "textflag.h"

// func block(dig *digest, p []byte)
TEXT ·block(SB), NOSPLIT|NOFRAME, $0-32
	MOVBZ  ·useAsm(SB), R4
	LMG    dig+0(FP), R1, R3            // R2 = &p[0], R3 = len(p)
	MOVBZ  $3, R0                       // SHA-512 function code
	CMPBEQ R4, $0, generic

loop:
	KIMD R0, R2      // compute intermediate message digest (KIMD)

loop:
	BEQ	X10, X11, done
	MOVBU	(X10), X15
	ADD	$1, X10
	BNE	X12, X15, loop
	ADD	$1, X14
	JMP	loop

done:
	MOV	X14, X10
	RET

TEXT ·CountString<ABIInternal>(SB),NOSPLIT,$0-32
	// X10 = s_base
	// X11 = s_len
	// X12 = byte to count
	AND	$0xff, X12
	MOV	ZERO, X14	// count
	ADD	X10, X11	// end

	PCALIGN	$16
loop:
	BEQ	X10, X11, done
	MOVBU	(X10), X15
	ADD	$1, X10

#define MSGSCHEDULE0(index) \
	MOVBU	((index*8)+0)(X29), X5; \
	MOVBU	((index*8)+1)(X29), X6; \
	MOVBU	((index*8)+2)(X29), X7; \
	MOVBU	((index*8)+3)(X29), X8; \
	SLL	$56, X5; \
	SLL	$48, X6; \
	OR	X5, X6, X5; \
	SLL	$40, X7; \
	OR	X5, X7, X5; \
	SLL	$32, X8; \
	OR	X5, X8, X5; \
	MOVBU	((index*8)+4)(X29), X9; \
	MOVBU	((index*8)+5)(X29), X6; \
	MOVBU	((index*8)+6)(X29), X7; \
	MOVBU	((index*8)+7)(X29), X8; \

	"MULLW",
	"MULLD",
	"MULHD",
	"MULHDU",
	"MLGR",
	"SUB",
	"SUBC",
	"SUBV",
	"SUBE",
	"SUBW",
	"NEG",
	"NEGW",
	"MOVWBR",
	"MOVB",
	"MOVBZ",
	"MOVH",
	"MOVHBR",
	"MOVHZ",
	"MOVW",
	"MOVWZ",
	"MOVD",
	"MOVDBR",
	"MOVDEQ",
	"MOVDGE",
	"MOVDGT",
	"MOVDLE",
	"MOVDLT",
	"MOVDNE",
	"LOCR",
	"LOCGR",
	"FLOGR",
	"POPCNT",
	"AND",
	"ANDW",
	"OR",
	"ORW",

	MOVV	8(R4), R16
	MOVV	8(R6), R17
	ADDV	$16, R4
	ADDV	$16, R6
	SUBVU	$1, R14
	BEQ	R16, R17, chunk16_loop
	SUBV	$8, R4
	SUBV	$8, R6

byte_loop:
	BEQ	R4, R12, samebytes
	MOVBU	(R4), R8
	ADDVU	$1, R4
	MOVBU	(R6), R9
	ADDVU	$1, R6
	BEQ	R8, R9, byte_loop

byte_cmp:
	SGTU	R8, R9, R4 // R12 = 1 if (R8 > R9)
	BNE	R0, R4, ret
	MOVV	$-1, R4
	JMP	ret

samebytes:
	SGTU	R5, R7, R8

	SUB	$16, X12
	BGE	X12, X23, loop16
	BEQZ	X12, eq

loop4_check:
	MOV	$4, X23
	BLT	X12, X23, loop1
loop4:
	MOVBU	0(X10), X19
	MOVBU	0(X11), X20
	MOVBU	1(X10), X21
	MOVBU	1(X11), X22
	BNE	X19, X20, not_eq
	BNE	X21, X22, not_eq
	MOVBU	2(X10), X14
	MOVBU	2(X11), X15
	MOVBU	3(X10), X16
	MOVBU	3(X11), X17
	BNE	X14, X15, not_eq
	BNE	X16, X17, not_eq
	ADD	$4, X10
	ADD	$4, X11
	SUB	$4, X12