// make sure both a and b are aligned.
	OR	R4, R6, R15
	AND	$7, R15
	BNE	R0, R15, byte_loop

	PCALIGN	$16
chunk16_loop:
	BEQ	R0, R14, byte_loop
	MOVV	(R4), R8
	MOVV	(R6), R9
	BNE	R8, R9, byte_loop
	MOVV	8(R4), R16
	MOVV	8(R6), R17
	ADDV	$16, R4
	ADDV	$16, R6
	SUBVU	$1, R14
	BEQ	R16, R17, chunk16_loop
	SUBV	$8, R4
	SUBV	$8, R6

byte_loop:
	BEQ	R4, R12, samebytes
	MOVBU	(R4), R8

	ROUND3(R4,R5,R6,R7, 8,0xfffa3942, 4);
	ROUND3(R7,R4,R5,R6,11,0x8771f681,11);
	ROUND3(R6,R7,R4,R5,14,0x6d9d6122,16);
	ROUND3(R5,R6,R7,R4, 1,0xfde5380c,23);
	ROUND3(R4,R5,R6,R7, 4,0xa4beea44, 4);
	ROUND3(R7,R4,R5,R6, 7,0x4bdecfa9,11);
	ROUND3(R6,R7,R4,R5,10,0xf6bb4b60,16);
	ROUND3(R5,R6,R7,R4,13,0xbebfbc70,23);
	ROUND3(R4,R5,R6,R7, 0,0x289b7ec6, 4);
	ROUND3(R7,R4,R5,R6, 3,0xeaa127fa,11);
	ROUND3(R6,R7,R4,R5, 6,0xd4ef3085,16);

	MOVD	ptr+0(FP), R3
	MOVBZ	val+8(FP), R4
	LWSYNC
again:
	LBAR	(R3), R6
	OR	R4, R6
	STBCCC	R6, (R3)
	BNE	again
	RET

// void ·And8(byte volatile*, byte);
TEXT ·And8(SB), NOSPLIT, $0-9
	MOVD	ptr+0(FP), R3
	MOVBZ	val+8(FP), R4
	LWSYNC
again:
	LBAR	(R3), R6
	AND	R4, R6
	STBCCC	R6, (R3)
	BNE	again
	RET

// func Or(addr *uint32, v uint32)

	MULU	R4, R5, R6		// a6101c00
	MULH	R4, R5	   		// a5901c00
	MULH	R4, R5, R6	   	// a6901c00
	MULHU	R4, R5			// a5101d00
	MULHU	R4, R5, R6		// a6101d00
	REM	R4, R5	  		// a5902000
	REM	R4, R5, R6	  	// a6902000
	REMU	R4, R5	   		// a5902100
	REMU	R4, R5, R6	   	// a6902100
	DIV	R4, R5	  		// a5102000
	DIV	R4, R5, R6	  	// a6102000
	DIVU	R4, R5	   		// a5102100
	DIVU	R4, R5, R6	   	// a6102100

	MOVV	ptr+0(FP), R4
	MOVW	delta+8(FP), R5
	DBAR
	LL	(R4), R6
	ADDU	R6, R5, R7
	MOVV	R7, R6
	SC	R7, (R4)
	BEQ	R7, -4(PC)
	MOVW	R6, ret+16(FP)
	DBAR
	RET

TEXT ·Xadd64(SB), NOSPLIT, $0-24
	MOVV	ptr+0(FP), R4
	MOVV	delta+8(FP), R5
	DBAR
	LLV	(R4), R6
	ADDVU	R6, R5, R7
	MOVV	R7, R6
	SCV	R7, (R4)
	BEQ	R7, -4(PC)
	MOVV	R6, ret+16(FP)
	DBAR
	RET

TEXT ·Xchg(SB), NOSPLIT, $0-20

	ADDV	$-7, R9, R6 // R6 is end pointer-7

	PCALIGN	$16
	SGTU	R6, R4, R8
	BEQ	R8, out
	MOVV	(R5), R7
	ADDV	$8, R5
	MOVV	R7, (R4)
	ADDV	$8, R4
	JMP	-6(PC)

out:
	BEQ	R4, R9, done
	MOVB	(R5), R7
	ADDV	$1, R5
	MOVB	R7, (R4)
	ADDV	$1, R4
	JMP	-5(PC)
done:
	RET

backward:
	ADDV	R6, R5 // from-end pointer
	ADDV	R4, R6, R9 // to-end pointer

	ADD	$32, R8
	ADD	$-32, R6
	CMP	R6, $8
	BLE	small
	// Case for 8 <= n < 32 bytes
	// Process 16 bytes if available
xor16:
	CMP	R6, $16
	BLT	xor8
	LXVD2X	(R4)(R8), VS32
	LXVD2X	(R5)(R8), VS33
	XXLXOR	VS32, VS33, VS32
	STXVD2X	VS32, (R3)(R8)
	ADD	$16, R8
	ADD	$-16, R6
small:
	CMP	R6, $0
	BC	12,2,LR		// BEQLR
xor8:
#ifdef GOPPC64_power10
	SLD	$56,R6,R17
	ADD	R4,R8,R18
	ADD	R5,R8,R19

	MOVD	ptr+0(FP), R4
	MOVW	new+8(FP), R3
	MOVW	(R4), R6
repeat:
	CS	R6, R3, (R4) // if R6==(R4) then (R4)=R3 else R6=(R4)
	BNE	repeat
	MOVW	R6, ret+16(FP)
	RET

// func Xchg64(ptr *uint64, new uint64) uint64
TEXT ·Xchg64(SB), NOSPLIT, $0-24
	MOVD	ptr+0(FP), R4
	MOVD	new+8(FP), R3
	MOVD	(R4), R6
repeat:
	CSG	R6, R3, (R4) // if R6==(R4) then (R4)=R3 else R6=(R4)
	BNE	repeat
	MOVD	R6, ret+16(FP)
	RET

	MOVD	(R1), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	SUB	$8, R2, R6	// offset of the last 8 bytes
	MOVD	(R0)(R6), R4
	MOVD	(R1)(R6), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	B	equal
lt_8:
	TBZ	$2, R2, lt_4
	MOVWU	(R0), R4
	MOVWU	(R1), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	SUB	$4, R2, R6	// offset of the last 4 bytes
	MOVWU	(R0)(R6), R4
	MOVWU	(R1)(R6), R5
	EOR	R4, R5
	CBNZ	R5, not_equal
	B	equal
lt_4:

	RLWNM $3, R4, $29, $31, R6      // 54861f7e
	RLWNM $0, R4, $29, $31, R6      // 5486077e
	RLWNM R0, R4, $29, $31, R6      // 5c86077e
	RLWNM R3, R4, $7, R6            // 5c861f7e
	RLWNM R3, R4, $29, $31, R6      // 5c861f7e
	RLWNMCC $3, R4, $7, R6          // 54861f7f
	RLWNMCC $3, R4, $29, $31, R6    // 54861f7f
	RLWNMCC R3, R4, $7, R6          // 5c861f7f
	RLWNMCC R3, R4, $29, $31, R6    // 5c861f7f
	RLDMI $0, R4, $7, R6            // 7886076c