// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !purego

package field

//go:noescape
func carryPropagate(v *Element)

func (v *Element) carryPropagate() *Element {
	carryPropagate(v)
	return v

//go:build !purego

#include "textflag.h"

// carryPropagate works exactly like carryPropagateGeneric and uses the
// same AND, ADD, and LSR+MADD instructions emitted by the compiler, but
// avoids loading R0-R4 twice and uses LDP and STP.
//
// See https://golang.org/issues/43145 for the main compiler issue.
//
// func carryPropagate(v *Element)
TEXT ·carryPropagate(SB),NOFRAME|NOSPLIT,$0-8
	MOVD v+0(FP), R20

// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !arm64 || purego

package field

func (v *Element) carryPropagate() *Element {
	return v.carryPropagateGeneric()

	// to obtain a result with limbs that are at most slightly larger than 2⁵¹,
	// to respect the Element invariant.
	//
	// Overall, the reduction works the same as carryPropagate, except with
	// wider inputs: we take the carry for each coefficient by shifting it right
	// by 51, and add it to the limb above it. The top carry is multiplied by 19

	ADDQ   R14, DI
	ANDQ   AX, R9
	ADDQ   SI, R9
	ANDQ   AX, R11
	ADDQ   R8, R11
	ANDQ   AX, R13
	ADDQ   R10, R13
	ANDQ   AX, R15
	ADDQ   R12, R15

	// Second reduction chain (carryPropagate)
	MOVQ   DI, SI
	SHRQ   $0x33, SI
	MOVQ   R9, R8
	SHRQ   $0x33, R8
	MOVQ   R11, R10
	SHRQ   $0x33, R10
	MOVQ   R13, R12
	SHRQ   $0x33, R12
	MOVQ   R15, R14
	SHRQ   $0x33, R14

	maskAndAdd(r1lo, maskLow51Bits, c0, 1)
	maskAndAdd(r2lo, maskLow51Bits, c1, 1)
	maskAndAdd(r3lo, maskLow51Bits, c2, 1)
	maskAndAdd(r4lo, maskLow51Bits, c3, 1)

	Comment("Second reduction chain (carryPropagate)")
	// c0 = r0 >> 51
	MOVQ(r0lo, c0)
	SHRQ(Imm(51), c0)
	// c1 = r1 >> 51
	MOVQ(r1lo, c1)
	SHRQ(Imm(51), c1)
	// c2 = r2 >> 51
	MOVQ(r2lo, c2)
	SHRQ(Imm(51), c2)
	// c3 = r3 >> 51