perf: fast path operations for small non-native values (#1326)

internal/stats/latest_stats.csv

@@ -153,14 +153,14 @@ pairing_bls12377,bls24_315,plonk,0,0
 pairing_bls12377,bls24_317,plonk,0,0
 pairing_bls12377,bw6_761,plonk,51280,51280
 pairing_bls12377,bw6_633,plonk,0,0
-pairing_bls12381,bn254,groth16,1429070,2382640
+pairing_bls12381,bn254,groth16,1419904,2366999
 pairing_bls12381,bls12_377,groth16,0,0
 pairing_bls12381,bls12_381,groth16,0,0
 pairing_bls12381,bls24_315,groth16,0,0
 pairing_bls12381,bls24_317,groth16,0,0
 pairing_bls12381,bw6_761,groth16,0,0
 pairing_bls12381,bw6_633,groth16,0,0
-pairing_bls12381,bn254,plonk,5629807,5285448
+pairing_bls12381,bn254,plonk,5593770,5250897
 pairing_bls12381,bls12_377,plonk,0,0
 pairing_bls12381,bls12_381,plonk,0,0
 pairing_bls12381,bls24_315,plonk,0,0
@@ -181,70 +181,70 @@ pairing_bls24315,bls24_315,plonk,0,0
 pairing_bls24315,bls24_317,plonk,0,0
 pairing_bls24315,bw6_761,plonk,0,0
 pairing_bls24315,bw6_633,plonk,141249,141249
-pairing_bn254,bn254,groth16,969638,1614382
+pairing_bn254,bn254,groth16,963003,1603091
 pairing_bn254,bls12_377,groth16,0,0
 pairing_bn254,bls12_381,groth16,0,0
 pairing_bn254,bls24_315,groth16,0,0
 pairing_bn254,bls24_317,groth16,0,0
 pairing_bn254,bw6_761,groth16,0,0
 pairing_bn254,bw6_633,groth16,0,0
-pairing_bn254,bn254,plonk,3798583,3560759
+pairing_bn254,bn254,plonk,3771397,3534755
 pairing_bn254,bls12_377,plonk,0,0
 pairing_bn254,bls12_381,plonk,0,0
 pairing_bn254,bls24_315,plonk,0,0
 pairing_bn254,bls24_317,plonk,0,0
 pairing_bn254,bw6_761,plonk,0,0
 pairing_bn254,bw6_633,plonk,0,0
-pairing_bw6761,bn254,groth16,3014749,4979960
+pairing_bw6761,bn254,groth16,2592181,4256159
 pairing_bw6761,bls12_377,groth16,0,0
 pairing_bw6761,bls12_381,groth16,0,0
 pairing_bw6761,bls24_315,groth16,0,0
 pairing_bw6761,bls24_317,groth16,0,0
 pairing_bw6761,bw6_761,groth16,0,0
 pairing_bw6761,bw6_633,groth16,0,0
-pairing_bw6761,bn254,plonk,11486969,10777222
+pairing_bw6761,bn254,plonk,9920293,9270827
 pairing_bw6761,bls12_377,plonk,0,0
 pairing_bw6761,bls12_381,plonk,0,0
 pairing_bw6761,bls24_315,plonk,0,0
 pairing_bw6761,bls24_317,plonk,0,0
 pairing_bw6761,bw6_761,plonk,0,0
 pairing_bw6761,bw6_633,plonk,0,0
-scalar_mul_G1_bn254,bn254,groth16,74345,117078
+scalar_mul_G1_bn254,bn254,groth16,69013,108022
 scalar_mul_G1_bn254,bls12_377,groth16,0,0
 scalar_mul_G1_bn254,bls12_381,groth16,0,0
 scalar_mul_G1_bn254,bls24_315,groth16,0,0
 scalar_mul_G1_bn254,bls24_317,groth16,0,0
 scalar_mul_G1_bn254,bw6_761,groth16,0,0
 scalar_mul_G1_bn254,bw6_633,groth16,0,0
-scalar_mul_G1_bn254,bn254,plonk,278909,261995
+scalar_mul_G1_bn254,bn254,plonk,260289,244439
 scalar_mul_G1_bn254,bls12_377,plonk,0,0
 scalar_mul_G1_bn254,bls12_381,plonk,0,0
 scalar_mul_G1_bn254,bls24_315,plonk,0,0
 scalar_mul_G1_bn254,bls24_317,plonk,0,0
 scalar_mul_G1_bn254,bw6_761,plonk,0,0
 scalar_mul_G1_bn254,bw6_633,plonk,0,0
-scalar_mul_P256,bn254,groth16,100828,161106
+scalar_mul_P256,bn254,groth16,93170,148354
 scalar_mul_P256,bls12_377,groth16,0,0
 scalar_mul_P256,bls12_381,groth16,0,0
 scalar_mul_P256,bls24_315,groth16,0,0
 scalar_mul_P256,bls24_317,groth16,0,0
 scalar_mul_P256,bw6_761,groth16,0,0
 scalar_mul_P256,bw6_633,groth16,0,0
-scalar_mul_P256,bn254,plonk,385060,359805
+scalar_mul_P256,bn254,plonk,355345,331788
 scalar_mul_P256,bls12_377,plonk,0,0
 scalar_mul_P256,bls12_381,plonk,0,0
 scalar_mul_P256,bls24_315,plonk,0,0
 scalar_mul_P256,bls24_317,plonk,0,0
 scalar_mul_P256,bw6_761,plonk,0,0
 scalar_mul_P256,bw6_633,plonk,0,0
-scalar_mul_secp256k1,bn254,groth16,75154,118312
+scalar_mul_secp256k1,bn254,groth16,69860,109339
 scalar_mul_secp256k1,bls12_377,groth16,0,0
 scalar_mul_secp256k1,bls12_381,groth16,0,0
 scalar_mul_secp256k1,bls24_315,groth16,0,0
 scalar_mul_secp256k1,bls24_317,groth16,0,0
 scalar_mul_secp256k1,bw6_761,groth16,0,0
 scalar_mul_secp256k1,bw6_633,groth16,0,0
-scalar_mul_secp256k1,bn254,plonk,281870,264753
+scalar_mul_secp256k1,bn254,plonk,263180,247131
 scalar_mul_secp256k1,bls12_377,plonk,0,0
 scalar_mul_secp256k1,bls12_381,plonk,0,0
 scalar_mul_secp256k1,bls24_315,plonk,0,0

std/internal/limbcomposition/composition.go

@@ -12,9 +12,6 @@ import (
 //
 //	res = \sum_{i=0}^{len(inputs)} inputs[i] * 2^{nbBits * i}
 func Recompose(inputs []*big.Int, nbBits uint, res *big.Int) error {
-	if len(inputs) == 0 {
-		return fmt.Errorf("zero length slice input")
-	}
 	if res == nil {
 		return fmt.Errorf("result not initialized")
 	}

std/math/emulated/custommod.go

@@ -14,6 +14,10 @@ import (
 //
 // NB! circuit complexity depends on T rather on the actual length of the modulus.
 func (f *Field[T]) ModMul(a, b *Element[T], modulus *Element[T]) *Element[T] {
+	// fast path when either of the inputs is zero then result is always zero
+	if len(a.Limbs) == 0 || len(b.Limbs) == 0 {
+		return f.Zero()
+	}
 	res := f.mulMod(a, b, 0, modulus)
 	return res
 }
@@ -33,9 +37,9 @@ func (f *Field[T]) ModAdd(a, b *Element[T], modulus *Element[T]) *Element[T] {
 	for nextOverflow, err = f.addPreCond(a, b); errors.As(err, &target); nextOverflow, err = f.addPreCond(a, b) {
 		if errors.As(err, &target) {
 			if !target.reduceRight {
-				a = f.mulMod(a, f.shortOne(), 0, modulus)
+				a = f.mulMod(a, f.One(), 0, modulus)
 			} else {
-				b = f.mulMod(b, f.shortOne(), 0, modulus)
+				b = f.mulMod(b, f.One(), 0, modulus)
 			}
 		}
 	}
@@ -93,6 +97,10 @@ func (f *Field[T]) ModAssertIsEqual(a, b *Element[T], modulus *Element[T]) {
 //
 // NB! circuit complexity depends on T rather on the actual length of the modulus.
 func (f *Field[T]) ModExp(base, exp, modulus *Element[T]) *Element[T] {
+	// fasth path when the base is zero then result is always zero
+	if len(base.Limbs) == 0 {
+		return f.Zero()
+	}
 	expBts := f.ToBits(exp)
 	n := len(expBts)
 	res := f.Select(expBts[0], base, f.One())

std/math/emulated/element.go

@@ -43,22 +43,31 @@ type Element[T FieldParams] struct {
 	evaluation  frontend.Variable `gnark:"-"`
 }
 
-// ValueOf returns an Element[T] from a constant value.
-// The input is converted to *big.Int and decomposed into limbs and packed into new Element[T].
+// ValueOf returns an Element[T] from a constant value. This method is used for
+// witness assignment. For in-circuit constant assignment use the
+// [Field.NewElement] method.
+//
+// The input is converted into limbs according to the parameters of the field
+// and returned as a new [Element[T]]. Note that it returns the value, not a
+// reference, which is more convenient for witness assignment.
 func ValueOf[T FieldParams](constant interface{}) Element[T] {
+	// in this method we set the isWitness flag to true, because we do not know
+	// the width of the input value. Even though it is valid to call this method
+	// in circuit without reference to `Field`, then the canonical way would be
+	// to call [Field.NewElement] method (which would set isWitness to false).
 	if constant == nil {
-		r := newConstElement[T](0)
+		r := newConstElement[T](0, true)
 		return *r
 	}
-	r := newConstElement[T](constant)
+	r := newConstElement[T](constant, true)
 	return *r
 }
 
 // newConstElement is shorthand for initialising new element using NewElement and
 // taking pointer to it. We only want to have a public method for initialising
 // an element which return a value because the user uses this only for witness
 // creation and it mess up schema parsing.
-func newConstElement[T FieldParams](v interface{}) *Element[T] {
+func newConstElement[T FieldParams](v interface{}, isWitness bool) *Element[T] {
 	var fp T
 	// convert to big.Int
 	bValue := utils.FromInterface(v)
@@ -68,9 +77,18 @@ func newConstElement[T FieldParams](v interface{}) *Element[T] {
 		bValue.Mod(&bValue, fp.Modulus())
 	}
 
-	// decompose into limbs
+	// decompose into limbs. When set with isWitness, then we do not know at
+	// compile time the width of the input, so we allocate the maximum number of
+	// limbs. However, in-circuit we already do (we set it from actual
+	// constant), thus we can allocate the exact number of limbs.
+	var nbLimbs int
+	if isWitness {
+		nbLimbs = int(fp.NbLimbs())
+	} else {
+		nbLimbs = (bValue.BitLen() + int(fp.BitsPerLimb()) - 1) / int(fp.BitsPerLimb())
+	}
 	// TODO @gbotrel use big.Int pool here
-	blimbs := make([]*big.Int, fp.NbLimbs())
+	blimbs := make([]*big.Int, nbLimbs)
 	for i := range blimbs {
 		blimbs[i] = new(big.Int)
 	}

std/math/emulated/element_test.go

@@ -1425,3 +1425,75 @@ func testPolyEvalNegativeCoefficient[T FieldParams](t *testing.T) {
 	err = test.IsSolved(&PolyEvalNegativeCoefficient[T]{Inputs: make([]Element[T], nbInputs)}, assignment, testCurve.ScalarField())
 	assert.NoError(err)
 }
+
+type FastPathsCircuit[T FieldParams] struct {
+	Rand Element[T]
+	Zero Element[T]
+}
+
+func (c *FastPathsCircuit[T]) Define(api frontend.API) error {
+	f, err := NewField[T](api)
+	if err != nil {
+		return err
+	}
+	// instead of using witness values, we need to create the elements
+	// in-circuit. In witness creation we always create elements with full
+	// number of limbs.
+
+	zero := f.Zero()
+
+	// mul
+	res := f.Mul(zero, &c.Rand)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+	res = f.Mul(&c.Rand, zero)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	res = f.MulMod(zero, &c.Rand)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+	res = f.MulMod(&c.Rand, zero)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	res = f.MulNoReduce(zero, &c.Rand)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+	res = f.MulNoReduce(&c.Rand, zero)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	// div
+	res = f.Div(zero, &c.Rand)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	// square root
+	res = f.Sqrt(zero)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	// exp
+	res = f.Exp(zero, &c.Rand)
+	f.AssertIsEqual(res, &c.Zero)
+	f.AssertIsEqual(res, zero)
+
+	return nil
+}
+
+func TestFasthPaths(t *testing.T) {
+	testFastPaths[Goldilocks](t)
+	testFastPaths[BN254Fr](t)
+	testFastPaths[emparams.Mod1e512](t)
+}
+
+func testFastPaths[T FieldParams](t *testing.T) {
+	assert := test.NewAssert(t)
+	var fp T
+	randVal, _ := rand.Int(rand.Reader, fp.Modulus())
+	circuit := &FastPathsCircuit[T]{}
+	assignment := &FastPathsCircuit[T]{Rand: ValueOf[T](randVal), Zero: ValueOf[T](0)}
+
+	assert.CheckCircuit(circuit, test.WithValidAssignment(assignment))
+}

std/math/emulated/emparams/emparams.go

@@ -43,6 +43,11 @@ func (twelveLimbPrimeField) NbLimbs() uint     { return 12 }
 func (twelveLimbPrimeField) BitsPerLimb() uint { return 64 }
 func (twelveLimbPrimeField) IsPrime() bool     { return true }
 
+type oneLimbPrimeField struct{}
+
+func (oneLimbPrimeField) NbLimbs() uint { return 1 }
+func (oneLimbPrimeField) IsPrime() bool { return true }
+
 // Goldilocks provides type parametrization for field emulation:
 //   - limbs: 1
 //   - limb width: 64 bits
@@ -51,11 +56,9 @@ func (twelveLimbPrimeField) IsPrime() bool     { return true }
 //
 //	0xffffffff00000001 (base 16)
 //	18446744069414584321 (base 10)
-type Goldilocks struct{}
+type Goldilocks struct{ oneLimbPrimeField }
 
-func (fp Goldilocks) NbLimbs() uint     { return 1 }
 func (fp Goldilocks) BitsPerLimb() uint { return 64 }
-func (fp Goldilocks) IsPrime() bool     { return true }
 func (fp Goldilocks) Modulus() *big.Int { return goldilocks.Modulus() }
 
 // Secp256k1Fp provides type parametrization for field emulation:
@@ -366,3 +369,35 @@ func (Mod1e256) Modulus() *big.Int {
 	val, _ := new(big.Int).SetString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16)
 	return val
 }
+
+// BabyBear provides type parametrization for field emulation:
+//   - limbs: 1
+//   - limb width: 31 bits
+//
+// The prime modulus for type parametrisation is:
+//
+//	15*2^27+1
+//	0x78000001 (base 16)
+//	2013265921 (base 10)
+//
+// The field has 2-adicity of 27.
+type BabyBear struct{ oneLimbPrimeField }
+
+func (BabyBear) BitsPerLimb() uint { return 31 }
+func (BabyBear) Modulus() *big.Int { return big.NewInt(2013265921) }
+
+// KoalaBear provides type parametrization for field emulation:
+//   - limbs: 1
+//   - limb width: 31 bits
+//
+// The prime modulus for type parametrisation is:
+//
+//	2^31-2^24+1
+//	0x7f000001 (base 16)
+//	2130706433 (base 10)
+//
+// The field has 2-adicity of 24.
+type KoalaBear struct{ oneLimbPrimeField }
+
+func (KoalaBear) BitsPerLimb() uint { return 31 }
+func (KoalaBear) Modulus() *big.Int { return big.NewInt(2130706433) }

std/math/emulated/field.go

@@ -129,46 +129,41 @@ func (f *Field[T]) NewElement(v interface{}) *Element[T] {
 	if e, ok := v.([]frontend.Variable); ok {
 		return f.packLimbs(e, true)
 	}
-	c := ValueOf[T](v)
-	return &c
+	// the input was not a variable, so it must be a constant. Create a new
+	// element from it while setting isWitness flag to false. This ensures that
+	// we use the minimal number of limbs necessary.
+	c := newConstElement[T](v, false)
+	return c
 }
 
 // Zero returns zero as a constant.
 func (f *Field[T]) Zero() *Element[T] {
 	f.zeroConstOnce.Do(func() {
-		f.zeroConst = newConstElement[T](0)
+		f.zeroConst = f.newInternalElement([]frontend.Variable{}, 0)
 	})
 	return f.zeroConst
 }
 
 // One returns one as a constant.
 func (f *Field[T]) One() *Element[T] {
 	f.oneConstOnce.Do(func() {
-		f.oneConst = newConstElement[T](1)
+		f.oneConst = f.newInternalElement([]frontend.Variable{1}, 0)
 	})
 	return f.oneConst
 }
 
-// shortOne returns one as a constant stored in a single limb.
-func (f *Field[T]) shortOne() *Element[T] {
-	f.shortOneConstOnce.Do(func() {
-		f.shortOneConst = f.newInternalElement([]frontend.Variable{1}, 0)
-	})
-	return f.shortOneConst
-}
-
 // Modulus returns the modulus of the emulated ring as a constant.
 func (f *Field[T]) Modulus() *Element[T] {
 	f.nConstOnce.Do(func() {
-		f.nConst = newConstElement[T](f.fParams.Modulus())
+		f.nConst = newConstElement[T](f.fParams.Modulus(), false)
 	})
 	return f.nConst
 }
 
 // modulusPrev returns modulus-1 as a constant.
 func (f *Field[T]) modulusPrev() *Element[T] {
 	f.nprevConstOnce.Do(func() {
-		f.nprevConst = newConstElement[T](new(big.Int).Sub(f.fParams.Modulus(), big.NewInt(1)))
+		f.nprevConst = newConstElement[T](new(big.Int).Sub(f.fParams.Modulus(), big.NewInt(1)), false)
 	})
 	return f.nprevConst
 }