Productionize KZG EIP-4844 (#304)

* KZG C API + namespace Ethereum BLS signature

* Headers for C API for EIP4844

* Read trusted setups without Nim allocator and exceptions

* fix ysrand on win and Mac

* stash: Update to Eth mainnet KZG trusted setup from ceremony [skip ci]

* update to KZG ceremony trusted setup

* fix kzg bench compilation

* fix sysrand rebase mac

* Provide C API to EIP4844

* kzg: Portable LF / CRLF parsing of trusted setup

* kzg: fix blob initialization

* kzg: oops, parallel verification wasn't fully turned on

benchmarks/bench_ethereum_bls_signatures.nim

@@ -159,7 +159,7 @@ proc benchVerifyMulti*(numSigs, iters: int) =
   bench("BLS verif of " & $numSigs & " msgs by "& $numSigs & " pubkeys", "BLS12_381", iters):
     for i in 0 ..< triplets.len:
       let ok = triplets[i].pubkey.verify(triplets[i].msg, triplets[i].sig)
-      doAssert ok == cttBLS_Success
+      doAssert ok == cttEthBls_Success
 
 proc benchVerifyBatched*(numSigs, iters: int) =
   ## Verification of N pubkeys signing for N messages
@@ -185,7 +185,7 @@ proc benchVerifyBatched*(numSigs, iters: int) =
 
   bench("BLS serial batch verify of " & $numSigs & " msgs by "& $numSigs & " pubkeys (with blinding)", "BLS12_381", iters):
     let ok = batch_verify(pubkeys, messages, signatures, secureBlindingBytes)
-    doAssert ok == cttBLS_Success
+    doAssert ok == cttEthBls_Success
 
 proc benchVerifyBatchedParallel*(numSigs, iters: int) =
   ## Verification of N pubkeys signing for N messages
@@ -220,7 +220,7 @@ proc benchVerifyBatchedParallel*(numSigs, iters: int) =
 
   bench("BLS parallel batch verify (" & $tp.numThreads & " threads) of " & $numSigs & " msgs by "& $numSigs & " pubkeys (with blinding)", "BLS12_381", iters):
     let ok = tp.batch_verify_parallel(pubkeys, messages, signatures, secureBlindingBytes)
-    doAssert ok == cttBLS_Success, "invalid status: " & $ok
+    doAssert ok == cttEthBls_Success, "invalid status: " & $ok
 
   tp.shutdown()
 

benchmarks/bench_ethereum_eip4844_kzg.nim

@@ -16,7 +16,9 @@ import
   ../constantine/platforms/primitives,
   # Helpers
   ../helpers/prng_unsafe,
-  ./bench_blueprint
+  ./bench_blueprint,
+  # Standard library
+  std/[os, strutils]
 
 proc separator*() = separator(180)
 
@@ -41,11 +43,18 @@ type
     # there is no short-circuit if they don't match
     challenge, eval_at_challenge: array[32, byte]
 
+proc randomize(rng: var RngState, blob: var Blob) =
+  for i in 0 ..< FIELD_ELEMENTS_PER_BLOB:
+    let t {.noInit.} = rng.random_unsafe(Fr[BLS12_381])
+    let offset = i*BYTES_PER_FIELD_ELEMENT
+    blob.toOpenArray(offset, offset+BYTES_PER_FIELD_ELEMENT-1)
+        .marshal(t, bigEndian)
+
 proc new(T: type BenchSet, ctx: ptr EthereumKZGContext): T =
   new(result)
   for i in 0 ..< result.N:
     let t {.noInit.} = rng.random_unsafe(Fr[BLS12_381])
-    result.blobs[i].marshal(t, bigEndian)
+    rng.randomize(result.blobs[i])
     discard ctx.blob_to_kzg_commitment(result.commitments[i], result.blobs[i].addr)
     discard ctx.compute_blob_kzg_proof(result.proofs[i], result.blobs[i].addr, result.commitments[i])
 
@@ -61,7 +70,7 @@ proc benchBlobToKzgCommitment(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("blob_to_kzg_commitment", "serial", iters):
       var commitment {.noInit.}: array[48, byte]
-      doAssert cttEthKZG_Success == ctx.blob_to_kzg_commitment(commitment, b.blobs[0].addr)
+      doAssert cttEthKzg_Success == ctx.blob_to_kzg_commitment(commitment, b.blobs[0].addr)
   let stopSerial = getMonotime()
 
   ## We require `tp` to be unintialized as even idle threads somehow reduce perf of serial benches
@@ -71,7 +80,7 @@ proc benchBlobToKzgCommitment(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("blob_to_kzg_commitment", $tp.numThreads & " threads", iters):
       var commitment {.noInit.}: array[48, byte]
-      doAssert cttEthKZG_Success == tp.blob_to_kzg_commitment_parallel(ctx, commitment, b.blobs[0].addr)
+      doAssert cttEthKzg_Success == tp.blob_to_kzg_commitment_parallel(ctx, commitment, b.blobs[0].addr)
   let stopParallel = getMonotime()
 
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
@@ -87,7 +96,7 @@ proc benchComputeKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: int)
     bench("compute_kzg_proof", "serial", iters):
       var proof {.noInit.}: array[48, byte]
       var eval_at_challenge {.noInit.}: array[32, byte]
-      doAssert cttEthKZG_Success == ctx.compute_kzg_proof(proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
+      doAssert cttEthKzg_Success == ctx.compute_kzg_proof(proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
   let stopSerial = getMonotime()
 
   ## We require `tp` to be unintialized as even idle threads somehow reduce perf of serial benches
@@ -98,7 +107,7 @@ proc benchComputeKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: int)
     bench("compute_kzg_proof", $tp.numThreads & " threads", iters):
       var proof {.noInit.}: array[48, byte]
       var eval_at_challenge {.noInit.}: array[32, byte]
-      doAssert cttEthKZG_Success == tp.compute_kzg_proof_parallel(ctx, proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
+      doAssert cttEthKzg_Success == tp.compute_kzg_proof_parallel(ctx, proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
   let stopParallel = getMonotime()
 
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
@@ -113,7 +122,7 @@ proc benchComputeBlobKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("compute_blob_kzg_proof", "serial", iters):
       var proof {.noInit.}: array[48, byte]
-      doAssert cttEthKZG_Success == ctx.compute_blob_kzg_proof(proof, b.blobs[0].addr, b.commitments[0])
+      doAssert cttEthKzg_Success == ctx.compute_blob_kzg_proof(proof, b.blobs[0].addr, b.commitments[0])
   let stopSerial = getMonotime()
 
   ## We require `tp` to be unintialized as even idle threads somehow reduce perf of serial benches
@@ -123,7 +132,7 @@ proc benchComputeBlobKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("compute_blob_kzg_proof", $tp.numThreads & " threads", iters):
       var proof {.noInit.}: array[48, byte]
-      doAssert cttEthKZG_Success == tp.compute_blob_kzg_proof_parallel(ctx, proof, b.blobs[0].addr, b.commitments[0])
+      doAssert cttEthKzg_Success == tp.compute_blob_kzg_proof_parallel(ctx, proof, b.blobs[0].addr, b.commitments[0])
   let stopParallel = getMonotime()
 
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
@@ -207,10 +216,25 @@ proc benchVerifyBlobKzgProofBatch(b: BenchSet, ctx: ptr EthereumKZGContext, iter
 
     i *= 2
 
+const TrustedSetupMainnet =
+  currentSourcePath.rsplit(DirSep, 1)[0] /
+  ".." / "constantine" /
+  "trusted_setups" /
+  "trusted_setup_ethereum_kzg4844_reference.dat"
+
+proc trusted_setup*(): ptr EthereumKZGContext =
+  ## This is a convenience function for the Ethereum mainnet testing trusted setups.
+  ## It is insecure and will be replaced once the KZG ceremony is done.
+
+  var ctx: ptr EthereumKZGContext
+  let tsStatus = ctx.trusted_setup_load(TrustedSetupMainnet, kReferenceCKzg4844)
+  doAssert tsStatus == tsSuccess, "\n[Trusted Setup Error] " & $tsStatus
+  echo "Trusted Setup loaded successfully"
+  return ctx
 
 const Iters = 100
 proc main() =
-  let ctx = load_ethereum_kzg_test_trusted_setup_mainnet()
+  let ctx = trusted_setup()
   let b = BenchSet[64].new(ctx)
   separator()
   benchBlobToKzgCommitment(b, ctx, Iters)

bindings/lib_constantine.nim

@@ -18,8 +18,10 @@ import
   ../constantine/threadpool,
   ./lib_hashes,
   ./lib_curves,
+  ../constantine/csprngs,
   # Protocols
   ../constantine/ethereum_bls_signatures,
+  ../constantine/ethereum_eip4844_kzg,
 
   # Ensure globals like proc from kernel32.dll are populated at library load time
   ./lib_autoload

constantine-rust/constantine-sys/src/bindings.rs

@@ -3870,11 +3870,11 @@ fn bindgen_test_layout_ctt_eth_bls_signature() {
 #[repr(u8)]
 #[derive(Copy, Clone, Hash, PartialEq, Eq)]
 pub enum ctt_eth_bls_status {
-    cttBLS_Success = 0,
-    cttBLS_VerificationFailure = 1,
-    cttBLS_PointAtInfinity = 2,
-    cttBLS_ZeroLengthAggregation = 3,
-    cttBLS_InconsistentLengthsOfInputs = 4,
+    cttEthBls_Success = 0,
+    cttEthBls_VerificationFailure = 1,
+    cttEthBls_PointAtInfinity = 2,
+    cttEthBls_ZeroLengthAggregation = 3,
+    cttEthBls_InconsistentLengthsOfInputs = 4,
 }
 extern "C" {
     #[must_use]

constantine/commitments/kzg_polynomial_commitments_parallel.nim

@@ -120,7 +120,7 @@ proc kzg_verify_batch_parallel*[bits: static int, F2; C: static Curve](
        proofs: ptr UncheckedArray[ECP_ShortW_Aff[Fp[C], G1]],
        linearIndepRandNumbers: ptr UncheckedArray[Fr[C]],
        n: int,
-       tauG2: ECP_ShortW_Aff[F2, G2]): bool {.tags:[HeapAlloc, Alloca, Vartime].} =
+       tauG2: ECP_ShortW_Aff[F2, G2]): bool =
   ## Verify multiple KZG proofs efficiently
   ##
   ## Parameters
@@ -204,7 +204,7 @@ proc kzg_verify_batch_parallel*[bits: static int, F2; C: static Curve](
 
     commits_min_evals.batchAffine(commits_min_evals_jac, n)
     freeHeapAligned(commits_min_evals_jac)
-    tp.multiScalarMul_vartime(sum_commit_minus_evals_G1, coefs, commits_min_evals, n)
+    tp.multiScalarMul_vartime_parallel(sum_commit_minus_evals_G1, coefs, commits_min_evals, n)
     freeHeapAligned(commits_min_evals)
 
   let sum_commit_minus_evals_G1_fv = tp.spawnAwaitable tp.compute_sum_commitments_minus_evals(
@@ -228,16 +228,17 @@ proc kzg_verify_batch_parallel*[bits: static int, F2; C: static Curve](
 
     syncScope:
       tp.parallelFor i in 0 ..< n:
+        captures: {rand_coefs, rand_coefs_fr, linearIndepRandNumbers, challenges}
         rand_coefs_fr[i].prod(linearIndepRandNumbers[i], challenges[i])
         rand_coefs[i].fromField(rand_coefs_fr[i])
 
-    tp.multiScalarMul_vartime(sum_rand_challenge_proofs, rand_coefs, proofs, n)
+    tp.multiScalarMul_vartime_parallel(sum_rand_challenge_proofs, rand_coefs, proofs, n)
 
     freeHeapAligned(rand_coefs_fr)
     freeHeapAligned(rand_coefs)
 
   let sum_rand_challenge_proofs_fv = tp.spawnAwaitable tp.compute_sum_rand_challenge_proofs(
-                                                   sum_rand_challenge_proofs,
+                                                   sum_rand_challenge_proofs.addr,
                                                    linearIndepRandNumbers,
                                                    challenges,
                                                    proofs,

constantine/csprngs.nim

@@ -0,0 +1,10 @@
+# Constantine
+# Copyright (c) 2018-2019    Status Research & Development GmbH
+# Copyright (c) 2020-Present Mamy André-Ratsimbazafy
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import csprngs/sysrand
+export sysrand

constantine/csprngs/sysrand.nim

@@ -17,6 +17,9 @@
 type
   CSPRNG    = object
 
+const prefix_ffi = "ctt_csprng_"
+import ../zoo_exports
+
 when defined(windows):
   # There are several Windows CSPRNG APIs:
   # - CryptGenRandom
@@ -51,9 +54,10 @@ when defined(windows):
     # BOOLEAN (to not be confused with winapi BOOL)
     # is `typedef BYTE BOOLEAN;` and so has the same representation as Nim bools.
 
-  proc sysrand*[T](buffer: var T): bool {.inline.} =
+  proc sysrand*(buffer: pointer, len: csize_t): bool {.libPrefix: prefix_ffi.} =
     ## Fills the buffer with cryptographically secure random data
-    return RtlGenRandom(buffer.addr, culong sizeof(T))
+    ## Returns true on success, false otherwise
+    return RtlGenRandom(buffer, culong len)
 
 elif defined(linux):
   proc syscall(sysno: clong): cint {.importc, header:"<unistd.h>", varargs.}
@@ -75,13 +79,14 @@ elif defined(linux):
   #
   # We choose to handle partial buffer fills to limit the number of syscalls
 
-  proc urandom(pbuffer: pointer, len: int): bool {.sideeffect, tags: [CSPRNG].} =
-
-    var cur = 0
+  proc sysrand*(buffer: pointer, len: csize_t): bool {.libPrefix: prefix_ffi, sideeffect, tags: [CSPRNG].} =
+    ## Fills the buffer with cryptographically secure random data
+    ## Returns true on success, false otherwise
+    var cur = csize_t 0
     while cur < len:
-      let bytesRead = syscall(SYS_getrandom, pbuffer, len-cur, 0)
+      let bytesRead = syscall(SYS_getrandom, buffer, len-cur, 0)
       if bytesRead > 0:
-        cur += bytesRead
+        cur += csize_t bytesRead
       elif bytesRead == 0:
         # According to documentation this should never happen,
         # either we read a positive number of bytes, or we have a negative error code
@@ -96,10 +101,6 @@ elif defined(linux):
 
     return true
 
-  proc sysrand*[T](buffer: var T): bool {.inline.} =
-    ## Fills the buffer with cryptographically secure random data
-    return urandom(buffer.addr, sizeof(T))
-
 elif defined(ios) or defined(macosx) or defined(macos):
   # There are 4 APIs we can use
   # - The getentropy(2) system call (similar to OpenBSD)
@@ -129,14 +130,20 @@ elif defined(ios) or defined(macosx) or defined(macos):
 
   func `==`(x, y: CCRNGStatus): bool {.borrow.}
 
-  proc CCRandomGenerateBytes(pbuffer: pointer, len: int): CCRNGStatus {.sideeffect, tags: [CSPRNG], importc, header: "<CommonCrypto/CommonRandom.h>".}
+  proc CCRandomGenerateBytes(pbuffer: pointer, len: csize_t): CCRNGStatus {.sideeffect, tags: [CSPRNG], importc, header: "<CommonCrypto/CommonRandom.h>".}
     # https://opensource.apple.com/source/CommonCrypto/CommonCrypto-60178.40.2/include/CommonRandom.h.auto.html
 
-  proc sysrand*[T](buffer: var T): bool {.inline.} =
+  proc sysrand*(buffer: pointer, len: csize_t): bool {.libPrefix: prefix_ffi.} =
     ## Fills the buffer with cryptographically secure random data
-    if kCCSuccess == CCRandomGenerateBytes(buffer.addr, sizeof(T)):
+    ## Returns true on success, false otherwise
+    if kCCSuccess == CCRandomGenerateBytes(buffer, len):
       return true
     return false
 
 else:
-  {.error: "The OS '" & $hostOS & "' has no CSPRNG configured.".}
+  {.error: "The OS '" & $hostOS & "' has no CSPRNG configured.".}
+
+proc sysrand*[T](buffer: var T): bool {.inline.} =
+  ## Fills the buffer with cryptographically secure random data
+  ## Returns true on success, false otherwise
+  return sysrand(buffer.addr, csize_t sizeof(T))