diff --git a/constantine/ethereum_eip4844_kzg.nim b/constantine/ethereum_eip4844_kzg.nim index 99c505206..1145dcccf 100644 --- a/constantine/ethereum_eip4844_kzg.nim +++ b/constantine/ethereum_eip4844_kzg.nim @@ -21,7 +21,7 @@ import ./serialization/[codecs_status_codes, codecs_bls12_381, endians], ./trusted_setups/ethereum_kzg_srs -export loadTrustedSetup, TrustedSetupStatus, EthereumKZGContext +export loadTrustedSetup_tsif, TrustedSetupStatus, EthereumKZGContext ## ############################################################ ## @@ -593,15 +593,13 @@ const TrustedSetupMainnet = "trusted_setups" / "trusted_setup_ethereum_kzg_test_mainnet.tsif" -proc load_ethereum_kzg_test_trusted_setup_mainnet*(): ptr EthereumKZGContext {.libPrefix: prefix_ffi.} = +proc load_ethereum_kzg_test_trusted_setup_mainnet*(): ptr EthereumKZGContext {.libPrefix: prefix_ffi, raises: [OSError, IOError].} = ## This is a convenience function for the Ethereum mainnet testing trusted setups. ## It is insecure and will be replaced once the KZG ceremony is done. let ctx = allocHeapAligned(EthereumKZGContext, alignment = 64) - - let tsStatus = ctx.loadTrustedSetup(TrustedSetupMainnet) + let tsStatus = ctx.loadTrustedSetup_tsif(TrustedSetupMainnet) doAssert tsStatus == tsSuccess, "\n[Trusted Setup Error] " & $tsStatus - echo "Trusted Setup loaded successfully" return ctx diff --git a/constantine/platforms/fileio.nim b/constantine/platforms/fileio.nim new file mode 100644 index 000000000..02f9e911d --- /dev/null +++ b/constantine/platforms/fileio.nim @@ -0,0 +1,129 @@ +# Constantine +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Copyright (c) 2020-Present Mamy André-Ratsimbazafy +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# ############################################################ +# +# File IO +# +# ############################################################ + +# We create our own File IO primitives on top of +# We do not use std/syncio or std/streams +# as we do not use Nim memory allocator and exceptions. + +# Ensure all exceptions are converted to error codes +{.push raises: [], checks: off.} + +# Datatypes +# ------------------------------------------------------------ + +type + CFile {.importc: "FILE", header: "", incompleteStruct.} = object + ## C Filestream API + File* = ptr CFile + + FileSeekFrom {.size: sizeof(cint).} = enum + ## SEEK_SET, SEEK_CUR and SEEK_END in stdio.h + kAbsolute + kCurrent + kEnd + + FileMode* = enum + # We always use binary mode for files for universality. + kRead # Open a file for reading in binary mode. File must exist. + kOverwrite # Open a file for overwriting in binary mode. If file exists it is cleared, otherwise it is created. + kAppend # Open a file for writing in binary mode. If file exists data is appended, otherwise it is created. + kReadWrite # Open a fil for read-write in binary mode. File must exist. + kReadOverwrite # Open a file for read-overwrite in binary mode. If file exists it is cleared, otherwise it is created. + +const + childProcNoInherit = block: + # Child processes should not inherit open files + when defined(windows): + "N" + elif defined(linux) or defined(bsd): + "e" + else: + "" + + MapFileMode = [ + # We want to ensure no dynamic alloc at runtime with strings + kRead: cstring("rb" & childProcNoInherit), + kOverwrite: cstring("wb" & childProcNoInherit), + kAppend: cstring("ab" & childProcNoInherit), + kReadWrite: cstring("rb+" & childProcNoInherit), + kReadOverwrite: cstring("wb+" & childProcNoInherit) + ] + +# Opening/Closing files +# ------------------------------------------------------------ + +proc c_fopen(filepath, mode: cstring): File {.importc: "fopen", header: "", sideeffect.} +proc c_fclose(f: File): cint {.importc: "fclose", header: "", sideeffect.} + +when defined(windows): + proc c_fileno(f: File): cint {.importc: "_fileno", header: "", sideeffect.} +else: + type + Mode {.importc: "mode_t", header: "".} = cint + Stat {.importc: "struct stat", header: "", final, pure.} = object + st_mode: Mode + proc is_dir(m: Mode): bool {.importc: "S_ISDIR", header: "".} + proc c_fileno(f: File): cint {.importc: "fileno", header: "", sideeffect.} + proc c_fstat(a1: cint, a2: var Stat): cint {.importc: "fstat", header: "", sideeffect.} + +proc close*(f: File) = + if not f.isNil: + discard f.c_fclose() + +proc open*(f: var File, filepath: cstring, mode = kRead): bool = + f = c_fopen(filepath, MapFileMode[mode]) + if f.isNil: + return false + + # Posix OSes can open directories, prevent that. + when defined(posix): + var stat {.noInit.}: Stat + if c_fstat(c_fileno(f), stat) >= 0 and stat.st_mode.is_dir: + f.close() + return false + + return true + + +# Navigating files +# ------------------------------------------------------------ + +when defined(windows): + func getFilePosition*(f: File): int64 {.importc: "_ftelli64", header: "".} + func setFilePosition*(f: File, offset: int64, relative = kAbsolute): cint {.importc: "_fseeki64", header: "".} +else: + func getFilePosition*(f: File): int64 {.importc: "ftello", header: "".} + func setFilePosition*(f: File, offset: int64, relative = kAbsolute): cint {.importc: "fseeko", header: "".} + +# Reading files +# ------------------------------------------------------------ + +func c_fread(buffer: pointer, len, count: csize_t, f: File): csize_t {.importc: "fread", header: "".} + +func readInto*(f: File, buffer: pointer, len: int): int = + ## Read data into buffer, return the number of bytes read + cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) + +func readInto*[T](f: File, buf: var T): bool = + ## Read data into buffer, + ## return true if the number of bytes read + ## matches the output type size + return f.readInto(buf.addr, sizeof(buf)) == sizeof(T) + +func read*(f: File, T: typedesc): T = + ## Interpret next bytes as type `T` + ## Panics if the number of bytes read does not match + ## the size of `T` + let ok = f.readInto(result) + doAssert ok, "Fatal error when reading '" & $T & "' from file." diff --git a/constantine/trusted_setups/ethereum_kzg_srs.nim b/constantine/trusted_setups/ethereum_kzg_srs.nim index 5ff061cdd..5033b266e 100644 --- a/constantine/trusted_setups/ethereum_kzg_srs.nim +++ b/constantine/trusted_setups/ethereum_kzg_srs.nim @@ -9,13 +9,11 @@ import ../math/config/curves, ../math/[ec_shortweierstrass, arithmetic, extension_fields], - ../platforms/abstractions, + ../platforms/[abstractions, fileio], ../serialization/endians, ../math/constants/zoo_generators, ../math/polynomials/polynomials, - ../math/io/io_fields, - - std/streams # TODO: use the C streams which do not allocate, are exception-free and are guaranteed not to bring unremovable runtime procs. + ../math/io/io_fields # Ensure all exceptions are converted to error codes {.push raises: [], checks: off.} @@ -98,166 +96,159 @@ type tsInvalidFile tsLowLevelReadError -proc skipMod64(f: FileStream): TrustedSetupStatus = +proc skipMod64(f: fileio.File): TrustedSetupStatus = ## Skip to a 64-byte boundary try: - let pos = f.getPosition() + let pos = f.getFilePosition() + if pos < 0: + return tsLowLevelReadError let posMod64 = pos and 63 - f.setPosition(pos+posMod64) + let status = f.setFilePosition(pos+posMod64) + if status != 0: + return tsLowLevelReadError return tsSuccess except IOError, OSError: return tsInvalidFile -proc loadTrustedSetup*(ctx: ptr EthereumKZGContext, filePath: string): TrustedSetupStatus = - # TODO, use C cstring so the library is exportable and usable from C +proc loadTrustedSetup_tsif*(ctx: ptr EthereumKZGContext, filepath: cstring): TrustedSetupStatus = + ## Load trusted setup in the TSIF format + ## Opening and closing the file is the responsibility of the caller static: doAssert cpuEndian == littleEndian, "Trusted setup creation is only supported on little-endian CPUs at the moment." - let f = try: openFileStream(filePath, fmRead) - except IOError, OSError: return tsMissingFile + var buf: array[32, byte] + var len = 0 - defer: - try: - f.close() - except Exception: # For some reason close can raise a bare Exception. - quit "Unrecoverable error while closing trusted setup file." + var f: fileio.File + let ok = f.open(filepath, kRead) + defer: f.close() - try: - var buf = newSeqOfCap[byte](32) - var len = 0 + if not ok: + return tsInvalidFile - buf.setLen(12) - len = f.readData(buf[0].addr, 12) - if len != 12: - return tsInvalidFile - if buf != static(@[byte 0xE2, 0x88, 0x83, 0xE2, 0x8B, 0x83, 0xE2, 0x88, 0x88, 0xE2, 0x88, 0x8E]): - # ∃⋃∈∎ in UTF-8 - return tsInvalidFile + len = f.readInto(buf[0].addr, 12) + if len != 12: + return tsInvalidFile + if buf.toOpenArray(0, len-1) != static(@[byte 0xE2, 0x88, 0x83, 0xE2, 0x8B, 0x83, 0xE2, 0x88, 0x88, 0xE2, 0x88, 0x8E]): + # ∃⋃∈∎ in UTF-8 + return tsInvalidFile - if f.readChar() != 'v': - return tsInvalidFile - if f.readUint8() != 1: - return tsUnsupportedFileVersion - if f.readChar() != '.': - return tsUnsupportedFileVersion - if f.readUint8() != 0: - return tsUnsupportedFileVersion - - buf.setLen(32) - len = f.readData(buf[0].addr, 32) + if f.read(char) != 'v': + return tsInvalidFile + if f.read(uint8) != 1: + return tsUnsupportedFileVersion + if f.read(char) != '.': + return tsUnsupportedFileVersion + if f.read(uint8) != 0: + return tsUnsupportedFileVersion + + len = f.readInto(buf[0].addr, 32) + if len != 32: + return tsInvalidFile + if buf.toOpenArray(0, 17) != asBytes"ethereum_deneb_kzg": + return tsWrongPreset + if buf.toOpenArray(18, 31) != default(array[18..31, byte]): + debugEcho buf.toOpenArray(18, 31) + return tsWrongPreset + + len = f.readInto(buf[0].addr, 15) + if len != 15: + return tsInvalidFile + if buf.toOpenArray(0, 8) != asBytes"bls12_381": + return tsWrongPreset + if buf.toOpenArray(9, 14) != default(array[9..14, byte]): + return tsWrongPreset + + let num_fields = f.read(uint8) + if num_fields != 3: + return tsWrongPreset + + block: # Read 1st metadata + len = f.readInto(buf[0].addr, 32) if len != 32: return tsInvalidFile - if buf.toOpenArray(0, 17) != asBytes"ethereum_deneb_kzg": + if buf.toOpenArray(0, 11) != asBytes"srs_lagrange": + return tsWrongPreset + if buf.toOpenArray(12, 14) != default(array[12..14, byte]): + return tsWrongPreset + if buf.toOpenArray(15, 19) != asBytes"g1brp": return tsWrongPreset - if buf.toOpenArray(18, 31) != default(array[18..31, byte]): - debugEcho buf.toOpenArray(18, 31) + let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) + if elemSize != uint32 sizeof(ECP_ShortW_Aff[Fp[BLS12_381], G1]): + return tsWrongPreset + let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) + if numElems != FIELD_ELEMENTS_PER_BLOB: return tsWrongPreset - buf.setLen(15) - len = f.readData(buf[0].addr, 15) - if len != 15: + block: # Read 2nd metadata + len = f.readInto(buf[0].addr, 32) + if len != 32: return tsInvalidFile - if buf.toOpenArray(0, 8) != asBytes"bls12_381": + if buf.toOpenArray(0, 11) != asBytes"srs_monomial": + return tsWrongPreset + if buf.toOpenArray(12, 14) != default(array[12..14, byte]): + return tsWrongPreset + if buf.toOpenArray(15, 19) != asBytes"g2asc": return tsWrongPreset - if buf.toOpenArray(9, 14) != default(array[9..14, byte]): + let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) + if elemSize != uint32 sizeof(ECP_ShortW_Aff[Fp2[BLS12_381], G2]): + return tsWrongPreset + let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) + if numElems != KZG_SETUP_G2_LENGTH: return tsWrongPreset - let num_fields = f.readUint8() - if num_fields != 3: + block: # Read 3rd metadata + len = f.readInto(buf[0].addr, 32) + if len != 32: + return tsInvalidFile + if buf.toOpenArray(0, 10) != asBytes"roots_unity": + return tsWrongPreset + if buf.toOpenArray(11, 14) != default(array[11..14, byte]): + return tsWrongPreset + if buf.toOpenArray(15, 19) != asBytes"frbrp": + return tsWrongPreset + let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) + if elemSize != uint32 sizeof(Fr[BLS12_381]): + return tsWrongPreset + let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) + if numElems != FIELD_ELEMENTS_PER_BLOB: return tsWrongPreset - block: # Read 1st metadata - buf.setLen(32) - len = f.readData(buf[0].addr, 32) - if len != 32: - return tsInvalidFile - if buf.toOpenArray(0, 11) != asBytes"srs_lagrange": - return tsWrongPreset - if buf.toOpenArray(12, 14) != default(array[12..14, byte]): - return tsWrongPreset - if buf.toOpenArray(15, 19) != asBytes"g1brp": - return tsWrongPreset - let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) - if elemSize != uint32 sizeof(ECP_ShortW_Aff[Fp[BLS12_381], G1]): - return tsWrongPreset - let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) - if numElems != FIELD_ELEMENTS_PER_BLOB: - return tsWrongPreset - - block: # Read 2nd metadata - buf.setLen(32) - len = f.readData(buf[0].addr, 32) - if len != 32: - return tsInvalidFile - if buf.toOpenArray(0, 11) != asBytes"srs_monomial": - return tsWrongPreset - if buf.toOpenArray(12, 14) != default(array[12..14, byte]): - return tsWrongPreset - if buf.toOpenArray(15, 19) != asBytes"g2asc": - return tsWrongPreset - let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) - if elemSize != uint32 sizeof(ECP_ShortW_Aff[Fp2[BLS12_381], G2]): - return tsWrongPreset - let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) - if numElems != KZG_SETUP_G2_LENGTH: - return tsWrongPreset - - block: # Read 3rd metadata - buf.setLen(32) - len = f.readData(buf[0].addr, 32) - if len != 32: - return tsInvalidFile - if buf.toOpenArray(0, 10) != asBytes"roots_unity": - return tsWrongPreset - if buf.toOpenArray(11, 14) != default(array[11..14, byte]): - return tsWrongPreset - if buf.toOpenArray(15, 19) != asBytes"frbrp": - return tsWrongPreset - let elemSize = uint32.fromBytes(buf.toOpenArray(20, 23), littleEndian) - if elemSize != uint32 sizeof(Fr[BLS12_381]): - return tsWrongPreset - let numElems = uint64.fromBytes(buf.toOpenArray(24, 31), littleEndian) - if numElems != FIELD_ELEMENTS_PER_BLOB: - return tsWrongPreset - - block: # Read 1st data, assume little-endian - let status64Balign = f.skipMod64() - if status64Balign != tsSuccess: - return status64Balign - - len = f.readData(ctx.srs_lagrange_g1.addr, sizeof(ctx.srs_lagrange_g1)) - if len != sizeof(ctx.srs_lagrange_g1): - return tsInvalidFile - - block: # Read 2nd data, assume little-endian - let status64Balign = f.skipMod64() - if status64Balign != tsSuccess: - return status64Balign - - len = f.readData(ctx.srs_monomial_g2.addr, sizeof(ctx.srs_monomial_g2)) - if len != sizeof(ctx.srs_monomial_g2): - return tsInvalidFile - - block: # Read 3rd data, assume little-endian - let status64Balign = f.skipMod64() - if status64Balign != tsSuccess: - return status64Balign - - len = f.readData(ctx.domain.rootsOfUnity.addr, sizeof(ctx.domain.rootsOfUnity)) - if len != sizeof(ctx.domain.rootsOfUnity): - return tsInvalidFile - - # Compute the inverse of the domain degree - ctx.domain.invMaxDegree.fromUint(ctx.domain.rootsOfUnity.len.uint64) - ctx.domain.invMaxDegree.inv_vartime() - - block: # Last sanity check - # When the srs is in monomial form we can check that - # the first point is the generator - if bool(ctx.srs_monomial_g2.coefs[0] != BLS12_381.getGenerator"G2"): - return tsWrongPreset + block: # Read 1st data, assume little-endian + let status64Balign = f.skipMod64() + if status64Balign != tsSuccess: + return status64Balign - return tsSuccess + len = f.readInto(ctx.srs_lagrange_g1.addr, sizeof(ctx.srs_lagrange_g1)) + if len != sizeof(ctx.srs_lagrange_g1): + return tsInvalidFile - except IOError, OSError: - return tsLowLevelReadError \ No newline at end of file + block: # Read 2nd data, assume little-endian + let status64Balign = f.skipMod64() + if status64Balign != tsSuccess: + return status64Balign + + len = f.readInto(ctx.srs_monomial_g2.addr, sizeof(ctx.srs_monomial_g2)) + if len != sizeof(ctx.srs_monomial_g2): + return tsInvalidFile + + block: # Read 3rd data, assume little-endian + let status64Balign = f.skipMod64() + if status64Balign != tsSuccess: + return status64Balign + + len = f.readInto(ctx.domain.rootsOfUnity.addr, sizeof(ctx.domain.rootsOfUnity)) + if len != sizeof(ctx.domain.rootsOfUnity): + return tsInvalidFile + + # Compute the inverse of the domain degree + ctx.domain.invMaxDegree.fromUint(ctx.domain.rootsOfUnity.len.uint64) + ctx.domain.invMaxDegree.inv_vartime() + + block: # Last sanity check + # When the srs is in monomial form we can check that + # the first point is the generator + if bool(ctx.srs_monomial_g2.coefs[0] != BLS12_381.getGenerator"G2"): + return tsWrongPreset + + return tsSuccess