Rework circuit downloader (#882)

* Introduces a start method to prover * Moves backend creation into start method * sets up three paths for backend initialization * Extracts backend initialization to backend-factory * Implements loading backend from cli files or previously downloaded local files * Wires up downloading and unzipping * functional implementation * Fixes testprover.nim * Sets up tests for backendfactory * includes libzip-dev * pulls in updated contracts * removes integration cli tests for r1cs, wasm, and zkey file arguments. * Fixes issue where inner-scope values are lost before returning * sets local proof verification for dist-test images * Adds two traces and bumps nim-ethers * Adds separate path for circuit files * Create circuit dir if not exists * fix: make sure requestStorage is mined * fix: correct place to plug confirm * test: fixing contracts tests * Restores gitmodules * restores nim-datastore reference * Sets up downloader exe * sets up tool skeleton * implements getting of circuit hash * Implements downloader tool * sets up test skeleton * Implements test for cirdl * includes testTools in testAll * Cleanup building.md * cleans up previous downloader implementation * cleans up testbackendfactory * moves start of prover into node.nim * Fills in arguments in example command * Initializes backend in prover constructor * Restores tests * Restores tests for cli instructions * Review comments by Dmitriy, part 1 * Quotes path in download instruction. * replaces curl with chronos http session * Moves cirdl build output to 'build' folder. * Fixes chronicles log output * Add cirdl support to the codex Dockerfile Signed-off-by: Slava <[email protected]> * Add cirdl support to the docker entrypoint Signed-off-by: Slava <[email protected]> * Add cirdl support to the release workflow Signed-off-by: Slava <[email protected]> * Disable verify_circuit flag for releases Signed-off-by: Slava <[email protected]> * Removes backendFactory placeholder type * wip * Replaces zip library with status-im/zippy library (which supports zip and tar) * Updates cirdl to not change circuitdir folder * Switches from zip to tar.gz * Review comments by Dmitriy * updates codex-contracts-eth * Adds testTools to CI * Adds check for access to config.circuitdir * Update fixture circuit zkey * Update matrix to run tools tests on Windows * Adds 'deps' dependency for cirdl * Adjust docker-entrypoint.sh to use CODEX_CIRCUIT_DIR env var * Review comments by Giuliano --------- Signed-off-by: Slava <[email protected]> Co-authored-by: Adam Uhlíř <[email protected]> Co-authored-by: Veaceslav Doina <[email protected]>
codex-storage · Sep 23, 2024 · 4e86307 · 4e86307
1 parent e8e9820
commit 4e86307
Show file tree

Hide file tree

Showing 30 changed files with 547 additions and 89 deletions.
diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml
@@ -53,7 +53,7 @@ jobs:
           node-version: 18.15
 
       - name: Start Ethereum node with Codex contracts
-        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all'
+        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
@@ -79,6 +79,11 @@ jobs:
           path: tests/integration/logs/
           retention-days: 1
 
+      ## Part 4 Tools ##
+      - name: Tools tests
+        if: matrix.tests == 'tools' || matrix.tests == 'all'
+        run: make -j${ncpu} testTools
+
   status:
     if: always()
     needs: [build]

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -33,6 +33,7 @@ jobs:
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
+          os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools},       nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:
     needs: matrix

diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml
@@ -24,7 +24,7 @@ jobs:
     name: Build and Push
     uses: ./.github/workflows/docker-reusable.yml
     with:
-      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true'
+      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true'
       nat_ip_auto: true
       tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
       tag_suffix: dist-tests

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,9 +10,10 @@ env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: pinned
   rust_version: 1.78.0
-  binary_base: codex
+  codex_binary_base: codex
+  cirdl_binary_base: cirdl
   build_dir: build
-  nim_flags: '-d:verify_circuit=true'
+  nim_flags: ''
   windows_libs: 'libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll'
 
 jobs:
@@ -70,13 +71,19 @@ jobs:
             macos*)   os_name="darwin"  ;;
             windows*) os_name="windows" ;;
           esac
-          binary="${{ env.binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
-          [[ ${os_name} == "windows" ]] && binary="${binary}.exe"
-          echo "binary=${binary}" >>$GITHUB_ENV
+          codex_binary="${{ env.codex_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
+          cirdl_binary="${{ env.cirdl_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
+          if [[ ${os_name} == "windows" ]]; then
+            codex_binary="${codex_binary}.exe"
+            cirdl_binary="${cirdl_binary}.exe"
+          fi
+          echo "codex_binary=${codex_binary}" >>$GITHUB_ENV
+          echo "cirdl_binary=${cirdl_binary}" >>$GITHUB_ENV
 
       - name: Release - Build
         run: |
-          make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.binary }} ${{ env.nim_flags }}"
+          make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.codex_binary }} ${{ env.nim_flags }}"
+          make cirdl NIMFLAGS="--out:${{ env.build_dir }}/${{ env.cirdl_binary }} ${{ env.nim_flags }}"
 
       - name: Release - Libraries
         run: |
@@ -86,11 +93,26 @@ jobs:
             done
           fi
 
-      - name: Release - Upload build artifacts
+      - name: Release - Upload codex build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ env.codex_binary }}
+          path: ${{ env.build_dir }}/${{ env.codex_binary_base }}*
+          retention-days: 1
+
+      - name: Release - Upload cirdl build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ env.cirdl_binary }}
+          path: ${{ env.build_dir }}/${{ env.cirdl_binary_base }}*
+          retention-days: 1
+
+      - name: Release - Upload windows libs
+        if: matrix.os == 'windows'
         uses: actions/upload-artifact@v4
         with:
-          name: release-${{ env.binary }}
-          path: ${{ env.build_dir }}/
+          name: release-${{ matrix.os }}-libs
+          path: ${{ env.build_dir }}/*.dll
           retention-days: 1
 
   # Release
@@ -115,32 +137,30 @@ jobs:
           }
 
           # Compress and prepare
-          for file in *; do
-            # Exclude libraries
-            if [[ "${file}" != *".dll"* ]]; then
-              if [[ "${file}" == *".exe"* ]]; then
-
-                # Windows - binary only
-                arc="${file%.*}.zip"
-                zip "${arc}" "${file}"
-                checksum "${arc}"
-
-                # Windows - binary and libs
-                arc="${file%.*}-libs.zip"
-                zip "${arc}" "${file}" ${{ env.windows_libs }}
-                rm -f "${file}" ${{ env.windows_libs }}
-                checksum "${arc}"
-              else
-
-                # Linux/macOS
-                arc="${file}.tar.gz"
-                chmod 755 "${file}"
-                tar cfz "${arc}" "${file}"
-                rm -f "${file}"
-                checksum "${arc}"
-              fi
+          for file in ${{ env.codex_binary_base }}* ${{ env.cirdl_binary_base }}*; do
+            if [[ "${file}" == *".exe"* ]]; then
+
+              # Windows - binary only
+              arc="${file%.*}.zip"
+              zip "${arc}" "${file}"
+              checksum "${arc}"
+
+              # Windows - binary and libs
+              arc="${file%.*}-libs.zip"
+              zip "${arc}" "${file}" ${{ env.windows_libs }}
+              rm -f "${file}"
+              checksum "${arc}"
+            else
+
+              # Linux/macOS
+              arc="${file}.tar.gz"
+              chmod 755 "${file}"
+              tar cfz "${arc}" "${file}"
+              rm -f "${file}"
+              checksum "${arc}"
             fi
           done
+          rm -f ${{ env.windows_libs }}
 
       - name: Release - Upload compressed artifacts and checksums
         uses: actions/upload-artifact@v4

diff --git a/.gitmodules b/.gitmodules
@@ -215,3 +215,6 @@
 [submodule "vendor/nim-leveldbstatic"]
 	path = vendor/nim-leveldbstatic
 	url = https://github.com/codex-storage/nim-leveldb.git
+[submodule "vendor/nim-zippy"]
+	path = vendor/nim-zippy
+	url = https://github.com/status-im/nim-zippy.git
diff --git a/BUILDING.md b/BUILDING.md
@@ -33,7 +33,7 @@ The current implementation of Codex's zero-knowledge proving circuit requires th
 On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run
 
 ```shell
-apt-get update && apt-get install build-essential cmake curl git rustc cargo
+$ apt-get update && apt-get install build-essential cmake curl git rustc cargo
 ```
 
 Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc.
@@ -157,6 +157,16 @@ In Bash run
 make test
 ```
 
+### Tools
+
+#### Circuit download tool
+
+To build the circuit download tool located in `tools/cirdl` run:
+
+```shell
+make cirdl
+```
+
 ### testAll
 
 #### Prerequisites

diff --git a/Makefile b/Makefile
@@ -74,6 +74,11 @@ all: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim codex $(NIM_PARAMS) build.nims
 
+# Build tools/cirdl
+cirdl: | deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim toolsCirdl $(NIM_PARAMS) build.nims
+
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
 
@@ -124,7 +129,12 @@ testAll: | build deps
 # Builds and runs Taiko L2 tests
 testTaiko: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) codex.nims
+		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
+
+# Builds and runs tool tests
+testTools: | cirdl
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
 
 # nim-libbacktrace
 LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0

diff --git a/build.nims b/build.nims
@@ -1,5 +1,6 @@
 mode = ScriptMode.Verbose
 
+import std/os except commandLineParams
 
 ### Helper functions
 proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
@@ -14,7 +15,11 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
     for i in 2..<paramCount():
       extra_params &= " " & paramStr(i)
 
-  let cmd = "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
+  let
+    # Place build output in 'build' folder, even if name includes a longer path.
+    outName = os.lastPathPart(name)
+    cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & name & ".nim"
+
   exec(cmd)
 
 proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
@@ -24,6 +29,9 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
 task codex, "build codex binary":
   buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
+task toolsCirdl, "build tools/cirdl binary":
+  buildBinary "tools/cirdl/cirdl"
+
 task testCodex, "Build & run Codex tests":
   test "testCodex", params = "-d:codex_enable_proof_failures=true"
 
@@ -40,10 +48,15 @@ task build, "build codex binary":
 task test, "Run tests":
   testCodexTask()
 
+task testTools, "Run Tools tests":
+  toolsCirdlTask()
+  test "testTools"
+
 task testAll, "Run all tests (except for Taiko L2 tests)":
   testCodexTask()
   testContractsTask()
   testIntegrationTask()
+  testToolsTask()
 
 task testTaiko, "Run Taiko L2 tests":
   codexTask()

diff --git a/codex.nim b/codex.nim
@@ -67,6 +67,9 @@ when isMainModule:
     # permissions are insecure.
     quit QuitFailure
 
+  if config.prover() and not(checkAndCreateDataDir((config.circuitDir).string)):
+    quit QuitFailure
+
   trace "Data dir initialized", dir = $config.dataDir
 
   if not(checkAndCreateDataDir((config.dataDir / "repo"))):

diff --git a/codex/codex.nim b/codex/codex.nim
@@ -273,32 +273,8 @@ proc new*(
     engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks)
     store = NetworkStore.new(engine, repoStore)
     prover = if config.prover:
-      if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and
-        endsWith($config.circomR1cs, ".r1cs"):
-        error "Circom R1CS file not accessible"
-        raise (ref Defect)(
-          msg: "r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
-
-      if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and
-        endsWith($config.circomWasm, ".wasm"):
-        error "Circom wasm file not accessible"
-        raise (ref Defect)(
-          msg: "wasm file not readable, doesn't exist or wrong extension (.wasm)")
-
-      let zkey = if not config.circomNoZkey:
-          if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and
-            endsWith($config.circomZkey, ".zkey"):
-            error "Circom zkey file not accessible"
-            raise (ref Defect)(
-              msg: "zkey file not readable, doesn't exist or wrong extension (.zkey)")
-
-          $config.circomZkey
-        else: ""
-
-      some Prover.new(
-        store,
-        CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey),
-        config.numProofSamples)
+      let backend = config.initializeBackend().expect("Unable to create prover backend.")
+      some Prover.new(store, backend, config.numProofSamples)
     else:
       none Prover
 

diff --git a/codex/conf.nim b/codex/conf.nim
@@ -62,6 +62,7 @@ const
   codex_enable_log_counter* {.booldefine.} = false
 
   DefaultDataDir* = defaultDataDir()
+  DefaultCircuitDir* = defaultDataDir() / "circuits"
 
 type
   StartUpCmd* {.pure.} = enum
@@ -303,23 +304,30 @@ type
         command }: PersistenceCmd
 
       of PersistenceCmd.prover:
+        circuitDir* {.
+          desc: "Directory where Codex will store proof circuit data"
+          defaultValue: DefaultCircuitDir
+          defaultValueDesc: $DefaultCircuitDir
+          abbr: "cd"
+          name: "circuit-dir" }: OutDir
+
         circomR1cs* {.
           desc: "The r1cs file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
-          defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
+          defaultValue: $DefaultCircuitDir / "proof_main.r1cs"
+          defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs"
           name: "circom-r1cs"
         .}: InputFile
 
         circomWasm* {.
           desc: "The wasm file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
+          defaultValue: $DefaultCircuitDir / "proof_main.wasm"
           defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
           name: "circom-wasm"
         .}: InputFile
 
         circomZkey* {.
           desc: "The zkey file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
+          defaultValue: $DefaultCircuitDir / "proof_main.zkey"
           defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
           name: "circom-zkey"
         .}: InputFile

diff --git a/codex/slots/proofs.nim b/codex/slots/proofs.nim
@@ -1,4 +1,5 @@
 import ./proofs/backends
 import ./proofs/prover
+import ./proofs/backendfactory
 
-export circomcompat, prover
+export circomcompat, prover, backendfactory