feat: upload katana dump (#489)

* add submodule hash to dump * store Katana dump as artifact * update ci * add features dump * rework CI layout of files * on pull request * typo * remove change log * modify time out * add submodules setup * launch CI * fix jobs deps * syntax issues * add checkout * move foundry * remove unused step * add caching of dump * reload submodules in tests * use artifacts for dump * seperate linters from test * path for artifact * remove artifacts workflow * yagni + remove comments
kkrt-labs · Aug 29, 2023 · d2089f5 · d2089f5
1 parent 69e80c5
commit d2089f5
Show file tree

Hide file tree

Showing 14 changed files with 388 additions and 192 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -1,106 +1,106 @@
 ---
-    name: Task - Benchmark
-
-    on:
-      push:
-        branches: [main]
+name: Task - Benchmark
 
-    permissions:
-      deployments: write
-      contents: write
-      pull-requests: write
-
-    jobs:
-      benchmark:
-        runs-on: ubuntu-latest-16-cores
-        timeout-minutes: 120
-        steps:
-          - uses: actions/checkout@v3
-            with:
-              clean: false
-
-          # install dependencies
-          - name: Install submodules and dependencies
-            run: |
-              git submodule update --init --recursive
-              sudo apt update
-              sudo apt install -y protobuf-compiler clang jq wget
+on:
+  workflow_call:
 
-          - uses: actions/cache@v3
-            id: cache
-            with:
-              path: |
-                ~/.cargo/bin/
-                ~/.cargo/registry/index/
-                ~/.cargo/registry/cache/
-                ~/.cargo/git/db/
-              key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}}
-
-          # build rpc and madara  
-          - name: Setup Rust
-            uses: actions-rs/toolchain@v1
-            with:
-                toolchain: stable
-          - name: Build RPC
-            run: |
-              cargo build --release
-          - name: Download Madara
-            if: steps.cache.outputs.cache-hit  != 'true'
-            run: |
-              cd ./lib/madara
-              mkdir -p target/release
-              cd target/release
-              wget https://github.com/keep-starknet-strange/madara/releases/download/v0.1.0.experimental.3/x86_64-unknown-linux-gnu-madara
-              chmod +x x86_64-unknown-linux-gnu-madara && mv x86_64-unknown-linux-gnu-madara madara
-          
-          # Python Setup and dependencies installation
-          - uses: actions/setup-python@v4
-            with:
-              python-version: "3.9.13"
-          - name: Load cached Poetry installation
-            id: cached-poetry
-            uses: actions/cache@v3
-            with:
-              path: ~/.local
-              key: poetry-${{ runner.os }}
-          - name: Install Poetry
-            if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-            uses: snok/install-poetry@v1
-            with:
-              virtualenvs-create: true
-              virtualenvs-in-project: true
-              installer-parallel: true
-          - name: install kakarot python dependencies
-            run: |-
-              cd lib/kakarot
-              poetry install 
-          - name: Load cached venv
-            id: cached-poetry-dependencies
-            uses: actions/cache@v3
-            with:
-              path: .venv
-              key: venv-${{ runner.os }}-${{ hashFiles('./lib/kakarot/poetry.lock') }}
+permissions:
+  deployments: write
+  contents: write
+  pull-requests: write
 
-          # benchmark rpc
-          - name: install benchmark projects
-            run: |-
-              cd ./benchmarking
-              npm i
-          - name: Run benchmark
-            run: |
-              cd benchmarking
-              ./scripts/benchmark.sh
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          clean: false
 
-          # compare benchmarks and publish
-          - name: Compare result
-            uses: benchmark-action/github-action-benchmark@v1
-            with:
-              tool: "customBiggerIsBetter"
-              output-file-path: ./benchmarking/reports/metrics.json
-              alert-threshold: "120%"
-              github-token: ${{ secrets.GITHUB_TOKEN }}
-              fail-on-alert: true
-              summary-always: ${{ github.ref == 'refs/heads/main' }}
-              comment-always: false
-              comment-on-alert: false
-              auto-push: ${{ github.ref == 'refs/heads/main' }}
+      # install dependencies
+      - name: Install submodules and dependencies
+        run: |
+          git submodule update --init --recursive
+          sudo apt update
+          sudo apt install -y protobuf-compiler clang jq wget
+
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}}
+
+      # build rpc and madara
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Build RPC
+        run: |
+          cargo build --release
+      - name: Download Madara
+        if: steps.cache.outputs.cache-hit  != 'true'
+        run: |
+          cd ./lib/madara
+          mkdir -p target/release
+          cd target/release
+          wget https://github.com/keep-starknet-strange/madara/releases/download/v0.1.0.experimental.3/x86_64-unknown-linux-gnu-madara
+          chmod +x x86_64-unknown-linux-gnu-madara && mv x86_64-unknown-linux-gnu-madara madara
+
+      # Python Setup and dependencies installation
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9.13'
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.local
+          key: poetry-${{ runner.os }}
+      - name: Install Poetry
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: install kakarot python dependencies
+        run: |-
+          cd lib/kakarot
+          poetry install
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key:
+            venv-${{ runner.os }}-${{ hashFiles('./lib/kakarot/poetry.lock') }}
+
+      # benchmark rpc
+      - name: install benchmark projects
+        run: |-
+          cd ./benchmarking
+          npm i
+      - name: Run benchmark
+        run: |
+          cd benchmarking
+          ./scripts/benchmark.sh
+
+      # compare benchmarks and publish
+      - name: Compare result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'customBiggerIsBetter'
+          output-file-path: ./benchmarking/reports/metrics.json
+          alert-threshold: '120%'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fail-on-alert: true
+          summary-always: ${{ github.ref == 'refs/heads/main' }}
+          comment-always: false
+          comment-on-alert: false
+          auto-push: ${{ github.ref == 'refs/heads/main' }}
diff --git a/.github/workflows/dump.yml b/.github/workflows/dump.yml
@@ -0,0 +1,31 @@
+name: Dump the Katana state into the .katana folder
+
+on:
+  workflow_call:
+
+jobs:
+  dump:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+      - name: Load cached submodules
+        id: cached-submodules
+        uses: actions/cache@v3
+        with:
+          path: ./lib/
+          key: ${{ runner.os }}-submodules
+      - name: Setup Rust Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          override: true
+      - name: Retrieve cached dependencies
+        uses: Swatinem/rust-cache@v2
+      - name: Generate the Katana dumped state
+        run: ./scripts/make_with_env.sh dump-katana
+      - name: Save dump as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: dump-katana
+          path: ./.katana/
diff --git a/.github/workflows/kakarot_rpc.yml b/.github/workflows/kakarot_rpc.yml
@@ -2,9 +2,7 @@
 name: RPC
 
 on:
-  push:
-    branches:
-      - 'main'
+  workflow_call:
 
 env:
   REGISTRY_IMAGE: ghcr.io/${{ github.repository }}/node

diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml
@@ -0,0 +1,51 @@
+name: linters
+
+on:
+  workflow_call:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          override: true
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  fmt:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          override: true
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --workspace --all-features --all-targets -- -D warnings
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -0,0 +1,27 @@
+---
+name: Workflow - Pull Request
+
+on: [pull_request]
+
+jobs:
+  submodules:
+    name: Set up submodules
+    uses: ./.github/workflows/submodules.yml
+
+  dump:
+    name: Dump katana state
+    uses: ./.github/workflows/dump.yml
+    needs: submodules
+
+  spell_check:
+    name: Spell check
+    uses: ./.github/workflows/spell_check.yml
+
+  linters:
+    name: Linters
+    uses: ./.github/workflows/linters.yml
+
+  tests:
+    name: Rust tests
+    uses: ./.github/workflows/test.yml
+    needs: [dump, submodules]
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -0,0 +1,28 @@
+---
+name: Workflow - Push
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  submodules:
+    name: Set up submodules
+    uses: ./.github/workflows/submodules.yml
+
+  dump:
+    name: Dump katana state
+    uses: ./.github/workflows/dump.yml
+    needs: submodules
+
+  spell_check:
+    name: Spell check
+    uses: ./.github/workflows/spell_check.yml
+
+  benchmark:
+    name: Erc20 benchmark
+    uses: ./.github/workflows/benchmark.yml
+
+  build:
+    name: Build binaries
+    uses: ./.github/workflows/kakarot_rpc.yml
diff --git a/.github/workflows/spell_check.yml b/.github/workflows/spell_check.yml
@@ -1,14 +1,15 @@
 name: Spell Check
 
-on: [push, pull_request]
+on:
+  workflow_call:
 
 jobs:
   spell-check:
     runs-on: ubuntu-latest
     timeout-minutes: 5
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: codespell-project/actions-codespell@v1
         with:
           check_filenames: true