Release 1.210.0

See release notes.

.github/workflows/coverage.yml

@@ -15,12 +15,20 @@ jobs:
     name: build - python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     env:
+      # > --------------------------------------------------
+      # > sccache
       # https://github.com/Mozilla-Actions/sccache-action
-      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_IDLE_TIMEOUT: 0
+      SCCACHE_DIRECT: "true"
       SCCACHE_CACHE_MULTIARCH: 1
+      SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache
       RUSTC_WRAPPER: "sccache"
       CC: "sccache clang"
-      CXX: "sccache clang"
+      CXX: "sccache clang++"
+      # Incrementally compiled crates cannot be cached by sccache
+      # https://github.com/mozilla/sccache#rust
+      CARGO_INCREMENTAL: 0
+      # > --------------------------------------------------
     services:
       redis:
         image: redis
@@ -59,9 +67,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Run sccache-cache
-        uses: mozilla-actions/[email protected]
-
       - name: Set up Rust toolchain
         run: |
           rustup toolchain add --profile minimal stable --component clippy,rustfmt
@@ -87,17 +92,21 @@ jobs:
           version: ${{ env.POETRY_VERSION }}
 
       - name: Install build dependencies
-        run: python -m pip install --upgrade pip setuptools wheel pre-commit
+        run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec
 
-      - name: Cached pre-commit
-        id: cached-pre-commit
+      - name: Cached sccache
+        id: cached-sccache
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pre-commit
-          key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('**/Cargo.lock', '**/poetry.lock') }}
+          restore-keys: |
+            sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-
+            sccache-${{ runner.os }}-${{ github.workflow }}-
+            sccache-${{ runner.os }}-
 
-      - name: Run pre-commit
-        run: pre-commit run --all-files
+      - name: Run sccache
+        uses: mozilla-actions/[email protected]
 
       - name: Set poetry cache-dir
         run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV
@@ -117,23 +126,24 @@ jobs:
           key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }}
           restore-keys: ${{ runner.os }}-large-files-
 
-      - name: Install Nautilus CLI and run init postgres
-        run: |
-          make install-cli
-          nautilus database init --schema ${{ github.workspace }}/schema
-        env:
-          POSTGRES_HOST: localhost
-          POSTGRES_PORT: 5432
-          POSTGRES_USERNAME: postgres
-          POSTGRES_PASSWORD: pass
-          POSTGRES_DATABASE: nautilus
-
-      - name: Run tests with coverage
-        run: make pytest-coverage
-
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v4
-        with:
-          # fail_ci_if_error: true  # leave commented until flakiness improves
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
+      # TODO: Pause the below for now pending inventigation on why it started failing?
+      # - name: Install Nautilus CLI and run init postgres
+      #   run: |
+      #     make install-cli
+      #     nautilus database init --schema ${{ github.workspace }}/schema
+      #   env:
+      #     POSTGRES_HOST: localhost
+      #     POSTGRES_PORT: 5432
+      #     POSTGRES_USERNAME: postgres
+      #     POSTGRES_PASSWORD: pass
+      #     POSTGRES_DATABASE: nautilus
+      #
+      # - name: Run tests with coverage
+      #   run: make pytest-coverage
+      #
+      # - name: Upload coverage report
+      #   uses: codecov/codecov-action@v4
+      #   with:
+      #     # fail_ci_if_error: true  # leave commented until flakiness improves
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     verbose: true

.github/workflows/performance.yml

@@ -0,0 +1,172 @@
+name: performance
+
+on:
+  push:
+    branches: [nightly]
+
+jobs:
+  performance-benchmarks:
+    runs-on: ubuntu-latest
+    env:
+      BUILD_MODE: release
+      RUST_BACKTRACE: 1
+      # > --------------------------------------------------
+      # > sccache
+      # https://github.com/Mozilla-Actions/sccache-action
+      SCCACHE_IDLE_TIMEOUT: 0
+      SCCACHE_DIRECT: "true"
+      SCCACHE_CACHE_MULTIARCH: 1
+      SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache
+      RUSTC_WRAPPER: "sccache"
+      CC: "sccache clang"
+      CXX: "sccache clang++"
+      # Incrementally compiled crates cannot be cached by sccache
+      # https://github.com/mozilla/sccache#rust
+      CARGO_INCREMENTAL: 0
+      # > --------------------------------------------------
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: nautilus
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+      - name: Install runner dependencies
+        run: sudo apt-get install -y curl clang git libssl-dev make pkg-config
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Cached sccache
+        id: cached-sccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('**/Cargo.lock', '**/poetry.lock') }}
+          restore-keys: |
+            sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-
+            sccache-${{ runner.os }}-${{ github.workflow }}-
+            sccache-${{ runner.os }}-
+
+      - name: Run sccache
+        uses: mozilla-actions/[email protected]
+
+      - name: Set up Rust toolchain
+        run: |
+          rustup toolchain add --profile minimal stable --component clippy,rustfmt
+
+      - name: Set up Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Get Python version
+        run: |
+          version=$(bash scripts/python-version.sh)
+          echo "PYTHON_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Get Poetry version from poetry-version
+        run: |
+          version=$(cat poetry-version)
+          echo "POETRY_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+
+      - name: Install build dependencies
+        run: python -m pip install --upgrade pip setuptools wheel poetry-plugin-export pre-commit msgspec
+
+      - name: Cached pre-commit
+        id: cached-pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Run pre-commit # Runs again here to check on Python 3.12
+        run: |
+          pre-commit run --all-files
+
+      - name: Cached cargo
+        id: cached-cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Cache Python site-packages
+        id: cached-site-packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.local/lib/python${{ env.PYTHON_VERSION }}/site-packages
+          key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-site-packages
+          restore-keys: |
+            ${{ runner.os }}-site-packages-
+
+      - name: Cached test data
+        id: cached-testdata-large
+        uses: actions/cache@v4
+        with:
+          path: tests/test_data/large
+          key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }}
+          restore-keys: ${{ runner.os }}-large-files-
+
+      - name: Install Nautilus CLI and run init postgres
+        run: |
+          make install-cli
+          nautilus database init --schema ${{ github.workspace }}/schema
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_USERNAME: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DATABASE: nautilus
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+
+      - name: Run nautilus_core tests
+        run: make cargo-test
+
+      - name: Build Python wheel
+        run: |
+          poetry build --format wheel
+          ls -lh dist/
+
+      - name: Install Python wheel
+        run: |
+          poetry export --with test --all-extras --format requirements.txt --output requirements-test.txt
+          python -m pip install -r requirements-test.txt
+          pip install "$(ls dist/*.whl)"
+
+      # Run codspeed once only
+      - name: Run benchmarks
+        uses: CodSpeedHQ/[email protected]
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: pytest tests/performance_tests --benchmark-disable-gc --codspeed