speediedan · speediedan · Sep 18, 2025 · Sep 17, 2025 · Sep 18, 2025 · Sep 18, 2025
diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml
@@ -65,6 +65,7 @@ jobs:
         HF_HUB_CACHE: /opt/interpretune/hf_cache/hub
         TRANSFORMERS_CACHE: /opt/interpretune/hf_cache/transformers
         XDG_CACHE_HOME: /opt/interpretune/hf_cache
+        APPLY_POST_UPGRADES: "false"
 
     workspace:
       clean: outputs
@@ -96,16 +97,21 @@ jobs:
 
       - bash: |
           . /tmp/venvs/it_dev/bin/activate
-          python -m pip install --upgrade pip setuptools wheel build
-          python -m pip install -r requirements/ci/requirements.txt -r requirements/platform_dependent.txt --no-warn-script-location
+          python -m pip install --upgrade pip setuptools setuptools-scm wheel build
+          python -m pip install -r requirements/ci/requirements.txt -r requirements/ci/platform_dependent.txt --no-warn-script-location
           python -m pip install -e '.[test,examples,lightning]'  --no-warn-script-location
-          python -m pip install --upgrade -r requirements/post_upgrades.txt  --no-warn-script-location
+          if ([ "${APPLY_POST_UPGRADES:-}" = "1" ] || [ "${APPLY_POST_UPGRADES:-}" = "true" ]) && [ -s requirements/ci/post_upgrades.txt ]; then
+            echo "Applying post-upgrades (requirements/ci/post_upgrades.txt)..."
+            python -m pip install --upgrade -r requirements/ci/post_upgrades.txt --cache-dir "$PIP_CACHE_DIR"
+          else
+            echo "Skipping post-upgrades (either disabled or file empty)."
+          fi
+          python -m pip list
         displayName: 'Install dependencies'
 
       - bash: |
           . /tmp/venvs/it_dev/bin/activate
-          python requirements/collect_env_details.py
-          python scripts/ci_print_env_versions.py
+          python requirements/utils/collect_env_details.py
         displayName: 'Env details and package versions'
 
       - bash: |

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -27,12 +27,12 @@ but remember, bugs with code are fixed faster!
 
 <!--
 Please copy and paste the output from our environment collection script:
-https://raw.githubusercontent.com/speediedan/interpretune/main/requirements/collect_env_details.py
+https://raw.githubusercontent.com/speediedan/interpretune/main/requirements/utils/collect_env_details.py
 (For security purposes, please check the contents of the script before running it)
 
 You can get the script and run it with:
 ```bash
-wget https://raw.githubusercontent.com/speediedan/interpretune/main/requirements/collect_env_details.py
+wget https://raw.githubusercontent.com/speediedan/interpretune/main/requirements/utils/collect_env_details.py
 python collect_env_details.py
 ```
 

diff --git a/.github/actions/install-ci-dependencies/action.yml b/.github/actions/install-ci-dependencies/action.yml
@@ -17,13 +17,13 @@ runs:
     - name: Set up venv and install ci dependencies
       shell: bash
       run: |
-        python -m pip install --upgrade pip setuptools wheel build --cache-dir "$PIP_CACHE_DIR"
+        python -m pip install --upgrade pip setuptools setuptools-scm wheel build --cache-dir "$PIP_CACHE_DIR"
         # Prefer CI pinned requirements if present
         if [ -f requirements/ci/requirements.txt ]; then
           pip install -r requirements/ci/requirements.txt --cache-dir "$PIP_CACHE_DIR"
           python -m pip install -e '.[test,examples,lightning]' --cache-dir "$PIP_CACHE_DIR"
         else
-          python -m pip install -e '.[test,examples,lightning]' -c requirements/ci_constraints.txt --cache-dir "$PIP_CACHE_DIR"
+          python -m pip install -e '.[test,examples,lightning]' --cache-dir "$PIP_CACHE_DIR"
         fi
         if [ "${{ inputs.show_pip_list }}" = "true" ]; then
           pip list
@@ -33,9 +33,9 @@ runs:
       shell: bash
       run: |
         # Install platform-dependent packages with flexible constraints to allow platform-specific resolution
-        if [ -f requirements/platform_dependent.txt ] && [ -s requirements/platform_dependent.txt ]; then
-          echo "Installing platform-dependent packages..."
-          python -m pip install -r requirements/platform_dependent.txt --cache-dir "$PIP_CACHE_DIR" || echo "Some platform-dependent packages may not be available on this platform, continuing..."
+        if [ -f requirements/ci/platform_dependent.txt ] && [ -s requirements/ci/platform_dependent.txt ]; then
+          echo "Installing platform-dependent packages from requirements/ci/platform_dependent.txt..."
+          python -m pip install -r requirements/ci/platform_dependent.txt --cache-dir "$PIP_CACHE_DIR" || echo "Some platform-dependent packages may not be available on this platform, continuing..."
         else
           echo "No platform-dependent packages to install."
         fi
@@ -45,9 +45,9 @@ runs:
       env:
         APPLY_POST_UPGRADES: ${{ inputs.apply_post_upgrades }}
       run: |
-        if ([ "${APPLY_POST_UPGRADES}" = "1" ] || [ "${APPLY_POST_UPGRADES}" = "true" ]) && [ -s requirements/post_upgrades.txt ]; then
-          echo "Applying post-upgrades..."
-          python -m pip install --upgrade -r requirements/post_upgrades.txt --cache-dir "$PIP_CACHE_DIR"
+        if ([ "${APPLY_POST_UPGRADES}" = "1" ] || [ "${APPLY_POST_UPGRADES}" = "true" ]) && [ -s requirements/ci/post_upgrades.txt ]; then
+          echo "Applying post-upgrades (requirements/ci/post_upgrades.txt)..."
+          python -m pip install --upgrade -r requirements/ci/post_upgrades.txt --cache-dir "$PIP_CACHE_DIR"
           if [ "${{ inputs.show_pip_list }}" = "true" ]; then
             pip list
           fi

diff --git a/.github/actions/regen-ci-reqs/action.yml b/.github/actions/regen-ci-reqs/action.yml
@@ -12,7 +12,7 @@ inputs:
   compare_paths:
     description: Space-separated list of files/paths to compare with git diff
     required: false
-    default: "requirements/ci/requirements.txt requirements/post_upgrades.txt requirements/platform_dependent.txt"
+    default: "requirements/ci/requirements.txt requirements/ci/post_upgrades.txt requirements/ci/platform_dependent.txt"
   patch_path:
     description: Path to write the patch file
     required: false
@@ -34,7 +34,7 @@ runs:
     - name: Run regen (pip-compile)
       shell: bash
       run: |
-        python requirements/regen_reqfiles.py --mode pip-compile --ci-output-dir=${{ inputs.ci_output_dir }}
+        python requirements/utils/regen_reqfiles.py --mode pip-compile --ci-output-dir=${{ inputs.ci_output_dir }}
 
     - id: check
       name: Check for diffs and write patch

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
@@ -35,11 +35,9 @@ Always install dependencies in order to avoid conflicts:
 
 ```bash
 # Basic development setup
-python -m pip install --upgrade pip setuptools wheel build
-python -m pip install -r requirements/ci/requirements.txt -r requirements/platform_dependent.txt
+python -m pip install --upgrade pip setuptools setuptools-scm wheel build
+python -m pip install -r requirements/ci/requirements.txt -r requirements/ci/platform_dependent.txt
 python -m pip install -e '.[test,examples,lightning]'
-# enabled until further notice (will be removed from instructions once no longer necessary)
-pip install --upgrade -r requirements/post_upgrades.txt
 
 # If circuit-tracer install fails, use the built-in tool after basic install:
 pip install interpretune[examples]
@@ -55,8 +53,8 @@ For complex setups, use the provided build script:
 # Standard development build (recommended for dev work)
 ./scripts/build_it_env.sh --repo_home=${PWD} --target_env_name=it_latest
 
-# Build without circuit-tracer commit pin
-./scripts/build_it_env.sh --repo_home=${PWD} --target_env_name=it_latest --no_commit_pin
+# Build with a circuit-tracer commit pin
+./scripts/build_it_env.sh --repo_home=${PWD} --target_env_name=it_latest
 ```
 
 ### Linting and Code Quality
@@ -128,7 +126,6 @@ src/it_examples/            # Example experiments
 - `.pre-commit-config.yaml` - Code quality hooks
 - `requirements/` - Pinned dependency files
   - `base.txt` - Core dependencies
-  - `ci_constraints.txt` - CI version pins
   - `test.txt`, `examples.txt`, etc. - Optional dependency groups
 
 ### Key Entry Points
@@ -183,8 +180,7 @@ When updating top-level requirements or periodically refreshing CI pins, use the
 Run these commands from your repo home after activating ensuring you've activated any relevant venv (e.g. `source ~/.venvs/${target_env_name}/bin/activate`):
 
 ```bash
-python requirements/regen_reqfiles.py && \
-python requirements/regen_reqfiles.py --mode pip-compile --ci-output-dir=requirements/ci
+python requirements/utils/regen_reqfiles.py --mode pip-compile --ci-output-dir=requirements/ci
 ```
 
 Notes:

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
@@ -171,7 +171,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.PIP_CACHE_DIR }}/wheels
-          key: ${{ runner.os }}-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/post_upgrades.txt') }}-${{ hashFiles('requirements/platform_dependent.txt') }}
+          key: ${{ runner.os }}-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/ci/post_upgrades.txt') }}-${{ hashFiles('requirements/ci/platform_dependent.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py${{ matrix.python-version }}-
 

diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml
@@ -47,7 +47,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.PIP_CACHE_DIR }}/wheels
-          key: ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/post_upgrades.txt') }}-${{ hashFiles('requirements/platform_dependent.txt') }}
+          key: ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/ci/post_upgrades.txt') }}-${{ hashFiles('requirements/ci/platform_dependent.txt') }}
           restore-keys: |
             ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-
 

diff --git a/.github/workflows/regen-ci-req-check.yml b/.github/workflows/regen-ci-req-check.yml
@@ -21,7 +21,7 @@ jobs:
         with:
           python_version: '3.12'
           ci_output_dir: requirements/ci
-          compare_paths: "requirements/ci/requirements.txt requirements/post_upgrades.txt requirements/platform_dependent.txt"
+          compare_paths: "requirements/ci/requirements.txt requirements/ci/post_upgrades.txt requirements/ci/platform_dependent.txt"
           patch_path: /tmp/regen_diff.patch
 
       - name: Create PR with updated pins

diff --git a/.github/workflows/regen-ci-req-report.yml b/.github/workflows/regen-ci-req-report.yml
@@ -62,7 +62,7 @@ jobs:
         with:
           python_version: '3.12'
           ci_output_dir: requirements/ci
-          compare_paths: "requirements/ci/requirements.txt requirements/post_upgrades.txt requirements/platform_dependent.txt"
+          compare_paths: "requirements/ci/requirements.txt requirements/ci/post_upgrades.txt requirements/ci/platform_dependent.txt"
           patch_path: /tmp/regen_diff.patch
 
       - name: Upload regen diff (if present)

diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml
@@ -73,14 +73,14 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.PIP_CACHE_DIR }}/wheels
-          key: ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/post_upgrades.txt') }}-${{ hashFiles('requirements/platform_dependent.txt') }}
+          key: ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-${{ hashFiles('requirements/ci/requirements.txt') }}-${{ hashFiles('requirements/ci/post_upgrades.txt') }}-${{ hashFiles('requirements/ci/platform_dependent.txt') }}
           restore-keys: |
             ubuntu-22.04-pip-wheels-${{ steps.set_time_period.outputs.TIME_PERIOD }}-py3.12-
 
       - name: Install CI dependencies
         uses: ./.github/actions/install-ci-dependencies
         with:
-          apply_post_upgrades: ${{ vars.APPLY_POST_UPGRADES || 'true' }}
+          apply_post_upgrades: ${{ vars.APPLY_POST_UPGRADES || 'false' }}
 
       - name: Check type stubs are up to date
         id: check_stubs

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -17,15 +17,8 @@ include requirements.txt
 # Excludes
 # -----------------------------------------------------------------------
 # ci requirements
-recursive-exclude requirements/ci *
-
-# misc dev req files
-exclude requirements/ci_constraints.txt
-exclude requirements/post_upgrades.txt
-exclude requirements/collect_env_details.py
-exclude requirements/regen_reqfiles.py
-exclude requirements/circuit_tracer_pin.txt
-exclude requirements/platform_dependent.txt
+prune requirements/ci
+prune requirements/utils
 
 # legacy/duplicate packaging config files from the sdist
 # Note: setuptools writes a small setup.cfg and interpretune.egg-info dir into the sdist release tree that would require

diff --git a/Makefile b/Makefile
@@ -3,6 +3,7 @@
 .PHONY: help lint test build sdist wheel clean
 
 export SPHINX_MOCK_REQUIREMENTS=1
+# TODO: remove this once CT is available via PyPI or our own package index
 export IT_USE_CT_COMMIT_PIN="1"
 
 help:
@@ -32,8 +33,6 @@ lint:
 test: clean
 	pip install -r requirements/devel.txt
 	# TODO: remove this once no longer necessary
-	pip install --upgrade -r requirements/post_upgrades.txt
-	# TODO: remove this once no longer necessary
 	@echo "Using IT_USE_CT_COMMIT_PIN for circuit-tracer installation"
 	interpretune-install-circuit-tracer
 	# run tests with coverage (cpu-only, running gpu standalone tests required for full coverage)

diff --git a/docs/CIRCUIT_TRACER_INSTALLATION.md b/docs/CIRCUIT_TRACER_INSTALLATION.md
@@ -51,7 +51,7 @@ This installs circuit-tracer from a specific tested commit for maximum compatibi
 ### Using Version-based Installation
 
 ```bash
-interpretune-install-circuit-tracer --no-commit-pin
+interpretune-install-circuit-tracer
 ```
 
 This attempts to install circuit-tracer from PyPI (currently not available, will fail).
@@ -78,8 +78,8 @@ For development environments, the build scripts handle circuit-tracer installati
 # Standard development build (uses commit pin)
 ./scripts/build_it_env.sh --repo_home=${HOME}/repos/interpretune --target_env_name=it_latest
 
-# Development build without commit pin
-./scripts/build_it_env.sh --repo_home=${HOME}/repos/interpretune --target_env_name=it_latest --no_commit_pin
+# Development build without CT commit pin
+./scripts/build_it_env.sh --repo_home=${HOME}/repos/interpretune --target_env_name=it_latest
 
 # Development build with local circuit-tracer source
 ./scripts/build_it_env.sh --repo_home=${HOME}/repos/interpretune --target_env_name=it_latest --ct_from_source=${HOME}/repos/circuit-tracer

diff --git a/pyproject.toml b/pyproject.toml
@@ -36,7 +36,7 @@ dependencies = [
     "sae_lens >= 6.3.1",  # use new sae_lens API
     "torch >=2.7.1",
     "tabulate >= 0.9.0",
-    "datasets >= 2.2.0",
+    "datasets >= 4.0.0",
     "jsonargparse[signatures] >= 4.35.0",  # replace this req once adding fts req
     # "finetuning-scheduler[possible_future_it_plugin] >= 2.5.0",
 ]
@@ -49,7 +49,6 @@ dependencies = [
 
 [project.entry-points."console_scripts"]
 interpretune = "interpretune.base.components.cli:bootstrap_cli"
-interpretune-install-circuit-tracer = "interpretune.tools.install_circuit_tracer:main"
 
 [project.optional-dependencies]
 
@@ -76,13 +75,13 @@ examples = [
 "evaluate",
 "scikit-learn",
 "neuronpedia",
-# TODO: add our packaged circuit-tracer dep (either origin or fork) once it is available and remove the
+# TODO: add our packaged circuit-tracer dep (either pypi or pypi fork) once it is available and remove the
 #       `install_circuit_tracer` tool
-# "circuit-tracer @ git+https://github.com/speediedan/circuit-tracer.git@6c74ea291c410bb3391e572cd6a8d020be714922",
+# "circuit-tracer @ git+https://github.com/speediedan/circuit-tracer.git@75fd21f666cbdaece57eca561b4342ad84746b40",
 ]
 
 docs = [
-"sphinx >=4.0,<5.0 ",
+"sphinx >= 4.0",
 "myst-parser >= 0.18.1",
 "nbsphinx >= 0.8.5",
 "pandoc >= 1.0",
@@ -131,7 +130,6 @@ namespaces = true
 [tool.setuptools_scm]
 fallback_version = "0.1.0.dev0"
 
-
 [tool.ruff]
 line-length = 120
 # Enable Pyflakes `E` and `F` codes by default.
@@ -245,5 +243,4 @@ platform_dependent = ["bitsandbytes"]
 
 # packages that should be applied as post-upgrades (package -> desired_version)
 [tool.ci_pinning.post_upgrades]
-datasets = "==4.0.0"
-fsspec = ">=2025.3.0"
+# no packages currently require post installation upgrades
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -2,5 +2,5 @@ transformer_lens >= 2.15.4
 sae_lens >= 6.3.1
 torch >=2.7.1
 tabulate >= 0.9.0
-datasets >= 2.2.0
+datasets >= 4.0.0
 jsonargparse[signatures] >= 4.35.0
diff --git a/requirements/ci/circuit_tracer_pin.txt b/requirements/ci/circuit_tracer_pin.txt
@@ -0,0 +1 @@
+75fd21f666cbdaece57eca561b4342ad84746b40
diff --git a/requirements/platform_dependent.txt → requirements/ci/platform_dependent.txt b/requirements/platform_dependent.txt → requirements/ci/platform_dependent.txt
diff --git a/requirements/ci/post_upgrades.txt b/requirements/ci/post_upgrades.txt
diff --git a/requirements/ci/requirements.in b/requirements/ci/requirements.in
@@ -2,6 +2,7 @@ transformer_lens >= 2.15.4
 sae_lens >= 6.3.1
 torch >=2.7.1
 tabulate >= 0.9.0
+datasets >= 4.0.0
 jsonargparse[signatures] >= 4.35.0
 finetuning-scheduler >= 2.5.0
 peft
@@ -28,4 +29,4 @@ psycopg
 toml
 pip-tools
 huggingface_hub[hf_xet]
-git+https://github.com/speediedan/circuit-tracer.git@6c74ea291c410bb3391e572cd6a8d020be714922#egg=circuit-tracer
+git+https://github.com/speediedan/circuit-tracer.git@75fd21f666cbdaece57eca561b4342ad84746b40#egg=circuit-tracer
diff --git a/requirements/ci/requirements.txt b/requirements/ci/requirements.txt
@@ -2,21 +2,27 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=requirements/ci/requirements.txt requirements/ci/requirements.in
+#    pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements/ci/requirements.txt --pip-args=None requirements/ci/requirements.in
 #
-circuit-tracer @ git+https://github.com/speediedan/circuit-tracer.git@6c74ea291c410bb3391e572cd6a8d020be714922
+circuit-tracer @ git+https://github.com/speediedan/circuit-tracer.git@75fd21f666cbdaece57eca561b4342ad84746b40
     # via -r requirements/ci/requirements.in
 coverage==7.10.6
     # via
     #   -r requirements/ci/requirements.in
     #   nbval
-evaluate==0.4.5
+datasets==4.1.1
+    # via
+    #   -r requirements/ci/requirements.in
+    #   evaluate
+    #   sae-lens
+    #   transformer-lens
+evaluate==0.4.6
     # via -r requirements/ci/requirements.in
 finetuning-scheduler==2.5.3
     # via -r requirements/ci/requirements.in
 gdown==5.2.0
     # via -r requirements/ci/requirements.in
-huggingface-hub[hf-xet]==0.34.5
+huggingface-hub[hf-xet]==0.35.0
     # via
     #   -r requirements/ci/requirements.in
     #   accelerate
@@ -27,7 +33,9 @@ huggingface-hub[hf-xet]==0.34.5
     #   tokenizers
     #   transformers
 ipywidgets==8.1.7
-    # via -r requirements/ci/requirements.in
+    # via
+    #   -r requirements/ci/requirements.in
+    #   circuit-tracer
 jsonargparse[signatures,typing-extensions]==4.41.0
     # via -r requirements/ci/requirements.in
 jupyterlab==4.4.7
@@ -60,7 +68,6 @@ pyright==1.1.405
 pytest==8.4.2
     # via
     #   -r requirements/ci/requirements.in
-    #   circuit-tracer
     #   nbval
     #   pytest-rerunfailures
 pytest-rerunfailures==16.0.1

diff --git a/requirements/ci_constraints.txt b/requirements/ci_constraints.txt
diff --git a/requirements/circuit_tracer_pin.txt b/requirements/circuit_tracer_pin.txt