diff --git a/.github/workflows/release-script.yml b/.github/workflows/release-script.yml
index cd79c9b68..7109d348c 100644
--- a/.github/workflows/release-script.yml
+++ b/.github/workflows/release-script.yml
@@ -1,8 +1,11 @@
 name: Generate Installer Script
 
 on:
-  schedule:
-    - cron: '10 * * * *'
+  workflow_dispatch: # Allows manual triggering of the workflow
+    inputs:
+      TESTING_HYDRA_EVAL_ID:
+        description: "Eval ID of Hydra job to use artifacts from for testing"
+        default: ""
 
 jobs:
   build:
@@ -12,9 +15,7 @@ jobs:
       uses: actions/checkout@v4
     - uses: cachix/install-nix-action@v25
     - name: Download Installer
+      # The script also depends on gh and git but those are both pre-installed on the runner
       run: nix-shell -p python3Packages.requests --run "python assemble_installer.py" -I nixpkgs=channel:nixos-23.11
-    - uses: stefanzweifel/git-auto-commit-action@v5
-      with:
-        commit_message: "Update installer script"
-        commit_user_name: "GitHub Actions"
-        branch: prerelease
\ No newline at end of file
+      env:
+        GH_TOKEN: ${{ github.token }}
diff --git a/assemble_installer.py b/assemble_installer.py
index ff9d40b9d..b5a5cc0c5 100644
--- a/assemble_installer.py
+++ b/assemble_installer.py
@@ -1,11 +1,29 @@
+import os
 import requests
 import subprocess
-import shutil 
+import shutil
 import sys
+import tempfile
+import tomllib
 
 response = requests.get('https://hydra.nixos.org/jobset/experimental-nix-installer/experimental-installer/evals', headers={'Accept': 'application/json'})
 
-hydra_eval = response.json()['evals'][0]
+evals = response.json()['evals']
+eval_id = int(os.getenv("TESTING_HYDRA_EVAL_ID"))
+if eval_id is not None:
+    ids = [eval['id'] for eval in evals]
+    hydra_eval = next( eval for eval in evals if eval['id'] == eval_id )
+else:
+    hydra_eval = evals[0]
+
+    rev = subprocess.run(
+        ["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, check=True, text=True
+    ).stdout.strip()
+
+    if not rev in hydra_eval["flake"]:
+        raise RuntimeError(
+            f"Expected flake with rev {rev} but found flake {hydra_eval['flake']}"
+        )
 
 installers = []
 
@@ -22,7 +40,35 @@
             subprocess.call(f"nix-store -r {installer_url}", shell=True)
         installers.append((installer_url, system))
     else:
+        print(
+            f"Build {build_id} not finished. Check status at https://hydra.nixos.org/eval/{hydra_eval['id']}#tabs-unfinished"
+        )
         sys.exit(0)
 
-for installer_url, system in installers:
-    shutil.copy(f"{installer_url}/bin/nix-installer", f"nix-installer-{system}")
+with open("Cargo.toml", "rb") as f:
+    cargo_toml = tomllib.load(f)
+version = cargo_toml["package"]["version"]
+
+with tempfile.TemporaryDirectory() as tmpdirname:
+    release_files = []
+    for installer_url, system in installers:
+        installer_file = f"{tmpdirname}/nix-installer-{system}"
+        release_files.append(installer_file)
+        print(f"Copying {installer_url} to {installer_file}")
+        shutil.copy(f"{installer_url}/bin/nix-installer", installer_file)
+    release_files.append("nix-installer.sh")
+    subprocess.run(
+        [
+            "gh",
+            "release",
+            "create",
+            "--notes",
+            f"Release experimental nix installer v{version}",
+            "--title",
+            f"v{version}",
+            "--draft",
+            version,
+            *release_files,
+        ],
+        check=True,
+    )
diff --git a/nix-installer.sh b/nix-installer.sh
index 97144a5df..390a8a807 100755
--- a/nix-installer.sh
+++ b/nix-installer.sh
@@ -25,7 +25,7 @@ fi
 set -u
 
 # If NIX_INSTALLER_FORCE_ALLOW_HTTP is unset or empty, default it.
-NIX_INSTALLER_BINARY_ROOT="${NIX_INSTALLER_BINARY_ROOT:-https://raw.githubusercontent.com/NixOS/experimental-nix-installer/prerelease}"
+NIX_INSTALLER_BINARY_ROOT="${NIX_INSTALLER_BINARY_ROOT:-https://github.com/NixOS/experimental-nix-installer/releases/download/0.27.0}"
 
 main() {
     downloader --check