diff --git a/.github/workflows/release-script.yml b/.github/workflows/release-script.yml
index e8b32627..26b548dc 100644
--- a/.github/workflows/release-script.yml
+++ b/.github/workflows/release-script.yml
@@ -1,9 +1,12 @@
 name: Generate Installer Script
 
 on:
-  schedule:
-    - cron: '10 * * * *'
   workflow_dispatch: # Allows manual triggering of the workflow
+    inputs:
+      testing_hydra_eval_id:
+        description: "Eval ID of Hydra job to use artifacts from for testing"
+        required: false
+        default: ""
 
 jobs:
   build:
@@ -12,13 +15,8 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
     - uses: cachix/install-nix-action@v25
-    - name: Download Installer
-      run: nix-shell -p python3Packages.requests --run "python assemble_installer.py" -I nixpkgs=channel:nixos-23.11
-    - uses: stefanzweifel/git-auto-commit-action@v5
-      with:
-        commit_message: "Update installer script"
-        commit_user_name: "GitHub Actions"
-        branch: prerelease
-        # assemble_installer.py already does a fetch and checkout
-        skip_fetch: true
-        skip_checkout: true
+    - name: Create draft release
+      # The script also depends on gh and git but those are both pre-installed on the runner
+      run: nix-shell -p python3Packages.requests --run "python assemble_installer.py ${{ github.event.inputs.testing_hydra_eval_id }}" -I nixpkgs=channel:nixos-23.11
+      env:
+        GH_TOKEN: ${{ github.token }}
diff --git a/assemble_installer.py b/assemble_installer.py
index bd19bd35..e01f7ea6 100644
--- a/assemble_installer.py
+++ b/assemble_installer.py
@@ -1,11 +1,37 @@
+import os
 import requests
 import subprocess
 import shutil
 import sys
+import tempfile
+import tomllib
+
+from string import Template
+
+# A Hydra eval id must be passed as the argument to this script or an empty
+# string to use latest eval on Hydra
+# TODO: using an empty string is not the cleanest
+# TODO: print script usage
+eval_id = sys.argv[1]
 
 response = requests.get('https://hydra.nixos.org/jobset/experimental-nix-installer/experimental-installer/evals', headers={'Accept': 'application/json'})
+evals = response.json()['evals']
+
+if eval_id is not None and eval_id != "":
+    eval_id_int = int(eval_id)
+    ids = [eval['id'] for eval in evals]
+    hydra_eval = next( eval for eval in evals if eval['id'] == eval_id_int )
+else:
+    hydra_eval = evals[0]
 
-hydra_eval = response.json()['evals'][0]
+    rev = subprocess.run(
+        ["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, check=True, text=True
+    ).stdout.strip()
+
+    if not rev in hydra_eval["flake"]:
+        raise RuntimeError(
+            f"Expected flake with rev {rev} but found flake {hydra_eval['flake']}"
+        )
 
 installers = []
 
@@ -22,10 +48,50 @@
             subprocess.call(f"nix-store -r {installer_url}", shell=True)
         installers.append((installer_url, system))
     else:
+        print(
+            f"Build {build_id} not finished. Check status at https://hydra.nixos.org/eval/{hydra_eval['id']}#tabs-unfinished"
+        )
         sys.exit(0)
 
-subprocess.run(["git", "fetch", "origin", "prerelease"], check=True)
-subprocess.run(["git", "checkout", "-b", "prerelease", "origin/prerelease"], check=True)
+with open("Cargo.toml", "rb") as f:
+    cargo_toml = tomllib.load(f)
+version = cargo_toml["package"]["version"]
+
+with tempfile.TemporaryDirectory() as tmpdirname:
+    release_files = []
+    for installer_url, system in installers:
+        installer_file = f"{tmpdirname}/nix-installer-{system}"
+        release_files.append(installer_file)
+        print(f"Copying {installer_url} to {installer_file}")
+        shutil.copy(f"{installer_url}/bin/nix-installer", installer_file)
+
+    # Subsitute version in nix-installer.sh
+    original_file = "nix-installer.sh"
+
+    with open(original_file, "r") as nix_installer_sh:
+        nix_installer_sh_contents = nix_installer_sh.read()
+
+    template = Template(nix_installer_sh_contents)
+    updated_content = template.safe_substitute(assemble_installer_templated_version=version)
+
+    # Write the modified content to the output file
+    substituted_file=f"{tmpdirname}/nix-installer.sh"
+    with open(substituted_file, "w", encoding="utf-8") as output_file:
+        output_file.write(updated_content)
+    release_files.append(substituted_file)
 
-for installer_url, system in installers:
-    shutil.copy(f"{installer_url}/bin/nix-installer", f"nix-installer-{system}")
+    subprocess.run(
+        [
+            "gh",
+            "release",
+            "create",
+            "--notes",
+            f"Release experimental nix installer v{version}",
+            "--title",
+            f"v{version}",
+            "--draft",
+            version,
+            *release_files,
+        ],
+        check=True,
+    )
diff --git a/nix-installer.sh b/nix-installer.sh
index 97144a5d..fae0974c 100755
--- a/nix-installer.sh
+++ b/nix-installer.sh
@@ -25,7 +25,7 @@ fi
 set -u
 
 # If NIX_INSTALLER_FORCE_ALLOW_HTTP is unset or empty, default it.
-NIX_INSTALLER_BINARY_ROOT="${NIX_INSTALLER_BINARY_ROOT:-https://raw.githubusercontent.com/NixOS/experimental-nix-installer/prerelease}"
+NIX_INSTALLER_BINARY_ROOT="${NIX_INSTALLER_BINARY_ROOT:-https://github.com/NixOS/experimental-nix-installer/releases/download/$assemble_installer_templated_version}"
 
 main() {
     downloader --check