Merge pull request #143 from figsoda/cargo-lock

add support for updating packages using importCargoLock

default.nix

@@ -4,18 +4,19 @@
 
 
 with pkgs;
-python3.pkgs.buildPythonApplication rec {
+python311.pkgs.buildPythonApplication rec {
   name = "nix-update";
   inherit src;
   buildInputs = [ makeWrapper ];
   nativeCheckInputs = [
-    python3.pkgs.pytest
-    python3.pkgs.black
+    python311.pkgs.pytest
+    python311.pkgs.black
     ruff
     glibcLocales
     mypy
-    # technically not a test input, but we need it for development in PATH
+    # technically not test inputs, but we need it for development in PATH
     pkgs.nixVersions.stable or nix_2_4
+    nix-prefetch-git
   ];
   checkPhase = ''
     echo -e "\x1b[32m## run black\x1b[0m"
@@ -28,7 +29,7 @@ python3.pkgs.buildPythonApplication rec {
   makeWrapperArgs = [
     "--prefix PATH"
     ":"
-    (lib.makeBinPath [ pkgs.nixVersions.stable or nix_2_4 nixpkgs-fmt nixpkgs-review ])
+    (lib.makeBinPath [ pkgs.nixVersions.stable or nix_2_4 nixpkgs-fmt nixpkgs-review nix-prefetch-git ])
   ];
   shellHook = ''
     # workaround because `python setup.py develop` breaks for me

nix_update/__init__.py

@@ -156,7 +156,10 @@ def format_commit_message(package: Package) -> str:
 def git_commit(git_dir: str, package: Package) -> None:
     msg = format_commit_message(package)
     new_version = package.new_version
-    run(["git", "-C", git_dir, "add", package.filename], stdout=None)
+    cmd = ["git", "-C", git_dir, "add", package.filename]
+    if package.cargo_lock:
+        cmd.append(package.cargo_lock)
+    run(cmd, stdout=None)
     if new_version and package.old_version != new_version.number:
         run(
             ["git", "-C", git_dir, "commit", "--verbose", "--message", msg], stdout=None

nix_update/eval.py

@@ -33,6 +33,7 @@ class Package:
     vendor_hash: Optional[str]
     vendor_sha256: Optional[str]
     cargo_deps: Optional[str]
+    cargo_lock: Optional[str]
     npm_deps: Optional[str]
     tests: List[str]
     has_update_script: bool
@@ -109,6 +110,11 @@ def eval_expression(
   vendor_hash = pkg.vendorHash or null;
   vendor_sha256 = pkg.vendorSha256 or null;
   cargo_deps = pkg.cargoDeps.outputHash or null;
+  cargo_lock =
+    if pkg ? cargoDeps.lockFile then
+      (sanitizePosition {{ file = pkg.cargoDeps.lockFile; }}).file
+    else
+      null;
   npm_deps = pkg.npmDeps.outputHash or null;
   tests = builtins.attrNames (pkg.passthru.tests or {{}});
   has_update_script = {has_update_script};

nix_update/update.py

@@ -1,8 +1,14 @@
 import fileinput
+import json
+import re
+import shutil
 import subprocess
 import tempfile
+import tomllib
+from concurrent.futures import ThreadPoolExecutor
 from os import path
-from typing import Dict, Optional
+from pathlib import Path
+from typing import Dict, Optional, Tuple
 
 from .errors import UpdateError
 from .eval import Package, eval_attr
@@ -27,7 +33,7 @@ def replace_version(package: Package) -> bool:
             for line in f:
                 if package.new_version.rev:
                     line = line.replace(package.rev, package.new_version.rev)
-                print(line.replace(old_version, new_version), end="")
+                print(line.replace(f'"{old_version}"', f'"{new_version}"'), end="")
     else:
         info(f"Not updating version, already {old_version}")
 
@@ -71,13 +77,17 @@ def replace_hash(filename: str, current: str, target: str) -> None:
                 print(line, end="")
 
 
-def nix_prefetch(opts: Options, attr: str) -> str:
-    expr = (
-        f'let flake = builtins.getFlake "{opts.import_path}"; in (flake.packages.${{builtins.currentSystem}}.{opts.attribute} or flake.{opts.attribute}).{attr}'
+def get_package(opts: Options) -> str:
+    return (
+        f'(let flake = builtins.getFlake "{opts.import_path}"; in flake.packages.${{builtins.currentSystem}}.{opts.attribute} or flake.{opts.attribute})'
         if opts.flake
-        else f"(import {opts.import_path} {disable_check_meta(opts)}).{opts.attribute}.{attr}"
+        else f"(import {opts.import_path} {disable_check_meta(opts)}).{opts.attribute}"
     )
 
+
+def nix_prefetch(opts: Options, attr: str) -> str:
+    expr = f"{get_package(opts)}.{attr}"
+
     extra_env: Dict[str, str] = {}
     tempdir: Optional[tempfile.TemporaryDirectory[str]] = None
     if extra_env.get("XDG_RUNTIME_DIR") is None:
@@ -116,6 +126,12 @@ def disable_check_meta(opts: Options) -> str:
     return f'(if (builtins.hasAttr "config" (builtins.functionArgs (import {opts.import_path}))) then {{ config.checkMeta = false; overlays = []; }} else {{ }})'
 
 
+def git_prefetch(x: Tuple[str, Tuple[str, str]]) -> Tuple[str, str]:
+    rev, (key, url) = x
+    res = run(["nix-prefetch-git", url, rev, "--fetch-submodules"])
+    return key, to_sri(json.loads(res.stdout)["sha256"])
+
+
 def update_src_hash(opts: Options, filename: str, current_hash: str) -> None:
     target_hash = nix_prefetch(opts, "src")
     replace_hash(filename, current_hash, target_hash)
@@ -131,6 +147,83 @@ def update_cargo_deps_hash(opts: Options, filename: str, current_hash: str) -> N
     replace_hash(filename, current_hash, target_hash)
 
 
+def update_cargo_lock(opts: Options, filename: str, dst: str) -> None:
+    res = run(
+        [
+            "nix",
+            "build",
+            "--impure",
+            "--print-out-paths",
+            "--expr",
+            f'{get_package(opts)}.overrideAttrs (_: {{ prePatch = "cp -r . $out; exit"; outputs = [ "out" ]; }})',
+        ]
+        + opts.extra_flags,
+    )
+    src = Path(res.stdout.strip()) / "Cargo.lock"
+    if not src.is_file():
+        return
+
+    shutil.copyfile(src, dst)
+    hashes = {}
+    with open(dst, "rb") as f:
+        lock = tomllib.load(f)
+        regex = re.compile(r"git\+([^?]+)(\?(rev|tag|branch)=.*)?#(.*)")
+        git_deps = {}
+        for pkg in lock["package"]:
+            if source := pkg.get("source"):
+                if match := regex.fullmatch(source):
+                    rev = match[4]
+                    if rev not in git_deps:
+                        git_deps[rev] = f"{pkg['name']}-{pkg['version']}", match[1]
+
+        for k, v in ThreadPoolExecutor().map(git_prefetch, git_deps.items()):
+            hashes[k] = v
+
+    with fileinput.FileInput(filename, inplace=True) as f:
+        short = re.compile(r"(\s*)cargoLock\.lockFile\s*=\s*(.+)\s*;\s*")
+        expanded = re.compile(r"(\s*)lockFile\s*=\s*(.+)\s*;\s*")
+
+        for line in f:
+            if match := short.fullmatch(line):
+                indent = match[1]
+                path = match[2]
+                print(f"{indent}cargoLock = {{")
+                print(f"{indent}  lockFile = {path};")
+                print_hashes(hashes, f"{indent}  ")
+                print(f"{indent}}};")
+                for line in f:
+                    print(line, end="")
+                return
+            elif match := expanded.fullmatch(line):
+                indent = match[1]
+                path = match[2]
+                print(line, end="")
+                print_hashes(hashes, indent)
+                brace = 0
+                for line in f:
+                    for c in line:
+                        if c == "{":
+                            brace -= 1
+                        if c == "}":
+                            brace += 1
+                        if brace == 1:
+                            print(line, end="")
+                            for line in f:
+                                print(line, end="")
+                            return
+            else:
+                print(line, end="")
+
+
+def print_hashes(hashes: Dict[str, str], indent: str) -> None:
+    if not hashes:
+        return
+    print(f"{indent}outputHashes = {{")
+    for k, v in hashes.items():
+        print(f'{indent}  "{k}" = "{v}";')
+    print(f"{indent}}};")
+
+
 def update_npm_deps_hash(opts: Options, filename: str, current_hash: str) -> None:
     target_hash = nix_prefetch(opts, "npmDeps")
     replace_hash(filename, current_hash, target_hash)
@@ -228,6 +321,9 @@ def update(opts: Options) -> Package:
         if package.cargo_deps:
             update_cargo_deps_hash(opts, package.filename, package.cargo_deps)
 
+        if package.cargo_lock:
+            update_cargo_lock(opts, package.filename, package.cargo_lock)
+
         if package.npm_deps:
             update_npm_deps_hash(opts, package.filename, package.npm_deps)
 

pyproject.toml

@@ -1,11 +1,12 @@
 [tool.ruff]
+target-version = "py311"
 line-length = 88
 
 select = ["E", "F", "I"]
 ignore = [ "E501" ]
 
 [tool.mypy]
-python_version = "3.10"
+python_version = "3.11"
 warn_redundant_casts = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true

tests/test_cargo_lock_expand.py

@@ -0,0 +1,38 @@
+import subprocess
+
+import conftest
+
+from nix_update import main
+
+
+def test_main(helpers: conftest.Helpers) -> None:
+    with helpers.testpkgs(init_git=True) as path:
+        main(
+            ["--file", str(path), "--commit", "cargoLockExpand", "--version", "v0.3.8"]
+        )
+        subprocess.run(
+            [
+                "nix",
+                "eval",
+                "--raw",
+                "--extra-experimental-features",
+                "nix-command",
+                "-f",
+                path,
+                "cargoLockExpand.cargoDeps",
+            ],
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        ).stdout.strip()
+        diff = subprocess.run(
+            ["git", "-C", path, "show"],
+            text=True,
+            stdout=subprocess.PIPE,
+            check=True,
+        ).stdout.strip()
+        print(diff)
+        assert "Cargo.lock" in diff
+        assert '+source = "git+' in diff
+        assert "outputHashes" in diff
+        assert "https://github.com/nix-community/nurl/compare/v0.3.7...v0.3.8" in diff

tests/test_cargo_lock_update.py

@@ -0,0 +1,46 @@
+import subprocess
+
+import conftest
+
+from nix_update import main
+
+
+def test_main(helpers: conftest.Helpers) -> None:
+    with helpers.testpkgs(init_git=True) as path:
+        main(
+            [
+                "--file",
+                str(path),
+                "--commit",
+                "cargoLockUpdate",
+                "--version",
+                "v0.0.255",
+            ]
+        )
+        subprocess.run(
+            [
+                "nix",
+                "eval",
+                "--raw",
+                "--extra-experimental-features",
+                "nix-command",
+                "-f",
+                path,
+                "cargoLockUpdate.cargoDeps",
+            ],
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        ).stdout.strip()
+        diff = subprocess.run(
+            ["git", "-C", path, "show"],
+            text=True,
+            stdout=subprocess.PIPE,
+            check=True,
+        ).stdout.strip()
+        print(diff)
+        assert "Cargo.lock" in diff
+        assert '+source = "git+' in diff
+        assert (
+            "https://github.com/charliermarsh/ruff/compare/v0.0.254...v0.0.255" in diff
+        )