Multiversion bundle config support

.github/workflows/CI.yml

@@ -21,16 +21,111 @@ permissions:
 jobs:
   finalize:
     timeout-minutes: 10
-    needs: [test-bundle]
+    needs: [test-bundle, test-multiversion]
     if: always()
     runs-on: ubuntu-latest
     steps:
       - run: |
-          echo test: ${{ needs.test-bundle.result }}
+          echo test-multiversion: ${{ needs.test-multiversion.result }}
+          echo test-bundle: ${{ needs.test-bundle.result }}
       - run: exit 1
         if: |
+          (needs.test-multiversion.result != 'success') ||
           (needs.test-bundle.result != 'success')
 
+  build-multiversion:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+        with:
+          persist-credentials: false
+
+      - uses: asdf-vm/actions/setup@05e0d2ed97b598bfce82fd30daf324ae0c4570e6
+      - run: |
+          asdf plugin-add julia
+          asdf install julia 1.10.1
+          asdf install julia 1.10.2
+
+      - run: julia --project=test/multiversion/envs/CommonMark_1.10.1 -e 'import Pkg; Pkg.instantiate()'
+        env:
+          ASDF_JULIA_VERSION: "1.10.1"
+          JULIA_PKG_PRECOMPILE_AUTO: "0"
+
+      - run: julia --project=test/multiversion/envs/CommonMark_1.10.2 -e 'import Pkg; Pkg.instantiate()'
+        env:
+          ASDF_JULIA_VERSION: "1.10.2"
+          JULIA_PKG_PRECOMPILE_AUTO: "0"
+
+      - run: asdf global julia 1.10.2
+
+      - run: julia --project -e 'import Pkg; Pkg.instantiate()'
+      - run: julia --project test/multiversion/bundle.jl
+
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        with:
+          name: multiversion
+          path: test/multiversion/build/CommonMarkStrippedRegistry
+
+  test-multiversion:
+    needs: [build-multiversion]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "1.10.1"
+          - "1.10.2"
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+
+    steps:
+      - uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874
+        with:
+          version: ${{ matrix.version }}
+
+      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
+        with:
+          name: multiversion
+          path: multiversion
+
+      - name: Initialize Julia Depot
+        run: julia -e 'import Pkg; Pkg.status()'
+
+      - name: Install bundle
+        run: julia multiversion/registry/install.jl
+
+      - name: Installing the same bundle again should uninstall and reinstall
+        run: julia multiversion/registry/install.jl
+
+      - name: Resolve environment and precompile
+        run: |
+          julia --project=@CommonMark_${{ matrix.version }} -e '
+            pushfirst!(LOAD_PATH, "@stdlib")
+            import Pkg
+            Pkg.status()
+            Pkg.resolve()
+            Pkg.precompile("CommonMark")
+          '
+
+      - name: Load stripped package and check for source stripping
+        run: julia --project=@CommonMark_${{ matrix.version }} -e 'import CommonMark; @show(first(functionloc(CommonMark.issafe, Tuple{Char}))) === nothing || exit(1)'
+
+      - name: Remove bundle
+        run: |
+          julia -e '
+            for depot in DEPOT_PATH
+              path = joinpath(depot, "registries", "CommonMarkStrippedRegistry", "remove.jl")
+              if isfile(path)
+                run(`julia $path`)
+                break
+              end
+            end
+          '
+
   test:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60

.gitignore

@@ -3,4 +3,5 @@
 *.pem
 *.pub
 test/build
+test/multiversion/build
 .DS_Store

src/PackageBundler.jl

@@ -30,7 +30,7 @@ environments = ["env1", "env2"]               # required
 outputs = "PackageBundle"                     # default: same as `name`
 key = "key"                                   # default: "key"
 clean = false                                 # default: false
-multiplexer = ""                              # default: ""
+multiplexers = ["asdf", "juliaup"]            # default: []
 
 [packages]
 "<uuid>" = "<name>"
@@ -80,13 +80,12 @@ should be cleaned before generating the bundle. If `clean` is `true` and the
 output directory exists it will be deleted before generating the bundle. `clean`
 has no effect if the output is a tarball or an `Artifacts.toml` file.
 
-The `multiplexer` field sets the environment variable that is used to specify
-the version of `julia` that is run, e.g `ASDF_JULIA_VERSION`, `JULIAUP_CHANNEL`,
-`MISE_JULIA_VERSION`, etc. This is only relevant if your bundle config and
-environments contains multiple different Julia versions that are all used by the
-same versions of packages that need to be stripped and serialized, since
-serialization requires the exact version of `julia` for deserializating the
-expressions correctly.
+The `multiplexers` field sets the allowed multiplexer programs to use to select
+specific versions of Julia for each environment based on it's `julia_version`.
+`"juliaup"`, `"asdf"`, and `"mise"` are supported. This is an ordered list of
+options, each of which are tried in turn until one is found that can be used to
+select the correct Julia version. If no multiplexer is found then the current
+`julia` is used.
 """
 function bundle(
     config::AbstractString = "PackageBundler.toml";
@@ -152,9 +151,10 @@ function bundle(
     outputs = String.(vcat(outputs))
     outputs = isempty(outputs) ? [name] : outputs
 
-    # Multiplexer. The environment variable used by the Julia version multiplexer
-    # to select the version of `julia` to run.
-    multiplexer = get(config, "multiplexer", "")::String
+    # Multiplexers. The list of multiplexer programs to try to use to select
+    # specific versions of Julia for each environment based on it's
+    # `julia_version`.
+    multiplexers = String.(get(Vector{String}, config, "multiplexers"))
 
     mktempdir() do temp_dir
         # Generate the bundle in a temp directory, afterwhich copy the result
@@ -167,7 +167,7 @@ function bundle(
             uuid = uuid,
             key_pair = (; private, public),
             handlers = handlers,
-            multiplexer = multiplexer,
+            multiplexers = multiplexers,
         )
         for output in outputs
             output = normpath(joinpath(dir, output))

src/code_stripping.jl

@@ -8,7 +8,7 @@ function _generate_stripped_bundle(;
     uuid::Union{AbstractString,Integer,Base.UUID},
     key_pair::@NamedTuple{private::String, public::String},
     handlers::Dict,
-    multiplexer::String,
+    multiplexers::Vector{String},
 )
     output_dir = abspath(output_dir)
 
@@ -27,7 +27,7 @@ function _generate_stripped_bundle(;
             stripped = stripped,
             key_pair = key_pair,
             handlers = handlers,
-            multiplexer = multiplexer,
+            multiplexers = multiplexers,
         )
 
         stripped_uuid_mapping = merge(stripped_uuid_mapping, new_stripped_uuid_mapping)
@@ -255,7 +255,7 @@ function _process_project_env(;
     stripped::Dict{String,String},
     key_pair::@NamedTuple{private::String, public::String},
     handlers::Dict,
-    multiplexer::String,
+    multiplexers::Vector{String},
 )
     project_dir = normpath(project_dir)
     output_dir = normpath(output_dir)
@@ -301,7 +301,7 @@ function _process_project_env(;
                 stripped_pkg_uuid_mapping,
                 key_pair,
                 handlers,
-                multiplexer,
+                multiplexers,
             )
             versions = get!(Dict{String,Any}, pkg_version_info, each_name)
             versions[version_info["project"]["version"]] = version_info
@@ -478,7 +478,7 @@ function _strip_package(
     uuid_replacements::Dict,
     key_pair::@NamedTuple{private::String, public::String},
     handlers::Dict,
-    multiplexer::String,
+    multiplexers::Vector{String},
 )
     @info "Stripping source code." package
 
@@ -588,10 +588,8 @@ function _strip_package(
                 TOML.print(io, Dict("julia_files" => julia_files, "handlers" => handlers))
             end
             script = joinpath(@__DIR__, "serializer.jl")
-            envs = isempty(multiplexer) ? [] : [multiplexer => "$julia_version"]
-            withenv(envs...) do
-                run(`$(Base.julia_exename()) --startup-file=no $(script) $(toml_file)`)
-            end
+            binary = _process_multiplexers(multiplexers, julia_version)
+            run(`$(binary) --startup-file=no $(script) $(toml_file)`)
         end
 
         # Sign all the files that have been created/modified during serialization.
@@ -632,3 +630,31 @@ function _stripped_source_path(
     project_root = dirname(project_toml)
     return joinpath("[bundled]", package_name, version, relpath(source_file, project_root))
 end
+
+function _process_multiplexers(multiplexers::Vector{String}, julia_version::VersionNumber)
+    for multiplexer in multiplexers
+        exists = Sys.which(multiplexer)
+        if !isnothing(exists)
+            if multiplexer == "juliaup"
+                return `julia +$(julia_version)`
+            elseif multiplexer == "asdf"
+                return withenv("ASDF_JULIA_VERSION" => "$(julia_version)") do
+                    path = readchomp(`asdf which julia`)
+                    return `$(path)`
+                end
+            elseif multiplexer == "mise"
+                return withenv("MISE_JULIA_VERSION" => "$(julia_version)") do
+                    path = readchomp(`mise which julia`)
+                    return `$(path)`
+                end
+            else
+                error("Unsupported multiplexer: $multiplexer")
+            end
+        end
+    end
+    if isempty(multiplexers)
+        return Base.julia_cmd()
+    else
+        error("no multiplexers found: $(repr(multiplexers))")
+    end
+end

test/multiversion/PackageBundler.toml

@@ -0,0 +1,13 @@
+name = "CommonMarkStrippedRegistry"
+uuid = "a0e467e0-faf1-11e4-9d3f-2f4b262e6b06"
+environments = [
+    "envs/CommonMark_1.10.1",
+    "envs/CommonMark_1.10.2"
+]
+outputs = ["build/CommonMarkStrippedRegistry"]
+key = "key"
+clean = true
+multiplexers = ["asdf", "juliaup"]
+
+[packages]
+"a80b9123-70ca-4bc0-993e-6e3bcb318db6" = "CommonMark"

test/multiversion/bundle.jl

@@ -0,0 +1,6 @@
+import PackageBundler
+
+cd(@__DIR__) do
+    PackageBundler.keypair()
+    PackageBundler.bundle()
+end

test/multiversion/envs/CommonMark_1.10.1/Manifest.toml

@@ -0,0 +1,79 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.10.1"
+manifest_format = "2.0"
+project_hash = "8d4ac4f014d6a6f19334c7b9fe0d116e6a06f436"
+
+[[deps.CommonMark]]
+deps = ["Crayons", "JSON", "PrecompileTools", "URIs"]
+git-tree-sha1 = "532c4185d3c9037c0237546d817858b23cf9e071"
+uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6"
+version = "0.8.12"
+
+[[deps.Crayons]]
+git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.1.1"
+
+[[deps.Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[deps.JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.4"
+
+[[deps.Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[deps.Parsers]]
+deps = ["Dates", "PrecompileTools", "UUIDs"]
+git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "2.8.1"
+
+[[deps.PrecompileTools]]
+deps = ["Preferences"]
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
+uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+version = "1.2.1"
+
+[[deps.Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.4.3"
+
+[[deps.Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[deps.TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.3"
+
+[[deps.URIs]]
+git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b"
+uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
+version = "1.5.1"
+
+[[deps.UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[deps.Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

test/multiversion/envs/CommonMark_1.10.1/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+CommonMark = "a80b9123-70ca-4bc0-993e-6e3bcb318db6"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"