From 8788c17484ef77a946a1083bb183c2bd802ba551 Mon Sep 17 00:00:00 2001 From: MichaelHatherly Date: Thu, 4 Apr 2024 18:02:27 +0100 Subject: [PATCH] Multiversion bundle config support --- .github/workflows/CI.yml | 99 ++++++++++++++++++- .gitignore | 1 + src/PackageBundler.jl | 24 ++--- src/code_stripping.jl | 44 +++++++-- test/multiversion/PackageBundler.toml | 13 +++ test/multiversion/bundle.jl | 6 ++ .../envs/CommonMark_1.10.1/Manifest.toml | 79 +++++++++++++++ .../envs/CommonMark_1.10.1/Project.toml | 3 + .../envs/CommonMark_1.10.2/Manifest.toml | 79 +++++++++++++++ .../envs/CommonMark_1.10.2/Project.toml | 3 + 10 files changed, 328 insertions(+), 23 deletions(-) create mode 100644 test/multiversion/PackageBundler.toml create mode 100644 test/multiversion/bundle.jl create mode 100644 test/multiversion/envs/CommonMark_1.10.1/Manifest.toml create mode 100644 test/multiversion/envs/CommonMark_1.10.1/Project.toml create mode 100644 test/multiversion/envs/CommonMark_1.10.2/Manifest.toml create mode 100644 test/multiversion/envs/CommonMark_1.10.2/Project.toml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4705c8f..55d6859 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,16 +21,111 @@ permissions: jobs: finalize: timeout-minutes: 10 - needs: [test-bundle] + needs: [test-bundle, test-multiversion] if: always() runs-on: ubuntu-latest steps: - run: | - echo test: ${{ needs.test-bundle.result }} + echo test-multiversion: ${{ needs.test-multiversion.result }} + echo test-bundle: ${{ needs.test-bundle.result }} - run: exit 1 if: | + (needs.test-multiversion.result != 'success') || (needs.test-bundle.result != 'success') + build-multiversion: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 + with: + persist-credentials: false + + - uses: asdf-vm/actions/setup@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 + - run: | + asdf plugin-add julia + asdf install julia 1.10.1 + asdf install julia 1.10.2 + + - run: julia --project=test/multiversion/envs/CommonMark_1.10.1 -e 'import Pkg; Pkg.instantiate()' + env: + ASDF_JULIA_VERSION: "1.10.1" + JULIA_PKG_PRECOMPILE_AUTO: "0" + + - run: julia --project=test/multiversion/envs/CommonMark_1.10.2 -e 'import Pkg; Pkg.instantiate()' + env: + ASDF_JULIA_VERSION: "1.10.2" + JULIA_PKG_PRECOMPILE_AUTO: "0" + + - run: asdf global julia 1.10.2 + + - run: julia --project -e 'import Pkg; Pkg.instantiate()' + - run: julia --project test/multiversion/bundle.jl + + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + name: multiversion + path: test/multiversion/build/CommonMarkStrippedRegistry + + test-multiversion: + needs: [build-multiversion] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + version: + - "1.10.1" + - "1.10.2" + os: + - macos-latest + - ubuntu-latest + - windows-latest + + steps: + - uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874 + with: + version: ${{ matrix.version }} + + - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + with: + name: multiversion + path: multiversion + + - name: Initialize Julia Depot + run: julia -e 'import Pkg; Pkg.status()' + + - name: Install bundle + run: julia multiversion/registry/install.jl + + - name: Installing the same bundle again should uninstall and reinstall + run: julia multiversion/registry/install.jl + + - name: Resolve environment and precompile + run: | + julia --project=@CommonMark_${{ matrix.version }} -e ' + pushfirst!(LOAD_PATH, "@stdlib") + import Pkg + Pkg.status() + Pkg.resolve() + Pkg.precompile("CommonMark") + ' + + - name: Load stripped package and check for source stripping + run: julia --project=@CommonMark_${{ matrix.version }} -e 'import CommonMark; @show(first(functionloc(CommonMark.issafe, Tuple{Char}))) === nothing || exit(1)' + + - name: Remove bundle + run: | + julia -e ' + for depot in DEPOT_PATH + path = joinpath(depot, "registries", "CommonMarkStrippedRegistry", "remove.jl") + if isfile(path) + run(`julia $path`) + break + end + end + ' + test: runs-on: ${{ matrix.os }} timeout-minutes: 60 diff --git a/.gitignore b/.gitignore index 9f8a8ef..0f79d46 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ *.pem *.pub test/build +test/multiversion/build .DS_Store diff --git a/src/PackageBundler.jl b/src/PackageBundler.jl index 8429dc9..a28fb86 100644 --- a/src/PackageBundler.jl +++ b/src/PackageBundler.jl @@ -30,7 +30,7 @@ environments = ["env1", "env2"] # required outputs = "PackageBundle" # default: same as `name` key = "key" # default: "key" clean = false # default: false -multiplexer = "" # default: "" +multiplexers = ["asdf", "juliaup"] # default: [] [packages] "" = "" @@ -80,13 +80,12 @@ should be cleaned before generating the bundle. If `clean` is `true` and the output directory exists it will be deleted before generating the bundle. `clean` has no effect if the output is a tarball or an `Artifacts.toml` file. -The `multiplexer` field sets the environment variable that is used to specify -the version of `julia` that is run, e.g `ASDF_JULIA_VERSION`, `JULIAUP_CHANNEL`, -`MISE_JULIA_VERSION`, etc. This is only relevant if your bundle config and -environments contains multiple different Julia versions that are all used by the -same versions of packages that need to be stripped and serialized, since -serialization requires the exact version of `julia` for deserializating the -expressions correctly. +The `multiplexers` field sets the allowed multiplexer programs to use to select +specific versions of Julia for each environment based on it's `julia_version`. +`"juliaup"`, `"asdf"`, and `"mise"` are supported. This is an ordered list of +options, each of which are tried in turn until one is found that can be used to +select the correct Julia version. If no multiplexer is found then the current +`julia` is used. """ function bundle( config::AbstractString = "PackageBundler.toml"; @@ -152,9 +151,10 @@ function bundle( outputs = String.(vcat(outputs)) outputs = isempty(outputs) ? [name] : outputs - # Multiplexer. The environment variable used by the Julia version multiplexer - # to select the version of `julia` to run. - multiplexer = get(config, "multiplexer", "")::String + # Multiplexers. The list of multiplexer programs to try to use to select + # specific versions of Julia for each environment based on it's + # `julia_version`. + multiplexers = String.(get(Vector{String}, config, "multiplexers")) mktempdir() do temp_dir # Generate the bundle in a temp directory, afterwhich copy the result @@ -167,7 +167,7 @@ function bundle( uuid = uuid, key_pair = (; private, public), handlers = handlers, - multiplexer = multiplexer, + multiplexers = multiplexers, ) for output in outputs output = normpath(joinpath(dir, output)) diff --git a/src/code_stripping.jl b/src/code_stripping.jl index 28048df..c813459 100644 --- a/src/code_stripping.jl +++ b/src/code_stripping.jl @@ -8,7 +8,7 @@ function _generate_stripped_bundle(; uuid::Union{AbstractString,Integer,Base.UUID}, key_pair::@NamedTuple{private::String, public::String}, handlers::Dict, - multiplexer::String, + multiplexers::Vector{String}, ) output_dir = abspath(output_dir) @@ -27,7 +27,7 @@ function _generate_stripped_bundle(; stripped = stripped, key_pair = key_pair, handlers = handlers, - multiplexer = multiplexer, + multiplexers = multiplexers, ) stripped_uuid_mapping = merge(stripped_uuid_mapping, new_stripped_uuid_mapping) @@ -255,7 +255,7 @@ function _process_project_env(; stripped::Dict{String,String}, key_pair::@NamedTuple{private::String, public::String}, handlers::Dict, - multiplexer::String, + multiplexers::Vector{String}, ) project_dir = normpath(project_dir) output_dir = normpath(output_dir) @@ -301,7 +301,7 @@ function _process_project_env(; stripped_pkg_uuid_mapping, key_pair, handlers, - multiplexer, + multiplexers, ) versions = get!(Dict{String,Any}, pkg_version_info, each_name) versions[version_info["project"]["version"]] = version_info @@ -478,7 +478,7 @@ function _strip_package( uuid_replacements::Dict, key_pair::@NamedTuple{private::String, public::String}, handlers::Dict, - multiplexer::String, + multiplexers::Vector{String}, ) @info "Stripping source code." package @@ -588,10 +588,8 @@ function _strip_package( TOML.print(io, Dict("julia_files" => julia_files, "handlers" => handlers)) end script = joinpath(@__DIR__, "serializer.jl") - envs = isempty(multiplexer) ? [] : [multiplexer => "$julia_version"] - withenv(envs...) do - run(`$(Base.julia_exename()) --startup-file=no $(script) $(toml_file)`) - end + binary = _process_multiplexers(multiplexers, julia_version) + run(`$(binary) --startup-file=no $(script) $(toml_file)`) end # Sign all the files that have been created/modified during serialization. @@ -632,3 +630,31 @@ function _stripped_source_path( project_root = dirname(project_toml) return joinpath("[bundled]", package_name, version, relpath(source_file, project_root)) end + +function _process_multiplexers(multiplexers::Vector{String}, julia_version::VersionNumber) + for multiplexer in multiplexers + exists = Sys.which(multiplexer) + if !isnothing(exists) + if multiplexer == "juliaup" + return `julia +$(julia_version)` + elseif multiplexer == "asdf" + return withenv("ASDF_JULIA_VERSION" => "$(julia_version)") do + path = readchomp(`asdf which julia`) + return `$(path)` + end + elseif multiplexer == "mise" + return withenv("MISE_JULIA_VERSION" => "$(julia_version)") do + path = readchomp(`mise which julia`) + return `$(path)` + end + else + error("Unsupported multiplexer: $multiplexer") + end + end + end + if isempty(multiplexers) + return Base.julia_cmd() + else + error("no multiplexers found: $(repr(multiplexers))") + end +end diff --git a/test/multiversion/PackageBundler.toml b/test/multiversion/PackageBundler.toml new file mode 100644 index 0000000..6b0a132 --- /dev/null +++ b/test/multiversion/PackageBundler.toml @@ -0,0 +1,13 @@ +name = "CommonMarkStrippedRegistry" +uuid = "a0e467e0-faf1-11e4-9d3f-2f4b262e6b06" +environments = [ + "envs/CommonMark_1.10.1", + "envs/CommonMark_1.10.2" +] +outputs = ["build/CommonMarkStrippedRegistry"] +key = "key" +clean = true +multiplexers = ["asdf", "juliaup"] + +[packages] +"a80b9123-70ca-4bc0-993e-6e3bcb318db6" = "CommonMark" diff --git a/test/multiversion/bundle.jl b/test/multiversion/bundle.jl new file mode 100644 index 0000000..fcb54e9 --- /dev/null +++ b/test/multiversion/bundle.jl @@ -0,0 +1,6 @@ +import PackageBundler + +cd(@__DIR__) do + PackageBundler.keypair() + PackageBundler.bundle() +end diff --git a/test/multiversion/envs/CommonMark_1.10.1/Manifest.toml b/test/multiversion/envs/CommonMark_1.10.1/Manifest.toml new file mode 100644 index 0000000..34f0ce9 --- /dev/null +++ b/test/multiversion/envs/CommonMark_1.10.1/Manifest.toml @@ -0,0 +1,79 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.1" +manifest_format = "2.0" +project_hash = "8d4ac4f014d6a6f19334c7b9fe0d116e6a06f436" + +[[deps.CommonMark]] +deps = ["Crayons", "JSON", "PrecompileTools", "URIs"] +git-tree-sha1 = "532c4185d3c9037c0237546d817858b23cf9e071" +uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" +version = "0.8.12" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/test/multiversion/envs/CommonMark_1.10.1/Project.toml b/test/multiversion/envs/CommonMark_1.10.1/Project.toml new file mode 100644 index 0000000..251c4b2 --- /dev/null +++ b/test/multiversion/envs/CommonMark_1.10.1/Project.toml @@ -0,0 +1,3 @@ +[deps] +CommonMark = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" diff --git a/test/multiversion/envs/CommonMark_1.10.2/Manifest.toml b/test/multiversion/envs/CommonMark_1.10.2/Manifest.toml new file mode 100644 index 0000000..94efc25 --- /dev/null +++ b/test/multiversion/envs/CommonMark_1.10.2/Manifest.toml @@ -0,0 +1,79 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.2" +manifest_format = "2.0" +project_hash = "8d4ac4f014d6a6f19334c7b9fe0d116e6a06f436" + +[[deps.CommonMark]] +deps = ["Crayons", "JSON", "PrecompileTools", "URIs"] +git-tree-sha1 = "532c4185d3c9037c0237546d817858b23cf9e071" +uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" +version = "0.8.12" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/test/multiversion/envs/CommonMark_1.10.2/Project.toml b/test/multiversion/envs/CommonMark_1.10.2/Project.toml new file mode 100644 index 0000000..251c4b2 --- /dev/null +++ b/test/multiversion/envs/CommonMark_1.10.2/Project.toml @@ -0,0 +1,3 @@ +[deps] +CommonMark = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"