Skip to content

Commit

Permalink
Multiversion bundle config support
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaelHatherly
MichaelHatherly committed Apr 5, 2024
1 parent 7f3c5ce commit 53d2076
Show file tree
Hide file tree
Showing 10 changed files with 363 additions and 123 deletions.
234 changes: 132 additions & 102 deletions .github/workflows/CI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,112 +19,142 @@ permissions:
contents: read

jobs:
finalize:
timeout-minutes: 10
needs: [test-bundle]
if: always()
# finalize:
# timeout-minutes: 10
# needs: [test-bundle]
# if: always()
# runs-on: ubuntu-latest
# steps:
# - run: |
# echo test: ${{ needs.test-bundle.result }}
# - run: exit 1
# if: |
# (needs.test-bundle.result != 'success')

multiversion-build:
runs-on: ubuntu-latest
steps:
- run: |
echo test: ${{ needs.test-bundle.result }}
- run: exit 1
if: |
(needs.test-bundle.result != 'success')
test:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
version:
- "1.10"
os:
- macos-latest
- ubuntu-latest
- windows-latest

steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
with:
persist-credentials: false
- uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874
with:
version: ${{ matrix.version }}
- uses: julia-actions/cache@dc1a3cdeacb521b0ca93cfc66143fcadb15a5bd0
- uses: julia-actions/julia-buildpkg@90dd6f23eb49626e4e6612cb9d64d456f86e6a1c

# `PackageBundler.bundle()` expects the environments to be bundled to have
# already been instantiated (hence downloaded) prior to running it. We
# don't need to precompile it though.
- run: julia --project=test/envs/CustomEnv -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.instantiate()'

- uses: julia-actions/julia-runtest@79a7e100883947123f8263c5f06e6c0ea3eb972f
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
with:
name: bundle-${{ matrix.os }}
path: test/build/LocalCustomRegistry

test-bundle:
needs: [test]
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
version:
- "1.10"
os:
- macos-latest
- ubuntu-latest
- windows-latest
bundle:
- bundle-macos-latest
- bundle-ubuntu-latest
- bundle-windows-latest

steps:
- uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874
with:
version: ${{ matrix.version }}

- uses: julia-actions/cache@dc1a3cdeacb521b0ca93cfc66143fcadb15a5bd0

- uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
with:
name: ${{ matrix.bundle }}
path: ${{ matrix.bundle }}

- name: Initialize Julia Depot
run: julia -e 'import Pkg; Pkg.status()'

- name: Install bundle
run: julia ${{ matrix.bundle }}/registry/install.jl

- name: Installing the same bundle again should uninstall and reinstall
run: julia ${{ matrix.bundle }}/registry/install.jl

- name: Resolve environment and precompile
run: |
julia --project=@CustomEnv -e '
pushfirst!(LOAD_PATH, "@stdlib")
import Pkg
Pkg.status()
Pkg.resolve()
Pkg.precompile("CairoMakie")
'
- name: Load packages
run: julia --project=@CustomEnv -e 'import CairoMakie'

- name: Remove bundle
run: |
julia -e '
for depot in DEPOT_PATH
path = joinpath(depot, "registries", "LocalCustomRegistry", "remove.jl")
if isfile(path)
run(`julia $path`)
break
end
end
'
- uses: asdf-vm/actions/setup@05e0d2ed97b598bfce82fd30daf324ae0c4570e6
- run: |
asdf plugin-add julia
asdf install julia 1.10.1
asdf install julia 1.10.2
- run: julia --project=test/multiversion/envs/CommonMark_1.10.1 -e 'import Pkg; Pkg.instantiate()'
env:
ASDF_JULIA_VERSION: "1.10.1"

- run: julia --project=test/multiversion/envs/CommonMark_1.10.2 -e 'import Pkg; Pkg.instantiate()'
env:
ASDF_JULIA_VERSION: "1.10.2"

- run: julia --project -e 'import Pkg; Pkg.instantiate()'
env:
ASDF_JULIA_VERSION: "1.10.2"

- run: asdf global julia 1.10.2

- run: julia --project test/multiversion/bundle.jl

# test:
# runs-on: ${{ matrix.os }}
# timeout-minutes: 60
# strategy:
# fail-fast: false
# matrix:
# version:
# - "1.10"
# os:
# - macos-latest
# - ubuntu-latest
# - windows-latest

# steps:
# - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
# with:
# persist-credentials: false
# - uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874
# with:
# version: ${{ matrix.version }}
# - uses: julia-actions/cache@dc1a3cdeacb521b0ca93cfc66143fcadb15a5bd0
# - uses: julia-actions/julia-buildpkg@90dd6f23eb49626e4e6612cb9d64d456f86e6a1c

# # `PackageBundler.bundle()` expects the environments to be bundled to have
# # already been instantiated (hence downloaded) prior to running it. We
# # don't need to precompile it though.
# - run: julia --project=test/envs/CustomEnv -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.instantiate()'

# - uses: julia-actions/julia-runtest@79a7e100883947123f8263c5f06e6c0ea3eb972f
# - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
# with:
# name: bundle-${{ matrix.os }}
# path: test/build/LocalCustomRegistry

# test-bundle:
# needs: [test]
# runs-on: ${{ matrix.os }}
# timeout-minutes: 60
# strategy:
# fail-fast: false
# matrix:
# version:
# - "1.10"
# os:
# - macos-latest
# - ubuntu-latest
# - windows-latest
# bundle:
# - bundle-macos-latest
# - bundle-ubuntu-latest
# - bundle-windows-latest

# steps:
# - uses: julia-actions/setup-julia@f2258781c657ad9b4b88072c5eeaf9ec8c370874
# with:
# version: ${{ matrix.version }}

# - uses: julia-actions/cache@dc1a3cdeacb521b0ca93cfc66143fcadb15a5bd0

# - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
# with:
# name: ${{ matrix.bundle }}
# path: ${{ matrix.bundle }}

# - name: Initialize Julia Depot
# run: julia -e 'import Pkg; Pkg.status()'

# - name: Install bundle
# run: julia ${{ matrix.bundle }}/registry/install.jl

# - name: Installing the same bundle again should uninstall and reinstall
# run: julia ${{ matrix.bundle }}/registry/install.jl

# - name: Resolve environment and precompile
# run: |
# julia --project=@CustomEnv -e '
# pushfirst!(LOAD_PATH, "@stdlib")
# import Pkg
# Pkg.status()
# Pkg.resolve()
# Pkg.precompile("CairoMakie")
# '

# - name: Load packages
# run: julia --project=@CustomEnv -e 'import CairoMakie'

# - name: Remove bundle
# run: |
# julia -e '
# for depot in DEPOT_PATH
# path = joinpath(depot, "registries", "LocalCustomRegistry", "remove.jl")
# if isfile(path)
# run(`julia $path`)
# break
# end
# end
# '
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@
*.pem
*.pub
test/build
test/multiversion/build
.DS_Store
24 changes: 12 additions & 12 deletions src/PackageBundler.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ environments = ["env1", "env2"] # required
outputs = "PackageBundle" # default: same as `name`
key = "key" # default: "key"
clean = false # default: false
multiplexer = "" # default: ""
multiplexers = ["asdf", "juliaup"] # default: []
[packages]
"<uuid>" = "<name>"
Expand Down Expand Up @@ -80,13 +80,12 @@ should be cleaned before generating the bundle. If `clean` is `true` and the
output directory exists it will be deleted before generating the bundle. `clean`
has no effect if the output is a tarball or an `Artifacts.toml` file.
The `multiplexer` field sets the environment variable that is used to specify
the version of `julia` that is run, e.g `ASDF_JULIA_VERSION`, `JULIAUP_CHANNEL`,
`MISE_JULIA_VERSION`, etc. This is only relevant if your bundle config and
environments contains multiple different Julia versions that are all used by the
same versions of packages that need to be stripped and serialized, since
serialization requires the exact version of `julia` for deserializating the
expressions correctly.
The `multiplexers` field sets the allowed multiplexer programs to use to select
specific versions of Julia for each environment based on it's `julia_version`.
`"juliaup"`, `"asdf"`, and `"mise"` are supported. This is an ordered list of
options, each of which are tried in turn until one is found that can be used to
select the correct Julia version. If no multiplexer is found then the current
`julia` is used.
"""
function bundle(
config::AbstractString = "PackageBundler.toml";
Expand Down Expand Up @@ -152,9 +151,10 @@ function bundle(
outputs = String.(vcat(outputs))
outputs = isempty(outputs) ? [name] : outputs

# Multiplexer. The environment variable used by the Julia version multiplexer
# to select the version of `julia` to run.
multiplexer = get(config, "multiplexer", "")::String
# Multiplexers. The list of multiplexer programs to try to use to select
# specific versions of Julia for each environment based on it's
# `julia_version`.
multiplexers = String.(get(Vector{String}, config, "multiplexers"))

mktempdir() do temp_dir
# Generate the bundle in a temp directory, afterwhich copy the result
Expand All @@ -167,7 +167,7 @@ function bundle(
uuid = uuid,
key_pair = (; private, public),
handlers = handlers,
multiplexer = multiplexer,
multiplexers = multiplexers,
)
for output in outputs
output = normpath(joinpath(dir, output))
Expand Down
44 changes: 35 additions & 9 deletions src/code_stripping.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ function _generate_stripped_bundle(;
uuid::Union{AbstractString,Integer,Base.UUID},
key_pair::@NamedTuple{private::String, public::String},
handlers::Dict,
multiplexer::String,
multiplexers::Vector{String},
)
output_dir = abspath(output_dir)

Expand All @@ -27,7 +27,7 @@ function _generate_stripped_bundle(;
stripped = stripped,
key_pair = key_pair,
handlers = handlers,
multiplexer = multiplexer,
multiplexers = multiplexers,
)

stripped_uuid_mapping = merge(stripped_uuid_mapping, new_stripped_uuid_mapping)
Expand Down Expand Up @@ -255,7 +255,7 @@ function _process_project_env(;
stripped::Dict{String,String},
key_pair::@NamedTuple{private::String, public::String},
handlers::Dict,
multiplexer::String,
multiplexers::Vector{String},
)
project_dir = normpath(project_dir)
output_dir = normpath(output_dir)
Expand Down Expand Up @@ -301,7 +301,7 @@ function _process_project_env(;
stripped_pkg_uuid_mapping,
key_pair,
handlers,
multiplexer,
multiplexers,
)
versions = get!(Dict{String,Any}, pkg_version_info, each_name)
versions[version_info["project"]["version"]] = version_info
Expand Down Expand Up @@ -478,7 +478,7 @@ function _strip_package(
uuid_replacements::Dict,
key_pair::@NamedTuple{private::String, public::String},
handlers::Dict,
multiplexer::String,
multiplexers::Vector{String},
)
@info "Stripping source code." package

Expand Down Expand Up @@ -588,10 +588,8 @@ function _strip_package(
TOML.print(io, Dict("julia_files" => julia_files, "handlers" => handlers))
end
script = joinpath(@__DIR__, "serializer.jl")
envs = isempty(multiplexer) ? [] : [multiplexer => "$julia_version"]
withenv(envs...) do
run(`$(Base.julia_exename()) --startup-file=no $(script) $(toml_file)`)
end
binary = _process_multiplexers(multiplexers, julia_version)
run(`$(binary) --startup-file=no $(script) $(toml_file)`)
end

# Sign all the files that have been created/modified during serialization.
Expand Down Expand Up @@ -632,3 +630,31 @@ function _stripped_source_path(
project_root = dirname(project_toml)
return joinpath("[bundled]", package_name, version, relpath(source_file, project_root))
end

function _process_multiplexers(multiplexers::Vector{String}, julia_version::VersionNumber)
for multiplexer in multiplexers
exists = Sys.which(multiplexer)
if !isnothing(exists)
if multiplexer == "juliaup"
return `julia +$(julia_version)`
elseif multiplexer == "asdf"
return withenv("ASDF_JULIA_VERSION" => "$(julia_version)") do
path = readchomp(`asdf which julia`)
return `$(path)`
end
elseif multiplexer == "mise"
return withenv("MISE_JULIA_VERSION" => "$(julia_version)") do
path = readchomp(`mise which julia`)
return `$(path)`
end
else
error("Unsupported multiplexer: $multiplexer")
end
end
end
if isempty(multiplexers)
return Base.julia_cmd()
else
error("no multiplexers found: $(repr(multiplexers))")
end
end
Loading

0 comments on commit 53d2076

Please sign in to comment.