From 498d0c30073779bc3a5353f76bd1d1f53b8dac4b Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Wed, 2 Sep 2026 13:06:31 +0300 Subject: [PATCH 01/14] resolving #33 --- src/bundlers/JuliaImg/JuliaImg.jl | 6 ++++-- src/bundlers/JuliaImg/SysImgTools.jl | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/bundlers/JuliaImg/JuliaImg.jl b/src/bundlers/JuliaImg/JuliaImg.jl index 18e4229..e546d13 100644 --- a/src/bundlers/JuliaImg/JuliaImg.jl +++ b/src/bundlers/JuliaImg/JuliaImg.jl @@ -242,7 +242,9 @@ function compile_sysimg(destination, project; # Precompile packages before sysimage creation to avoid segfaults. The sysimage builder's # aggressive AOT compilation can trigger LLVM codegen bugs on certain constant expressions # (e.g., matrix inversions in Colors.jl) that don't occur during regular precompilation. - run(`$julia_cmd --startup-file=no --pkgimages=no --project=$project --eval "import Pkg; Pkg.precompile( $(repr(string.(sysimg_packages))) )"`) + withenv("JULIA_LOAD_PATH"=>nothing, "JULIA_DEPOT_PATH"=>nothing) do + run(`$julia_cmd --startup-file=no --pkgimages=no --project=$project --eval "import Pkg; Pkg.precompile( $(repr(string.(sysimg_packages))) )"`) + end base_sysimg = "$destination/lib/julia/sys" * ".$(Libdl.dlext)" tmp_sysimg = tempname() * ".$(Libdl.dlext)" @@ -281,7 +283,7 @@ function compile_pkgimgs(destination, project; push!(LOAD_PATH, "@stdlib", joinpath(Sys.STDLIB, $(repr(stdlib_project_name)))) push!(DEPOT_PATH, joinpath(dirname(Sys.BINDIR), "share/julia")) """ - withenv("JULIA_PROJECT" => project, "USER_DATA" => mktempdir(), "JULIA_CPU_TARGET" => cpu_target) do + withenv("JULIA_PROJECT" => project, "USER_DATA" => mktempdir(), "JULIA_CPU_TARGET" => cpu_target, "JULIA_LOAD_PATH"=>nothing, "JULIA_DEPOT_PATH"=>nothing) do if use_pkg run(`$julia_cmd --eval "$init_script; import AppEnv; import Pkg; Pkg.precompile( $(repr(string.(precompiled_modules))) ) "`) else diff --git a/src/bundlers/JuliaImg/SysImgTools.jl b/src/bundlers/JuliaImg/SysImgTools.jl index 532dd22..5b9fe10 100644 --- a/src/bundlers/JuliaImg/SysImgTools.jl +++ b/src/bundlers/JuliaImg/SysImgTools.jl @@ -80,7 +80,9 @@ function create_sysimg_object_file(script::String, @debug "running $cmd" spinner = TerminalSpinners.Spinner(msg = "Compiling system image") - @monitor_oom TerminalSpinners.@spin spinner run(cmd) + withenv("JULIA_LOAD_PATH"=>nothing, "JULIA_DEPOT_PATH"=>nothing) do + @monitor_oom TerminalSpinners.@spin spinner run(cmd) + end return end @@ -89,7 +91,9 @@ function link_image(object_files, sysimg_path; julia_cmd = nothing) if isnothing(julia_cmd) Base.Linking.link_image(object_files, sysimg_path) else - run(`$julia_cmd --startup-file=no --eval "Base.Linking.link_image($(repr(object_files)), $(repr(sysimg_path)))"`) + withenv("JULIA_LOAD_PATH"=>nothing, "JULIA_DEPOT_PATH"=>nothing) do + run(`$julia_cmd --startup-file=no --eval "Base.Linking.link_image($(repr(object_files)), $(repr(sysimg_path)))"`) + end end end From fa22471e83d6aa222b7dcf961c6a544def8608af Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Wed, 2 Sep 2026 19:39:18 +0300 Subject: [PATCH 02/14] adding skipsign --- LocalPreferences.toml | 1 + src/bundle.jl | 4 ++-- src/main.jl | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/LocalPreferences.toml b/LocalPreferences.toml index 021b9de..5a1ed0a 100644 --- a/LocalPreferences.toml +++ b/LocalPreferences.toml @@ -7,6 +7,7 @@ app_description = "A longer description of the app" windowed = false compress = true selfsign = false +skipsign = false overwrite_target = false msix_path_length_threshold = 260 diff --git a/src/bundle.jl b/src/bundle.jl index f0b632e..0e24d4b 100644 --- a/src/bundle.jl +++ b/src/bundle.jl @@ -76,7 +76,7 @@ function MSIX(; skip_unicode_paths = preferences["msix_skip_unicode_paths"], selfsign = preferences["selfsign"], publisher = preferences["msix_publisher"] |> normalize_publisher, #get_publisher(pfx_cert, selfsign), - pfx_cert = get_path(prefix, "msix/certificate.pfx"), # We actually want the warning + pfx_cert = preferences["skipsign"] ? nothing : get_path(prefix, "msix/certificate.pfx"), # We actually want the warning windowed = preferences["windowed"], compress = preferences["compress"], arch = Sys.ARCH, @@ -252,7 +252,7 @@ function DMG(; entitlements = get_path(prefix, "dmg/Entitlements.plist"), dsstore = get_path(prefix, ["dmg/DS_Store.toml", "dmg/DS_Store"]), selfsign = preferences["selfsign"], - pfx_cert = get_path(prefix, "dmg/certificate.pfx"), + pfx_cert = preferences["skipsign"] ? nothing : get_path(prefix, "dmg/certificate.pfx"), shallow_signing = preferences["dmg_shallow_signing"], hardened_runtime = preferences["dmg_hardened_runtime"], sandboxed_runtime = preferences["dmg_sandboxed_runtime"], diff --git a/src/main.jl b/src/main.jl index a27aa89..f7b998a 100644 --- a/src/main.jl +++ b/src/main.jl @@ -56,6 +56,7 @@ function main_build(ARGS; sources_dir) #compress = config[:compress] #windowed = config[:windowed] selfsign = preferences["selfsign"] + skipsign = preferences["skipsign"] overwrite_target = preferences["overwrite_target"] bundler = preferences["bundler"] @@ -100,7 +101,7 @@ function main_build(ARGS; sources_dir) msix = MSIX(sources_dir; arch = target_arch, preferences) - if selfsign + if selfsign || skipsign password = "" elseif isnothing(msix.pfx_cert) error("No pfx certificate found and selfsign is disabled. Enable self signing with `--selfsign` or generate pfx certificates") @@ -115,7 +116,7 @@ function main_build(ARGS; sources_dir) dmg = DMG(sources_dir; arch = target_arch, preferences) - if selfsign + if selfsign || skipsign password = "" elseif isnothing(dmg.pfx_cert) error("No pfx certificate found and selfsign is disabled. Enable self signing with `--selfsign` or generate pfx certificates") @@ -295,6 +296,8 @@ function parse_args(raw_args) #; preferences = Base.get_preferences()["AppBundle config[:target_name] = args[i] elseif arg == "--selfsign" preferences["selfsign"] = true + elseif arg == "--skipsign" + preferences["skipsign"] = true elseif arg == "--password" i += 1 config[:password] = args[i] |> strip From 5e60f4109c9ffc03077b9365283a83e57a5c68a8 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Fri, 4 Sep 2026 20:15:11 +0300 Subject: [PATCH 03/14] AppImage prototype --- recipes/appimage/main.desktop | 9 ++ src/AppBundler.jl | 1 + src/AppImage/AppImagePack.jl | 241 ++++++++++++++++++++++++++++++++++ test/appimage.jl | 143 ++++++++++++++++++++ 4 files changed, 394 insertions(+) create mode 100644 recipes/appimage/main.desktop create mode 100644 src/AppImage/AppImagePack.jl create mode 100644 test/appimage.jl diff --git a/recipes/appimage/main.desktop b/recipes/appimage/main.desktop new file mode 100644 index 0000000..e7a5fba --- /dev/null +++ b/recipes/appimage/main.desktop @@ -0,0 +1,9 @@ +[Desktop Entry] +Type=Application +Name={{APP_DISPLAY_NAME}} +Comment={{APP_SUMMARY}} +Exec=AppRun +Icon={{APP_NAME}} +Categories=Utility; +Terminal=true +#X-AppImage-Version=$(APP_VERSION) \ No newline at end of file diff --git a/src/AppBundler.jl b/src/AppBundler.jl index d70edad..2bdf534 100644 --- a/src/AppBundler.jl +++ b/src/AppBundler.jl @@ -28,6 +28,7 @@ include("DMG/HFS.jl") include("DMG/DMGPack.jl") include("Snap/SnapPack.jl") +include("AppImage/AppImagePack.jl") include("MSIX/MSIXPack.jl") include("MSIX/MSIXIcons.jl") diff --git a/src/AppImage/AppImagePack.jl b/src/AppImage/AppImagePack.jl new file mode 100644 index 0000000..803b5bc --- /dev/null +++ b/src/AppImage/AppImagePack.jl @@ -0,0 +1,241 @@ +""" + AppImage + +Pack an AppDir into a type-2 AppImage and unpack it again. + +The format is a prebuilt static ELF runtime with a SquashFS image concatenated +onto it. The runtime derives the payload offset from its own ELF size, so +packing is a compress-and-concatenate with no header fixup; unpacking has to +recover that offset before `unsquashfs` can find the superblock. +""" +module AppImagePack + +import squashfs_tools_jll: mksquashfs, unsquashfs +using Downloads +using Scratch: @get_scratch! + +export pack, unpack + +const RUNTIME_TAG = "continuous" + +"Runtime architectures published by AppImage/type2-runtime." +const RUNTIME_ARCHS = ("x86_64", "i686", "aarch64", "armhf") + +runtime_url(arch) = "https://github.com/AppImage/type2-runtime/releases/download/" * + "$(RUNTIME_TAG)/runtime-$(arch)" + +""" + host_arch() -> String + +The runtime architecture matching the host, using AppImage's naming rather +than Julia's. Only meaningful for a Linux host; cross-packaging should pass +`arch` explicitly. +""" +function host_arch() + a = Sys.ARCH + a === :x86_64 && return "x86_64" + a === :i686 && return "i686" + a === :aarch64 && return "aarch64" + a === :arm && return "armhf" + a === :armv7l && return "armhf" + error("no AppImage runtime for host architecture $a; pass arch= explicitly") +end + +""" + runtime_cache() -> String + +Scratch space holding downloaded runtimes between builds, keyed to this +package so `Pkg.gc()` can reclaim it and `Scratch.delete_scratch!` can clear +it. The path is resolved on each call rather than stored in a `const`, since +baking a scratch path into precompiled code is not safe. + +Runtimes are re-downloaded if the space is garbage collected, so treat this +purely as a cache. +""" +runtime_cache() = @get_scratch!("runtimes") + +# ---------------------------------------------------------------- runtime -- + +""" + fetch_runtime(arch, cachedir) -> String + +Download the prebuilt type-2 runtime for `arch` (cached in `cachedir`), and +check that it is an ELF carrying the AppImage magic `AI\\x02` at offset 8. +""" +function fetch_runtime(arch::AbstractString, cachedir::AbstractString) + arch in RUNTIME_ARCHS || + error("unknown runtime architecture $arch; expected one of $(join(RUNTIME_ARCHS, ", "))") + + mkpath(cachedir) + dest = joinpath(cachedir, "runtime-$(arch)") + if !isfile(dest) || filesize(dest) == 0 + @info "Downloading runtime-$(arch)" + tmp = dest * ".part" + try + Downloads.download(runtime_url(arch), tmp) + mv(tmp, dest; force = true) + finally + rm(tmp; force = true) + end + else + @info "Using cached runtime-$(arch)" + end + + header = open(io -> read(io, 11), dest, "r") + length(header) == 11 || error("runtime is truncated") + header[1:4] == UInt8[0x7f, 0x45, 0x4c, 0x46] || error("runtime is not an ELF file") + header[9:11] == UInt8[0x41, 0x49, 0x02] || + error("runtime lacks the AI\\x02 magic at offset 8") + + return dest +end + +# ------------------------------------------------------------------ offset -- + +""" + elf_size(path) -> Int + +Size of the ELF image at `path`, computed from its header as +`e_shoff + e_shentsize * e_shnum`. For an AppImage runtime this is where the +appended SquashFS payload begins -- the same arithmetic the runtime itself +performs at startup, so it does not require executing the file. +""" +function elf_size(path::AbstractString) + hdr = open(io -> read(io, 64), path, "r") + length(hdr) >= 64 || error("$path is too short to be an ELF file") + hdr[1:4] == UInt8[0x7f, 0x45, 0x4c, 0x46] || error("$path is not an ELF file") + hdr[6] == 0x01 || error("only little-endian ELF is supported") + + # e_shoff / e_shentsize / e_shnum sit at different offsets for ELF32/ELF64. + read_at(T, off) = only(reinterpret(T, hdr[off+1 : off+sizeof(T)])) + + if hdr[5] == 0x02 # ELFCLASS64 + shoff = Int(read_at(UInt64, 0x28)) + shentsize = Int(read_at(UInt16, 0x3A)) + shnum = Int(read_at(UInt16, 0x3C)) + elseif hdr[5] == 0x01 # ELFCLASS32 + shoff = Int(read_at(UInt32, 0x20)) + shentsize = Int(read_at(UInt16, 0x2E)) + shnum = Int(read_at(UInt16, 0x30)) + else + error("unknown ELF class $(hdr[5])") + end + + return shoff + shentsize * shnum +end + +""" + payload_offset(appimage) -> Int + +Byte offset of the SquashFS payload inside `appimage`, verified against the +`hsqs` superblock magic. +""" +function payload_offset(appimage::AbstractString) + off = elf_size(appimage) + magic = open(appimage, "r") do io + seek(io, off) + read(io, 4) + end + magic == UInt8[0x68, 0x73, 0x71, 0x73] || + error("no SquashFS superblock at offset $off; is $appimage a type-2 AppImage?") + return off +end + +# --------------------------------------------------------------- squashfs -- + +""" + compress(appdir, payload; comp="zstd") -> String + +Compress `appdir` into a SquashFS image at `payload`. Timestamps are pinned to +the epoch and ownership forced to root so repeated builds are bit-identical; +`-all-root` overrides ownership only, not the exec bit or symlinks already +staged in `appdir`. +""" +function compress(appdir::AbstractString, payload::AbstractString; + comp::AbstractString = "zstd") + rm(payload; force = true) + args = String[ + "-noappend", "-no-progress", "-quiet", + "-all-root", "-no-xattrs", + "-comp", comp, "-b", "128K", + "-mkfs-time", "0", "-all-time", "0", + ] + @info "Packing SquashFS payload ($comp)" + run(`$(mksquashfs()) $appdir $payload $args`) + return payload +end + +# ----------------------------------------------------------------- concat -- + +""" + concatenate(runtime, payload, output) -> String + +`runtime * payload`, streamed in chunks. The runtime derives the payload +offset from its own ELF size, so no header fixup is needed. +""" +function concatenate(runtime::AbstractString, payload::AbstractString, + output::AbstractString) + buf = Vector{UInt8}(undef, 1024 * 1024) + open(output, "w") do out + for part in (runtime, payload) + open(part, "r") do io + while !eof(io) + n = readbytes!(io, buf) + write(out, view(buf, 1:n)) + end + end + end + end + chmod(output, 0o755) + return output +end + +# ------------------------------------------------------------ pack/unpack -- + +""" + pack(source, destination; arch=host_arch(), comp="zstd", cache=runtime_cache()) -> String + +Package the AppDir at `source` into an AppImage at `destination`. + +`source` must already be a valid AppDir: an executable `AppRun` at its root, +plus the `.desktop` file, icon and `.DirIcon` the format expects. Nothing here +edits the tree. +""" +function pack(source::AbstractString, destination::AbstractString; + arch::AbstractString = host_arch(), + comp::AbstractString = "zstd", + cache::AbstractString = runtime_cache()) + + isdir(source) || error("AppDir not found: $source") + isfile(joinpath(source, "AppRun")) || error("$source has no AppRun at its root") + + runtime = fetch_runtime(arch, cache) + + mktempdir() do work + payload = compress(source, joinpath(work, "payload.squashfs"); comp = comp) + concatenate(runtime, payload, destination) + end + + @info "Built $(basename(destination))" bytes = filesize(destination) payload_offset = filesize(runtime) + + return destination +end + +""" + unpack(source, destination) -> String + +Extract the AppDir from the AppImage at `source` into `destination`. + +`unsquashfs` cannot find the superblock on its own here, since the payload is +preceded by the runtime -- hence the explicit `-offset`. +""" +function unpack(source::AbstractString, destination::AbstractString) + isfile(source) || error("AppImage not found: $source") + offset = payload_offset(source) + + run(`$(unsquashfs()) -offset $offset -force -quiet -no-progress -dest $destination $source`) + + return destination +end + +end # module diff --git a/test/appimage.jl b/test/appimage.jl new file mode 100644 index 0000000..e9776c7 --- /dev/null +++ b/test/appimage.jl @@ -0,0 +1,143 @@ +#!/usr/bin/env julia +# +# example.jl -- stage a hello-world AppDir and package it with `AppImage.pack`. +# +# `pack` deliberately does not touch the tree it is given, so building a valid +# AppDir is this script's job: an executable `AppRun` at the root, plus the +# `.desktop` file, icon and `.DirIcon` the format expects. +# +# The exec bit and the root symlinks are set with `chmod`/`symlink`, which +# requires a genuine POSIX filesystem -- fine for a Linux-hosted build, but not +# portable to a Windows host. Packaging cross-platform would stage the same +# tree without those calls and hand the metadata to mksquashfs as a +# pseudo-file instead. +# +# julia --project example.jl +# +# Set APPIMAGE_ARCH to cross-target another runtime (i686, aarch64, armhf). + +using AppBundler.AppImagePack +using Base64 + +const APP_NAME = "HelloWorld" +const APP_ID = "helloworld" # desktop file / icon basename +const APP_VERSION = "1.0.0" + +# ----------------------------------------------------------------- assets -- + +const APPRUN = """ +#!/bin/bash +echo "Hello World" +echo +echo " APPDIR = \${APPDIR:-}" +echo " APPIMAGE = \${APPIMAGE:-}" +echo " OWD = \${OWD:-}" +""" + +const DESKTOP = """ +[Desktop Entry] +Type=Application +Name=$(APP_NAME) +Comment=Minimal AppImage built from Julia +Exec=AppRun +Icon=$(APP_ID) +Categories=Utility; +Terminal=true +X-AppImage-Version=$(APP_VERSION) +""" + +# 128x128 PNG, embedded so the script carries no side assets. +const ICON_PNG_BASE64 = replace(""" +iVBORw0KGgoAAAANSUhEUgAAAIAAAACABAMAAAAxEHz4AAAAHlBMVEUmMkoFBQV90aUjM0klMkmC26tmp40lMkoAAAAAAACkdcZr +AAAACHRSTlP+AP8Od///ouV+0goAAAMPSURBVHja7ZnPaxNBFMe/O5M0aaJ2Y64KWwterLKh4EGaEvHgRaSnnoQGxJMggv03hN48 +iJBePQV66s2TqKBZqL+gtAREvDRmFY0k6e542E2ySTY68ybHzCkb8j68z9uZt5sZIwe9wTBFgEsBJPqfhGHiglSMX48FGKywvC0F +SBUbdWH0wsIiMn9pdVs+8RvvMALAxddK6lvPXBMAwOeD+mUP1Gr38v6n6F0Q5h3V6j+2hxSu7infv1QmkgFXj0d7cwBwlylT6MkA +cPYFBdAu92sgSJMYiVNGkIFVpq2jk0U3yIAfE1ci900wAOeoSzlhAgwQJSqgbQEM4NvkbmICDEKjHdkujJywHDIgcRoMhnZPNOnR +3AUDbDqgbeu3deTyOtHXc5oZaCs40320zQAzwAwwXcCcLkBsaALO18paAGZ7v7UAdx283dABNAAoSIwBkg4A7ycd0OUAsG/RFQ5t +AF6yTAb4eahIxMyDp1CRiFsLX1Qk4gBdFYnY1bhjy0vEAjq78hLx/SCUaNEbSiBRWycDQokPZSpAXmJiT5SVmAgIJRgZgK4DAPsa +bb2u+VyYW9cEbDoAUCADkg0A4Bn5DYgRgVsOAH6pQs0gFKhSayArMElBWmBSBoHA5Sr1NoYCZ9S2gcYEuhXqRAoFyFOZKQjEApYU +BOIA9zwFgRiAmkDMXQi+iAgMb9Bdqf4vgw4ArEQEuGoNdgGejVx7qgDDRqEC6TEO6ORXnuu9aFaymm+q3YomYHSTQRfg0ZrqYBz9 +42r2n2kGmAGmBbDBgIcaAAd8Xki+EMaOH7oKFhjxeCkYqToYdAw8gMl0tYlDBHvrdMADaNbAccEA3yYDaiYYsPCNGs+DqWycUAFr +4RnL9xJxFrgIztrSxy0a4U9vNQraeiq56J21tdqE+MSC3+8HGUoKRR/onze+V6/Coz0RATRvHqoKfGymIy3NfKMokSqGRythBnAP +1lRySF1zerMxBKTF19Xbr2Tjt359tsI1aETO3tmiKRffPILb+2kE4JqU2RRpqqZCH9J7sAydjf0FmzPAK8E+tygAAAAASUVORK5CYII= +""", "\n" => "") + +# ----------------------------------------------------------------- AppDir -- + +""" + build_appdir(root) -> String + +Lay out a complete AppDir under `root`, ready to hand to `AppImage.pack`. +""" +function build_appdir(root::AbstractString) + appdir = joinpath(root, "$(APP_NAME).AppDir") + rm(appdir; force = true, recursive = true) + + deskdir = joinpath(appdir, "usr", "share", "applications") + icondir = joinpath(appdir, "usr", "share", "icons", "hicolor", "128x128", "apps") + mkpath(deskdir) + mkpath(icondir) + + apprun = joinpath(appdir, "AppRun") + write(apprun, APPRUN) + chmod(apprun, 0o755) + + desktop_rel = joinpath("usr", "share", "applications", "$(APP_ID).desktop") + icon_rel = joinpath("usr", "share", "icons", "hicolor", "128x128", "apps", "$(APP_ID).png") + + write(joinpath(appdir, desktop_rel), DESKTOP) + write(joinpath(appdir, icon_rel), base64decode(ICON_PNG_BASE64)) + + # Root-level entries the AppImage format expects. + symlink(desktop_rel, joinpath(appdir, "$(APP_ID).desktop")) + symlink(icon_rel, joinpath(appdir, "$(APP_ID).png")) + symlink("$(APP_ID).png", joinpath(appdir, ".DirIcon")) + + return appdir +end + +# ------------------------------------------------------------------- main -- + +""" + verify(appimage) + +Round-trip the result through `AppImage.unpack` and check the entries the +runtime depends on actually survived into the payload. +""" +function verify(appimage::AbstractString) + mktempdir() do dir + extracted = unpack(appimage, joinpath(dir, "AppDir")) + + apprun = joinpath(extracted, "AppRun") + isfile(apprun) || error("AppRun missing from payload") + (stat(apprun).mode & 0o111) != 0 || error("AppRun is not executable in the payload") + + for entry in (".DirIcon", "$(APP_ID).desktop", "$(APP_ID).png") + islink(joinpath(extracted, entry)) || error("root entry $entry missing or not a symlink") + end + + @info "Payload verified" entries = sort(readdir(extracted; join = false)) + end + return nothing +end + +function main(; workdir::AbstractString = joinpath(@__DIR__, "build"), + arch::AbstractString = get(ENV, "APPIMAGE_ARCH", AppImagePack.host_arch())) + mkpath(workdir) + + appdir = build_appdir(workdir) + output = joinpath(@__DIR__, "$(APP_NAME)-$(APP_VERSION)-$(arch).AppImage") + + pack(appdir, output; arch = arch) + verify(output) + + println() + println("Try it:") + println(" ", output) + println(" APPIMAGE_EXTRACT_AND_RUN=1 ", output, " # no FUSE (containers, CI)") + + return output +end + + +main() From 8a198be7e0c66491d144739f05c827c1d99389e1 Mon Sep 17 00:00:00 2001 From: s-celles Date: Sat, 12 Sep 2026 22:21:19 +0300 Subject: [PATCH 04/14] Add AppImage docs and CLI target (from #46) --- docs/src/appimage.md | 130 +++++++++++++++++++++ docs/src/reference.md | 14 ++- recipes/appimage/AppRun.sh | 8 ++ recipes/appimage/juliaimg_AppRun.sh | 20 ++++ recipes/appimage/main.desktop | 10 +- recipes/appimage/metainfo.xml | 16 +++ src/AppImage/AppImagePack.jl | 7 ++ src/bundle.jl | 168 ++++++++++++++++++++++++++++ src/main.jl | 3 +- src/recipes.jl | 57 ++++++++++ 10 files changed, 423 insertions(+), 10 deletions(-) create mode 100644 docs/src/appimage.md create mode 100644 recipes/appimage/AppRun.sh create mode 100644 recipes/appimage/juliaimg_AppRun.sh create mode 100644 recipes/appimage/metainfo.xml diff --git a/docs/src/appimage.md b/docs/src/appimage.md new file mode 100644 index 0000000..075f2e9 --- /dev/null +++ b/docs/src/appimage.md @@ -0,0 +1,130 @@ +# AppImage + +An AppImage is a single executable file that is **mounted** rather than installed. For a Julia +application this matters more than it might seem: a bundled distribution is tens of thousands of +small files, and writing all of them onto a network filesystem is the slow part of every other +format. On HPC clusters that cost — sometimes called the hydration problem — is what stops people +distributing Julia applications as archives at all. An AppImage never unpacks, so it does not pay it. + +``` +appbundler build . --target-bundle=appimage --build-dir=build +``` + +produces `build/--.AppImage`, which a user makes executable and runs. There is +nothing to install and nothing to clean up afterwards. + +## How the format works + +An AppImage is a **runtime ELF followed by a squashfs image**. The runtime knows the filesystem +begins immediately after itself, mounts it through FUSE, and executes `AppRun` from the mount +point. That is the whole format — there is no container, no manifest, and no package database. + +AppBundler stages an AppDir, then has `mksquashfs` write the filesystem straight into the output +file after a reserved prefix, and finally writes the runtime into that prefix. Building the +squashfs separately and concatenating would need a second full-size temporary copy, which for a +Julia distribution is hundreds of megabytes. + +The staged AppDir looks like this: + +``` +AppRun entry point, execs bin/julia +.desktop Icon=, no path, no extension +.png +.DirIcon what file managers read for the thumbnail +usr/share/applications/.desktop menu integration once installed +usr/share/icons/hicolor/256x256/apps/.png +usr/share/metainfo/.appdata.xml AppStream metadata +bin/ lib/ share/ etc/ the Julia distribution +``` + +The payload is compressed with **zstd** by default, and `gzip` is the only other choice. The +runtime bundles squashfuse built against zstd and zlib alone, so an image compressed any other way +cannot be mounted — `mksquashfs` happily produces `xz` and `lz4` images, and the runtime answers +`Failed to extract AppImage` for both. AppBundler therefore rejects them at configuration time +rather than letting you ship an AppImage nobody can run. Of the two that work, zstd decompresses +far faster, which is the point of mounting rather than unpacking. + +## Obtaining the runtime + +The runtime is a ~900 KB static binary. There is no jll for it yet — packaging one means first +packaging `libfuse` and `squashfuse`, neither of which exists in Yggdrasil — so for now point +AppBundler at a runtime you obtained yourself: + +```toml +# LocalPreferences.toml +[AppBundler] +appimage_runtime = "/path/to/runtime-x86_64" +``` + +or pass it directly: + +```julia +AppImage(app_dir; runtime = "/path/to/runtime-x86_64") +``` + +Signed runtimes are published at +[AppImage/type2-runtime](https://github.com/AppImage/type2-runtime/releases). Prefer a dated tag +over `continuous` so your builds stay reproducible. + +Once `AppImageRuntime_jll` is registered, leaving `appimage_runtime` empty will pick it up +automatically. Note that a jll provides the artifact for the host platform, so building an AppImage +for another architecture still needs an explicit path. + +## Build-time disk space + +The AppDir is staged uncompressed before it is packed, so building an AppImage of a Julia +distribution needs a couple of gigabytes of working space. Julia stages into `TMPDIR`, which on +many systems is a `tmpfs` sized well below that, and `mksquashfs` reports exhaustion only as +`FATAL ERROR: Probably out of space on output filesystem`. If you hit that, point `TMPDIR` at a +real filesystem: + +``` +TMPDIR=/var/tmp appbundler build . --target-bundle=appimage --build-dir=build +``` + +## Where the depot goes + +The mounted filesystem is read-only and disappears when the application exits, so the Julia depot +has to live somewhere else. Two options, selected with the `appimage_depot` preference: + +- **`"app"`** (default) — `AppRun` points `USER_DATA` at `$XDG_DATA_HOME/`, falling back to + `~/.local/share/`. The application gets a persistent per-user depot and the host's `~/.julia` + is never touched. +- **`"julia"`** — the stock depot is left as Julia found it, with the bundled `share/julia` + appended so the shipped packages stay resolvable. Choose this where users expect the application + to see the environments they already have, which is common on HPC. + +## FUSE + +Mounting needs `fusermount` on the machine running the AppImage. Where it is missing the runtime +says so and suggests `--appimage-extract-and-run`, which extracts to a temporary directory first — +correct, but it gives up the property that made the format attractive. + +If your target machines lack FUSE, the payload can be read directly, since it is an ordinary +squashfs at a known offset: + +```julia +using AppBundler +AppBundler.AppImagePack.unpack("MyApp-1.0.0-x86_64.AppImage", "extracted/") +``` + +or, outside Julia: + +``` +unsquashfs -o $(./MyApp-1.0.0-x86_64.AppImage --appimage-offset) -d extracted MyApp-1.0.0-x86_64.AppImage +``` + +`--appimage-offset` and `AppBundler.AppImagePack.offset` return the same number: the size of the +runtime prefix. + +## API + +```@docs +AppBundler.AppImagePack +AppBundler.AppImagePack.pack +AppBundler.AppImagePack.offset +AppBundler.AppImagePack.unpack +AppBundler.AppImageRuntime +AppBundler.AppImageRuntime.resolve +AppBundler.bundle(::AppBundler.JuliaImgBundle, ::AppBundler.AppImage, ::String) +``` diff --git a/docs/src/reference.md b/docs/src/reference.md index f3a7928..f7c7967 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -21,14 +21,17 @@ Both specs can be staged directly into a directory for inspection before packagi ## Bundle Formats -A bundle format defines the packaging target. The three supported formats are `DMG` (macOS), `MSIX` (Windows), and `Snap` (Linux). They are instantiated from a project directory: +A bundle format defines the packaging target. The supported formats are `DMG` (macOS), `MSIX` (Windows), and `Snap` and `AppImage` (Linux). They are instantiated from a project directory: ```julia -dmg = DMG(project; arch = Sys.ARCH, kwargs...) -msix = MSIX(project; arch = Sys.ARCH, kwargs...) -snap = Snap(project; arch = Sys.ARCH, kwargs...) +dmg = DMG(project; arch = Sys.ARCH, kwargs...) +msix = MSIX(project; arch = Sys.ARCH, kwargs...) +snap = Snap(project; arch = Sys.ARCH, kwargs...) +appimage = AppImage(project; arch = Sys.ARCH, kwargs...) ``` +`AppImage` produces a single mountable file rather than an installed tree; see [AppImage](@ref). + Each format reads configuration file overrides from the corresponding `project/meta/` directory and carries architecture information that determines the destination platform. Bundle formats can also be staged independently via `stage(format, destination)` to produce the directory structure before compression and signing. ### Low-Level Bundle API @@ -51,6 +54,7 @@ AppBundler.JuliaCBundle AppBundler.DMG AppBundler.MSIX AppBundler.Snap +AppBundler.AppImage ``` ## Functions @@ -60,5 +64,7 @@ AppBundler.stage(::AppBundler.JuliaImg.JuliaImgBundle, ::String) AppBundler.stage(::AppBundler.JuliaC.JuliaCBundle, ::String) AppBundler.stage(::AppBundler.MSIX, ::String) AppBundler.bundle(::Function, ::AppBundler.DMG, ::String) +AppBundler.bundle(::Function, ::AppBundler.MSIX, ::String) +AppBundler.bundle(::AppBundler.JuliaC.JuliaCBundle, ::AppBundler.AppImage, ::String) AppBundler.bundle(::AppBundler.JuliaImgBundle, ::AppBundler.DMG, ::String) ``` diff --git a/recipes/appimage/AppRun.sh b/recipes/appimage/AppRun.sh new file mode 100644 index 0000000..e676ac9 --- /dev/null +++ b/recipes/appimage/AppRun.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Generic entry point for a non-Julia payload. + +if [ -z "${APPDIR}" ]; then + APPDIR=$(cd -P "$(dirname "$(readlink -f "$0")")" >/dev/null 2>&1 && pwd) +fi + +exec "${APPDIR}/bin/{{APP_NAME}}" "$@" diff --git a/recipes/appimage/juliaimg_AppRun.sh b/recipes/appimage/juliaimg_AppRun.sh new file mode 100644 index 0000000..d7e96eb --- /dev/null +++ b/recipes/appimage/juliaimg_AppRun.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Entry point the AppImage runtime executes after mounting the bundled filesystem. +# +# $APPDIR is set by the runtime and points at the mount point. It is absent when the AppDir is +# executed directly (during a build, or after --appimage-extract), so fall back to resolving it +# from this script's own location. + +if [ -z "${APPDIR}" ]; then + APPDIR=$(cd -P "$(dirname "$(readlink -f "$0")")" >/dev/null 2>&1 && pwd) +fi + +{{#APP_DEPOT}} +# Persist the depot under a per-user directory rather than the mount point, which is read-only and +# disappears when the AppImage exits. AppEnv reads USER_DATA in startup.jl to place the depot. +if [ -z "${USER_DATA}" ]; then + export USER_DATA="${XDG_DATA_HOME:-${HOME}/.local/share}/{{APP_NAME}}" +fi +{{/APP_DEPOT}} + +exec "${APPDIR}/bin/julia" {{#MODULE_NAME}}--eval="using {{MODULE_NAME}}" -- {{/MODULE_NAME}}"$@" diff --git a/recipes/appimage/main.desktop b/recipes/appimage/main.desktop index e7a5fba..8f39fe8 100644 --- a/recipes/appimage/main.desktop +++ b/recipes/appimage/main.desktop @@ -1,9 +1,9 @@ [Desktop Entry] -Type=Application Name={{APP_DISPLAY_NAME}} -Comment={{APP_SUMMARY}} -Exec=AppRun +Exec={{APP_NAME}} Icon={{APP_NAME}} +Version={{APP_VERSION}} +Comment={{APP_SUMMARY}} +Terminal={{#WINDOWED}}false{{/WINDOWED}}{{^WINDOWED}}true{{/WINDOWED}} +Type=Application Categories=Utility; -Terminal=true -#X-AppImage-Version=$(APP_VERSION) \ No newline at end of file diff --git a/recipes/appimage/metainfo.xml b/recipes/appimage/metainfo.xml new file mode 100644 index 0000000..e5fd6ea --- /dev/null +++ b/recipes/appimage/metainfo.xml @@ -0,0 +1,16 @@ + + + {{BUNDLE_IDENTIFIER}} + {{APP_DISPLAY_NAME}} + {{APP_SUMMARY}} + +

{{APP_DESCRIPTION}}

+ + CC0-1.0 + LicenseRef-proprietary + {{PUBLISHER_DISPLAY_NAME}} + {{APP_NAME}}.desktop + + + + diff --git a/src/AppImage/AppImagePack.jl b/src/AppImage/AppImagePack.jl index 803b5bc..b460a2b 100644 --- a/src/AppImage/AppImagePack.jl +++ b/src/AppImage/AppImagePack.jl @@ -21,6 +21,9 @@ const RUNTIME_TAG = "continuous" "Runtime architectures published by AppImage/type2-runtime." const RUNTIME_ARCHS = ("x86_64", "i686", "aarch64", "armhf") + +# ToDo: +# - replace this with Artifacts.toml source runtime_url(arch) = "https://github.com/AppImage/type2-runtime/releases/download/" * "$(RUNTIME_TAG)/runtime-$(arch)" @@ -56,6 +59,9 @@ runtime_cache() = @get_scratch!("runtimes") # ---------------------------------------------------------------- runtime -- + +# It is not possible to use Artifacts.toml because it expects artifacts to be placed in tarballs +# The best option now is to wait for https://github.com/JuliaPackaging/Yggdrasil/pull/14695 to be merged """ fetch_runtime(arch, cachedir) -> String @@ -81,6 +87,7 @@ function fetch_runtime(arch::AbstractString, cachedir::AbstractString) @info "Using cached runtime-$(arch)" end + # Belongs to the tests header = open(io -> read(io, 11), dest, "r") length(header) == 11 || error("runtime is truncated") header[1:4] == UInt8[0x7f, 0x45, 0x4c, 0x46] || error("runtime is not an ELF file") diff --git a/src/bundle.jl b/src/bundle.jl index 0e24d4b..88a577f 100644 --- a/src/bundle.jl +++ b/src/bundle.jl @@ -178,6 +178,11 @@ function Snap(overlay; preferences = preferences(), kwargs...) return snap end + + + + + # TODO: mention that application needs to be notarized by Apple. That can be done outside the build process by stapling already signed DMG archive. """ @@ -606,3 +611,166 @@ function bundle(setup::Function, snap::Snap, destination::String; force = false) return end + + +""" + AppImage([overlay]; arch, compress, compression, depot, runtime, kwargs...) + +Create an AppImage configuration object for Linux application packaging. + +An AppImage is a runtime ELF followed by a squashfs image. The runtime **mounts** that filesystem +rather than extracting it, so a bundled Julia distribution — tens of thousands of small files — +never lands on disk. That is what makes the format worth having on HPC, where unpacking a tarball +onto a network filesystem is the expensive part. + +Mounting requires `fusermount` on the target machine. Where it is missing the runtime falls back to +`--appimage-extract-and-run`, and [`AppBundler.AppImagePack.unpack`](@ref) reads the payload without +running the runtime at all. + +# Arguments +- `overlay`: Path to a project directory containing `Project.toml`, optional `LocalPreferences.toml`, and optional `meta/appimage/` overrides + +# Keyword Arguments +- `prefix = joinpath(dirname(@__DIR__), "recipes")`: Base directory or array of directories to search for configuration files in sequential order +- `icon = get_path(prefix, ["appimage/icon.png", "icon.png"])`: Path to application icon file +- `desktop_launcher = get_path(prefix, "appimage/main.desktop")`: Path to the desktop entry template +- `metainfo = get_path(prefix, "appimage/metainfo.xml")`: Path to the AppStream metadata template +- `main_launcher`: Path to the `AppRun` template; resolved from prefix using the bundler predicate +- `startup_file = get_path(prefix, "appimage/startup.jl")`: Startup file used when `depot = "julia"`, which sets up the load path without replacing `DEPOT_PATH` +- `depot`: Where a Julia payload keeps its depot. `"app"` (default) points `USER_DATA` at + `\$XDG_DATA_HOME/`, giving a persistent per-user depot that leaves the host `~/.julia` + untouched; `"julia"` keeps the stock depot; defaults to the `appimage_depot` preference +- `compression`: squashfs compressor, one of `:zstd` (default), `:gzip` or `:xz`; defaults to the + `appimage_compression` preference +- `runtime`: Path to the AppImage runtime. When unset, `AppImageRuntime_jll` is used if installed; + defaults to the `appimage_runtime` preference +- `windowed`: If `true`, the application runs without a console window; defaults to `windowed` preference +- `compress`: If `true`, pack the AppDir into an `.AppImage`; defaults to `compress` preference +- `arch = Sys.ARCH`: Target CPU architecture +- `predicate`: Bundler predicate used for hook selection; defaults to `bundler` preference +- `parameters`: Dictionary of parameters for Mustache template rendering. When `overlay` is provided, pre-populated from `Project.toml` and preferences + +# Examples +```julia +AppImage(app_dir) +AppImage(app_dir; depot = "julia") # keep the stock ~/.julia depot +AppImage(app_dir; runtime = "/path/to/runtime-x86_64") # until AppImageRuntime_jll exists +``` +""" +struct AppImage + icon::String + desktop_launcher::String + metainfo::String + main_launcher::Union{String, Nothing} + startup_file::Union{String, Nothing} + depot::String + compression::Symbol + runtime::Union{String, Nothing} + windowed::Bool + compress::Bool + arch::Symbol + predicate::String + parameters::Dict{String, Any} +end + +const APPIMAGE_DEPOTS = ["app", "julia"] + +function AppImage(; + prefix = joinpath(dirname(@__DIR__), "recipes"), + preferences = preferences(), + predicate = preferences["bundler"], + icon = get_path(prefix, ["appimage/icon.png", "icon.png"]), + desktop_launcher = get_path(prefix, "appimage/main.desktop"), + metainfo = get_path(prefix, "appimage/metainfo.xml"), + main_launcher = get_path(prefix, hook("appimage/AppRun.sh", predicate); warn = false), + startup_file = get_path(prefix, "appimage/startup.jl"; warn = false), + depot = get(preferences, "appimage_depot", "app"), + compression = Symbol(get(preferences, "appimage_compression", "zstd")), + runtime = get(preferences, "appimage_runtime", ""), + windowed = preferences["windowed"], + compress = preferences["compress"], + arch = Sys.ARCH, + parameters = Dict{String, Any}("WINDOWED" => windowed) + ) + + depot in APPIMAGE_DEPOTS || + error("`appimage_depot` must be one of: " * join(APPIMAGE_DEPOTS, ", ") * ". Got `$depot`.") + + compression in AppImagePack.COMPRESSORS || + error("`appimage_compression` must be one of: " * + join(AppImagePack.COMPRESSORS, ", ") * ". Got `$compression`.") + + # The AppRun template branches on this rather than on the preference string, so the + # rendered launcher only carries the lines that apply. + parameters["APP_DEPOT"] = depot == "app" + + return AppImage(icon, desktop_launcher, metainfo, main_launcher, startup_file, depot, compression, + isempty(something(runtime, "")) ? nothing : runtime, + windowed, compress, arch, predicate, parameters) +end + +function AppImage(overlay; preferences = preferences(), kwargs...) + + prefix = [overlay, joinpath(overlay, "meta"), joinpath(dirname(@__DIR__), "recipes")] + appimage = AppImage(; prefix, preferences, kwargs...) + get_bundle_parameters!(appimage.parameters, joinpath(overlay, "Project.toml"); preferences) + + return appimage +end + +function stage(appimage::AppImage, destination::String) + + (; predicate, parameters) = appimage + app_name = parameters["APP_NAME"] + bundle_identifier = get(parameters, "BUNDLE_IDENTIFIER", app_name) + + # The spec looks for the desktop entry and the icon at the AppDir root, and the `Icon=` key + # names the icon with no path and no extension. + install(appimage.icon, joinpath(destination, "$app_name.png")) + install(appimage.desktop_launcher, joinpath(destination, "$app_name.desktop"); parameters, predicate) + + # `.DirIcon` is what file managers read for the thumbnail. A copy rather than a symlink, since + # squashfs preserves symlinks but some extraction paths do not follow them. + cp(joinpath(destination, "$app_name.png"), joinpath(destination, ".DirIcon"); force = true) + + # Freedesktop locations, so an AppImage the user installs integrates with the menu + install(appimage.icon, joinpath(destination, "usr/share/icons/hicolor/256x256/apps/$app_name.png")) + install(appimage.desktop_launcher, joinpath(destination, "usr/share/applications/$app_name.desktop"); parameters, predicate) + install(appimage.metainfo, joinpath(destination, "usr/share/metainfo/$bundle_identifier.appdata.xml"); parameters, predicate) + + if !isnothing(appimage.main_launcher) + install(appimage.main_launcher, joinpath(destination, "AppRun"); parameters, executable = true, predicate) + end + + return +end + +function bundle(setup::Function, appimage::AppImage, destination::String; force = false) + + if ispath(destination) + if force + rm(destination; force=true, recursive=true) + else + error("Destination $destination already exists. Use `force = true` argument.") + end + end + + # Resolve the runtime before doing the expensive staging work, so a missing one fails in + # seconds rather than after a full image build. + runtime = appimage.compress ? AppImageRuntime.resolve(appimage.arch; runtime = appimage.runtime) : nothing + + appdir = appimage.compress ? mktempdir() : destination + + @info "Initializing AppDir staging layout..." + stage(appimage, appdir) + + @info "Installing app into staging area..." + setup(appdir) + + if appimage.compress + @info "Packaging AppDir into AppImage..." + AppImagePack.pack(appdir, destination, runtime; compression = appimage.compression) + end + + return +end diff --git a/src/main.jl b/src/main.jl index f7b998a..baa3c7a 100644 --- a/src/main.jl +++ b/src/main.jl @@ -34,8 +34,9 @@ end suffix(msix::MSIX) = msix.compress ? ".msix" : "" suffix(dmg::DMG) = dmg.compress ? ".dmg" : "" suffix(snap::Snap) = snap.compress ? ".snap" : "" +suffix(appimage::AppImage) = appimage.compress ? ".AppImage" : "" -function canonical_target_name(spec::Union{MSIX, DMG, Snap}) +function canonical_target_name(spec::Union{MSIX, DMG, Snap, AppImage}) version = spec.parameters["APP_VERSION"] app_name = spec.parameters["APP_NAME"] return "$(app_name)-$version-$(spec.arch)" diff --git a/src/recipes.jl b/src/recipes.jl index 33347e9..9165da9 100644 --- a/src/recipes.jl +++ b/src/recipes.jl @@ -156,3 +156,60 @@ function bundle(product::JuliaCBundle, msix::MSIX, destination::String; password return end + +""" + bundle(product::JuliaImgBundle, appimage::AppImage, destination::String; force = false) + +Stage a Julia application into a mountable AppImage. + +The distribution is placed at the AppDir root — `bin/`, `lib/`, `share/`, `etc/` — exactly as the +Snap target stages it, and `AppRun` execs `bin/julia` from the mount point. Because the mounted +filesystem is read-only and vanishes when the application exits, the depot is redirected to a +per-user directory; see the `depot` field of [`AppImage`](@ref). +""" +function bundle(product::JuliaImgBundle, appimage::AppImage, destination::String; force = false) + + if !Sys.islinux() + @warn "AppImages only run on Linux and the payload is staged for the host platform" + end + + bundle(appimage, destination; force) do appdir + + app_name = appimage.parameters["APP_NAME"] + bundle_identifier = appimage.parameters["BUNDLE_IDENTIFIER"] + + stage(product, appdir; platform = Linux(appimage.arch), runtime_mode = "SANDBOX", app_name, bundle_identifier) + + # "julia" mode needs a startup file that leaves DEPOT_PATH alone; every set_depot_path_* + # in AppEnv begins with `empty!(DEPOT_PATH)`, so AppEnv.init() cannot be used there. + startup_file = appimage.depot == "julia" ? appimage.startup_file : product.startup_file + + isnothing(startup_file) && + error("No startup.jl available for `appimage_depot = \"julia\"`.") + + install(startup_file, joinpath(appdir, "etc/julia/startup.jl"); + parameters = appimage.parameters, force = true) + end + + return +end + +""" + bundle(product::JuliaCBundle, appimage::AppImage, destination::String; force = false) + +Package a juliac-compiled executable as a mountable AppImage. +""" +function bundle(product::JuliaCBundle, appimage::AppImage, destination::String; force = false) + + if !Sys.islinux() + @warn "AppImages only run on Linux and juliac compiles with the host toolchain" + end + + bundle(appimage, destination; force) do appdir + app_name = appimage.parameters["APP_NAME"] + bundle_identifier = appimage.parameters["BUNDLE_IDENTIFIER"] + stage(product, appdir; runtime_mode = "SANDBOX", app_name, bundle_identifier) + end + + return +end From b962cbb993e78937a8494829af0abe7c834e0c29 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Sun, 13 Sep 2026 00:15:57 +0300 Subject: [PATCH 05/14] making AppImage constructor to work with bundle --- LocalPreferences.toml | 3 + src/AppBundler.jl | 1 + src/AppImage/AppImagePack.jl | 109 ++-------------- src/AppImage/AppImageRuntime.jl | 91 ++++++++++++++ src/bundle.jl | 34 ++--- src/main.jl | 3 +- test/appimage.jl | 7 +- test/bundle.jl | 215 +++++++++++++++++--------------- 8 files changed, 237 insertions(+), 226 deletions(-) create mode 100644 src/AppImage/AppImageRuntime.jl diff --git a/LocalPreferences.toml b/LocalPreferences.toml index 5a1ed0a..c9417c2 100644 --- a/LocalPreferences.toml +++ b/LocalPreferences.toml @@ -21,6 +21,9 @@ dmg_hardened_runtime = true dmg_sandboxed_runtime = false dmg_compression = "lzma" +appimage_compression = "zstd" + + bundler = "juliaimg" juliaimg_mainless = false diff --git a/src/AppBundler.jl b/src/AppBundler.jl index 2bdf534..8aafda5 100644 --- a/src/AppBundler.jl +++ b/src/AppBundler.jl @@ -28,6 +28,7 @@ include("DMG/HFS.jl") include("DMG/DMGPack.jl") include("Snap/SnapPack.jl") +include("AppImage/AppImageRuntime.jl") include("AppImage/AppImagePack.jl") include("MSIX/MSIXPack.jl") diff --git a/src/AppImage/AppImagePack.jl b/src/AppImage/AppImagePack.jl index b460a2b..69c8cc7 100644 --- a/src/AppImage/AppImagePack.jl +++ b/src/AppImage/AppImagePack.jl @@ -11,91 +11,11 @@ recover that offset before `unsquashfs` can find the superblock. module AppImagePack import squashfs_tools_jll: mksquashfs, unsquashfs -using Downloads -using Scratch: @get_scratch! +import ..AppImageRuntime: get_runtime export pack, unpack -const RUNTIME_TAG = "continuous" - -"Runtime architectures published by AppImage/type2-runtime." -const RUNTIME_ARCHS = ("x86_64", "i686", "aarch64", "armhf") - - -# ToDo: -# - replace this with Artifacts.toml source -runtime_url(arch) = "https://github.com/AppImage/type2-runtime/releases/download/" * - "$(RUNTIME_TAG)/runtime-$(arch)" - -""" - host_arch() -> String - -The runtime architecture matching the host, using AppImage's naming rather -than Julia's. Only meaningful for a Linux host; cross-packaging should pass -`arch` explicitly. -""" -function host_arch() - a = Sys.ARCH - a === :x86_64 && return "x86_64" - a === :i686 && return "i686" - a === :aarch64 && return "aarch64" - a === :arm && return "armhf" - a === :armv7l && return "armhf" - error("no AppImage runtime for host architecture $a; pass arch= explicitly") -end - -""" - runtime_cache() -> String - -Scratch space holding downloaded runtimes between builds, keyed to this -package so `Pkg.gc()` can reclaim it and `Scratch.delete_scratch!` can clear -it. The path is resolved on each call rather than stored in a `const`, since -baking a scratch path into precompiled code is not safe. - -Runtimes are re-downloaded if the space is garbage collected, so treat this -purely as a cache. -""" -runtime_cache() = @get_scratch!("runtimes") - -# ---------------------------------------------------------------- runtime -- - - -# It is not possible to use Artifacts.toml because it expects artifacts to be placed in tarballs -# The best option now is to wait for https://github.com/JuliaPackaging/Yggdrasil/pull/14695 to be merged -""" - fetch_runtime(arch, cachedir) -> String - -Download the prebuilt type-2 runtime for `arch` (cached in `cachedir`), and -check that it is an ELF carrying the AppImage magic `AI\\x02` at offset 8. -""" -function fetch_runtime(arch::AbstractString, cachedir::AbstractString) - arch in RUNTIME_ARCHS || - error("unknown runtime architecture $arch; expected one of $(join(RUNTIME_ARCHS, ", "))") - - mkpath(cachedir) - dest = joinpath(cachedir, "runtime-$(arch)") - if !isfile(dest) || filesize(dest) == 0 - @info "Downloading runtime-$(arch)" - tmp = dest * ".part" - try - Downloads.download(runtime_url(arch), tmp) - mv(tmp, dest; force = true) - finally - rm(tmp; force = true) - end - else - @info "Using cached runtime-$(arch)" - end - - # Belongs to the tests - header = open(io -> read(io, 11), dest, "r") - length(header) == 11 || error("runtime is truncated") - header[1:4] == UInt8[0x7f, 0x45, 0x4c, 0x46] || error("runtime is not an ELF file") - header[9:11] == UInt8[0x41, 0x49, 0x02] || - error("runtime lacks the AI\\x02 magic at offset 8") - - return dest -end +const COMPRESSORS = [:zstd, :gzip] # ------------------------------------------------------------------ offset -- @@ -159,16 +79,12 @@ the epoch and ownership forced to root so repeated builds are bit-identical; staged in `appdir`. """ function compress(appdir::AbstractString, payload::AbstractString; - comp::AbstractString = "zstd") + compression::Symbol = :zstd) rm(payload; force = true) - args = String[ - "-noappend", "-no-progress", "-quiet", - "-all-root", "-no-xattrs", - "-comp", comp, "-b", "128K", - "-mkfs-time", "0", "-all-time", "0", - ] - @info "Packing SquashFS payload ($comp)" - run(`$(mksquashfs()) $appdir $payload $args`) + + @info "Packing SquashFS payload ($compression)" + run(`$(mksquashfs()) $appdir $payload -noappend -all-root -no-xattrs -comp $compression -b 128K -mkfs-time 0 -all-time 0`) + return payload end @@ -208,18 +124,13 @@ Package the AppDir at `source` into an AppImage at `destination`. plus the `.desktop` file, icon and `.DirIcon` the format expects. Nothing here edits the tree. """ -function pack(source::AbstractString, destination::AbstractString; - arch::AbstractString = host_arch(), - comp::AbstractString = "zstd", - cache::AbstractString = runtime_cache()) +function pack(source::AbstractString, destination::AbstractString; compression::Symbol = :zstd, runtime = get_runtime(Sys.ARCH)) isdir(source) || error("AppDir not found: $source") isfile(joinpath(source, "AppRun")) || error("$source has no AppRun at its root") - runtime = fetch_runtime(arch, cache) - mktempdir() do work - payload = compress(source, joinpath(work, "payload.squashfs"); comp = comp) + payload = compress(source, joinpath(work, "payload.squashfs"); compression) concatenate(runtime, payload, destination) end @@ -240,7 +151,7 @@ function unpack(source::AbstractString, destination::AbstractString) isfile(source) || error("AppImage not found: $source") offset = payload_offset(source) - run(`$(unsquashfs()) -offset $offset -force -quiet -no-progress -dest $destination $source`) + run(`$(unsquashfs()) -offset $offset -force -dest $destination $source`) return destination end diff --git a/src/AppImage/AppImageRuntime.jl b/src/AppImage/AppImageRuntime.jl new file mode 100644 index 0000000..1e43687 --- /dev/null +++ b/src/AppImage/AppImageRuntime.jl @@ -0,0 +1,91 @@ +module AppImageRuntime + +using Downloads +using Scratch: @get_scratch! + +const RUNTIME_TAG = "continuous" + +"Runtime architectures published by AppImage/type2-runtime." +const RUNTIME_ARCHS = ("x86_64", "i686", "aarch64", "armhf") + +# ToDo: +# - replace this with Artifacts.toml source +runtime_url(arch) = "https://github.com/AppImage/type2-runtime/releases/download/" * + "$(RUNTIME_TAG)/runtime-$(arch)" + +""" + host_arch() -> String + +The runtime architecture matching the host, using AppImage's naming rather +than Julia's. Only meaningful for a Linux host; cross-packaging should pass +`arch` explicitly. +""" +function arch_string(a::Symbol) + a === :x86_64 && return "x86_64" + a === :i686 && return "i686" + a === :aarch64 && return "aarch64" + a === :arm && return "armhf" + a === :armv7l && return "armhf" + error("no AppImage runtime for host architecture $a; pass arch= explicitly") +end + +""" + runtime_cache() -> String + +Scratch space holding downloaded runtimes between builds, keyed to this +package so `Pkg.gc()` can reclaim it and `Scratch.delete_scratch!` can clear +it. The path is resolved on each call rather than stored in a `const`, since +baking a scratch path into precompiled code is not safe. + +Runtimes are re-downloaded if the space is garbage collected, so treat this +purely as a cache. +""" +runtime_cache() = @get_scratch!("runtimes") + +# ---------------------------------------------------------------- runtime -- + + +# It is not possible to use Artifacts.toml because it expects artifacts to be placed in tarballs +# The best option now is to wait for https://github.com/JuliaPackaging/Yggdrasil/pull/14695 to be merged +""" + fetch_runtime(arch, cachedir) -> String + +Download the prebuilt type-2 runtime for `arch` (cached in `cachedir`), and +check that it is an ELF carrying the AppImage magic `AI\\x02` at offset 8. +""" +function get_runtime(arch::AbstractString; cachedir::AbstractString = runtime_cache()) + arch in RUNTIME_ARCHS || + error("unknown runtime architecture $arch; expected one of $(join(RUNTIME_ARCHS, ", "))") + + mkpath(cachedir) + dest = joinpath(cachedir, "runtime-$(arch)") + if !isfile(dest) || filesize(dest) == 0 + @info "Downloading runtime-$(arch)" + tmp = dest * ".part" + try + Downloads.download(runtime_url(arch), tmp) + mv(tmp, dest; force = true) + finally + rm(tmp; force = true) + end + else + @info "Using cached runtime-$(arch)" + end + + # Belongs to the tests + header = open(io -> read(io, 11), dest, "r") + length(header) == 11 || error("runtime is truncated") + header[1:4] == UInt8[0x7f, 0x45, 0x4c, 0x46] || error("runtime is not an ELF file") + header[9:11] == UInt8[0x41, 0x49, 0x02] || + error("runtime lacks the AI\\x02 magic at offset 8") + + return dest +end + +function get_runtime(arch::Symbol; cachedir::AbstractString = runtime_cache()) + return get_runtime(arch_string(arch); cachedir) +end + +export get_runtime + +end diff --git a/src/bundle.jl b/src/bundle.jl index 88a577f..e011247 100644 --- a/src/bundle.jl +++ b/src/bundle.jl @@ -252,7 +252,7 @@ function DMG(; prefix = joinpath(dirname(@__DIR__), "recipes"), preferences = preferences(), predicate = preferences["bundler"], - icon = get_path(prefix, ["dmg/icon.icns", "dmg/icon.png", "icon.icns"]), + icon = get_path(prefix, ["dmg/icon.icns", "icon.icns"]), # The "dmg/icon.png" is not yet supported info_config = get_path(prefix, "dmg/Info.plist"), entitlements = get_path(prefix, "dmg/Entitlements.plist"), dsstore = get_path(prefix, ["dmg/DS_Store.toml", "dmg/DS_Store"]), @@ -661,14 +661,12 @@ struct AppImage icon::String desktop_launcher::String metainfo::String - main_launcher::Union{String, Nothing} - startup_file::Union{String, Nothing} - depot::String + main_launcher::String # It is always AppRun.sh. compression::Symbol - runtime::Union{String, Nothing} windowed::Bool compress::Bool arch::Symbol + runtime::String predicate::String parameters::Dict{String, Any} end @@ -683,30 +681,19 @@ function AppImage(; desktop_launcher = get_path(prefix, "appimage/main.desktop"), metainfo = get_path(prefix, "appimage/metainfo.xml"), main_launcher = get_path(prefix, hook("appimage/AppRun.sh", predicate); warn = false), - startup_file = get_path(prefix, "appimage/startup.jl"; warn = false), - depot = get(preferences, "appimage_depot", "app"), compression = Symbol(get(preferences, "appimage_compression", "zstd")), - runtime = get(preferences, "appimage_runtime", ""), windowed = preferences["windowed"], compress = preferences["compress"], arch = Sys.ARCH, + runtime = AppImageRuntime.get_runtime(arch), parameters = Dict{String, Any}("WINDOWED" => windowed) ) - depot in APPIMAGE_DEPOTS || - error("`appimage_depot` must be one of: " * join(APPIMAGE_DEPOTS, ", ") * ". Got `$depot`.") - compression in AppImagePack.COMPRESSORS || error("`appimage_compression` must be one of: " * join(AppImagePack.COMPRESSORS, ", ") * ". Got `$compression`.") - # The AppRun template branches on this rather than on the preference string, so the - # rendered launcher only carries the lines that apply. - parameters["APP_DEPOT"] = depot == "app" - - return AppImage(icon, desktop_launcher, metainfo, main_launcher, startup_file, depot, compression, - isempty(something(runtime, "")) ? nothing : runtime, - windowed, compress, arch, predicate, parameters) + return AppImage(icon, desktop_launcher, metainfo, main_launcher, compression, windowed, compress, arch, runtime, predicate, parameters) end function AppImage(overlay; preferences = preferences(), kwargs...) @@ -731,16 +718,15 @@ function stage(appimage::AppImage, destination::String) # `.DirIcon` is what file managers read for the thumbnail. A copy rather than a symlink, since # squashfs preserves symlinks but some extraction paths do not follow them. - cp(joinpath(destination, "$app_name.png"), joinpath(destination, ".DirIcon"); force = true) + #cp(joinpath(destination, "$app_name.png"), joinpath(destination, ".DirIcon"); force = true) + install(appimage.icon, joinpath(destination, ".DirIcon")) # Freedesktop locations, so an AppImage the user installs integrates with the menu install(appimage.icon, joinpath(destination, "usr/share/icons/hicolor/256x256/apps/$app_name.png")) install(appimage.desktop_launcher, joinpath(destination, "usr/share/applications/$app_name.desktop"); parameters, predicate) install(appimage.metainfo, joinpath(destination, "usr/share/metainfo/$bundle_identifier.appdata.xml"); parameters, predicate) - if !isnothing(appimage.main_launcher) - install(appimage.main_launcher, joinpath(destination, "AppRun"); parameters, executable = true, predicate) - end + install(appimage.main_launcher, joinpath(destination, "AppRun"); parameters, executable = true, predicate) return end @@ -757,7 +743,7 @@ function bundle(setup::Function, appimage::AppImage, destination::String; force # Resolve the runtime before doing the expensive staging work, so a missing one fails in # seconds rather than after a full image build. - runtime = appimage.compress ? AppImageRuntime.resolve(appimage.arch; runtime = appimage.runtime) : nothing + #runtime = appimage.compress ? AppImageRuntime.resolve(appimage.arch; runtime = appimage.runtime) : nothing appdir = appimage.compress ? mktempdir() : destination @@ -769,7 +755,7 @@ function bundle(setup::Function, appimage::AppImage, destination::String; force if appimage.compress @info "Packaging AppDir into AppImage..." - AppImagePack.pack(appdir, destination, runtime; compression = appimage.compression) + AppImagePack.pack(appdir, destination; compression = appimage.compression, runtime = appimage.runtime) end return diff --git a/src/main.jl b/src/main.jl index baa3c7a..709ea6e 100644 --- a/src/main.jl +++ b/src/main.jl @@ -337,7 +337,8 @@ Options: --build-dir DIR Output directory for the bundle (default: temporary directory) Use '@temp' to explicitly request a temp dir - --target-bundle {dmg|snap|msix} Package format to produce + --target-bundle {dmg|snap|appimage|msix} + Package format to produce (default: platform native — dmg on macOS, snap on Linux, msix on Windows) --target-arch {x86_64|aarch64} Target CPU architecture diff --git a/test/appimage.jl b/test/appimage.jl index e9776c7..e0abde5 100644 --- a/test/appimage.jl +++ b/test/appimage.jl @@ -121,14 +121,13 @@ function verify(appimage::AbstractString) return nothing end -function main(; workdir::AbstractString = joinpath(@__DIR__, "build"), - arch::AbstractString = get(ENV, "APPIMAGE_ARCH", AppImagePack.host_arch())) +function main(; workdir::AbstractString = joinpath(@__DIR__, "build")) mkpath(workdir) appdir = build_appdir(workdir) - output = joinpath(@__DIR__, "$(APP_NAME)-$(APP_VERSION)-$(arch).AppImage") + output = joinpath(@__DIR__, "$(APP_NAME)-$(APP_VERSION)-$(Sys.ARCH).AppImage") - pack(appdir, output; arch = arch) + pack(appdir, output) verify(output) println() diff --git a/test/bundle.jl b/test/bundle.jl index 4a3580e..ec2da13 100644 --- a/test/bundle.jl +++ b/test/bundle.jl @@ -1,6 +1,6 @@ using Test -import AppBundler: stage, bundle, MSIX, DMG, Snap, MSIXPack +import AppBundler: stage, bundle, MSIX, DMG, Snap, MSIXPack, AppImage import AppBundler using osslsigncode_jll @@ -41,142 +41,161 @@ end predicate = "juliaimg" -@time @testset "MSIX bundling tests" begin +# @time @testset "MSIX bundling tests" begin - msix = MSIX(joinpath(@__DIR__, "../examples/GtkApp"); selfsign=true, predicate, windowed = true) +# msix = MSIX(joinpath(@__DIR__, "../examples/GtkApp"); selfsign=true, predicate, windowed = true) - @test hash_stage() do dest - stage(msix, dest) - end == "4351935f32e1b0036bbf31c0f496b5734dd6a9496e8a0005675a097233a6d07e" +# @test hash_stage() do dest +# stage(msix, dest) +# end == "4351935f32e1b0036bbf31c0f496b5734dd6a9496e8a0005675a097233a6d07e" - @test hash_stage() do stage_dir +# @test hash_stage() do stage_dir - dest = joinpath(mktempdir(), "gtkapp.msix") - bundle(msix, dest) do app_stage - @info "The MSIX app stage is $app_stage" - touch(joinpath(app_stage, "MRF_signal_Δθ_23_NTRs_500.mrd")) - end +# dest = joinpath(mktempdir(), "gtkapp.msix") +# bundle(msix, dest) do app_stage +# @info "The MSIX app stage is $app_stage" +# touch(joinpath(app_stage, "MRF_signal_Δθ_23_NTRs_500.mrd")) +# end - verify_msix_signature(dest) +# verify_msix_signature(dest) - MSIXPack.repack(dest, tempname()) # useful for debugging MSIX configuration issues +# MSIXPack.repack(dest, tempname()) # useful for debugging MSIX configuration issues - AppBundler.MSIXPack.unpack(dest, stage_dir) +# AppBundler.MSIXPack.unpack(dest, stage_dir) - rm(joinpath(stage_dir, "AppxSignature.p7x")) # Signatures are always nondeterministic +# rm(joinpath(stage_dir, "AppxSignature.p7x")) # Signatures are always nondeterministic - # @test hash_file(joinpath(stage_dir, "AppxBlockMap.xml")) == "70ff6695ec913326f645c1cd30e48f75f57545ee4ae546db5843bf0779e6ee7e" - rm(joinpath(stage_dir, "AppxBlockMap.xml")) # AppxBlockMap.xml has a slight nondeterminism +# # @test hash_file(joinpath(stage_dir, "AppxBlockMap.xml")) == "70ff6695ec913326f645c1cd30e48f75f57545ee4ae546db5843bf0779e6ee7e" +# rm(joinpath(stage_dir, "AppxBlockMap.xml")) # AppxBlockMap.xml has a slight nondeterminism - end == "4351935f32e1b0036bbf31c0f496b5734dd6a9496e8a0005675a097233a6d07e" -end +# end == "4351935f32e1b0036bbf31c0f496b5734dd6a9496e8a0005675a097233a6d07e" +# end if Sys.isunix() - # ------------------- DMG ------------- + # # ------------------- DMG ------------- - @time @testset "DMG bundling tests" begin + # @time @testset "DMG bundling tests" begin - dmg = DMG(joinpath(@__DIR__, "../examples/GtkApp"); hfsplus = true, selfsign = true, predicate, arch = :x86_64) + # dmg = DMG(joinpath(@__DIR__, "../examples/GtkApp"); hfsplus = true, selfsign = true, predicate, arch = :x86_64) - @test hash_stage() do dest - stage(dmg, joinpath(dest, "GtkApp.app"); dsstore=true) - AppBundler.DMGPack.replace_binary_with_hash(joinpath(dest, "GtkApp.app/Contents/MacOS/gtkapp")) - rm("$dest/Applications") - end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" + # @test hash_stage() do dest + # stage(dmg, joinpath(dest, "GtkApp.app"); dsstore=true) + # AppBundler.DMGPack.replace_binary_with_hash(joinpath(dest, "GtkApp.app/Contents/MacOS/gtkapp")) + # rm("$dest/Applications") + # end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" - @test hash_stage() do stage_dir + # @test hash_stage() do stage_dir - dest = joinpath(mktempdir(), "gtkapp.dmg") - bundle(dmg, dest) do app_stage - @info "The DMG app stage is $app_stage" - end + # dest = joinpath(mktempdir(), "gtkapp.dmg") + # bundle(dmg, dest) do app_stage + # @info "The DMG app stage is $app_stage" + # end - if Sys.isapple() - @info "Verifying integrity of the DMG archive" - run(`hdiutil verify $dest`) - end + # if Sys.isapple() + # @info "Verifying integrity of the DMG archive" + # run(`hdiutil verify $dest`) + # end - AppBundler.DMGPack.unpack(dest, stage_dir) + # AppBundler.DMGPack.unpack(dest, stage_dir) - if Sys.isapple() - # This check is also important for stagging - @info "Verifying that the application is correctly codesigned" - run(`codesign --verify --deep --verbose=4 "$stage_dir/GtkApp.app"`) + # if Sys.isapple() + # # This check is also important for stagging + # @info "Verifying that the application is correctly codesigned" + # run(`codesign --verify --deep --verbose=4 "$stage_dir/GtkApp.app"`) - @info "Verifying if the application has hardened runtime enabled" - io = IOBuffer() - run(pipeline(`codesign -dvv $stage_dir/GtkApp.app`, stderr=io)) - output = String(take!(io)) + # @info "Verifying if the application has hardened runtime enabled" + # io = IOBuffer() + # run(pipeline(`codesign -dvv $stage_dir/GtkApp.app`, stderr=io)) + # output = String(take!(io)) - @test occursin(r"Timestamp=", output) - @test occursin(r"flags=0x[0-9a-f]+\(runtime\)", output) - end + # @test occursin(r"Timestamp=", output) + # @test occursin(r"flags=0x[0-9a-f]+\(runtime\)", output) + # end - @show AppBundler.DMGPack.replace_binary_with_hash(joinpath(stage_dir, "GtkApp.app/Contents/MacOS/gtkapp")) - rm("$stage_dir/GtkApp.app/Contents/_CodeSignature"; recursive=true) + # @show AppBundler.DMGPack.replace_binary_with_hash(joinpath(stage_dir, "GtkApp.app/Contents/MacOS/gtkapp")) + # rm("$stage_dir/GtkApp.app/Contents/_CodeSignature"; recursive=true) - end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" + # end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" - if Sys.isapple() - @test hash_stage() do stage_dir + # if Sys.isapple() + # @test hash_stage() do stage_dir - dmg = DMG(joinpath(@__DIR__, "../examples/GtkApp"); hfsplus = false, selfsign = true, predicate, arch = :x86_64) - dest = joinpath(mktempdir(), "gtkapp.dmg") - bundle(dmg, dest) do app_stage - @info "The DMG app stage is $app_stage" - end + # dmg = DMG(joinpath(@__DIR__, "../examples/GtkApp"); hfsplus = false, selfsign = true, predicate, arch = :x86_64) + # dest = joinpath(mktempdir(), "gtkapp.dmg") + # bundle(dmg, dest) do app_stage + # @info "The DMG app stage is $app_stage" + # end - @info "Verifying integrity of the DMG archive" - run(`hdiutil verify $dest`) - - @info "Verifying contents of DMG archive" - mount_point = mount_dmg(dest) - try - @info "Verifying that the application is correctly codesigned" - run(`codesign --verify --deep --strict --verbose=4 "$mount_point/GtkApp.app"`) - - @info "Verifying if the application has hardened runtime enabled" - io = IOBuffer() - run(pipeline(`codesign -dvv $mount_point/GtkApp.app`, stderr=io)) - output = String(take!(io)) - - @test occursin(r"Timestamp=", output) - @test occursin(r"flags=0x[0-9a-f]+\(runtime\)", output) + # @info "Verifying integrity of the DMG archive" + # run(`hdiutil verify $dest`) + + # @info "Verifying contents of DMG archive" + # mount_point = mount_dmg(dest) + # try + # @info "Verifying that the application is correctly codesigned" + # run(`codesign --verify --deep --strict --verbose=4 "$mount_point/GtkApp.app"`) + + # @info "Verifying if the application has hardened runtime enabled" + # io = IOBuffer() + # run(pipeline(`codesign -dvv $mount_point/GtkApp.app`, stderr=io)) + # output = String(take!(io)) + + # @test occursin(r"Timestamp=", output) + # @test occursin(r"flags=0x[0-9a-f]+\(runtime\)", output) - cp(mount_point, stage_dir; force=true) - finally - unmount_dmg(mount_point) - end + # cp(mount_point, stage_dir; force=true) + # finally + # unmount_dmg(mount_point) + # end - @show AppBundler.DMGPack.replace_binary_with_hash(joinpath(stage_dir, "GtkApp.app/Contents/MacOS/gtkapp")) - rm("$stage_dir/GtkApp.app/Contents/_CodeSignature"; recursive=true) + # @show AppBundler.DMGPack.replace_binary_with_hash(joinpath(stage_dir, "GtkApp.app/Contents/MacOS/gtkapp")) + # rm("$stage_dir/GtkApp.app/Contents/_CodeSignature"; recursive=true) - end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" - end - end + # end == "340323df33e9f976003cb5b8e6059f3a09226c6eb93d489a406feae39ef3345d" + # end + # end - # -------------------- SNAP ----------------- + # # -------------------- SNAP ----------------- - @time @testset "Snap bundling tests" begin + # @time @testset "Snap bundling tests" begin - snap = Snap(joinpath(@__DIR__, "../examples/GtkApp"); predicate, configure_hook = nothing, windowed = true) + # snap = Snap(joinpath(@__DIR__, "../examples/GtkApp"); predicate, configure_hook = nothing, windowed = true) - @test hash_stage() do dest - stage(snap, dest) - end == "f64997788eca9a5d020c4fe73921d4085fc07ea2266b1401276162efd4695678" + # @test hash_stage() do dest + # stage(snap, dest) + # end == "f64997788eca9a5d020c4fe73921d4085fc07ea2266b1401276162efd4695678" - @test hash_stage() do stage_dir + # @test hash_stage() do stage_dir - dest = joinpath(mktempdir(), "gtkapp.snap") - bundle(snap, dest) do app_stage - @info "The Snap app stage is $app_stage" - end + # dest = joinpath(mktempdir(), "gtkapp.snap") + # bundle(snap, dest) do app_stage + # @info "The Snap app stage is $app_stage" + # end - AppBundler.SnapPack.unpack(dest, stage_dir) + # AppBundler.SnapPack.unpack(dest, stage_dir) + + # end == "f64997788eca9a5d020c4fe73921d4085fc07ea2266b1401276162efd4695678" + # end + + + # -------------------- AppImage ----------------- + + appimage = AppImage(joinpath(@__DIR__, "../examples/GtkApp"); predicate, windowed = false, arch = Sys.ARCH) + + + @test hash_stage() do stage_dir + + dest = joinpath(mktempdir(), "gtkapp.appimage") + + bundle(appimage, dest) do app_stage + @info "The AppImage app stage is $app_stage" + end + + AppBundler.AppImagePack.unpack(dest, stage_dir) + + end == "f90ec4725a7a9ea9ffd4d2d8e86a8b8780731d2f520d3861443f863d40292352" - end == "f64997788eca9a5d020c4fe73921d4085fc07ea2266b1401276162efd4695678" - end end From dbe0ab89253abedf11822fd6ee4a9191d5586e37 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Sun, 13 Sep 2026 14:04:00 +0300 Subject: [PATCH 06/14] AppImage debugging --- src/main.jl | 5 +++++ src/recipes.jl | 8 ++++---- test/appimage2e.jl | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 test/appimage2e.jl diff --git a/src/main.jl b/src/main.jl index 709ea6e..9134106 100644 --- a/src/main.jl +++ b/src/main.jl @@ -133,6 +133,11 @@ function main_build(ARGS; sources_dir) snap = Snap(sources_dir; arch = target_arch, preferences) bundle(spec, snap, target_path(snap); force = overwrite_target) + elseif :appimage == target_bundle + + appimage = AppImage(sources_dir; arch = target_arch, preferences) + bundle(spec, appimage, target_path(appimage); force = overwrite_target) + else error("Got unsupported bundle type $target_bundle") end diff --git a/src/recipes.jl b/src/recipes.jl index 9165da9..da644cb 100644 --- a/src/recipes.jl +++ b/src/recipes.jl @@ -182,12 +182,12 @@ function bundle(product::JuliaImgBundle, appimage::AppImage, destination::String # "julia" mode needs a startup file that leaves DEPOT_PATH alone; every set_depot_path_* # in AppEnv begins with `empty!(DEPOT_PATH)`, so AppEnv.init() cannot be used there. - startup_file = appimage.depot == "julia" ? appimage.startup_file : product.startup_file + # startup_file = appimage.depot == "julia" ? appimage.startup_file : product.startup_file - isnothing(startup_file) && - error("No startup.jl available for `appimage_depot = \"julia\"`.") + # isnothing(startup_file) && + # error("No startup.jl available for `appimage_depot = \"julia\"`.") - install(startup_file, joinpath(appdir, "etc/julia/startup.jl"); + install(product.startup_file, joinpath(appdir, "etc/julia/startup.jl"); parameters = appimage.parameters, force = true) end diff --git a/test/appimage2e.jl b/test/appimage2e.jl new file mode 100644 index 0000000..f305678 --- /dev/null +++ b/test/appimage2e.jl @@ -0,0 +1,20 @@ +using AppBundler +using Test + +build_dir = joinpath(@__DIR__, "../build") +args(app_dir) = ["build", app_dir, "--build-dir=$build_dir", "--force", "--selfsign", "--target-bundle=appimage"] + +# @testset "GtkApp" begin +# app_dir = joinpath(dirname(@__DIR__), "examples/GTKApp") +# AppBundler.main(args(app_dir)) +# end + +# @testset "CmdApp" begin +# app_dir = joinpath(dirname(@__DIR__), "examples/CmdApp") +# AppBundler.main(args(app_dir)) +# end + +@testset "ModJulia" begin + app_dir = joinpath(dirname(@__DIR__), "examples/modjulia") + AppBundler.main(args(app_dir)) +end From 3c0e315a4d812d26c51b6f51bd8cb63e65304cc1 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 15:09:02 +0300 Subject: [PATCH 07/14] Claude Opus 5; Make -D overrides escaping-free: coerce values against LocalPreferences.toml types, normalize arguments --- src/AppBundler.jl | 2 + src/CLIParser.jl | 217 ++++++++++++++++++++++++++++++ src/main.jl | 125 +++++++----------- test/appimage2e.jl | 22 ++-- test/argument_parser.jl | 283 ++++++++++++++++++++++++++++++++++++++++ test/integrity.jl | 3 - 6 files changed, 561 insertions(+), 91 deletions(-) create mode 100644 src/CLIParser.jl create mode 100644 test/argument_parser.jl diff --git a/src/AppBundler.jl b/src/AppBundler.jl index 8aafda5..7b62b35 100644 --- a/src/AppBundler.jl +++ b/src/AppBundler.jl @@ -45,6 +45,8 @@ using .JuliaImg.Resources: merge_directories#, install include("utils.jl") include("bundle.jl") include("recipes.jl") + +include("CLIParser.jl") include("main.jl") #bundle_app(app_dir, bundle_dir; version = VERSION) = bundle_app(HostPlatform(), app_dir, bundle_dir; version) diff --git a/src/CLIParser.jl b/src/CLIParser.jl new file mode 100644 index 0000000..4e05db2 --- /dev/null +++ b/src/CLIParser.jl @@ -0,0 +1,217 @@ +module CLIParser + +""" + is_balanced(s) -> Bool + +True when `s` contains no unclosed quote and no unclosed bracket, i.e. the +shell did not split a `-D` value across multiple argv entries. +""" +function is_balanced(s::AbstractString) + quote_char = nothing + depth = 0 + escaped = false + for c in s + if escaped + escaped = false + elseif c == '\\' + escaped = true + elseif quote_char !== nothing + c == quote_char && (quote_char = nothing) + elseif c == '"' || c == '\'' + quote_char = c + elseif c == '[' || c == '{' + depth += 1 + elseif c == ']' || c == '}' + depth -= 1 + end + end + return quote_char === nothing && depth <= 0 +end + + +ends_open(s) = endswith(rstrip(s), ',') + +function heal_args(raw_args) + out = String[] + i = 1 + while i <= length(raw_args) + tok = String(raw_args[i]) + if occursin('=', tok) + while !is_balanced(tok) || ends_open(tok) + i += 1 + i > length(raw_args) && error(""" + Unterminated value: $tok + It ends with a comma, or is missing a closing ']' or '"'. + Write the value without spaces, or quote the whole option: + -Dkey=a,-b or -D 'key=[a, -b]' + """) + tok *= " " * raw_args[i] + end + end + push!(out, tok) + i += 1 + end + return out +end + + +const Arg = Pair{String, Union{String, Nothing}} + +isoption(tok) = startswith(tok, "--") || startswith(tok, "-D") + +""" + normalize_args(raw_args) -> Vector{Arg} + +Turn raw ARGS into `option => value` pairs. Attached and detached forms are +equivalent, and only the first `=` separates option from value: + + --password=foo=bar ⇒ "--password" => "foo=bar" + --password foo=bar ⇒ "--password" => "foo=bar" + -Dbundler=juliaimg ⇒ "-D" => "bundler=juliaimg" + -D bundler=juliaimg ⇒ "-D" => "bundler=juliaimg" + --selfsign ⇒ "--selfsign" => nothing + +An option takes the following token as its value unless that token is itself an +option. Tokens appearing where no option is open are emitted as `token => +nothing`. + +Values are healed first: a token containing `=` keeps absorbing following +tokens while it has an unclosed quote or bracket, or ends in a comma. +""" +function normalize_args(raw_args) + tokens = heal_args(raw_args) + out = Arg[] + + i = 1 + while i <= length(tokens) + tok = tokens[i] + + if !isoption(tok) + push!(out, tok => nothing) # positional, or a stray value + i += 1 + continue + end + + if startswith(tok, "-D") && !startswith(tok, "--") + option = "-D" + value = length(tok) > 2 ? tok[3:end] : nothing + else + j = findfirst('=', tok) + option = j === nothing ? tok : tok[1:prevind(tok, j)] + value = j === nothing ? nothing : tok[nextind(tok, j):end] + end + + # Detached form: adopt the next token unless it is another option. + if value === nothing && i < length(tokens) && !isoption(tokens[i+1]) + i += 1 + value = tokens[i] + end + + push!(out, option => (value === nothing ? nothing : unquote(value))) + i += 1 + end + + return out +end + + +### Extra argument coercion according to schema + +function parse_extra_args(args::Vector{String}, schema::Dict) + + overrides = Dict{String, Any}() + + for arg in args + j = findfirst('=', arg) + key = j === nothing ? strip(arg) : strip(arg[1:prevind(arg, j)]) + raw = j === nothing ? nothing : strip(arg[nextind(arg, j):end]) + + haskey(schema, key) || error(unknown_key_message(key, schema)) + + default = schema[key] + + if raw === nothing + default isa Bool || error("preference '$key' expects $(type_name(default)); " * + "bare keys are only allowed for booleans. Use -D$key=.") + overrides[key] = true + else + overrides[key] = coerce(unquote(raw), default, key) + end + end + + return overrides +end + + +""" + coerce(value, default, key) -> Any + +Interpret `value` according to the type of `default`. The string is never +inspected to guess a type; the schema decides. +""" +function coerce(value::AbstractString, default::AbstractVector, key) + body = strip(value) + if startswith(body, '[') && endswith(body, ']') + body = strip(body[nextind(body, 1):prevind(body, lastindex(body))]) + end + isempty(body) && return similar(default, 0) + + elem_default = isempty(default) ? "" : first(default) + return [coerce(unquote(strip(p)), elem_default, key) for p in split(body, ',')] +end + +coerce(value::AbstractString, ::AbstractString, key) = String(value) + +function coerce(value::AbstractString, ::Bool, key) + value in ("true", "false") || + error("preference '$key' expects true or false, got '$value'") + return value == "true" +end + +function coerce(value::AbstractString, ::Integer, key) + n = tryparse(Int, value) + n === nothing && error("preference '$key' expects an integer, got '$value'") + return n +end + +function coerce(value::AbstractString, ::AbstractFloat, key) + x = tryparse(Float64, value) + x === nothing && error("preference '$key' expects a number, got '$value'") + return x +end + +"""Remove one layer of matching outer quotes, if the shell left any behind.""" +function unquote(s::AbstractString) + length(s) >= 2 || return s + (s[1] == s[end] && (s[1] == '"' || s[1] == '\'')) || return s + return s[nextind(s, 1):prevind(s, lastindex(s))] +end + +type_name(::AbstractString) = "a string" +type_name(::Bool) = "true or false" +type_name(::Integer) = "an integer" +type_name(::AbstractFloat) = "a number" +type_name(::AbstractVector) = "a list" + +function unknown_key_message(key, schema) + near = [k for k in keys(schema) if edit_distance(key, k) <= max(2, length(key) ÷ 4)] + msg = "unknown preference '$key'" + isempty(near) || (msg *= "\n did you mean " * join(("'$k'" for k in sort(near)), ", ", " or ") * "?") + return msg +end + +function edit_distance(a, b) + prev = collect(0:length(b)) + curr = similar(prev) + for (i, ca) in enumerate(a) + curr[1] = i + for (j, cb) in enumerate(b) + curr[j+1] = min(prev[j+1] + 1, curr[j] + 1, prev[j] + (ca != cb)) + end + prev, curr = curr, prev + end + return prev[end] +end + + +end diff --git a/src/main.jl b/src/main.jl index 9134106..f78e03f 100644 --- a/src/main.jl +++ b/src/main.jl @@ -48,9 +48,24 @@ function main_build(ARGS; sources_dir) project_preferences = Resources.get_project_preferences(sources_dir) preferences = merge(project_preferences["AppBundler"], preference_overrides) + + if config[:build_dir] == "@temp" + build_dir = mktempdir() + else + build_dir = abspath(expanduser(config[:build_dir])) + if !isdir(build_dir) + parent = dirname(build_dir) + if isdir(parent) || isempty(parent) # Allow relative paths + mkpath(build_dir) # Use mkpath instead of mkdir + else + error("Parent directory '$parent' does not exist. Aborting...") + end + end + end + target_arch = config[:target_arch] target_bundle = config[:target_bundle] - build_dir = config[:build_dir] + #build_dir = config[:build_dir] password = config[:password] # Theese could be substituted with preferences @@ -227,29 +242,12 @@ function get_bundle_parameters!(parameters::Dict{String, Any}, project_toml; pre return parameters end -# ToDo: Revise this function for accepting values that contain " " -# ToDo: Add tests for this funciton -function normalize_args(args) - normalized = String[] - for arg in args - if startswith(arg, "--") && contains(arg, '=') - flag, value = split(arg, '=', limit=2) - push!(normalized, flag) - push!(normalized, strip(value, ['"', '\''])) - elseif startswith(arg, "-D") - push!(normalized, "-D") - push!(normalized, arg[3:end]) - else - push!(normalized, arg) - end - end - return normalized -end +require(option, value) = value === nothing ? error("$option requires a value") : value +forbid(option, value) = value === nothing || error("$option does not take a value, got '$value'") -#function parse_args(raw_args; preferences = Base.get_preferences()["AppBundler"]) -function parse_args(raw_args) #; preferences = Base.get_preferences()["AppBundler"]) +function parse_args(raw_args) - args = normalize_args(raw_args) + args = CLIParser.normalize_args(raw_args) # Default values config = Dict( @@ -260,72 +258,45 @@ function parse_args(raw_args) #; preferences = Base.get_preferences()["AppBundle :password => nothing ) - preference_overrides = [] + preference_overrides = String[] preferences = Dict() - i = 1 - while i <= length(args) - arg = args[i] - if arg in ["--help", "-h"] + for (option, value) in args + if option in ["--help", "-h"] print_help() exit(0) - elseif arg == "--build-dir" - i += 1 - if i > length(args) - error("--build-dir requires a value") - end - build_dir = expanduser(args[i]) - if build_dir == "@temp" - config[:build_dir] = mktempdir() - else - if !isdir(build_dir) - parent = dirname(build_dir) - if isdir(parent) || isempty(parent) # Allow relative paths - mkpath(build_dir) # Use mkpath instead of mkdir - else - error("Parent directory '$parent' does not exist. Aborting...") - end - end - config[:build_dir] = abspath(build_dir) # Store absolute path - end - elseif arg == "-D" - i += 1 - push!(preference_overrides, args[i]) - elseif arg == "--force" - preferences["overwrite_target"] = true - elseif arg == "--debug" + elseif option == "--build-dir" + config[:build_dir] = require(option, value) + elseif option == "-D" + push!(preference_overrides, require(option, value)) + elseif option == "--target-name" + config[:target_name] = require(option, value) + elseif option == "--password" + config[:password] = strip(require(option, value)) + elseif option == "--target-arch" + config[:target_arch] = Symbol(require(option, value)) + elseif option == "--target-bundle" + config[:target_bundle] = Symbol(require(option, value)) + elseif option == "--force" + forbid(option, value); preferences["overwrite_target"] = true + elseif option == "--selfsign" + forbid(option, value) + preferences["selfsign"] = true + elseif option == "--skipsign" + forbid(option, value) + preferences["skipsign"] = true + elseif option == "--debug" + forbid(option, value) preferences["compress"] = false preferences["selfsign"] = true preferences["windowed"] = false - elseif arg == "--target-name" - i += 1 - config[:target_name] = args[i] - elseif arg == "--selfsign" - preferences["selfsign"] = true - elseif arg == "--skipsign" - preferences["skipsign"] = true - elseif arg == "--password" - i += 1 - config[:password] = args[i] |> strip - elseif arg == "--target-arch" - i += 1 - if i > length(args) - error("--target-arch requires a value") - end - config[:target_arch] = Symbol(args[i]) - elseif arg == "--target-bundle" - i += 1 - if i > length(args) - error("--target-bundle requires a value") - end - config[:target_bundle] = Symbol(args[i]) else - @warn "Unknown argument: $arg" + @warn "Unknown argument: $option" end - i += 1 end - preference_overrides_dict = TOML.parse(join(preference_overrides, "\n")) + schema = TOML.parse(String(read(joinpath(pkgdir(@__MODULE__), "LocalPreferences.toml"))))["AppBundler"] + preference_overrides_dict = CLIParser.parse_extra_args(preference_overrides, schema) merged_preferences = merge(preferences, preference_overrides_dict) return config, merged_preferences diff --git a/test/appimage2e.jl b/test/appimage2e.jl index f305678..fd6136a 100644 --- a/test/appimage2e.jl +++ b/test/appimage2e.jl @@ -4,17 +4,17 @@ using Test build_dir = joinpath(@__DIR__, "../build") args(app_dir) = ["build", app_dir, "--build-dir=$build_dir", "--force", "--selfsign", "--target-bundle=appimage"] -# @testset "GtkApp" begin -# app_dir = joinpath(dirname(@__DIR__), "examples/GTKApp") -# AppBundler.main(args(app_dir)) -# end +@testset "GtkApp" begin + app_dir = joinpath(dirname(@__DIR__), "examples/GTKApp") + AppBundler.main(args(app_dir)) +end + +# # @testset "CmdApp" begin +# # app_dir = joinpath(dirname(@__DIR__), "examples/CmdApp") +# # AppBundler.main(args(app_dir)) +# # end -# @testset "CmdApp" begin -# app_dir = joinpath(dirname(@__DIR__), "examples/CmdApp") +# @testset "ModJulia" begin +# app_dir = joinpath(dirname(@__DIR__), "examples/modjulia") # AppBundler.main(args(app_dir)) # end - -@testset "ModJulia" begin - app_dir = joinpath(dirname(@__DIR__), "examples/modjulia") - AppBundler.main(args(app_dir)) -end diff --git a/test/argument_parser.jl b/test/argument_parser.jl new file mode 100644 index 0000000..9dba6ca --- /dev/null +++ b/test/argument_parser.jl @@ -0,0 +1,283 @@ +# Currently there is a limitation +# appbundler build . --build-dir=build --selfsign -Dbundler=\"juliaimg\" -Djuliaimg_selective_assets=true -Djuliaimg_sysimg=[\"QMLApp\"] + +using AppBundler + +using Test + +using AppBundler: parse_args, CLIParser +using AppBundler.CLIParser: normalize_args + +# Original tests from integrity.jl +@test normalize_args(["--password=dfdfsdf"]) == ["--password" => "dfdfsdf"] +@test normalize_args(["--password=\"dfdfsdf\""]) == ["--password" => "dfdfsdf"] + + +@testset "normalize_args" begin + + @testset "attached and detached forms are equivalent" begin + @test normalize_args(["--password", "hunter2"]) == ["--password" => "hunter2"] + @test normalize_args(["--password=hunter2"]) == ["--password" => "hunter2"] + @test normalize_args(["--password", "foo=bar"]) == ["--password" => "foo=bar"] + @test normalize_args(["--password=foo=bar"]) == ["--password" => "foo=bar"] + @test normalize_args(["-Dbundler=juliaimg"]) == ["-D" => "bundler=juliaimg"] + @test normalize_args(["-D", "bundler=juliaimg"]) == ["-D" => "bundler=juliaimg"] + end + + @testset "valueless options" begin + @test normalize_args(["--selfsign"]) == ["--selfsign" => nothing] + @test normalize_args(["--selfsign", "--force"]) == + ["--selfsign" => nothing, "--force" => nothing] + @test normalize_args(["-D"]) == ["-D" => nothing] + @test normalize_args(["-Dselfsign"]) == ["-D" => "selfsign"] + end + + @testset "an option is never adopted as a value" begin + @test normalize_args(["--password", "--selfsign"]) == + ["--password" => nothing, "--selfsign" => nothing] + @test normalize_args(["-D", "--selfsign"]) == + ["-D" => nothing, "--selfsign" => nothing] + @test normalize_args(["--password", "-Dfoo=bar"]) == + ["--password" => nothing, "-D" => "foo=bar"] + end + + @testset "a non-option always is" begin + @test normalize_args(["--password", "-secret"]) == ["--password" => "-secret"] + @test normalize_args(["--target-name", "My App"]) == ["--target-name" => "My App"] + @test normalize_args(["--password", "a", "b"]) == + ["--password" => "a", "b" => nothing] + # Ambiguity the rule buys: a boolean option adopts a following + # positional. Caught downstream by the caller's arity check. + @test normalize_args(["--selfsign", "build"]) == ["--selfsign" => "build"] + @test normalize_args(["build", ".", "--selfsign"]) == + ["build" => nothing, "." => nothing, "--selfsign" => nothing] + end + + @testset "healing: unclosed bracket or quote" begin + @test normalize_args(["-Dsysimg=[QMLApp,", "AppEnv]"]) == + ["-D" => "sysimg=[QMLApp, AppEnv]"] + @test normalize_args(["-Dsysimg=\"[QMLApp,", "AppEnv]\""]) == + ["-D" => "sysimg=\"[QMLApp, AppEnv]\""] + @test normalize_args(["-Ds=[A,", "B,", "C]"]) == ["-D" => "s=[A, B, C]"] + # Absorbed tokens may themselves look like options. + @test normalize_args(["-Dsysimg=[a,", "-b]"]) == ["-D" => "sysimg=[a, -b]"] + @test normalize_args(["-Dsysimg=[a,", "--selfsign]"]) == + ["-D" => "sysimg=[a, --selfsign]"] + # Already balanced — nothing to heal. + @test normalize_args(["-Dsysimg=[\"QMLApp\"]"]) == ["-D" => "sysimg=[\"QMLApp\"]"] + # A quoted bracket is content, not an open delimiter. + @test normalize_args(["-Dbundler=\"[sdfsdffsdf\"", "--force"]) == + ["-D" => "bundler=\"[sdfsdffsdf\"", "--force" => nothing] + end + + @testset "healing: trailing comma" begin + @test normalize_args(["-Dsysimg=QMLApp,", "AppEnv"]) == + ["-D" => "sysimg=QMLApp, AppEnv"] + @test normalize_args(["-Dsysimg=a,", "--selfsign"]) == + ["-D" => "sysimg=a, --selfsign"] + @test normalize_args(["-Dsysimg=a,", "[b,", "c]"]) == ["-D" => "sysimg=a, [b, c]"] + @test normalize_args(["-Dsysimg=a,b"]) == ["-D" => "sysimg=a,b"] + end + + @testset "tokens that must not be healed" begin + # No '=' in the token, so it is a plain value. + @test normalize_args(["--password", "ab[cd", "--selfsign"]) == + ["--password" => "ab[cd", "--selfsign" => nothing] + # Balanced with no trailing comma: no continuation signal exists, so the + # attached form silently truncates. Use the detached form for spaces. + @test normalize_args(["--target-name=My", "App"]) == + ["--target-name" => "My", "App" => nothing] + # A single quote in ARGS is an apostrophe, never a grouping quote. + @test normalize_args(["--target-name", "Bob's App"]) == + ["--target-name" => "Bob's App"] + end + + @testset "unterminated values" begin + @test_throws ErrorException normalize_args(["-Dsysimg=[QMLApp,"]) + @test_throws ErrorException normalize_args(["-Dsysimg=a,"]) + @test_throws ErrorException normalize_args(["-Dbundler=\"juliaimg"]) + # Absorbs to the end of ARGS and is still unbalanced. + @test_throws ErrorException normalize_args(["-Dsysimg=[a,", "--selfsign"]) + end + + @testset "empty input" begin + @test normalize_args(String[]) == [] + end + + @testset "empty attached values" begin + # An attached empty value is "" rather than nothing, so a `require`-style + # check accepts it. Legitimate for -D (empty list); questionable for + # path-like flags, where it reaches mkpath(""). + @test normalize_args(["-Dsysimg="]) == ["-D" => "sysimg="] + @test normalize_args(["--build-dir="]) == ["--build-dir" => ""] + @test normalize_args(["--"]) == ["--" => nothing] + end +end + +prefs_of(v...) = parse_args(String[v...])[2] +config_of(v...) = parse_args(String[v...])[1] + +@testset "parse_args" begin + + @testset "the motivating invocation, short form" begin + prefs = prefs_of("--selfsign", + "-Dbundler=juliaimg", + "-Djuliaimg_selective_assets=true", + "-Djuliaimg_sysimg=QMLApp") + @test prefs["bundler"] == "juliaimg" + @test prefs["juliaimg_selective_assets"] === true + @test prefs["juliaimg_sysimg"] == ["QMLApp"] + @test prefs["selfsign"] === true + end + + @testset "the motivating invocation, escaped form still works" begin + prefs = prefs_of("-Dbundler=\"juliaimg\"", + "-Djuliaimg_selective_assets=true", + "-Djuliaimg_sysimg=[\"QMLApp\"]") + @test prefs["bundler"] == "juliaimg" + @test prefs["juliaimg_selective_assets"] === true + @test prefs["juliaimg_sysimg"] == ["QMLApp"] + end + + @testset "all list spellings converge" begin + # Single tokens — the shell kept these intact. + for raw in ("-Djuliaimg_sysimg=[\"QMLApp\",\"AppEnv\"]", + "-Djuliaimg_sysimg=[QMLApp,AppEnv]", + "-Djuliaimg_sysimg=QMLApp,AppEnv") + @test prefs_of(raw)["juliaimg_sysimg"] == ["QMLApp", "AppEnv"] + end + # Split by the shell on the unquoted space — healing must rejoin them. + @test prefs_of("-Djuliaimg_sysimg=[QMLApp,", "AppEnv]")["juliaimg_sysimg"] == + ["QMLApp", "AppEnv"] + @test prefs_of("-Djuliaimg_sysimg=\"[QMLApp,", "AppEnv]\"")["juliaimg_sysimg"] == + ["QMLApp", "AppEnv"] + end + + @testset "a bare scalar promotes to a one-element list" begin + @test prefs_of("-Djuliaimg_sysimg=QMLApp")["juliaimg_sysimg"] == ["QMLApp"] + @test prefs_of("-Djuliaimg_sysimg=\"QMLApp\"")["juliaimg_sysimg"] == ["QMLApp"] + end + + @testset "empty lists" begin + for raw in ("-Djuliaimg_sysimg=", "-Djuliaimg_sysimg=[]") + @test isempty(prefs_of(raw)["juliaimg_sysimg"]) + end + end + + @testset "strings are never type-guessed" begin + @test prefs_of("-Dbundler=2026")["bundler"] === "2026" + @test prefs_of("-Dbundler=true")["bundler"] === "true" + @test prefs_of("-Dbundler=")["bundler"] === "" + # Only the first '=' splits key from value. + @test prefs_of("-Dbundler=a=b")["bundler"] == "a=b" + end + + @testset "booleans" begin + @test prefs_of("-Dselfsign=true")["selfsign"] === true + @test prefs_of("-Dselfsign=false")["selfsign"] === false + @test prefs_of("-Dselfsign=\"true\"")["selfsign"] === true + @test prefs_of("-Dselfsign")["selfsign"] === true # bare key, Bool only + @test_throws Exception prefs_of("-Dselfsign=yes") + @test_throws Exception prefs_of("-Dselfsign=1") + @test_throws Exception prefs_of("-Dbundler") # bare key, not a Bool + end + + @testset "attached and detached -D forms agree" begin + @test prefs_of("-Dbundler=juliaimg")["bundler"] == + prefs_of("-D", "bundler=juliaimg")["bundler"] == "juliaimg" + @test prefs_of("-D", "juliaimg_sysimg=[QMLApp,", "AppEnv]")["juliaimg_sysimg"] == + ["QMLApp", "AppEnv"] + end + + @testset "-D overrides win over flag-set preferences" begin + @test prefs_of("--selfsign", "-Dselfsign=false")["selfsign"] === false + # --debug sets compress=false; the override wins whichever order they appear. + @test prefs_of("-Dcompress=true", "--debug")["compress"] === true + @test prefs_of("--debug", "-Dcompress=true")["compress"] === true + end + + @testset "repeated keys: last one wins" begin + @test prefs_of("-Dbundler=first", "-Dbundler=second")["bundler"] == "second" + end + + @testset "flags unrelated to -D are unaffected by healing" begin + config, prefs = parse_args(String["--target-name", "My App", + "--target-arch", "aarch64", + "--target-bundle", "dmg", + "--password", " hunter2 ", + "--force", + "-Djuliaimg_sysimg=[QMLApp,", "AppEnv]"]) + @test config[:target_name] == "My App" + @test config[:target_arch] === :aarch64 + @test config[:target_bundle] === :dmg + @test config[:password] == "hunter2" + @test prefs["overwrite_target"] === true + @test prefs["juliaimg_sysimg"] == ["QMLApp", "AppEnv"] + end + + @testset "values with no '=' are never absorbed" begin + # Guards --password 'ab[cd' from swallowing the next argument. + config, prefs = parse_args(String["--password", "ab[cd", "--selfsign"]) + @test config[:password] == "ab[cd" + @test prefs["selfsign"] === true + # A single quote in ARGS is an apostrophe, never a grouping quote. + @test config_of("--target-name", "Bob's App", "--force")[:target_name] == "Bob's App" + end + + @testset "a quoted bracket is content, not an open delimiter" begin + # Must not absorb --force looking for a closing ']'. + prefs = prefs_of("-Dbundler=\"[sdfsdffsdf\"", "--force") + @test prefs["bundler"] == "[sdfsdffsdf" + @test prefs["overwrite_target"] === true + end + + @testset "an unterminated list aborts the whole invocation" begin + # Must not silently swallow --selfsign into the list. + @test_throws Exception prefs_of("-Djuliaimg_sysimg=[QMLApp,", "--selfsign") + @test_throws Exception prefs_of("-Djuliaimg_sysimg=[QMLApp,") + @test_throws Exception prefs_of("-Dbundler=\"juliaimg") + end + + @testset "typos in preference names abort rather than build wrong" begin + @test_throws Exception prefs_of("-Djuliaimg_selective_asset=true") + @test_throws Exception prefs_of("-Dbundlr=juliaimg") + err = try + prefs_of("-Dbundlr=juliaimg"); nothing + catch e + sprint(showerror, e) + end + @test err !== nothing && occursin("bundler", err) + end + + @testset "defaults with no arguments" begin + config, prefs = parse_args(String[]) + @test isdir(config[:build_dir]) + @test config[:target_name] === nothing + @test config[:password] === nothing + @test config[:target_arch] === Sys.ARCH + @test isempty(prefs) + end + + @testset "unknown flags warn but do not abort" begin + @test_logs (:warn,) match_mode = :any begin + @test prefs_of("--nonsense", "-Dbundler=juliaimg")["bundler"] == "juliaimg" + end + end +end + + +# Currently failing — these branches do not bounds-check, so they throw +# BoundsError instead of a usable message. Remove @test_broken once the +# `i > length(args)` guard is applied uniformly (see --build-dir for the shape). +@testset "missing values report properly" begin + for flag in ("--target-name", "--password", "--target-arch", + "--target-bundle", "--build-dir", "-D") + @testset "$flag" begin + @test_throws ErrorException parse_args(String[flag]) + @test_throws ErrorException parse_args(String[flag, "--selfsign"]) + end + end +end + + + diff --git a/test/integrity.jl b/test/integrity.jl index 7a2e3b3..84f48c6 100644 --- a/test/integrity.jl +++ b/test/integrity.jl @@ -1,9 +1,6 @@ using AppBundler using Test -@test AppBundler.normalize_args(["--password=dfdfsdf"]) == ["--password", "dfdfsdf"] -@test AppBundler.normalize_args(["--password=\"dfdfsdf\""]) == ["--password", "dfdfsdf"] - app_dir = joinpath(dirname(@__DIR__), "examples/GLApp") build_dir = mktempdir() target_name = "glapp" From f6eb5a24ff5f25731805312c44d7e72a55f831dc Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 16:26:43 +0300 Subject: [PATCH 08/14] Claude Opus 5: find untested edge cases --- src/CLIParser.jl | 102 +++++++++++++++++++++++++++++++++++----- src/main.jl | 8 +++- test/argument_parser.jl | 44 +++++++++++++++++ 3 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/CLIParser.jl b/src/CLIParser.jl index 4e05db2..c8f91f6 100644 --- a/src/CLIParser.jl +++ b/src/CLIParser.jl @@ -13,8 +13,8 @@ function is_balanced(s::AbstractString) for c in s if escaped escaped = false - elseif c == '\\' - escaped = true + # elseif c == '\\' + # escaped = true elseif quote_char !== nothing c == quote_char && (quote_char = nothing) elseif c == '"' || c == '\'' @@ -57,7 +57,8 @@ end const Arg = Pair{String, Union{String, Nothing}} -isoption(tok) = startswith(tok, "--") || startswith(tok, "-D") + +isoption(tok; short_options = Dict()) = startswith(tok, "--") || startswith(tok, "-D") || haskey(short_options, tok) """ normalize_args(raw_args) -> Vector{Arg} @@ -70,15 +71,17 @@ equivalent, and only the first `=` separates option from value: -Dbundler=juliaimg ⇒ "-D" => "bundler=juliaimg" -D bundler=juliaimg ⇒ "-D" => "bundler=juliaimg" --selfsign ⇒ "--selfsign" => nothing + -h ⇒ "--help" => nothing An option takes the following token as its value unless that token is itself an -option. Tokens appearing where no option is open are emitted as `token => -nothing`. +option. Tokens appearing where no option is open are emitted as +`token => nothing`. Values are healed first (see `heal_args`) and have one +matching pair of outer quotes removed. -Values are healed first: a token containing `=` keeps absorbing following -tokens while it has an unclosed quote or bracket, or ends in a comma. +The `-D` payload is left untouched — `unquote` runs later, per list element, +during type coercion. """ -function normalize_args(raw_args) +function normalize_args(raw_args; short_options = Dict()) tokens = heal_args(raw_args) out = Arg[] @@ -86,8 +89,20 @@ function normalize_args(raw_args) while i <= length(tokens) tok = tokens[i] - if !isoption(tok) - push!(out, tok => nothing) # positional, or a stray value + if !isoption(tok; short_options) + push!(out, tok => nothing) # positional, or a stray value + i += 1 + continue + end + + if haskey(short_options, tok) # boolean short flag, takes no value + push!(out, short_options[tok] => nothing) + i += 1 + continue + end + + if tok == "--" # end-of-options marker + push!(out, "--" => nothing) i += 1 continue end @@ -102,12 +117,17 @@ function normalize_args(raw_args) end # Detached form: adopt the next token unless it is another option. - if value === nothing && i < length(tokens) && !isoption(tokens[i+1]) + if value === nothing && i < length(tokens) && !isoption(tokens[i+1]; short_options) i += 1 value = tokens[i] end - push!(out, option => (value === nothing ? nothing : unquote(value))) + if option == "-D" + push!(out, "-D" => value) # payload stays raw + else + push!(out, option => (value === nothing ? nothing : unquote(value))) + end + i += 1 end @@ -115,6 +135,64 @@ function normalize_args(raw_args) end + +# """ +# normalize_args(raw_args) -> Vector{Arg} + +# Turn raw ARGS into `option => value` pairs. Attached and detached forms are +# equivalent, and only the first `=` separates option from value: + +# --password=foo=bar ⇒ "--password" => "foo=bar" +# --password foo=bar ⇒ "--password" => "foo=bar" +# -Dbundler=juliaimg ⇒ "-D" => "bundler=juliaimg" +# -D bundler=juliaimg ⇒ "-D" => "bundler=juliaimg" +# --selfsign ⇒ "--selfsign" => nothing + +# An option takes the following token as its value unless that token is itself an +# option. Tokens appearing where no option is open are emitted as `token => +# nothing`. + +# Values are healed first: a token containing `=` keeps absorbing following +# tokens while it has an unclosed quote or bracket, or ends in a comma. +# """ + +# function normalize_args(raw_args) +# tokens = heal_args(raw_args) +# out = Arg[] + +# i = 1 +# while i <= length(tokens) +# tok = tokens[i] + +# if !isoption(tok) +# push!(out, tok => nothing) # positional, or a stray value +# i += 1 +# continue +# end + +# if startswith(tok, "-D") && !startswith(tok, "--") +# option = "-D" +# value = length(tok) > 2 ? tok[3:end] : nothing +# else +# j = findfirst('=', tok) +# option = j === nothing ? tok : tok[1:prevind(tok, j)] +# value = j === nothing ? nothing : tok[nextind(tok, j):end] +# end + +# # Detached form: adopt the next token unless it is another option. +# if value === nothing && i < length(tokens) && !isoption(tokens[i+1]) +# i += 1 +# value = tokens[i] +# end + +# push!(out, option => (value === nothing ? nothing : unquote(value))) +# i += 1 +# end + +# return out +# end + + ### Extra argument coercion according to schema function parse_extra_args(args::Vector{String}, schema::Dict) diff --git a/src/main.jl b/src/main.jl index f78e03f..bd6d665 100644 --- a/src/main.jl +++ b/src/main.jl @@ -242,12 +242,16 @@ function get_bundle_parameters!(parameters::Dict{String, Any}, project_toml; pre return parameters end +# Short options, mapped to their long form. Listed explicitly because a leading +# single dash is otherwise a value: `--password -secret` must keep -secret. +const SHORT_OPTIONS = Dict("-h" => "--help") + require(option, value) = value === nothing ? error("$option requires a value") : value forbid(option, value) = value === nothing || error("$option does not take a value, got '$value'") function parse_args(raw_args) - args = CLIParser.normalize_args(raw_args) + args = CLIParser.normalize_args(raw_args; short_options = SHORT_OPTIONS) # Default values config = Dict( @@ -262,7 +266,7 @@ function parse_args(raw_args) preferences = Dict() for (option, value) in args - if option in ["--help", "-h"] + if option == "--help" print_help() exit(0) elseif option == "--build-dir" diff --git a/test/argument_parser.jl b/test/argument_parser.jl index 9dba6ca..7dc43c8 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -279,5 +279,49 @@ end end end +@test normalize_args(["--password=''x''"]) == ["--password" => "'x'"] +@test prefs_of("-Dbundler=''x''")["bundler"] == "'x'" +@test prefs_of("-Dbundler=\"\\\"x\\\"\"")["bundler"] == "\\\"x\\\"" +@test normalize_args(["-Dpath=\"C:\\\"", "--selfsign"]) == + ["-D" => "path=\"C:\\\"", "--selfsign" => nothing] +@test prefs_of("-Dbundler=a\\b")["bundler"] == "a\\b" +@test normalize_args(["-Dpath=\"C:\\\"", "--selfsign"]) == ["-D" => "path=\"C:\\\"", "--selfsign" => nothing] +normalize_args(["-Dbundler=\"juliaimg\""]) == ["-D" => "bundler=\"juliaimg\""] + +prefs = prefs_of("-Djuliaimg_sysimg=a,", "--selfsign") +@test prefs["juliaimg_sysimg"] == ["a", "--selfsign"] +@test !haskey(prefs, "selfsign") # deliberately NOT set + +@test prefs_of("-Djuliaimg_sysimg=[a,b,]")["juliaimg_sysimg"] == ["a", "b", ""] +@test prefs_of("-Djuliaimg_sysimg=[,a]")["juliaimg_sysimg"] == ["", "a"] + +@test normalize_args(["-D="]) == ["-D" => "="] +@test_throws Exception prefs_of("-D=") # empty key +@test_throws Exception prefs_of("-D", "=value") +@test prefs_of("-D", "selfsign")["selfsign"] === true # detached bare bool + + +@test normalize_args(["-Dkey={a,", "b}"]) == ["-D" => "key={a, b}"] +@test_throws Exception prefs_of("-Dkey={a=1}") + + +# --skipsign never appears in any test +@test prefs_of("--skipsign")["skipsign"] === true + +# repeated non-D flags +@test config_of("--target-arch", "x86_64", "--target-arch", "aarch64")[:target_arch] === :aarch64 + +# attached --password keeps the strip() +@test config_of("--password= hunter2 ")[:password] == "hunter2" + +# normalize_args must not mutate its input +raw = ["-Dsysimg=[a,", "b]"]; before = copy(raw) +normalize_args(raw); @test raw == before + +# --debug sets windowed, which is never asserted +@test prefs_of("--debug")["windowed"] === false + +@test prefs_of("-Dbundler=\"sdsd,sds\"")["bundler"] == "sdsd,sds" +@test prefs_of("-Djuliaimg_sysimg=\"sdsd,sds\"")["juliaimg_sysimg"] == ["sdsd", "sds"] From b96752e6a5acfc3153fd388fbda2f40263bcb19d Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 16:59:55 +0300 Subject: [PATCH 09/14] Changing trailing coma to error --- src/CLIParser.jl | 77 +++++++++++++++++++++++++++++++++++------ src/main.jl | 2 ++ test/argument_parser.jl | 12 +++---- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/CLIParser.jl b/src/CLIParser.jl index c8f91f6..21170c9 100644 --- a/src/CLIParser.jl +++ b/src/CLIParser.jl @@ -31,22 +31,44 @@ end ends_open(s) = endswith(rstrip(s), ',') + +""" + heal_args(raw_args) -> Vector{String} + +Rejoin option values that the shell split on whitespace. A token containing `=` +absorbs following tokens while it has an unclosed quote or bracket — the closing +delimiter marks the end of the value, so absorbed tokens may look like anything: +`-Dsysimg=[a, --selfsign]` yields a two-element list. + +A comma is a separator, not a continuation signal. A value left ending in one is +rejected: write the list without spaces, or bracket it so its end is explicit. +""" function heal_args(raw_args) out = String[] i = 1 while i <= length(raw_args) tok = String(raw_args[i]) if occursin('=', tok) - while !is_balanced(tok) || ends_open(tok) + while !is_balanced(tok) i += 1 - i > length(raw_args) && error(""" - Unterminated value: $tok - It ends with a comma, or is missing a closing ']' or '"'. - Write the value without spaces, or quote the whole option: - -Dkey=a,-b or -D 'key=[a, -b]' - """) + if i > length(raw_args) + error(""" + Unterminated value: $tok + Missing a closing ']' or '"'. Write the list without spaces, + or bracket it: + -Dsysimg=a,b or -Dsysimg=[a, b] + """) + end tok *= " " * raw_args[i] end + if ends_open(tok) + error(""" + Trailing comma in value: $tok + A space after a comma ends the value. Remove the spaces, or + bracket the list so its end is explicit: + -Dsysimg=a,b or -Dsysimg=[a, b] + """) + end end push!(out, tok) i += 1 @@ -54,12 +76,36 @@ function heal_args(raw_args) return out end +# function heal_args(raw_args) +# out = String[] +# i = 1 +# while i <= length(raw_args) +# tok = String(raw_args[i]) +# if occursin('=', tok) +# while !is_balanced(tok) || ends_open(tok) +# i += 1 +# i > length(raw_args) && error(""" +# Unterminated value: $tok +# It ends with a comma, or is missing a closing ']' or '"'. +# Write the value without spaces, or quote the whole option: +# -Dkey=a,-b or -D 'key=[a, -b]' +# """) +# tok *= " " * raw_args[i] +# end +# end +# push!(out, tok) +# i += 1 +# end +# return out +# end + const Arg = Pair{String, Union{String, Nothing}} isoption(tok; short_options = Dict()) = startswith(tok, "--") || startswith(tok, "-D") || haskey(short_options, tok) + """ normalize_args(raw_args) -> Vector{Arg} @@ -76,10 +122,19 @@ equivalent, and only the first `=` separates option from value: An option takes the following token as its value unless that token is itself an option. Tokens appearing where no option is open are emitted as `token => nothing`. Values are healed first (see `heal_args`) and have one -matching pair of outer quotes removed. - -The `-D` payload is left untouched — `unquote` runs later, per list element, -during type coercion. +matching pair of outer quotes removed. The `-D` payload is left untouched — +`unquote` runs later, per list element, during type coercion. + +Which tokens count as options is asymmetric, and it constrains what a detached +value can be. Any token starting with `--` is an option, so a value beginning +with `--` can never be adopted: `--target-name --weird` yields two valueless +options rather than a name. Short options are matched exactly instead, so only +the literal token `-h` is an option while `-hunter2` and `-secret` remain +values. The attached form bypasses this check entirely and is the escape hatch +for both cases — write `--target-name=--weird` or `--password=-h`. None of it +applies inside a healed value, where an open bracket or trailing comma has +already joined the tokens before this function sees them, so +`-Dsysimg=[a, --selfsign]` keeps `--selfsign` as a list element. """ function normalize_args(raw_args; short_options = Dict()) tokens = heal_args(raw_args) diff --git a/src/main.jl b/src/main.jl index bd6d665..49d745f 100644 --- a/src/main.jl +++ b/src/main.jl @@ -11,6 +11,8 @@ function (@main)(ARGS) if command in ["--help", "-h"] + # one may want to print a generic help here and then point user down to + # build --help and etc for concrete information print_help() elseif command == "build" diff --git a/test/argument_parser.jl b/test/argument_parser.jl index 7dc43c8..eabaa10 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -71,11 +71,11 @@ using AppBundler.CLIParser: normalize_args end @testset "healing: trailing comma" begin - @test normalize_args(["-Dsysimg=QMLApp,", "AppEnv"]) == + @test_throws ErrorException normalize_args(["-Dsysimg=QMLApp,", "AppEnv"]) == ["-D" => "sysimg=QMLApp, AppEnv"] - @test normalize_args(["-Dsysimg=a,", "--selfsign"]) == + @test_throws ErrorException normalize_args(["-Dsysimg=a,", "--selfsign"]) == ["-D" => "sysimg=a, --selfsign"] - @test normalize_args(["-Dsysimg=a,", "[b,", "c]"]) == ["-D" => "sysimg=a, [b, c]"] + @test_throws ErrorException normalize_args(["-Dsysimg=a,", "[b,", "c]"]) == ["-D" => "sysimg=a, [b, c]"] @test normalize_args(["-Dsysimg=a,b"]) == ["-D" => "sysimg=a,b"] end @@ -290,9 +290,9 @@ end normalize_args(["-Dbundler=\"juliaimg\""]) == ["-D" => "bundler=\"juliaimg\""] -prefs = prefs_of("-Djuliaimg_sysimg=a,", "--selfsign") -@test prefs["juliaimg_sysimg"] == ["a", "--selfsign"] -@test !haskey(prefs, "selfsign") # deliberately NOT set +@test_throws ErrorException prefs_of("-Djuliaimg_sysimg=a,", "--selfsign") +#@test prefs["juliaimg_sysimg"] == ["a", "--selfsign"] +#@test !haskey(prefs, "selfsign") # deliberately NOT set @test prefs_of("-Djuliaimg_sysimg=[a,b,]")["juliaimg_sysimg"] == ["a", "b", ""] @test prefs_of("-Djuliaimg_sysimg=[,a]")["juliaimg_sysimg"] == ["", "a"] From 6d15288e59487ce70bbd2125fe4d1b3228aaea96 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 18:48:55 +0300 Subject: [PATCH 10/14] fixing edge cases --- src/CLIParser.jl | 146 ++++++++++++++-------------------------- src/main.jl | 3 + test/argument_parser.jl | 8 +++ 3 files changed, 60 insertions(+), 97 deletions(-) diff --git a/src/CLIParser.jl b/src/CLIParser.jl index 21170c9..dfdcdaa 100644 --- a/src/CLIParser.jl +++ b/src/CLIParser.jl @@ -1,37 +1,69 @@ module CLIParser -""" - is_balanced(s) -> Bool +# """ +# is_balanced(s) -> Bool + +# True when `s` contains no unclosed quote and no unclosed bracket, i.e. the +# shell did not split a `-D` value across multiple argv entries. +# """ +# function is_balanced(s::AbstractString) +# quote_char = nothing +# depth = 0 +# escaped = false +# for c in s +# if escaped +# escaped = false +# # elseif c == '\\' +# # escaped = true +# elseif quote_char !== nothing +# c == quote_char && (quote_char = nothing) +# elseif c == '"' || c == '\'' +# quote_char = c +# elseif c == '[' || c == '{' +# depth += 1 +# elseif c == ']' || c == '}' +# depth -= 1 +# end +# end +# return quote_char === nothing && depth <= 0 +# end + -True when `s` contains no unclosed quote and no unclosed bracket, i.e. the -shell did not split a `-D` value across multiple argv entries. -""" function is_balanced(s::AbstractString) + j = findfirst('=', s) + j === nothing && return true + return value_balanced(SubString(s, nextind(s, j))) +end + +function value_balanced(v::AbstractString) quote_char = nothing depth = 0 - escaped = false - for c in s - if escaped - escaped = false - # elseif c == '\\' - # escaped = true - elseif quote_char !== nothing - c == quote_char && (quote_char = nothing) - elseif c == '"' || c == '\'' + at_element_start = true + for c in v + if quote_char !== nothing + if c == quote_char + quote_char = nothing + at_element_start = false + end + elseif at_element_start && (c == '"' || c == '\'') quote_char = c elseif c == '[' || c == '{' depth += 1 + at_element_start = true elseif c == ']' || c == '}' depth -= 1 + at_element_start = false + elseif c == ',' + at_element_start = true + elseif !isspace(c) + at_element_start = false end end return quote_char === nothing && depth <= 0 end - ends_open(s) = endswith(rstrip(s), ',') - """ heal_args(raw_args) -> Vector{String} @@ -76,29 +108,6 @@ function heal_args(raw_args) return out end -# function heal_args(raw_args) -# out = String[] -# i = 1 -# while i <= length(raw_args) -# tok = String(raw_args[i]) -# if occursin('=', tok) -# while !is_balanced(tok) || ends_open(tok) -# i += 1 -# i > length(raw_args) && error(""" -# Unterminated value: $tok -# It ends with a comma, or is missing a closing ']' or '"'. -# Write the value without spaces, or quote the whole option: -# -Dkey=a,-b or -D 'key=[a, -b]' -# """) -# tok *= " " * raw_args[i] -# end -# end -# push!(out, tok) -# i += 1 -# end -# return out -# end - const Arg = Pair{String, Union{String, Nothing}} @@ -190,64 +199,6 @@ function normalize_args(raw_args; short_options = Dict()) end - -# """ -# normalize_args(raw_args) -> Vector{Arg} - -# Turn raw ARGS into `option => value` pairs. Attached and detached forms are -# equivalent, and only the first `=` separates option from value: - -# --password=foo=bar ⇒ "--password" => "foo=bar" -# --password foo=bar ⇒ "--password" => "foo=bar" -# -Dbundler=juliaimg ⇒ "-D" => "bundler=juliaimg" -# -D bundler=juliaimg ⇒ "-D" => "bundler=juliaimg" -# --selfsign ⇒ "--selfsign" => nothing - -# An option takes the following token as its value unless that token is itself an -# option. Tokens appearing where no option is open are emitted as `token => -# nothing`. - -# Values are healed first: a token containing `=` keeps absorbing following -# tokens while it has an unclosed quote or bracket, or ends in a comma. -# """ - -# function normalize_args(raw_args) -# tokens = heal_args(raw_args) -# out = Arg[] - -# i = 1 -# while i <= length(tokens) -# tok = tokens[i] - -# if !isoption(tok) -# push!(out, tok => nothing) # positional, or a stray value -# i += 1 -# continue -# end - -# if startswith(tok, "-D") && !startswith(tok, "--") -# option = "-D" -# value = length(tok) > 2 ? tok[3:end] : nothing -# else -# j = findfirst('=', tok) -# option = j === nothing ? tok : tok[1:prevind(tok, j)] -# value = j === nothing ? nothing : tok[nextind(tok, j):end] -# end - -# # Detached form: adopt the next token unless it is another option. -# if value === nothing && i < length(tokens) && !isoption(tokens[i+1]) -# i += 1 -# value = tokens[i] -# end - -# push!(out, option => (value === nothing ? nothing : unquote(value))) -# i += 1 -# end - -# return out -# end - - ### Extra argument coercion according to schema function parse_extra_args(args::Vector{String}, schema::Dict) @@ -346,5 +297,6 @@ function edit_distance(a, b) return prev[end] end +export normalize_args, parse_extra_args end diff --git a/src/main.jl b/src/main.jl index 49d745f..ac6923e 100644 --- a/src/main.jl +++ b/src/main.jl @@ -2,6 +2,9 @@ import TOML import LibGit2 function (@main)(ARGS) + + + @show ARGS if length(ARGS) == 0 error("No command provided. See `--help` for available commands.") diff --git a/test/argument_parser.jl b/test/argument_parser.jl index eabaa10..c904e1f 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -325,3 +325,11 @@ normalize_args(raw); @test raw == before @test prefs_of("-Dbundler=\"sdsd,sds\"")["bundler"] == "sdsd,sds" @test prefs_of("-Djuliaimg_sysimg=\"sdsd,sds\"")["juliaimg_sysimg"] == ["sdsd", "sds"] + + +@test normalize_args(["--target-name=\"Bob's Tool\""]) == ["--target-name" => "Bob's Tool"] +@test normalize_args(["--target-name", "Bob's Tool"]) == ["--target-name" => "Bob's Tool"] +@test normalize_args(["--target-name=Bob's Tool"]) == ["--target-name" => "Bob's Tool"] + +@test_throws Exception normalize_args(["--description=Tool for X, Y,"]) +@test normalize_args(["--description=\"Tool for X, Y,\""]) == ["--description" => "Tool for X, Y,"] From 285db5676fc9bfe48d41720f9e0405744e22add6 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 21:39:35 +0300 Subject: [PATCH 11/14] refactoring --- src/AppBundler.jl | 2 +- src/{CLIParser.jl => ArgTools.jl} | 51 ++++++++++++------------------- src/main.jl | 11 ++----- test/argument_parser.jl | 4 +-- 4 files changed, 26 insertions(+), 42 deletions(-) rename src/{CLIParser.jl => ArgTools.jl} (91%) diff --git a/src/AppBundler.jl b/src/AppBundler.jl index 7b62b35..1daa812 100644 --- a/src/AppBundler.jl +++ b/src/AppBundler.jl @@ -46,7 +46,7 @@ include("utils.jl") include("bundle.jl") include("recipes.jl") -include("CLIParser.jl") +include("ArgTools.jl") include("main.jl") #bundle_app(app_dir, bundle_dir; version = VERSION) = bundle_app(HostPlatform(), app_dir, bundle_dir; version) diff --git a/src/CLIParser.jl b/src/ArgTools.jl similarity index 91% rename from src/CLIParser.jl rename to src/ArgTools.jl index dfdcdaa..71c53ec 100644 --- a/src/CLIParser.jl +++ b/src/ArgTools.jl @@ -1,33 +1,4 @@ -module CLIParser - -# """ -# is_balanced(s) -> Bool - -# True when `s` contains no unclosed quote and no unclosed bracket, i.e. the -# shell did not split a `-D` value across multiple argv entries. -# """ -# function is_balanced(s::AbstractString) -# quote_char = nothing -# depth = 0 -# escaped = false -# for c in s -# if escaped -# escaped = false -# # elseif c == '\\' -# # escaped = true -# elseif quote_char !== nothing -# c == quote_char && (quote_char = nothing) -# elseif c == '"' || c == '\'' -# quote_char = c -# elseif c == '[' || c == '{' -# depth += 1 -# elseif c == ']' || c == '}' -# depth -= 1 -# end -# end -# return quote_char === nothing && depth <= 0 -# end - +module ArgTools function is_balanced(s::AbstractString) j = findfirst('=', s) @@ -297,6 +268,24 @@ function edit_distance(a, b) return prev[end] end -export normalize_args, parse_extra_args +function parse_args(raw_args; schema = Dict(), short_options = Dict()) + args = normalize_args(raw_args; short_options) + + options = Arg[] + defines = String[] + + for (key, value) in args + if key == "-D" + value === nothing && error("-D expects key=value, e.g. -Dbundler=juliaimg") + push!(defines, value) + else + push!(options, key => value) + end + end + + return options, parse_extra_args(defines, schema) +end + +export parse_args end diff --git a/src/main.jl b/src/main.jl index ac6923e..44c24c7 100644 --- a/src/main.jl +++ b/src/main.jl @@ -256,7 +256,8 @@ forbid(option, value) = value === nothing || error("$option does not take a val function parse_args(raw_args) - args = CLIParser.normalize_args(raw_args; short_options = SHORT_OPTIONS) + schema = TOML.parse(String(read(joinpath(pkgdir(@__MODULE__), "LocalPreferences.toml"))))["AppBundler"] + args, preference_overrides = ArgTools.parse_args(raw_args; schema, short_options = SHORT_OPTIONS) # Default values config = Dict( @@ -267,7 +268,6 @@ function parse_args(raw_args) :password => nothing ) - preference_overrides = String[] preferences = Dict() for (option, value) in args @@ -276,8 +276,6 @@ function parse_args(raw_args) exit(0) elseif option == "--build-dir" config[:build_dir] = require(option, value) - elseif option == "-D" - push!(preference_overrides, require(option, value)) elseif option == "--target-name" config[:target_name] = require(option, value) elseif option == "--password" @@ -304,10 +302,7 @@ function parse_args(raw_args) end end - schema = TOML.parse(String(read(joinpath(pkgdir(@__MODULE__), "LocalPreferences.toml"))))["AppBundler"] - preference_overrides_dict = CLIParser.parse_extra_args(preference_overrides, schema) - merged_preferences = merge(preferences, preference_overrides_dict) - + merged_preferences = merge(preferences, preference_overrides) return config, merged_preferences end diff --git a/test/argument_parser.jl b/test/argument_parser.jl index c904e1f..c5680de 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -5,8 +5,8 @@ using AppBundler using Test -using AppBundler: parse_args, CLIParser -using AppBundler.CLIParser: normalize_args +using AppBundler: parse_args +using AppBundler.ArgTools: normalize_args # Original tests from integrity.jl @test normalize_args(["--password=dfdfsdf"]) == ["--password" => "dfdfsdf"] From f9a9d72d531c656e8bec0584ab59a6eb44fd1ffb Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 21:43:12 +0300 Subject: [PATCH 12/14] Claude 5: Clean up the module ArgTools --- src/ArgTools.jl | 264 ++++++++++++++++++++++++++++++------------------ 1 file changed, 165 insertions(+), 99 deletions(-) diff --git a/src/ArgTools.jl b/src/ArgTools.jl index 71c53ec..d43aa13 100644 --- a/src/ArgTools.jl +++ b/src/ArgTools.jl @@ -1,23 +1,56 @@ +""" + ArgTools + +Command-line argument handling: rejoin values the shell split on whitespace, +turn tokens into `option => value` pairs, and coerce `-Dkey=value` payloads +against a schema of default values. +""" module ArgTools -function is_balanced(s::AbstractString) +export parse_args + +"""An option and its value; `nothing` for flags, markers and bare tokens.""" +const Arg = Pair{String, Union{String, Nothing}} + + +### Scanning + +""" + splitpair(s) -> (key, value) + +Split `s` at its first `=`. Later `=` characters belong to the value, which is +`nothing` when there is no `=` at all. +""" +function splitpair(s::AbstractString) j = findfirst('=', s) - j === nothing && return true - return value_balanced(SubString(s, nextind(s, j))) + j === nothing && return SubString(s, 1), nothing + return SubString(s, 1, prevind(s, j)), SubString(s, nextind(s, j)) end -function value_balanced(v::AbstractString) - quote_char = nothing +""" + scan(v) -> (breaks, open_quote, depth) + +Walk a value once, tracking quoting and bracket nesting. `breaks` holds the +indices of the top-level commas — the element separators — while `open_quote` +and `depth` describe the state left at the end: a quote still open or a positive +depth means the value is unterminated. + +A quote only opens at the start of an element (after `[`, `{`, `,`, or leading +whitespace), so an apostrophe inside a word is an ordinary character. +""" +function scan(v::AbstractString) + breaks = Int[] + open_quote = nothing depth = 0 at_element_start = true - for c in v - if quote_char !== nothing - if c == quote_char - quote_char = nothing + for (i, c) in pairs(v) + if open_quote !== nothing + if c == open_quote + open_quote = nothing at_element_start = false end elseif at_element_start && (c == '"' || c == '\'') - quote_char = c + open_quote = c elseif c == '[' || c == '{' depth += 1 at_element_start = true @@ -25,15 +58,52 @@ function value_balanced(v::AbstractString) depth -= 1 at_element_start = false elseif c == ',' + depth == 0 && push!(breaks, i) at_element_start = true elseif !isspace(c) at_element_start = false end end - return quote_char === nothing && depth <= 0 + return breaks, open_quote, depth end -ends_open(s) = endswith(rstrip(s), ',') +"""Whether a value closes every quote and bracket it opens.""" +function balanced(v::AbstractString) + _, open_quote, depth = scan(v) + return open_quote === nothing && depth <= 0 +end + +"""Whether the value part of `token` (everything after the first `=`) is balanced.""" +function token_balanced(token::AbstractString) + _, value = splitpair(token) + return value === nothing || balanced(value) +end + +""" + split_elements(body) -> Vector{String} + +Split a list body on its top-level commas. Commas inside quotes or nested +brackets belong to an element. +""" +function split_elements(body::AbstractString) + breaks, _, _ = scan(body) + parts = String[] + start = firstindex(body) + for i in breaks + push!(parts, body[start:prevind(body, i)]) + start = nextind(body, i) + end + push!(parts, body[start:end]) + return parts +end + + +### Healing + +const LIST_HINT = """ + Write the list without spaces, or bracket it so its end is explicit: + -Dsysimg=a,b or -Dsysimg=[a, b] + """ """ heal_args(raw_args) -> Vector{String} @@ -47,47 +117,43 @@ A comma is a separator, not a continuation signal. A value left ending in one is rejected: write the list without spaces, or bracket it so its end is explicit. """ function heal_args(raw_args) + tokens = map(String, raw_args) out = String[] + i = 1 - while i <= length(raw_args) - tok = String(raw_args[i]) - if occursin('=', tok) - while !is_balanced(tok) + while i <= length(tokens) + token = tokens[i] + + if occursin('=', token) + while !token_balanced(token) i += 1 - if i > length(raw_args) - error(""" - Unterminated value: $tok - Missing a closing ']' or '"'. Write the list without spaces, - or bracket it: - -Dsysimg=a,b or -Dsysimg=[a, b] - """) - end - tok *= " " * raw_args[i] - end - if ends_open(tok) - error(""" - Trailing comma in value: $tok - A space after a comma ends the value. Remove the spaces, or - bracket the list so its end is explicit: - -Dsysimg=a,b or -Dsysimg=[a, b] - """) + i > length(tokens) && error(""" + Unterminated value: $token + Missing a closing ']' or '"'. + """ * LIST_HINT) + token *= " " * tokens[i] end + endswith(rstrip(token), ',') && error(""" + Trailing comma in value: $token + A space after a comma ends the value. + """ * LIST_HINT) end - push!(out, tok) + + push!(out, token) i += 1 end + return out end -const Arg = Pair{String, Union{String, Nothing}} - - -isoption(tok; short_options = Dict()) = startswith(tok, "--") || startswith(tok, "-D") || haskey(short_options, tok) +### Normalisation +isoption(token, short_options) = + startswith(token, "--") || startswith(token, "-D") || haskey(short_options, token) """ - normalize_args(raw_args) -> Vector{Arg} + normalize_args(raw_args; short_options) -> Vector{Arg} Turn raw ARGS into `option => value` pairs. Attached and detached forms are equivalent, and only the first `=` separates option from value: @@ -106,83 +172,79 @@ matching pair of outer quotes removed. The `-D` payload is left untouched — `unquote` runs later, per list element, during type coercion. Which tokens count as options is asymmetric, and it constrains what a detached -value can be. Any token starting with `--` is an option, so a value beginning -with `--` can never be adopted: `--target-name --weird` yields two valueless -options rather than a name. Short options are matched exactly instead, so only -the literal token `-h` is an option while `-hunter2` and `-secret` remain -values. The attached form bypasses this check entirely and is the escape hatch -for both cases — write `--target-name=--weird` or `--password=-h`. None of it -applies inside a healed value, where an open bracket or trailing comma has +value can be. Any token starting with `--` or `-D` is an option, so a value +beginning with either can never be adopted: `--target-name --weird` yields two +valueless options rather than a name. Short options are matched exactly instead, +so only the literal token `-h` is an option while `-hunter2` and `-secret` +remain values. The attached form bypasses this check entirely and is the escape +hatch for both cases — write `--target-name=--weird` or `--password=-h`. None of +it applies inside a healed value, where an open bracket or trailing comma has already joined the tokens before this function sees them, so `-Dsysimg=[a, --selfsign]` keeps `--selfsign` as a list element. """ -function normalize_args(raw_args; short_options = Dict()) +function normalize_args(raw_args; short_options = Dict{String, String}()) tokens = heal_args(raw_args) out = Arg[] i = 1 while i <= length(tokens) - tok = tokens[i] + token = tokens[i] + i += 1 - if !isoption(tok; short_options) - push!(out, tok => nothing) # positional, or a stray value - i += 1 + if !isoption(token, short_options) + push!(out, token => nothing) # positional, or a stray value continue - end - - if haskey(short_options, tok) # boolean short flag, takes no value - push!(out, short_options[tok] => nothing) - i += 1 + elseif haskey(short_options, token) # boolean short flag, takes no value + push!(out, String(short_options[token]) => nothing) continue - end - - if tok == "--" # end-of-options marker + elseif token == "--" # end-of-options marker push!(out, "--" => nothing) - i += 1 continue end - if startswith(tok, "-D") && !startswith(tok, "--") + if startswith(token, "-D") option = "-D" - value = length(tok) > 2 ? tok[3:end] : nothing + value = length(token) > 2 ? token[3:end] : nothing else - j = findfirst('=', tok) - option = j === nothing ? tok : tok[1:prevind(tok, j)] - value = j === nothing ? nothing : tok[nextind(tok, j):end] + key, raw = splitpair(token) + option = String(key) + value = raw === nothing ? nothing : String(raw) end # Detached form: adopt the next token unless it is another option. - if value === nothing && i < length(tokens) && !isoption(tokens[i+1]; short_options) - i += 1 + if value === nothing && i <= length(tokens) && !isoption(tokens[i], short_options) value = tokens[i] + i += 1 end - if option == "-D" - push!(out, "-D" => value) # payload stays raw + if option == "-D" || value === nothing + push!(out, option => value) # -D payload stays raw else - push!(out, option => (value === nothing ? nothing : unquote(value))) + push!(out, option => String(unquote(value))) end - - i += 1 end return out end +"""Remove one layer of matching outer quotes, if the shell left any behind.""" +function unquote(s::AbstractString) + length(s) >= 2 || return s + (s[1] == s[end] && (s[1] == '"' || s[1] == '\'')) || return s + return s[nextind(s, 1):prevind(s, lastindex(s))] +end -### Extra argument coercion according to schema -function parse_extra_args(args::Vector{String}, schema::Dict) +### Coercion against the schema +function parse_extra_args(defines, schema::Dict) overrides = Dict{String, Any}() - for arg in args - j = findfirst('=', arg) - key = j === nothing ? strip(arg) : strip(arg[1:prevind(arg, j)]) - raw = j === nothing ? nothing : strip(arg[nextind(arg, j):end]) + for define in defines + key, raw = splitpair(define) + key = String(strip(key)) haskey(schema, key) || error(unknown_key_message(key, schema)) - default = schema[key] if raw === nothing @@ -190,14 +252,13 @@ function parse_extra_args(args::Vector{String}, schema::Dict) "bare keys are only allowed for booleans. Use -D$key=.") overrides[key] = true else - overrides[key] = coerce(unquote(raw), default, key) + overrides[key] = coerce(unquote(strip(raw)), default, key) end end return overrides end - """ coerce(value, default, key) -> Any @@ -212,7 +273,7 @@ function coerce(value::AbstractString, default::AbstractVector, key) isempty(body) && return similar(default, 0) elem_default = isempty(default) ? "" : first(default) - return [coerce(unquote(strip(p)), elem_default, key) for p in split(body, ',')] + return [coerce(unquote(strip(p)), elem_default, key) for p in split_elements(body)] end coerce(value::AbstractString, ::AbstractString, key) = String(value) @@ -223,35 +284,33 @@ function coerce(value::AbstractString, ::Bool, key) return value == "true" end -function coerce(value::AbstractString, ::Integer, key) - n = tryparse(Int, value) +function coerce(value::AbstractString, default::Integer, key) + n = tryparse(typeof(default), value) n === nothing && error("preference '$key' expects an integer, got '$value'") return n end -function coerce(value::AbstractString, ::AbstractFloat, key) - x = tryparse(Float64, value) +function coerce(value::AbstractString, default::AbstractFloat, key) + x = tryparse(typeof(default), value) x === nothing && error("preference '$key' expects a number, got '$value'") return x end -"""Remove one layer of matching outer quotes, if the shell left any behind.""" -function unquote(s::AbstractString) - length(s) >= 2 || return s - (s[1] == s[end] && (s[1] == '"' || s[1] == '\'')) || return s - return s[nextind(s, 1):prevind(s, lastindex(s))] -end +coerce(::AbstractString, default, key) = + error("preference '$key' has a default of type $(typeof(default)), which cannot be set from the command line") type_name(::AbstractString) = "a string" type_name(::Bool) = "true or false" type_name(::Integer) = "an integer" type_name(::AbstractFloat) = "a number" type_name(::AbstractVector) = "a list" +type_name(x) = "a $(typeof(x))" function unknown_key_message(key, schema) - near = [k for k in keys(schema) if edit_distance(key, k) <= max(2, length(key) ÷ 4)] + threshold = max(2, length(key) ÷ 4) + near = sort!([string(k) for k in keys(schema) if edit_distance(key, k) <= threshold]) msg = "unknown preference '$key'" - isempty(near) || (msg *= "\n did you mean " * join(("'$k'" for k in sort(near)), ", ", " or ") * "?") + isempty(near) || (msg *= "\n did you mean " * join(("'$k'" for k in near), ", ", " or ") * "?") return msg end @@ -268,13 +327,22 @@ function edit_distance(a, b) return prev[end] end -function parse_args(raw_args; schema = Dict(), short_options = Dict()) - args = normalize_args(raw_args; short_options) +### Entry point + +""" + parse_args(raw_args; schema, short_options) -> (options, overrides) + +Split `raw_args` into `option => value` pairs and `-Dkey=value` preference +overrides. `schema` maps preference names to defaults whose types drive +coercion; `short_options` maps single-dash aliases to their long form, as in +`Dict("-h" => "--help")`. +""" +function parse_args(raw_args; schema::Dict = Dict{String, Any}(), short_options = Dict{String, String}()) options = Arg[] defines = String[] - for (key, value) in args + for (key, value) in normalize_args(raw_args; short_options) if key == "-D" value === nothing && error("-D expects key=value, e.g. -Dbundler=juliaimg") push!(defines, value) @@ -286,6 +354,4 @@ function parse_args(raw_args; schema = Dict(), short_options = Dict()) return options, parse_extra_args(defines, schema) end -export parse_args - end From 263de7c80396c78da66fe9726ad11fdedd912194 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 22:52:07 +0300 Subject: [PATCH 13/14] Claude 5: fixing edge cases --- src/ArgTools.jl | 163 ++++++++++++++++++++++++++++------------ test/argument_parser.jl | 13 +++- 2 files changed, 128 insertions(+), 48 deletions(-) diff --git a/src/ArgTools.jl b/src/ArgTools.jl index d43aa13..aceba17 100644 --- a/src/ArgTools.jl +++ b/src/ArgTools.jl @@ -1,3 +1,6 @@ + + + """ ArgTools @@ -67,18 +70,6 @@ function scan(v::AbstractString) return breaks, open_quote, depth end -"""Whether a value closes every quote and bracket it opens.""" -function balanced(v::AbstractString) - _, open_quote, depth = scan(v) - return open_quote === nothing && depth <= 0 -end - -"""Whether the value part of `token` (everything after the first `=`) is balanced.""" -function token_balanced(token::AbstractString) - _, value = splitpair(token) - return value === nothing || balanced(value) -end - """ split_elements(body) -> Vector{String} @@ -105,16 +96,66 @@ const LIST_HINT = """ -Dsysimg=a,b or -Dsysimg=[a, b] """ +""" + value_part(token) -> SubString or nothing + +The part of `token` that list syntax governs. A token starting with a quote is +a value in its own right, so it is taken whole — the quote belongs to the value +and not to any key. Otherwise the value is everything after the first `=`, and a +token without one has no value at all. +""" +function value_part(token::AbstractString) + isempty(token) && return nothing + (token[1] == '"' || token[1] == '\'') && return SubString(token, 1) + return last(splitpair(token)) +end + +""" + absorb(tokens, i) -> (token, i) + +Grow `tokens[i]` until its value closes every quote and bracket it opened, +returning the healed token and the index of the last token consumed. +""" +function absorb(tokens, i) + token = tokens[i] + occursin('=', token) || return token, i + + while true + _, open_quote, depth = scan(something(value_part(token))) + + depth < 0 && error(""" + Unexpected ']' or '}' in value: $token + """ * LIST_HINT) + open_quote === nothing && depth == 0 && break + + i += 1 + i > length(tokens) && error(""" + Unterminated value: $token + Missing a closing ']' or '"'. + """ * LIST_HINT) + token *= " " * tokens[i] + end + + endswith(rstrip(token), ',') && error(""" + Trailing comma in value: $token + A space after a comma ends the value. + """ * LIST_HINT) + + return token, i +end + """ heal_args(raw_args) -> Vector{String} -Rejoin option values that the shell split on whitespace. A token containing `=` -absorbs following tokens while it has an unclosed quote or bracket — the closing -delimiter marks the end of the value, so absorbed tokens may look like anything: -`-Dsysimg=[a, --selfsign]` yields a two-element list. +Rejoin values that the shell split on whitespace. A token containing `=` +absorbs following tokens while its value has an unclosed quote or bracket — the +closing delimiter marks the end of the value, so absorbed tokens may look like +anything: `-Dsysimg=[a, --selfsign]` yields a two-element list. A comma is a separator, not a continuation signal. A value left ending in one is -rejected: write the list without spaces, or bracket it so its end is explicit. +rejected, as is one carrying a bracket that closes nothing: both are malformed +lists rather than requests to keep absorbing. Quoting the whole token opts out +of list syntax — `"a=b,"` is one string, comma and all. """ function heal_args(raw_args) tokens = map(String, raw_args) @@ -122,23 +163,7 @@ function heal_args(raw_args) i = 1 while i <= length(tokens) - token = tokens[i] - - if occursin('=', token) - while !token_balanced(token) - i += 1 - i > length(tokens) && error(""" - Unterminated value: $token - Missing a closing ']' or '"'. - """ * LIST_HINT) - token *= " " * tokens[i] - end - endswith(rstrip(token), ',') && error(""" - Trailing comma in value: $token - A space after a comma ends the value. - """ * LIST_HINT) - end - + token, i = absorb(tokens, i) push!(out, token) i += 1 end @@ -171,6 +196,11 @@ option. Tokens appearing where no option is open are emitted as matching pair of outer quotes removed. The `-D` payload is left untouched — `unquote` runs later, per list element, during type coercion. +Any token carrying an `=` is subject to list syntax, wherever it appears, so a +detached value ending in a comma is rejected like the malformed list it looks +like. Quoting the value opts out of that: `--password "a=b,"` is scanned as one +string and passes, while `--password a=b,` errors. + Which tokens count as options is asymmetric, and it constrains what a detached value can be. Any token starting with `--` or `-D` is an option, so a value beginning with either can never be adopted: `--target-name --weird` yields two @@ -234,6 +264,29 @@ function unquote(s::AbstractString) return s[nextind(s, 1):prevind(s, lastindex(s))] end +""" + isquoted(s) -> Bool + +Whether `s` is one quoted span and nothing else: a quote at the first character +whose first match is the last character. `"a,b"` qualifies; `"a","b"` does not, +its closing quote falling in the middle. This is the question `unquote` should +be asking but cannot, since it only compares the two end characters. +""" +function isquoted(s::AbstractString) + length(s) >= 2 || return false + q = first(s) + (q == '"' || q == '\'') || return false + i = nextind(s, firstindex(s)) + while i <= lastindex(s) + s[i] == q && return i == lastindex(s) + i = nextind(s, i) + end + return false +end + +"""Whether `s` is written as an explicit bracketed list.""" +islist(s::AbstractString) = startswith(s, '[') && endswith(s, ']') + ### Coercion against the schema @@ -252,7 +305,7 @@ function parse_extra_args(defines, schema::Dict) "bare keys are only allowed for booleans. Use -D$key=.") overrides[key] = true else - overrides[key] = coerce(unquote(strip(raw)), default, key) + overrides[key] = coerce(strip(raw), default, key) end end @@ -264,41 +317,57 @@ end Interpret `value` according to the type of `default`. The string is never inspected to guess a type; the schema decides. + +Quotes come off at the leaf, once. A list therefore sees its payload as written: +a wholly quoted one is a single element, and quotes inside a bracketed one +protect the commas they enclose. """ function coerce(value::AbstractString, default::AbstractVector, key) body = strip(value) - if startswith(body, '[') && endswith(body, ']') - body = strip(body[nextind(body, 1):prevind(body, lastindex(body))]) + elem_default = isempty(default) ? "" : first(default) + + if isquoted(body) + inner = strip(unquote(body)) + isempty(inner) && return similar(default, 0) + islist(inner) || return [coerce(inner, elem_default, key)] + body = inner end + + islist(body) && (body = strip(body[nextind(body, 1):prevind(body, lastindex(body))])) isempty(body) && return similar(default, 0) - elem_default = isempty(default) ? "" : first(default) - return [coerce(unquote(strip(p)), elem_default, key) for p in split_elements(body)] + return [coerce(p, elem_default, key) for p in split_elements(body)] end -coerce(value::AbstractString, ::AbstractString, key) = String(value) +coerce(value::AbstractString, ::AbstractString, key) = String(unwrap(value)) function coerce(value::AbstractString, ::Bool, key) - value in ("true", "false") || - error("preference '$key' expects true or false, got '$value'") - return value == "true" + v = unwrap(value) + v in ("true", "false") || + error("preference '$key' expects true or false, got '$v'") + return v == "true" end function coerce(value::AbstractString, default::Integer, key) - n = tryparse(typeof(default), value) - n === nothing && error("preference '$key' expects an integer, got '$value'") + v = unwrap(value) + n = tryparse(typeof(default), v) + n === nothing && error("preference '$key' expects an integer, got '$v'") return n end function coerce(value::AbstractString, default::AbstractFloat, key) - x = tryparse(typeof(default), value) - x === nothing && error("preference '$key' expects a number, got '$value'") + v = unwrap(value) + x = tryparse(typeof(default), v) + x === nothing && error("preference '$key' expects a number, got '$v'") return x end coerce(::AbstractString, default, key) = error("preference '$key' has a default of type $(typeof(default)), which cannot be set from the command line") +"""Trim a scalar and take one layer of quotes off it — the last step before parsing.""" +unwrap(value::AbstractString) = unquote(strip(value)) + type_name(::AbstractString) = "a string" type_name(::Bool) = "true or false" type_name(::Integer) = "an integer" diff --git a/test/argument_parser.jl b/test/argument_parser.jl index c5680de..655b32e 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -324,7 +324,7 @@ normalize_args(raw); @test raw == before @test prefs_of("--debug")["windowed"] === false @test prefs_of("-Dbundler=\"sdsd,sds\"")["bundler"] == "sdsd,sds" -@test prefs_of("-Djuliaimg_sysimg=\"sdsd,sds\"")["juliaimg_sysimg"] == ["sdsd", "sds"] +@test prefs_of("-Djuliaimg_sysimg=\"sdsd,sds\"")["juliaimg_sysimg"] == ["sdsd,sds"] @test normalize_args(["--target-name=\"Bob's Tool\""]) == ["--target-name" => "Bob's Tool"] @@ -333,3 +333,14 @@ normalize_args(raw); @test raw == before @test_throws Exception normalize_args(["--description=Tool for X, Y,"]) @test normalize_args(["--description=\"Tool for X, Y,\""]) == ["--description" => "Tool for X, Y,"] + + +@test_throws Exception normalize_args(["--password", "a=b,"]) +@test normalize_args(["--password", "\"a=b,\""]) == ["--password" => "a=b,"] + +@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a --selfsign]"] +@test normalize_args(["--filter", "key=\"[a\"", "--selfsign]"]) == ["--filter" => "key=\"[a\"", "--selfsign]" => nothing] + +@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a --selfsign]"] + + From 500d1ecfbb2fa789414c9d25dbcc234a90595892 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Mon, 14 Sep 2026 23:54:10 +0300 Subject: [PATCH 14/14] Claude Opus 5: make healing logic to work only on elements that are vectors according to schema --- src/ArgTools.jl | 96 ++++++++++++++++++++++++----------------- test/argument_parser.jl | 23 +++++----- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/src/ArgTools.jl b/src/ArgTools.jl index aceba17..0a6afcd 100644 --- a/src/ArgTools.jl +++ b/src/ArgTools.jl @@ -1,6 +1,3 @@ - - - """ ArgTools @@ -97,31 +94,46 @@ const LIST_HINT = """ """ """ - value_part(token) -> SubString or nothing + list_payload(token, prev, schema) -> value or nothing + +The value of `token`, if and only if it is a `-D` payload whose key the schema +declares a list: `-Dsysimg=...`, or `sysimg=...` following a bare `-D`. For +anything else — another option, a positional, a detached value, or a `-D` key +that is unknown or holds a scalar — the answer is `nothing` and the token is +left exactly as the shell delivered it. -The part of `token` that list syntax governs. A token starting with a quote is -a value in its own right, so it is taken whole — the quote belongs to the value -and not to any key. Otherwise the value is everything after the first `=`, and a -token without one has no value at all. +This is what makes healing a completion rather than a guess. An open `[` is only +read as "more is coming" where a list was expected, and there it cannot mean +anything else. """ -function value_part(token::AbstractString) - isempty(token) && return nothing - (token[1] == '"' || token[1] == '\'') && return SubString(token, 1) - return last(splitpair(token)) +function list_payload(token, prev, schema) + if startswith(token, "-D") + body = SubString(token, 3) + elseif prev == "-D" + body = SubString(token, 1) + else + return nothing + end + + key, value = splitpair(body) + value === nothing && return nothing + get(schema, String(strip(key)), nothing) isa AbstractVector || return nothing + return value end """ - absorb(tokens, i) -> (token, i) + absorb(tokens, i, prev, schema) -> (token, i) -Grow `tokens[i]` until its value closes every quote and bracket it opened, +Grow `tokens[i]` until its list value closes every quote and bracket it opened, returning the healed token and the index of the last token consumed. """ -function absorb(tokens, i) +function absorb(tokens, i, prev, schema) token = tokens[i] - occursin('=', token) || return token, i + value = list_payload(token, prev, schema) + value === nothing && return token, i while true - _, open_quote, depth = scan(something(value_part(token))) + _, open_quote, depth = scan(value) depth < 0 && error(""" Unexpected ']' or '}' in value: $token @@ -134,6 +146,7 @@ function absorb(tokens, i) Missing a closing ']' or '"'. """ * LIST_HINT) token *= " " * tokens[i] + value = something(list_payload(token, prev, schema)) # the gate cannot change as the token grows end endswith(rstrip(token), ',') && error(""" @@ -145,25 +158,29 @@ function absorb(tokens, i) end """ - heal_args(raw_args) -> Vector{String} + heal_args(raw_args, schema) -> Vector{String} + +Rejoin list values that the shell split on whitespace. A `-D` payload whose key +the schema declares a list absorbs following tokens while its value has an +unclosed quote or bracket — the closing delimiter marks the end of the value, so +absorbed tokens may look like anything: `-Dsysimg=[a, --selfsign]` yields a +two-element list. -Rejoin values that the shell split on whitespace. A token containing `=` -absorbs following tokens while its value has an unclosed quote or bracket — the -closing delimiter marks the end of the value, so absorbed tokens may look like -anything: `-Dsysimg=[a, --selfsign]` yields a two-element list. +Everything else passes through untouched. A value only completes itself where a +list was expected, so `--filter key=[a` keeps its bracket and cannot swallow the +option after it, and `-Dbundler=[a` is a string that happens to start with one. A comma is a separator, not a continuation signal. A value left ending in one is rejected, as is one carrying a bracket that closes nothing: both are malformed -lists rather than requests to keep absorbing. Quoting the whole token opts out -of list syntax — `"a=b,"` is one string, comma and all. +lists rather than requests to keep absorbing. """ -function heal_args(raw_args) +function heal_args(raw_args, schema) tokens = map(String, raw_args) out = String[] i = 1 while i <= length(tokens) - token, i = absorb(tokens, i) + token, i = absorb(tokens, i, isempty(out) ? "" : out[end], schema) push!(out, token) i += 1 end @@ -178,9 +195,9 @@ isoption(token, short_options) = startswith(token, "--") || startswith(token, "-D") || haskey(short_options, token) """ - normalize_args(raw_args; short_options) -> Vector{Arg} + normalize_args(tokens; short_options) -> Vector{Arg} -Turn raw ARGS into `option => value` pairs. Attached and detached forms are +Turn tokens into `option => value` pairs. Attached and detached forms are equivalent, and only the first `=` separates option from value: --password=foo=bar ⇒ "--password" => "foo=bar" @@ -192,14 +209,14 @@ equivalent, and only the first `=` separates option from value: An option takes the following token as its value unless that token is itself an option. Tokens appearing where no option is open are emitted as -`token => nothing`. Values are healed first (see `heal_args`) and have one -matching pair of outer quotes removed. The `-D` payload is left untouched — -`unquote` runs later, per list element, during type coercion. +`token => nothing`. Values have one matching pair of outer quotes removed. The +`-D` payload is left untouched — `unquote` runs later, per list element, during +type coercion. -Any token carrying an `=` is subject to list syntax, wherever it appears, so a -detached value ending in a comma is rejected like the malformed list it looks -like. Quoting the value opts out of that: `--password "a=b,"` is scanned as one -string and passes, while `--password a=b,` errors. +This function pairs tokens and nothing else: it takes each one as given, so a +value keeps its brackets, commas and quotes, and never reaches across a shell +split. Rejoining split list values is `heal_args`, a separate pass that +`parse_args` runs first because it needs the schema. Which tokens count as options is asymmetric, and it constrains what a detached value can be. Any token starting with `--` or `-D` is an option, so a value @@ -212,8 +229,8 @@ it applies inside a healed value, where an open bracket or trailing comma has already joined the tokens before this function sees them, so `-Dsysimg=[a, --selfsign]` keeps `--selfsign` as a list element. """ -function normalize_args(raw_args; short_options = Dict{String, String}()) - tokens = heal_args(raw_args) +function normalize_args(tokens; short_options = Dict{String, String}()) + tokens = map(String, tokens) out = Arg[] i = 1 @@ -404,14 +421,15 @@ end Split `raw_args` into `option => value` pairs and `-Dkey=value` preference overrides. `schema` maps preference names to defaults whose types drive -coercion; `short_options` maps single-dash aliases to their long form, as in +coercion — and, before that, decide which values may be rejoined across a shell +split. `short_options` maps single-dash aliases to their long form, as in `Dict("-h" => "--help")`. """ function parse_args(raw_args; schema::Dict = Dict{String, Any}(), short_options = Dict{String, String}()) options = Arg[] defines = String[] - for (key, value) in normalize_args(raw_args; short_options) + for (key, value) in normalize_args(heal_args(raw_args, schema); short_options) if key == "-D" value === nothing && error("-D expects key=value, e.g. -Dbundler=juliaimg") push!(defines, value) diff --git a/test/argument_parser.jl b/test/argument_parser.jl index 655b32e..7a25d6c 100644 --- a/test/argument_parser.jl +++ b/test/argument_parser.jl @@ -5,8 +5,11 @@ using AppBundler using Test -using AppBundler: parse_args -using AppBundler.ArgTools: normalize_args +using AppBundler: parse_args, ArgTools +#using AppBundler.ArgTools: normalize_args + + +normalize_args(raw_args) = ArgTools.normalize_args(ArgTools.heal_args(raw_args, Dict("sysimg"=>[], "bundler"=>"juliaimg", "s"=>[], "juliaimg_sysimg"=>[], "key"=>[]))) # Original tests from integrity.jl @test normalize_args(["--password=dfdfsdf"]) == ["--password" => "dfdfsdf"] @@ -95,7 +98,7 @@ using AppBundler.ArgTools: normalize_args @testset "unterminated values" begin @test_throws ErrorException normalize_args(["-Dsysimg=[QMLApp,"]) @test_throws ErrorException normalize_args(["-Dsysimg=a,"]) - @test_throws ErrorException normalize_args(["-Dbundler=\"juliaimg"]) + @test normalize_args(["-Dbundler=\"juliaimg"]) == ["-D" => "bundler=\"juliaimg"] # Absorbs to the end of ARGS and is still unbalanced. @test_throws ErrorException normalize_args(["-Dsysimg=[a,", "--selfsign"]) end @@ -235,7 +238,7 @@ config_of(v...) = parse_args(String[v...])[1] # Must not silently swallow --selfsign into the list. @test_throws Exception prefs_of("-Djuliaimg_sysimg=[QMLApp,", "--selfsign") @test_throws Exception prefs_of("-Djuliaimg_sysimg=[QMLApp,") - @test_throws Exception prefs_of("-Dbundler=\"juliaimg") + #@test_throws Exception prefs_of("-Dbundler=\"juliaimg") end @testset "typos in preference names abort rather than build wrong" begin @@ -331,16 +334,14 @@ normalize_args(raw); @test raw == before @test normalize_args(["--target-name", "Bob's Tool"]) == ["--target-name" => "Bob's Tool"] @test normalize_args(["--target-name=Bob's Tool"]) == ["--target-name" => "Bob's Tool"] -@test_throws Exception normalize_args(["--description=Tool for X, Y,"]) +@test normalize_args(["--description=Tool for X, Y,"]) == ["--description" => "Tool for X, Y,"] @test normalize_args(["--description=\"Tool for X, Y,\""]) == ["--description" => "Tool for X, Y,"] -@test_throws Exception normalize_args(["--password", "a=b,"]) +@test normalize_args(["--password", "a=b,"]) == ["--password" => "a=b,"] @test normalize_args(["--password", "\"a=b,\""]) == ["--password" => "a=b,"] -@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a --selfsign]"] -@test normalize_args(["--filter", "key=\"[a\"", "--selfsign]"]) == ["--filter" => "key=\"[a\"", "--selfsign]" => nothing] - -@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a --selfsign]"] - +@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a", "--selfsign]" => nothing] +@test normalize_args(["--filter", "key=\"[a\"", "--selfsign]"]) == ["--filter" => "key=\"[a\"", "--selfsign]" => nothing] +@test normalize_args(["--filter", "key=[a", "--selfsign]"]) == ["--filter" => "key=[a", "--selfsign]" => nothing]