From 0a013da1e7f4cde51f7b4d9fa06e836a4b9c93b1 Mon Sep 17 00:00:00 2001 From: Michael Adams Date: Sat, 4 Nov 2023 13:26:04 -0700 Subject: [PATCH 1/4] Added preliminary support for building and deploying JasPer for a WebAssembly target that supports WASI. --- .github/workflows/release.yml | 2 + CMakeLists.txt | 11 ++ build/build_wasi_jasper | 161 ++++++++++++++++++++++ build/install_wasi_sdk | 53 +++++++ build/install_wasmtime | 74 ++++++++++ build/make_release | 138 +++++++++++++++++-- build/wasm_cc | 46 +++++++ src/libjasper/base/jas_stream.c | 2 + src/libjasper/include/jasper/jas_stream.h | 3 + 9 files changed, 477 insertions(+), 13 deletions(-) create mode 100755 build/build_wasi_jasper create mode 100755 build/install_wasi_sdk create mode 100755 build/install_wasmtime create mode 100755 build/wasm_cc diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ae52716..52c1619b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,8 @@ on: - version-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - version-[0-9]+.[0-9]+.[0-9]+-alpha[0-9]+ - version-[0-9]+.[0-9]+.[0-9]+-beta[0-9]+ + - experimental-version-* + - mdadams-experimental-version-* jobs: build: diff --git a/CMakeLists.txt b/CMakeLists.txt index 14dc4836..69741b92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,6 +191,15 @@ option(JAS_ENABLE_DANGEROUS_INTERNAL_TESTING_MODE option(JAS_ENABLE_CXX "Enable C++ code (for testing)." OFF) option(JAS_ENABLE_CONFORMANCE_TESTS "Enable conformance tests." OFF) +option(JAS_WASM "Enable WebAssembly mode" OFF) +if(JAS_WASM) + set(JAS_CROSSCOMPILING 1) + set(JAS_STDC_VERSION "201112L") + set(JAS_ENABLE_MULTITHREADING_SUPPORT 0) + set(JAS_ENABLE_SHARED 0) + add_compile_definitions(JAS_WASI_LIBC) +endif() + ################################################################################ # ################################################################################ @@ -412,6 +421,7 @@ if((DEFINED JAS_CROSSCOMPILING AND JAS_CROSSCOMPILING) OR # cross-compiling. If cross-compiling, the value of JAS_STDC_VERSION # will need to be set manually from the command line # (e.g., using -DJAS_STDC_VERSION=YYYYMML) or by changing the line below. +if(NOT JAS_WASM) set(JAS_STDC_VERSION "0L" CACHE INTERNAL "The value of __STDC_VERSION__.") if (JAS_STDC_VERSION STREQUAL "0L") message(FATAL_ERROR @@ -421,6 +431,7 @@ if((DEFINED JAS_CROSSCOMPILING AND JAS_CROSSCOMPILING) OR "option -DJAS_STDC_VERSION=...) or modify the CMakeLists.txt " "appropriately.") endif() +endif() else() jas_get_stdc_version(status JAS_STDC_VERSION) if(NOT status) diff --git a/build/build_wasi_jasper b/build/build_wasi_jasper new file mode 100755 index 00000000..c8fee496 --- /dev/null +++ b/build/build_wasi_jasper @@ -0,0 +1,161 @@ +#! /usr/bin/env bash + +panic() +{ + echo "ERROR: $*" 1>& 2 + exit 1 +} + +perform_cleanup() +{ + if [ -n "$tmp_dir" -a -d "$tmp_dir" ]; then + rm -rf "$tmp_dir" || \ + echo "warning: cannot remove temporary directory $tmp_dir" + fi +} + +usage() +{ + cat <<- EOF + usage: + $0 [options] -p \$out_dir + + Examples + ======== + + $0 -C -t /tmp/mdadams/wasi_jasper-XXXXXX -o /tmp/mdadams/wasi_jasper + EOF + if [ $# -ne 0 ]; then + echo "BAD USAGE: $*" + fi + exit 2 +} + +self_dir="$(dirname "$0")" || panic +install_wasi_sdk="$self_dir/install_wasi_sdk" +install_wasmtime="$self_dir/install_wasmtime" +wasm_cc="$self_dir/wasm_cc" +wasm_cxx="$self_dir/wasm_cxx" + +wasi_sdk_version=20.0 +wasmtime_version=12.0.1 + +tmp_dir_template="${JAS_TMP_DIR:-/tmp}/wasi_jasper-XXXXXX" +cleanup=1 +source_dir="$self_dir/.." +out_dir= +version=unknown-version + +while getopts ":hCt:s:o:V:" option; do + case "$option" in + C) + cleanup=0;; + t) + tmp_dir_template="$OPTARG";; + s) + source_dir="$OPTARG";; + o) + out_dir="$OPTARG";; + V) + version="$OPTARG";; + h) + usage;; + *) + usage "invalid option $option";; + esac +done +shift $((OPTIND - 1)) + +if [ -z "$tmp_dir_template" ]; then + usage "no temporary directory pathname template specified" +fi + +if [ -z "$source_dir" ]; then + usage "no source directory specified" +fi + +if [ -z "$out_dir" ]; then + usage "no output directory specified" +fi + +tmp_dir="$(mktemp -d "$tmp_dir_template")" || \ + panic "cannot create temporary directory" +if [ "$cleanup" -ne 0 ]; then + trap perform_cleanup EXIT +fi + +sde_dir="$tmp_dir/wasm" +wasi_sdk_dir="$sde_dir/wasi_sdk" +wasmtime_dir="$sde_dir/wasmtime" + +"$install_wasi_sdk" -d "$wasi_sdk_dir" -v "$wasi_sdk_version" || \ + panic "cannot install WASI SDK" +"$install_wasmtime" -d "$wasmtime_dir" -v "$wasmtime_version" || \ + panic "cannot install wasmtime" +wasmtime="$wasmtime_dir/bin/wasmtime" + +export WASI_SDK_ROOT_DIR="$wasi_sdk_dir" +export CC="$wasm_cc" +export CXX="$wasm_cxx" + +#source_dir="$tmp_dir/git" +build_dir="$tmp_dir/build" +install_dir="$tmp_dir/install" +readme_file="$install_dir/README.txt" + +configure_options=( + -DCMAKE_INSTALL_PREFIX="$install_dir" + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_VERBOSE_MAKEFILE=1 + -DJAS_CROSSCOMPILING=1 + -DJAS_STDC_VERSION=201112L + -DJAS_ENABLE_MULTITHREADING_SUPPORT=0 + -DJAS_ENABLE_SHARED=0 + -DJAS_ENABLE_CXX=0 + -DJAS_USE_JAS_INIT=1 + -DJAS_STRICT=0 + -DJAS_WASM=1 + -DJAS_ENABLE_DOC=0 +) + +cmake -H"$source_dir" -B"$build_dir" "${configure_options[@]}" || \ + panic "configure failed" + +cmake --build "$build_dir" || \ + panic "build failed" + +cmake --build "$build_dir" --target install || \ + panic "install failed" + +"$wasmtime" "$install_dir/bin/imginfo" < "$source_dir/data/images/goldenears.jp2" || panic "jasper failed" + +cat > "$readme_file" <<- EOF +JasPer WebAssembly Edition ($version) + +This directory contains several JasPer programs built for a WebAssembly +target that uses WASI for interfacing to the host environment. + +The command-line interface (CLI) for these programs is identical to +their non-WebAssembly counterparts. For detailed information on the +CLI for these programs, refer to the online manual at: + + https://jasper-software.github.io/jasper-manual/latest/html/index.html + +In order to run these WebAssembly programs, you will need a WebAssembly +runtime with WASI support, such as: + + Wasmtime (https://wasmtime.dev) + Wasmer (https://wasmer.io) + WAMR (https://bytecodealliance.github.io/wamr.dev) + WasmEdge (https://wasmedge.org) +EOF +[ $? -eq 0 ] || panic "cannot make readme file" + +if [ ! -d "$out_dir" ]; then + mkdir -p "$out_dir" || panic +fi + +for file in jasper imginfo imgcmp; do + cp -a "$install_dir/bin/$file" "$out_dir" || panic +done +cp -a "$readme_file" "$out_dir" || panic diff --git a/build/install_wasi_sdk b/build/install_wasi_sdk new file mode 100755 index 00000000..93dec8bf --- /dev/null +++ b/build/install_wasi_sdk @@ -0,0 +1,53 @@ +#! /usr/bin/env bash + +panic() +{ + echo "ERROR: $*" + exit 1 +} + +wasi_sdk_version= +install_dir= + +while getopts :d:v: option; do + case "$option" in + d) + install_dir="$OPTARG";; + v) + wasi_sdk_version="$OPTARG";; + esac +done + +if [ -z "$install_dir" ]; then + panic "no install directory specified" +fi +if [ -z "$wasi_sdk_version" ]; then + panic "no version specified" +fi + +if [ ! -d "$install_dir" ]; then + mkdir -p "$install_dir" || \ + panic "cannot make directory $install_dir" +fi + +#source_dir="$install_dir/.source" +#build_dir="$install_dir/.build" +#build_dir_2="$install_dir/.build2" + +wasi_sdk_archive_url="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-$wasi_sdk_version-linux.tar.gz" + +wasi_sdk_archive="$install_dir/.wasi-sdk.tar.xz" + +if [ ! -d "$install_dir" ]; then + mkdir -p "$install_dir" || \ + panic "cannot make directory $install_dir" +fi + +wget -O "$wasi_sdk_archive" "$wasi_sdk_archive_url" || \ + panic "wget failed" +tar -x -z -f "$wasi_sdk_archive" --strip-components=1 -C "$install_dir" || \ + panic "tar failed" + +#rm -rf "$source_dir" || panic +#rm -rf "$build_dir" || panic +#rm -rf "$build_dir_2" || panic diff --git a/build/install_wasmtime b/build/install_wasmtime new file mode 100755 index 00000000..d9ba3359 --- /dev/null +++ b/build/install_wasmtime @@ -0,0 +1,74 @@ +#! /usr/bin/env bash + +# Wasmtime release artifacts for download: +# - library and headers for C API: +# wasmtime-$version-x86_64-linux-c-api.tar.xz +# - program to run WASM applications: +# wasmtime-$version-x86_64-linux.tar.xz + +panic() +{ + echo "ERROR: $*" + exit 1 +} + +usage() +{ + cat <<- EOF + usage: $0 [options] + -d \$install_dir + -v \$version + EOF + exit 2 +} + +wasmtime_version= +install_dir= + +while getopts :d:v: option; do + case "$option" in + d) + install_dir="$OPTARG";; + v) + wasmtime_version="$OPTARG";; + *) + usage "invalid option $OPTARG";; + esac +done + +if [ -z "$install_dir" ]; then + panic "no install directory specified" +fi +if [ -z "$wasmtime_version" ]; then + panic "no version specified" +fi + +if [ ! -d "$install_dir" ]; then + mkdir -p "$install_dir" || \ + panic "cannot make directory $install_dir" +fi + +#source_dir="$install_dir/.source" +#build_dir="$install_dir/.build" +#build_dir_2="$install_dir/.build2" + +wasmtime_archive="$install_dir/.wasmtime.tar.xz" +wasmtime_capi_archive="$install_dir/.wasmtime-capi.tar.xz" +wasmtime_archive_url="https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime_version/wasmtime-v$wasmtime_version-x86_64-linux.tar.xz" +wasmtime_capi_archive_url="https://github.com/bytecodealliance/wasmtime/releases/download/v$wasmtime_version/wasmtime-v$wasmtime_version-x86_64-linux-c-api.tar.xz" + +wget -O "$wasmtime_archive" "$wasmtime_archive_url" || \ + panic "wget failed" +wget -O "$wasmtime_capi_archive" "$wasmtime_capi_archive_url" || \ + panic "wget failed" +if [ ! -d "$install_dir/bin" ]; then + mkdir -p "$install_dir/bin" +fi +tar -x -J -f "$wasmtime_archive" --strip-components=1 -C "$install_dir/bin" || \ + panic "tar failed" +tar -x -J -f "$wasmtime_capi_archive" --strip-components=1 -C "$install_dir" || \ + panic "tar failed" + +#rm -rf "$source_dir" || panic +#rm -rf "$build_dir" || panic +#rm -rf "$build_dir_2" || panic diff --git a/build/make_release b/build/make_release index 1302fab3..fc7b1a16 100755 --- a/build/make_release +++ b/build/make_release @@ -1,30 +1,55 @@ #! /usr/bin/env bash -cmd_dir="$(dirname "$0")" || panic +################################################################################ panic() { - echo "ERROR: $@" + echo "ERROR: $*" 1>& 2 exit 1 } warn() { - echo "WARNING: $@" + echo "WARNING: $*" 1>& 2 } usage() { - echo "bad usage: $@" + cat <<- EOF + usage: + $0 [options] + + Example + ======= + + $0 -w $self_dir/.. \\ + -W 1 \\ + -t /tmp/jasper/tmp \\ + -o /tmp/jasper/output \\ + -r refs/tags/mdadams-experimental-version-wasm1 + EOF + if [ $# -ne 0 ]; then + echo "BAD USAGE: $*" + fi exit 2 } +################################################################################ + +self_dir="$(dirname "$0")" || panic +build_wasi_jasper="$self_dir/build_wasi_jasper" + +################################################################################ +# Command-Line Processing +################################################################################ + tmp_dir= workspace_dir= out_dir= github_ref= +enable_wasm=0 -while getopts t:w:o:r: opt; do +while getopts :t:w:o:r:W:h opt; do case "$opt" in w) workspace_dir="$OPTARG";; @@ -34,14 +59,16 @@ while getopts t:w:o:r: opt; do out_dir="$OPTARG";; r) github_ref="$OPTARG";; - \?) - usage "invalid option $opt" - break;; + W) + enable_wasm="$OPTARG";; + h) + usage;; + *) + usage "invalid option $OPTARG";; esac done shift $((OPTIND - 1)) - if [ -z "$github_ref" ]; then usage "no GitHub ref specified" fi @@ -67,14 +94,54 @@ echo "temporary directory: $tmp_dir" echo "workspace directory: $workspace_dir" echo "GitHub ref: $github_ref" +################################################################################ +# +################################################################################ + commit="$(git -C "$workspace_dir" rev-parse HEAD)" || \ panic "cannot get commit" #tag="$(git -C "$workspace_dir" describe "$commit")" || \ # panic "cannot get tag" tag="$(awk -v FS="/" '{print $3;}' <<< "$github_ref")" || \ panic "cannot get tag" -version="$(awk -v FS="-" '{print $2;}' <<< "$tag")" || \ - panic "cannot get version" + +release_type= +grep -q -E '^version-.*$' <<< "$tag" && release_type=normal +if [ -z "$release_type" ]; then + grep -q -E '^experimental-version-.*$' <<< "$tag" && \ + release_type=experimental +fi +if [ -z "$release_type" ]; then + grep -q -E '^mdadams-experimental-version-.*$' <<< "$tag" && \ + release_type=mdadams +fi +case "$release_type" in +normal) + version="$(awk -v FS="-" '{print $2;}' <<< "$tag")" || \ + panic "cannot get version" + ;; +experimental) + version="$(awk -v FS="-" '{print $3;}' <<< "$tag")" || \ + panic "cannot get version" + ;; +mdadams) + version="$(awk -v FS="-" '{print $4;}' <<< "$tag")" || \ + panic "cannot get version" + ;; +*) + #version="$tag" + panic "unknown release type" + ;; +esac + +cat <<- EOF +release type: $release_type +EOF + +################################################################################ +# +################################################################################ + name="jasper-$version" source_dir="$tmp_dir/$name" @@ -108,8 +175,26 @@ git -C "$workspace_dir" log --stat -M -C --name-status --no-color | \ fmt --split-only > "$changelog_file" || \ panic "cannot generate changelog" -"$cmd_dir"/extract_release_notes < "$news_file" > "$release_notes_file" || \ - panic +case "$release_type" in +normal) + "$self_dir"/extract_release_notes \ + < "$news_file" \ + > "$release_notes_file" \ + || \ + panic + ;; +experimental|mdadams) + cat > "$release_notes_file" <<- EOF + WARNING: DO NOT USE THIS RELEASE! + + This is an experimental release whose sole purpose is to test some of + the software build/deployment infrastructure employed in this project. + EOF + ;; +*) + panic "unknown release type" + ;; +esac cmake \ -G "Unix Makefiles" \ @@ -159,3 +244,30 @@ done tar -C "$tmp_dir" -czf - "$name" > "$archive_file" || \ panic "cannot make archive" + +################################################################################ + +if [ "$enable_wasm" -ne 0 ]; then + + name="jasper-WebAssembly-$version" + + stage_dir="$tmp_dir/wasm" + archive_file="$out_dir/$name.tar.gz" + + echo "stage directory: $stage_dir" + echo "archive file: $archive_file" + + JAS_TMP_DIR="$tmp_dir" \ + "$build_wasi_jasper" -V "$version" -o "$stage_dir/$name" || \ + panic "cannot build WASI jasper" + + ls -al "$stage_dir" + (cd "$stage_dir" && ls -al) + + tar -C "$stage_dir" -c -z -f - "$name" > "$archive_file" || \ + panic "cannot make archive" + +fi + +################################################################################ + diff --git a/build/wasm_cc b/build/wasm_cc new file mode 100755 index 00000000..043dde79 --- /dev/null +++ b/build/wasm_cc @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +eecho() +{ + echo "$*" 1>&2 +} + +panic() +{ + eecho "$*" + exit 1 +} + +#mode=zig +mode=wasi_sdk + +#eecho "wasm_cc mode: $mode" + +case "$mode" in + +wasi_sdk) + if [ -z "$WASI_SDK_ROOT_DIR" ]; then + panic "WASI_SDK_ROOT_DIR not set" + fi + wasi_sdk_path="$WASI_SDK_ROOT_DIR" + clang=$wasi_sdk_path/bin/clang + if [ ! -x "$clang" ]; then + panic "clang not found ($clang)" + fi + exec "$clang" --sysroot=$wasi_sdk_path/share/wasi-sysroot "$@" + ;; + +zig) + if [ -z "$WASM_ROOT_DIR" ]; then + panic "WASM_ROOT_DIR not set" + fi + wasm_root_dir="$WASM_ROOT_DIR" + #zig="$(type -P zig)" || zig=false + zig="$wasm_root_dir/zig/zig" + if [ ! -x "$zig" ]; then + panic "zig not found ($zig)" + fi + exec "$zig" cc --target=wasm32-wasi "$@" + ;; + +esac diff --git a/src/libjasper/base/jas_stream.c b/src/libjasper/base/jas_stream.c index 141db585..8263fee6 100644 --- a/src/libjasper/base/jas_stream.c +++ b/src/libjasper/base/jas_stream.c @@ -459,6 +459,8 @@ static int easy_mkstemp(char *buffer, size_t size) return open(buffer, O_CREAT | O_EXCL | O_RDWR | O_TRUNC | O_BINARY | O_CLOEXEC, JAS_STREAM_PERMS); +#elif defined(JAS_WASI_LIBC) + return -1; #else #ifdef JAS_HAVE_MKOSTEMP return mkostemp(buffer, O_CLOEXEC); diff --git a/src/libjasper/include/jasper/jas_stream.h b/src/libjasper/include/jasper/jas_stream.h index d8938d17..f947988d 100644 --- a/src/libjasper/include/jasper/jas_stream.h +++ b/src/libjasper/include/jasper/jas_stream.h @@ -258,6 +258,9 @@ typedef struct { typedef struct { int fd; int flags; +#if defined(JAS_WASI_LIBC) +#define L_tmpnam 4096 +#endif char pathname[L_tmpnam + 1]; } jas_stream_fileobj_t; From 7976385b6111f4a8ac7f26d28355769b30528556 Mon Sep 17 00:00:00 2001 From: Michael Adams Date: Sat, 4 Nov 2023 14:42:30 -0700 Subject: [PATCH 2/4] Made some changes to the GitHub Actions release workflow. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52c1619b..a2a1e3d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: ############################################################ - name: Build shell: bash - run: build/make_release -r ${{github.ref}} -w ${{github.workspace}} -t ${{runner.temp}}/tmp -o ${{runner.temp}}/out + run: build/make_release -W 1 -r ${{github.ref}} -w ${{github.workspace}} -t ${{runner.temp}}/tmp -o ${{runner.temp}}/out ############################################################ - name: Deploy Release uses: softprops/action-gh-release@v1 From a886dcd98f8bffb3c0bcd96b5fd26022e2d692b0 Mon Sep 17 00:00:00 2001 From: Michael Adams Date: Sat, 4 Nov 2023 14:58:04 -0700 Subject: [PATCH 3/4] Made some further build changes to experiment with WebAssembly support. --- .github/workflows/release.yml | 3 ++- build/build_wasi_jasper | 4 ++++ build/github/prebuild | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2a1e3d0..05ece77f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,8 @@ on: jobs: build: - runs-on: ubuntu-latest + #runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: ############################################################ - name: Checkout diff --git a/build/build_wasi_jasper b/build/build_wasi_jasper index c8fee496..e5aad739 100755 --- a/build/build_wasi_jasper +++ b/build/build_wasi_jasper @@ -97,6 +97,10 @@ wasmtime="$wasmtime_dir/bin/wasmtime" export WASI_SDK_ROOT_DIR="$wasi_sdk_dir" export CC="$wasm_cc" export CXX="$wasm_cxx" +export PATH="$wasi_sdk_dir/bin:$PATH" + +lld="$(type -P lld)" || lld= +echo "lld program: $lld" #source_dir="$tmp_dir/git" build_dir="$tmp_dir/build" diff --git a/build/github/prebuild b/build/github/prebuild index cccedd79..dd852669 100755 --- a/build/github/prebuild +++ b/build/github/prebuild @@ -13,6 +13,7 @@ panic() # Initialize the list of required packages. packages=() packages+=(doxygen) +#packages+=(lld) # Install the packages. sudo apt-get install -y "${packages[@]}" || \ From 304ffce7498eb38ec782785e853a8ac4419f4184 Mon Sep 17 00:00:00 2001 From: Michael Adams Date: Sat, 4 Nov 2023 15:03:01 -0700 Subject: [PATCH 4/4] Removed a comment from a GitHub Actions workflow file. --- build/github/prebuild | 1 - 1 file changed, 1 deletion(-) diff --git a/build/github/prebuild b/build/github/prebuild index dd852669..cccedd79 100755 --- a/build/github/prebuild +++ b/build/github/prebuild @@ -13,7 +13,6 @@ panic() # Initialize the list of required packages. packages=() packages+=(doxygen) -#packages+=(lld) # Install the packages. sudo apt-get install -y "${packages[@]}" || \