From 10574870458b7f4169d8a2b8fb173b52147594a3 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Thu, 4 May 2023 17:50:50 +0200 Subject: [PATCH 1/8] build: ignore '.sass-cache' Signed-off-by: Roman Volosatovs --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2f6879fb..d124ee38 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,5 @@ host_config.json # rebar lock file rebar.lock + +.sass-cache From a4497deb42f6374ad8255eff3c2c780f38a5a59e Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Wed, 3 May 2023 01:35:50 +0200 Subject: [PATCH 2/8] build,ci: provide (mostly) static builds via nix and burrito Signed-off-by: Roman Volosatovs --- .github/actions/install-nix/action.yml | 18 + .github/workflows/release.yml | 641 ++++++------ .github/workflows/static_macos | 101 -- .gitignore | 2 + Dockerfile.localstatic | 55 - Makefile | 4 + flake.lock | 466 +++++++++ flake.nix | 982 ++++++++++++++++++ host_core/Dockerfile | 61 +- host_core/Dockerfile.static | 27 - host_core/Makefile | 32 +- host_core/config/dev.exs | 2 + host_core/config/prod.exs | 4 + host_core/config/release_prod.exs | 7 - host_core/lib/copy_nif.ex | 66 ++ host_core/mix.exs | 89 +- host_core/mix.lock | 1 + .../hostcore_wasmcloud_native/Cross.toml | 11 - .../hostcore_wasmcloud_native/README.md | 6 - wasmcloud_host/Dockerfile | 144 +-- wasmcloud_host/Dockerfile.static | 35 - wasmcloud_host/Makefile | 55 +- wasmcloud_host/config/config.exs | 4 +- wasmcloud_host/config/prod.exs | 4 + wasmcloud_host/config/release_prod.exs | 9 - wasmcloud_host/mix.exs | 81 +- wasmcloud_host/mix.lock | 7 + 27 files changed, 2035 insertions(+), 879 deletions(-) create mode 100644 .github/actions/install-nix/action.yml delete mode 100755 .github/workflows/static_macos delete mode 100644 Dockerfile.localstatic create mode 100644 Makefile create mode 100644 flake.lock create mode 100644 flake.nix delete mode 100644 host_core/Dockerfile.static delete mode 100644 host_core/config/release_prod.exs create mode 100644 host_core/lib/copy_nif.ex delete mode 100644 host_core/native/hostcore_wasmcloud_native/Cross.toml delete mode 100644 wasmcloud_host/Dockerfile.static delete mode 100644 wasmcloud_host/config/release_prod.exs diff --git a/.github/actions/install-nix/action.yml b/.github/actions/install-nix/action.yml new file mode 100644 index 00000000..2197099f --- /dev/null +++ b/.github/actions/install-nix/action.yml @@ -0,0 +1,18 @@ +name: install Nix + +inputs: + cachixAuthToken: + description: auth token for https://app.cachix.org/organization/wasmcloud/cache/wasmcloud + +runs: + using: composite + steps: + - uses: DeterminateSystems/nix-installer-action@v2 + with: + extra-conf: | + accept-flake-config = true + - uses: cachix/cachix-action@v12 + continue-on-error: true + with: + name: wasmcloud + authToken: '${{ inputs.cachixAuthToken }}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 420c5ff7..57132ec1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,380 +2,393 @@ name: WasmCloud Release on: push: + branches: [main] tags: ["v*"] # Push events to matching v*, i.e. v1.0, v20.15.10 workflow_dispatch: # Allow manual creation of artifacts without a release pull_request: branches: [main] + env: - otp-version: 25 - elixir-version: 1.14.3 + FALLBACK_SECRET: 3ImiTAMO0TTD7wrACHrCA+ggkzpw6zGWvE3gtQwlXE6vmnDT9yGP5/WKpLWEJ8fF jobs: - create-static-macos: - if: ${{ startswith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - name: Static macOS release + nif-darwin: strategy: - matrix: - os: [macos-11] - include: - - os: macos-11 - rust-target: x86_64-apple-darwin - release-tarball: x86_64-macos-static.tar.gz - working-directory: wasmcloud_host - runs-on: ${{ matrix.os }} - env: - working-directory: ${{ matrix.working-directory }} - MIX_ENV: release_prod - SECRET_KEY_BASE: ${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} + matrix: + target: + - aarch64-apple-darwin + - x86_64-apple-darwin + + name: nif-${{ matrix.target }} + runs-on: macos-latest steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - - name: run macos script - shell: bash - run: | - bash .github/workflows/static_macos - tar -czvf ${{ matrix.release-tarball }} -C wasmcloud_host/_build/prod/rel/bakeware wasmcloud_host - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.release-tarball }} - path: ${{ matrix.release-tarball }} + - uses: actions/checkout@v3 + - run: rustup toolchain install stable --profile minimal + - run: rustup target add ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + if: ${{ !startswith(github.ref, 'refs/tags/') }} # Do not cache on tag pushes + with: + workspaces: "host_core/native/hostcore_wasmcloud_native" + - name: Build NIF + run: | + cd ./host_core/native/hostcore_wasmcloud_native + cargo build --target ${{ matrix.target }} --release + - uses: actions/upload-artifact@v3 + with: + name: nif-${{ matrix.target }} + path: ./host_core/native/hostcore_wasmcloud_native/target/${{ matrix.target }}/release/libhostcore_wasmcloud_native.dylib + if-no-files-found: error - create-mix-releases: - # Run on tag push or on manual dispatch. Release will not be created for manual dispatch - if: ${{ (startswith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && startswith(github.repository.name, 'wasmcloud') }} - name: Mix release + nif-windows: + name: nif-x86_64-pc-windows-msvc + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - run: rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@v2 + if: ${{ !startswith(github.ref, 'refs/tags/') }} # Do not cache on tag pushes + with: + workspaces: "host_core/native/hostcore_wasmcloud_native" + - name: Build NIF + run: | + cd ./host_core/native/hostcore_wasmcloud_native + cargo build --release + - uses: actions/upload-artifact@v3 + with: + name: nif-x86_64-pc-windows-msvc + path: host_core/native/hostcore_wasmcloud_native/target/release/hostcore_wasmcloud_native.dll + if-no-files-found: error + + burrito-darwin: strategy: - fail-fast: false matrix: - os: [ubuntu-20.04, windows-2019, macos-11] - working-directory: [wasmcloud_host, host_core] - include: - - os: ubuntu-20.04 - rust-target: x86_64-unknown-linux-gnu - release-tarball: x86_64-linux.tar.gz - working-directory: wasmcloud_host - - os: windows-2019 - rust-target: x86_64-pc-windows-gnu - release-tarball: x86_64-windows.tar.gz - working-directory: wasmcloud_host - - os: macos-11 - rust-target: x86_64-apple-darwin - release-tarball: x86_64-macos.tar.gz - working-directory: wasmcloud_host - - os: ubuntu-20.04 - rust-target: x86_64-unknown-linux-gnu - release-tarball: x86_64-linux-core.tar.gz - working-directory: host_core - - os: windows-2019 - rust-target: x86_64-pc-windows-gnu - release-tarball: x86_64-windows-core.tar.gz - working-directory: host_core - - os: macos-11 - rust-target: x86_64-apple-darwin - release-tarball: x86_64-macos-core.tar.gz - working-directory: host_core - runs-on: ${{ matrix.os }} - env: - working-directory: ${{ matrix.working-directory }} - MIX_ENV: release_prod - SECRET_KEY_BASE: ${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} + config: + - package: host_core + arch: aarch64 + - package: host_core + arch: x86_64 + + - package: wasmcloud_host + arch: aarch64 + - package: wasmcloud_host + arch: x86_64 + needs: + - nif-darwin + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-nix + with: + cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - # Install Rust - - run: rustup toolchain install stable --profile minimal + - uses: actions/download-artifact@v3 + with: + path: /tmp/nif/lib + name: nif-${{ matrix.config.arch }}-apple-darwin - # Install erlang/OTP and elixir - - name: Install erlang and elixir - if: ${{ startswith(matrix.os, 'ubuntu') || startswith(matrix.os, 'windows') }} - uses: erlef/setup-beam@v1 - with: - otp-version: "=${{ env.otp-version }}" - elixir-version: "${{ env.elixir-version }}" - install-hex: true - install-rebar: true - - name: Install erlang and elixir - if: ${{ startswith(matrix.os, 'macos') }} - run: | - brew install erlang - brew install elixir + - name: Wrap ${{ matrix.config.package }} ${{ matrix.config.arch }} Darwin burrito + env: + SECRET_KEY_BASE: ${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} + run: | + printf '%s' "${SECRET_KEY_BASE:-$FALLBACK_SECRET}" | nix build \ + --override-input secret-key-base 'file:///dev/stdin' \ + --override-input hostcore_wasmcloud_native-${{ matrix.config.arch }}-apple-darwin-mac /tmp/nif \ + -L .#${{ matrix.config.package }}-burrito-${{ matrix.config.arch }}-darwin - - name: Retrieve Mix Dependencies Cache - if: ${{ !startswith(matrix.os, 'windows') }} # Windows gets angry about not owning files if you restore from cache - uses: actions/cache@v2 - id: mix-cache #id to use in retrieve action - with: - path: | - ${{ env.working-directory }}/deps - key: ${{ runner.os }}-${{ env.ERLANG_VERSION }}-${{ env.ELIXIR_VERSION }}-mix-${{ hashFiles('${{ env.working-directory }}/mix.exs', '${{ env.working-directory }}/mix.lock') }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.config.package }}_${{ matrix.config.arch }}_darwin + path: ./result/bin/${{ matrix.config.package }}_${{ matrix.config.arch }}_darwin + if-no-files-found: error - - name: Install Mix Dependencies - working-directory: ${{env.working-directory}} - shell: bash - run: | - mix local.rebar --force - mix local.hex --force - mix deps.get + burrito-linux: + strategy: + matrix: + config: + - package: host_core + arch: aarch64 + libc: gnu + - package: host_core + arch: aarch64 + libc: musl + - package: host_core + arch: x86_64 + libc: gnu + - package: host_core + arch: x86_64 + libc: musl - - name: Compile Elixir - working-directory: ${{env.working-directory}} - shell: bash - run: | - mix compile + - package: wasmcloud_host + arch: aarch64 + libc: gnu + - package: wasmcloud_host + arch: aarch64 + libc: musl + - package: wasmcloud_host + arch: x86_64 + libc: gnu + - package: wasmcloud_host + arch: x86_64 + libc: musl - - name: Compile Phoenix Assets - if: ${{ env.working-directory == 'wasmcloud_host' }} - working-directory: ${{env.working-directory}} - shell: bash - run: | - make esbuild + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-nix + with: + cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - name: Create Mix Release - working-directory: ${{env.working-directory}} - shell: bash - run: | - mix release - cd _build/${{ env.MIX_ENV }}/rel/${{env.working-directory}} - tar -czvf ${{ matrix.release-tarball }} bin erts-* lib releases - mv ${{ matrix.release-tarball }} ../../../../ + - name: Wrap ${{ matrix.config.package }} ${{ matrix.config.arch }} Linux ${{ matrix.config.libc }} burrito + env: + SECRET_KEY_BASE: ${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} + run: | + printf '%s' "${SECRET_KEY_BASE:-$FALLBACK_SECRET}" | nix build \ + --override-input secret-key-base 'file:///dev/stdin' \ + -L .#${{ matrix.config.package }}-burrito-${{ matrix.config.arch }}-linux-${{ matrix.config.libc }} - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.release-tarball }} - path: ${{env.working-directory}}/${{ matrix.release-tarball }} - - # release-hostcore-docker: - # if: startswith(github.ref, 'refs/tags/') # Only run on tag push - # name: Release Image (host_core) - # runs-on: ubuntu-22.04 - # env: - # MIX_ENV: release_prod - # working-directory: wasmcloud_host - # SECRET_KEY_BASE: ${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} - # app-name: host_core - # arm-tarball: aarch64-linux-musl-core.tar.gz - # builder-image: elixir:1.13.4-alpine - # release-image: alpine:3.16 - # steps: - # - uses: actions/checkout@v3 - # - name: Determine version - # run: echo "app-version=${GITHUB_REF_NAME#v}" > $GITHUB_ENV - - # # Download the NIF in the path that Docker expects for x86 - # - uses: actions/download-artifact@v3 - # with: - # path: ./host_core/priv/built/x86_64 - # name: x86_64-unknown-linux-musl - # # Download the NIF in the path that Docker expects for aarch64 - # - uses: actions/download-artifact@v3 - # with: - # path: ./host_core/priv/built/aarch64 - # name: aarch64-unknown-linux-musl - - # - name: Login to AzureCR - # uses: azure/docker-login@v1 - # with: - # login-server: ${{ secrets.AZURECR_PUSH_URL }} - # username: ${{ secrets.AZURECR_PUSH_USER }} - # password: ${{ secrets.AZURECR_PUSH_PASSWORD }} - # - name: Login to DockerHub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_PUSH_USER }} - # password: ${{ secrets.DOCKERHUB_PUSH_PASSWORD }} - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # id: buildx-builder - # - name: Build and release docker image - # uses: docker/build-push-action@v3 - # with: - # builder: ${{ steps.buildx-builder.outputs.name }} - # push: false - # context: ${{ env.app-name}}/ - # file: ${{ env.app-name}}/Dockerfile - # platforms: linux/amd64,linux/arm64 - # build-args: | - # BUILDER_IMAGE=${{ env.builder-image }} - # RELEASE_IMAGE=${{ env.release-image }} - # APP_NAME=${{ env.app-name }} - # APP_VSN=${{ env.app-version }} - # MIX_ENV=${{ env.MIX_ENV }} - # tags: | - # wasmcloud.azurecr.io/${{ env.app-name }}:${{ env.app-version }} - # wasmcloud.azurecr.io/${{ env.app-name }}:latest - # wasmcloud/${{ env.app-name }}:${{ env.app-version }} - # wasmcloud/${{ env.app-name }}:latest - - # - name: Retrieve aarch64 tarball from Docker - # run: | - # docker run --rm --platform linux/arm64/v8 -iv ${PWD}:/aarch64temp wasmcloud/${{ env.app-name }}:${{ env.app-version}} sh -s < $GITHUB_ENV - # Compile phoenix assets for wasmcloud_host outside of image (dart-sass doesn't support arm linux) - - name: Install erlang and elixir - uses: erlef/setup-beam@v1 - with: - otp-version: "=${{ env.otp-version }}" - elixir-version: "${{ env.elixir-version }}" - install-hex: true - install-rebar: true - - name: Install Mix Dependencies - working-directory: ${{env.working-directory}} - run: | - mix local.rebar --force - mix local.hex --force - mix deps.get - - name: Compile Phoenix Assets - working-directory: ${{env.working-directory}} - shell: bash - run: | - make esbuild - - name: Login to AzureCR + if: startswith(github.ref, 'refs/tags/') # Only run on tag push uses: azure/docker-login@v1 with: login-server: ${{ secrets.AZURECR_PUSH_URL }} username: ${{ secrets.AZURECR_PUSH_USER }} password: ${{ secrets.AZURECR_PUSH_PASSWORD }} + - name: Login to DockerHub + if: startswith(github.ref, 'refs/tags/') # Only run on tag push uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_PUSH_USER }} password: ${{ secrets.DOCKERHUB_PUSH_PASSWORD }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 id: buildx-builder - - name: Build and release docker image - uses: docker/build-push-action@v3 - with: - builder: ${{ steps.buildx-builder.outputs.name }} - push: true - context: . - file: ${{ env.app-name }}/Dockerfile - platforms: linux/amd64,linux/arm64 - build-args: | - BUILDER_IMAGE=${{ env.builder-image }} - RELEASE_IMAGE=${{ env.release-image }} - APP_NAME=${{ env.app-name }} - APP_VSN=${{ env.app-version }} - SECRET_KEY_BASE=${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} - MIX_ENV=release_prod - tags: | - wasmcloud.azurecr.io/${{ env.app-name }}:latest - wasmcloud.azurecr.io/${{ env.app-name }}:${{ env.app-version }} - wasmcloud/${{ env.app-name }}:${{ env.app-version }} - wasmcloud/${{ env.app-name }}:latest - - name: Build common static image - uses: docker/build-push-action@v3 + + - uses: actions/download-artifact@v3 with: - builder: ${{ steps.buildx-builder.outputs.name }} - push: true - context: . - file: Dockerfile.localstatic - platforms: linux/amd64 #,linux/arm64 - tags: | - wasmcloud.azurecr.io/wasmcloud_static_base:${{ env.app-version }} - - name: Build and release static docker image + name: ${{ matrix.config.app-name }}_${{ matrix.config.target }} + path: ${{ matrix.config.app-name }} + - run: chmod +x ${{ matrix.config.app-name }}/${{ matrix.config.app-name }}_${{ matrix.config.target }} + + - name: Build and release docker image uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx-builder.outputs.name }} - push: true - context: . - file: ${{ env.app-name }}/Dockerfile.static - platforms: linux/amd64 #,linux/arm64 + push: ${{ startswith(github.ref, 'refs/tags/') }} + context: ${{ matrix.config.app-name }} + file: ${{ matrix.config.app-name }}/Dockerfile + platforms: ${{ matrix.config.platform }} build-args: | - BASEIMAGE=wasmcloud.azurecr.io/wasmcloud_static_base:${{ env.app-version }} - APP_NAME=${{ env.app-name }} - APP_VSN=${{ env.app-version }} - SECRET_KEY_BASE=${{ secrets.WASMCLOUD_HOST_SECRET_KEY_BASE }} - MIX_ENV=release_prod + BIN=${{ matrix.config.app-name }}_${{ matrix.config.target }} tags: | - wasmcloud.azurecr.io/${{ env.app-name }}:${{ env.app-version }}-static - wasmcloud/${{ env.app-name }}:${{ env.app-version }}-static - - - name: Retrieve aarch64, aarch64-static, and amd64-static tarballs from Docker - run: | - docker run --rm --platform linux/arm64/v8 -iv ${PWD}:/aarch64temp wasmcloud/${{ env.app-name }}:${{ env.app-version }} sh -s <> /etc/locale.gen \ - && locale-gen - -# compile specific openssl -RUN curl https://www.openssl.org/source/openssl-$SSL_VERSION.tar.gz -O \ - && tar -xzf openssl-$SSL_VERSION.tar.gz \ - && cd openssl-$SSL_VERSION \ - && ./config \ - && make depend \ - && make \ - && mkdir -p /root/openssl-$SSL_VERSION/lib \ - && cp -r /root/openssl-$SSL_VERSION/libc* /root/openssl-$SSL_VERSION/lib/ \ - && cp -r /root/openssl-$SSL_VERSION/libs* /root/openssl-$SSL_VERSION/lib/ - -# compile erlang, statically linking in ssl -RUN mkdir -p /root/source \ - && git clone https://github.com/erlang/otp /root/source/otp \ - && cd /root/source/otp \ - && git checkout $OTP_VERSION \ - && cd /root/source/otp \ - && ./otp_build autoconf \ - && ./configure --with-ssl=/root/openssl-$SSL_VERSION --disable-dynamic-ssl-lib \ - && make -j4 \ - && make install - -# compile elixir -RUN mkdir -p /root/source \ - && git clone https://github.com/elixir-lang/elixir.git /root/source/elixir \ - && cd /root/source/elixir \ - && git checkout $ELIXIR_VERSION \ - && make clean \ - && make install \ - && mix local.hex --force \ - && mix local.rebar --force - -# install rust -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..a130739b --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.PHONY: release + +release: + nix develop -c 'release' diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..9460998b --- /dev/null +++ b/flake.lock @@ -0,0 +1,466 @@ +{ + "nodes": { + "crane": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ], + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1681478485, + "narHash": "sha256-SIqtWDgUOQZAt/Ue/4LL3p9KJzWAx4MkJqFG3Qt2YAc=", + "owner": "rvolosatovs", + "repo": "crane", + "rev": "412731073de0a1896068ac1dd2392ed66f916af9", + "type": "github" + }, + "original": { + "owner": "rvolosatovs", + "ref": "feat/wit", + "repo": "crane", + "type": "github" + } + }, + "erlang": { + "flake": false, + "locked": { + "lastModified": 1683213918, + "narHash": "sha256-cuFeHwyAyS8R7rXcJJxzE7LTZGgDr4x3qMZx63PIowM=", + "owner": "erlang", + "repo": "otp", + "rev": "0418c10ec37fa374867eae49f2f776c72c55d623", + "type": "github" + }, + "original": { + "owner": "erlang", + "ref": "OTP-25.3.2", + "repo": "otp", + "type": "github" + } + }, + "erlang-aarch64-apple-darwin-mac": { + "flake": false, + "locked": { + "narHash": "sha256-H+6RDn9I7FwVefqu4IMLlTk9qgADBn/FoAoZ3msUL70=", + "type": "tarball", + "url": "https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-aarch64-apple-darwin-mac.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-aarch64-apple-darwin-mac.tar.gz" + } + }, + "erlang-x86_64-apple-darwin-mac": { + "flake": false, + "locked": { + "narHash": "sha256-uMOK2BtHcTSonjvR/wL3gXw1Yl8sw9b4LrdBS57lLfE=", + "type": "tarball", + "url": "https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-x86_64-apple-darwin-mac.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-x86_64-apple-darwin-mac.tar.gz" + } + }, + "erts-x86_64-pc-windows-msvc": { + "flake": false, + "locked": { + "narHash": "sha256-Q6ZVhfoU1JW6cHKtLYVT13BufXhc6V9QLBvh+gEPBck=", + "type": "file", + "url": "https://github.com/erlang/otp/releases/download/OTP-25.3.2/otp_win64_25.3.2.exe" + }, + "original": { + "type": "file", + "url": "https://github.com/erlang/otp/releases/download/OTP-25.3.2/otp_win64_25.3.2.exe" + } + }, + "fenix": { + "inputs": { + "nixpkgs": "nixpkgs", + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1682038649, + "narHash": "sha256-HwGwWLMKdIT24xhDf+mRoCehA8yUlLmuJgS9JeMt4IM=", + "owner": "nix-community", + "repo": "fenix", + "rev": "37b3a6dad6d6060bd305eb7d3628d3b476c87bb6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "monthly", + "repo": "fenix", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1678901627, + "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "hostcore_wasmcloud_native-aarch64-apple-darwin-mac": { + "flake": false, + "locked": { + "narHash": "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY=", + "type": "file", + "url": "file:/dev/null" + }, + "original": { + "type": "file", + "url": "file:/dev/null" + } + }, + "hostcore_wasmcloud_native-x86_64-apple-darwin-mac": { + "flake": false, + "locked": { + "narHash": "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY=", + "type": "file", + "url": "file:/dev/null" + }, + "original": { + "type": "file", + "url": "file:/dev/null" + } + }, + "hostcore_wasmcloud_native-x86_64-pc-windows-msvc": { + "flake": false, + "locked": { + "narHash": "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY=", + "type": "file", + "url": "file:/dev/null" + }, + "original": { + "type": "file", + "url": "file:/dev/null" + } + }, + "nix-filter": { + "locked": { + "lastModified": 1681154353, + "narHash": "sha256-MCJ5FHOlbfQRFwN0brqPbCunLEVw05D/3sRVoNVt2tI=", + "owner": "numtide", + "repo": "nix-filter", + "rev": "f529f42792ade8e32c4be274af6b6d60857fbee7", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "nix-filter", + "type": "github" + } + }, + "nix-flake-tests": { + "locked": { + "lastModified": 1677844186, + "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=", + "owner": "antifuchs", + "repo": "nix-flake-tests", + "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb", + "type": "github" + }, + "original": { + "owner": "antifuchs", + "repo": "nix-flake-tests", + "type": "github" + } + }, + "nix-log": { + "inputs": { + "nix-flake-tests": "nix-flake-tests", + "nixify": [ + "nixify" + ], + "nixlib": "nixlib" + }, + "locked": { + "lastModified": 1681933283, + "narHash": "sha256-phDsQdaoUEI4DUTErR6Tz7lS0y3kXvDwwbqtxpzd0eo=", + "owner": "rvolosatovs", + "repo": "nix-log", + "rev": "833d31e3c1a677eac81ba87e777afa5076071d66", + "type": "github" + }, + "original": { + "owner": "rvolosatovs", + "repo": "nix-log", + "type": "github" + } + }, + "nixify": { + "inputs": { + "crane": [ + "crane" + ], + "fenix": [ + "fenix" + ], + "flake-utils": "flake-utils_2", + "nix-filter": "nix-filter", + "nix-log": [ + "nix-log" + ], + "nixlib": [ + "nixlib" + ], + "nixpkgs": [ + "nixpkgs" + ], + "rust-overlay": "rust-overlay_2" + }, + "locked": { + "lastModified": 1683214284, + "narHash": "sha256-BdiPhr466QvAZw2egOHhyPRpOMPIaFqcUH5x8MG35/U=", + "owner": "rvolosatovs", + "repo": "nixify", + "rev": "15a598543c9b1fe5e700311a99dacc87c26c5daf", + "type": "github" + }, + "original": { + "owner": "rvolosatovs", + "repo": "nixify", + "type": "github" + } + }, + "nixlib": { + "locked": { + "lastModified": 1679791877, + "narHash": "sha256-tTV1Mf0hPWIMtqyU16Kd2JUBDWvfHlDC9pF57vcbgpQ=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "cc060ddbf652a532b54057081d5abd6144d01971", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixlib_2": { + "locked": { + "lastModified": 1681001314, + "narHash": "sha256-5sDnCLdrKZqxLPK4KA8+f4A3YKO/u6ElpMILvX0g72c=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "367c0e1086a4eb4502b24d872cea2c7acdd557f4", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1683014792, + "narHash": "sha256-6Va9iVtmmsw4raBc3QKvQT2KT/NGRWlvUlJj46zN8B8=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "1a411f23ba299db155a5b45d5e145b85a7aafc42", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-old": { + "locked": { + "lastModified": 1651477912, + "narHash": "sha256-YDFgJElf0ZL977+fo6ueQAmpb6lwrw1lzDlxLEINVXE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "66b0db71f463164486a36dded50bedee185e45c2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-20.09-darwin", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1683020768, + "narHash": "sha256-ZyZl6k9NWS5QPwD3NoAVz/eSgodQDvl+y+fu8MVbrHc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "44f30edf5661d86fb3a95841c35127f3d0ea8b0f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "erlang": "erlang", + "erlang-aarch64-apple-darwin-mac": "erlang-aarch64-apple-darwin-mac", + "erlang-x86_64-apple-darwin-mac": "erlang-x86_64-apple-darwin-mac", + "erts-x86_64-pc-windows-msvc": "erts-x86_64-pc-windows-msvc", + "fenix": "fenix", + "hostcore_wasmcloud_native-aarch64-apple-darwin-mac": "hostcore_wasmcloud_native-aarch64-apple-darwin-mac", + "hostcore_wasmcloud_native-x86_64-apple-darwin-mac": "hostcore_wasmcloud_native-x86_64-apple-darwin-mac", + "hostcore_wasmcloud_native-x86_64-pc-windows-msvc": "hostcore_wasmcloud_native-x86_64-pc-windows-msvc", + "nix-log": "nix-log", + "nixify": "nixify", + "nixlib": "nixlib_2", + "nixpkgs": "nixpkgs_2", + "nixpkgs-old": "nixpkgs-old", + "secret-key-base": "secret-key-base" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1680267680, + "narHash": "sha256-atC3zkM5nBXdBFE1+Xoxpm/Ye42j/Rq12IR0qi5+/ao=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "853fb44a24b8d3341f52747caa949013121b24b4", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": [ + "crane", + "flake-utils" + ], + "nixpkgs": [ + "crane", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1680488274, + "narHash": "sha256-0vYMrZDdokVmPQQXtFpnqA2wEgCCUXf5a3dDuDVshn0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "7ec2ff598a172c6e8584457167575b3a1a5d80d8", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { + "inputs": { + "flake-utils": [ + "nixify", + "flake-utils" + ], + "nixpkgs": [ + "nixify", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1681352318, + "narHash": "sha256-+kwy7bTsuW8GYrRqWRQ8T5hg6duZb5IJiHlKo1J+v9g=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "aeaa11c65a5c5cebaa51652353ab3c497b9a7bbf", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "secret-key-base": { + "flake": false, + "locked": { + "narHash": "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY=", + "type": "file", + "url": "file:/dev/null" + }, + "original": { + "type": "file", + "url": "file:/dev/null" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..5fc6a90d --- /dev/null +++ b/flake.nix @@ -0,0 +1,982 @@ +{ + nixConfig.extra-substituters = [ + "https://wasmcloud.cachix.org" + "https://nix-community.cachix.org" + "https://cache.garnix.io" + ]; + nixConfig.extra-trusted-public-keys = [ + "wasmcloud.cachix.org-1:9gRBzsKh+x2HbVVspreFg/6iFRiD4aOcUQfXVDl3hiM=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=" + ]; + + inputs.crane.inputs.nixpkgs.follows = "nixpkgs"; + inputs.crane.url = github:rvolosatovs/crane/feat/wit; + inputs.erlang-aarch64-apple-darwin-mac.flake = false; + inputs.erlang-aarch64-apple-darwin-mac.url = https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-aarch64-apple-darwin-mac.tar.gz; + inputs.erlang-x86_64-apple-darwin-mac.flake = false; + inputs.erlang-x86_64-apple-darwin-mac.url = https://github.com/rvolosatovs/otp/releases/download/OTP-25.3.2/erts-x86_64-apple-darwin-mac.tar.gz; + inputs.erlang.flake = false; + inputs.erlang.url = github:erlang/otp/OTP-25.3.2; + inputs.erts-x86_64-pc-windows-msvc.flake = false; + inputs.erts-x86_64-pc-windows-msvc.url = https://github.com/erlang/otp/releases/download/OTP-25.3.2/otp_win64_25.3.2.exe; + inputs.fenix.url = github:nix-community/fenix/monthly; + inputs.nix-log.inputs.nixify.follows = "nixify"; + inputs.nix-log.url = github:rvolosatovs/nix-log; + inputs.nixify.inputs.crane.follows = "crane"; + inputs.nixify.inputs.fenix.follows = "fenix"; + inputs.nixify.inputs.nix-log.follows = "nix-log"; + inputs.nixify.inputs.nixlib.follows = "nixlib"; + inputs.nixify.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixify.url = github:rvolosatovs/nixify; + inputs.nixlib.url = github:nix-community/nixpkgs.lib; + inputs.nixpkgs-old.url = github:NixOS/nixpkgs/nixpkgs-20.09-darwin; + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixpkgs-unstable; + + # These inputs are "virtual", exposed just to be overriden + inputs.hostcore_wasmcloud_native-aarch64-apple-darwin-mac.flake = false; + inputs.hostcore_wasmcloud_native-aarch64-apple-darwin-mac.url = file:/dev/null; + inputs.hostcore_wasmcloud_native-x86_64-apple-darwin-mac.flake = false; + inputs.hostcore_wasmcloud_native-x86_64-apple-darwin-mac.url = file:/dev/null; + inputs.hostcore_wasmcloud_native-x86_64-pc-windows-msvc.flake = false; + inputs.hostcore_wasmcloud_native-x86_64-pc-windows-msvc.url = file:/dev/null; + inputs.secret-key-base.flake = false; + inputs.secret-key-base.url = file:/dev/null; + + outputs = { + self, + erlang, + erlang-aarch64-apple-darwin-mac, + erlang-x86_64-apple-darwin-mac, + erts-x86_64-pc-windows-msvc, + fenix, + hostcore_wasmcloud_native-aarch64-apple-darwin-mac, + hostcore_wasmcloud_native-x86_64-apple-darwin-mac, + hostcore_wasmcloud_native-x86_64-pc-windows-msvc, + nix-log, + nixify, + nixlib, + nixpkgs-old, + secret-key-base, + ... + }: + with nixlib.lib; + with builtins; + with nix-log.lib; + with nixify.lib; + with nixify.lib.rust.targets; let + version = "0.62.1"; + + otpVersion = "25.3.2"; + + mkStdenv = { + pkgs, + pkgsCross ? pkgs, + }: let + pkgsOld = nixpkgs-old.legacyPackages.${pkgs.stdenv.hostPlatform.system}; + in + if pkgsCross.stdenv.hostPlatform.isGnu && pkgsCross.stdenv.hostPlatform.isAarch64 + then let + cc = + if pkgs.stdenv.hostPlatform.isGnu && pkgs.stdenv.hostPlatform.isAarch64 + # Pull in newer gcc to enable support for aarch64 atomics (e.g. `__aarch64_ldadd4_relax`) + then pkgsOld.gcc10Stdenv.cc + else pkgsOld.pkgsCross.aarch64-multiplatform.gcc10Stdenv.cc; + in + pkgsCross.overrideCC pkgsCross.stdenv cc + else if pkgsCross.stdenv.hostPlatform.isGnu && pkgsCross.stdenv.hostPlatform.isx86_64 + then let + cc = + if pkgs.stdenv.hostPlatform.isGnu && pkgs.stdenv.hostPlatform.isx86_64 + then pkgsOld.stdenv.cc + else pkgsOld.pkgsCross.gnu64.stdenv.cc; + in + pkgsCross.overrideCC pkgsCross.stdenv cc + else pkgsCross.stdenv; + + # Given a package set, constructs and attribute set of packages, overlays, etc. + # See `nixify.lib.rust.mkAttrs` https://github.com/rvolosatovs/nixify/blob/0dc7973067d597457452a050de9b03d979564df4/lib/rust/mkAttrs.nix#L423-L435 + # mkNifAttrs :: PackageSet -> AttributeSet + mkNifAttrs = rust.mkAttrs { + src = ./host_core/native/hostcore_wasmcloud_native; + + doCheck = false; + + targets.armv7-unknown-linux-musleabihf = false; + targets.wasm32-wasi = false; + + buildOverrides = { + pkgs, + pkgsCross ? pkgs, + ... + } @ args: { + depsBuildBuild ? [], + buildInputs ? [], + ... + }: let + crossPlatform = pkgsCross.stdenv.hostPlatform; + in + { + buildInputs = + buildInputs + ++ optional pkgs.stdenv.buildPlatform.isDarwin pkgs.libiconv; + + depsBuildBuild = + depsBuildBuild + ++ [ + pkgs.protobuf # build dependency of prost-build v0.9.0 + ] + ++ optionals pkgsCross.stdenv.hostPlatform.isDarwin [ + pkgsCross.darwin.apple_sdk.frameworks.CoreFoundation + pkgsCross.libiconv + ]; + } + // optionalAttrs (crossPlatform.isGnu && crossPlatform.isx86_64) { + # TODO: Downgrade aarch64 libc + stdenv = mkStdenv { + inherit + pkgs + pkgsCross + ; + }; + + meta.broken = crossPlatform.isGnu && pkgs.stdenv.hostPlatform.isDarwin; # downgrading glibc breaks Darwin support here + }; + }; + + # mkErlang :: AttributeSet -> Package + mkErlang = { + pkgs, + pkgsCross ? pkgs, + }: let + openssl = pkgsCross.pkgsStatic.openssl_3_0.override { + static = true; # upstream darwin builds are not actually static, force static for all + }; + + opensslLib = getOutput "out" openssl; + opensslIncl = getDev openssl; + + nativeErlang = pkgs.erlang_25; + yielding_c_fun = "${nativeErlang}/lib/erlang/erts-13.2.1/bin/yielding_c_fun"; + + nativeBuildInputs = [ + pkgs.autoconf + pkgs.gnum4 + pkgs.libxml2 + pkgs.libxslt + pkgs.makeWrapper + pkgs.perl + ]; + + crossPlatform = pkgsCross.stdenv.hostPlatform; + stdenv = mkStdenv { + inherit + pkgs + pkgsCross + ; + }; + + configureFlags = + [ + "--disable-dynamic-ssl-lib" + "--enable-builtin-zlib" + "--enable-deterministic-build" + "--enable-hipe" + "--enable-kernel-poll" + "--enable-smp-support" + "--enable-static-drivers" + "--enable-static-nifs" + "--enable-threads" + "--with-ssl-incl=${opensslIncl}" + "--with-ssl=${opensslLib}" + "--without-cdv" + "--without-debugger" + "--without-et" + "--without-javac" + "--without-observer" + "--without-odbc" + "--without-termcap" + "--without-wx" + "LIBS=${opensslLib}/lib/libcrypto.a" + ] + ++ optional crossPlatform.isDarwin "--enable-darwin-64bit"; + in + stdenv.mkDerivation ( + { + inherit + nativeBuildInputs + ; + + src = erlang; + name = "erlang"; + + dontInstall = true; + dontPatchShebangs = true; + + # NOTE: parallel Linux builds seems to be flaky + preConfigure = optionalString crossPlatform.isDarwin '' + export MAKEFLAGS+=" -j$NIX_BUILD_CORES" + ''; + + configureFlags = + configureFlags + ++ optionals crossPlatform.isDarwin [ + "AR=/usr/bin/ar" + "CC=/usr/bin/clang" + "CXX=/usr/bin/clang++" + "LD=/usr/bin/ld" + "RANLIB=/usr/bin/ranlib" + ]; + + depsBuildBuild = [ + openssl + ]; + + postPatch = '' + patchShebangs make + ''; + + configureScript = "./otp_build configure"; + buildPhase = "./otp_build release -a $out"; + } + // optionalAttrs crossPlatform.isLinux { + NIX_CFLAGS_COMPILE = [ + "-static-libgcc" + "-static-libstdc++" + ]; + NIX_CFLAGS_LINK = [ + "-static-libgcc" + "-static-libstdc++" + ]; + } + // optionalAttrs (pkgs.stdenv.hostPlatform.config != crossPlatform.config) { + erl_xcomp_sysroot = pkgs.symlinkJoin { + name = "erlang-${crossPlatform.config}-sysroot"; + paths = [ + opensslLib + opensslIncl + ]; + }; + + configureFlags = + configureFlags + ++ optional (crossPlatform.isDarwin && crossPlatform.isAarch64) "--xcomp-conf=xcomp/erl-xcomp-aarch64-darwin.conf"; + + postPatch = assert pathExists yielding_c_fun; '' + substituteInPlace erts/emulator/Makefile.in \ + --replace 'YCF_EXECUTABLE_PATH=`utils/find_cross_ycf`' 'YCF_EXECUTABLE_PATH=${yielding_c_fun}' + ''; + + nativeBuildInputs = + nativeBuildInputs + ++ [ + nativeErlang + ]; + } + ); + + mkBeam = pkgs: pkgs.beam.packagesWith pkgs.beam.interpreters.erlang; + + mkErlangEnv = {SECRET_KEY_BASE ? ""}: + { + LANG = "C.UTF-8"; + LC_TYPE = "C.UTF-8"; + } + // optionalAttrs (SECRET_KEY_BASE != "") { + inherit + SECRET_KEY_BASE + ; + }; + + mkMixDeps = { + installPhase ? null, + pkgs, + pname, + SECRET_KEY_BASE ? "", + sha256 ? fakeHash, + src, + version, + }: + (mkBeam pkgs).fetchMixDeps ({ + inherit + sha256 + src + version + ; + pname = "mix-${pname}-deps"; + + env = mkErlangEnv { + inherit + SECRET_KEY_BASE + ; + }; + + dontFixup = true; + } + // optionalAttrs (installPhase != null) { + inherit installPhase; + }); + + # mkBurritoBuildInputs :: PackageSet -> [ Package ] + mkBurritoBuildInputs = pkgs: [ + pkgs.p7zip + pkgs.xz + pkgs.zig_0_10 + ]; + + # mkBurrito :: AttributeSet -> Package + mkBurrito = { + buildInputs ? [], + BURRITO_TARGET ? "", + esbuild ? null, + mixFodDeps, + pkgs, + pname, + preBuild ? null, + preConfigure ? null, + sass ? null, + SECRET_KEY_BASE ? "", + src, + version, + }: let + beam = mkBeam pkgs; + + packages = self.packages.${pkgs.stdenv.system}; + + NIF_AARCH64_DARWIN = "${hostcore_wasmcloud_native-aarch64-apple-darwin-mac}/lib/libhostcore_wasmcloud_native.dylib"; + NIF_X86_64_DARWIN = "${hostcore_wasmcloud_native-x86_64-apple-darwin-mac}/lib/libhostcore_wasmcloud_native.dylib"; + NIF_X86_64_WINDOWS = "${hostcore_wasmcloud_native-x86_64-pc-windows-msvc}/lib/hostcore_wasmcloud_native.dll"; + + env = + mkErlangEnv { + inherit + SECRET_KEY_BASE + ; + } + // optionalAttrs (sass != null) { + MIX_SASS_PATH = "${sass}/bin/sass"; + } + // optionalAttrs (esbuild != null) { + MIX_ESBUILD_PATH = "${esbuild}/bin/esbuild"; + } + // optionalAttrs (BURRITO_TARGET != "") { + inherit + BURRITO_TARGET + ; + } + // filterAttrs (k: _: BURRITO_TARGET == "" || hasSuffix (toUpper BURRITO_TARGET) k) { + ERTS_AARCH64_DARWIN = packages.erts-aarch64-apple-darwin-mac; + ERTS_AARCH64_LINUX_GNU = packages.erts-aarch64-unknown-linux-gnu-fhs; + ERTS_AARCH64_LINUX_MUSL = packages.erts-aarch64-unknown-linux-musl-fhs; + ERTS_X86_64_DARWIN = packages.erts-x86_64-apple-darwin-mac; + ERTS_X86_64_WINDOWS = packages.erts-x86_64-pc-windows-msvc; + ERTS_X86_64_LINUX_GNU = packages.erts-x86_64-unknown-linux-gnu-fhs; + ERTS_X86_64_LINUX_MUSL = packages.erts-x86_64-unknown-linux-musl-fhs; + + NIF_AARCH64_DARWIN = assert pathExists NIF_AARCH64_DARWIN; NIF_AARCH64_DARWIN; + NIF_AARCH64_LINUX_GNU = "${packages.hostcore_wasmcloud_native-aarch64-unknown-linux-gnu-fhs}/lib/libhostcore_wasmcloud_native.so"; + NIF_AARCH64_LINUX_MUSL = "${packages.hostcore_wasmcloud_native-aarch64-unknown-linux-musl-fhs}/lib/libhostcore_wasmcloud_native.so"; + NIF_X86_64_DARWIN = assert pathExists NIF_X86_64_DARWIN; NIF_X86_64_DARWIN; + NIF_X86_64_WINDOWS = assert pathExists NIF_X86_64_WINDOWS; NIF_X86_64_WINDOWS; + NIF_X86_64_LINUX_GNU = "${packages.hostcore_wasmcloud_native-x86_64-unknown-linux-gnu-fhs}/lib/libhostcore_wasmcloud_native.so"; + NIF_X86_64_LINUX_MUSL = "${packages.hostcore_wasmcloud_native-x86_64-unknown-linux-musl-fhs}/lib/libhostcore_wasmcloud_native.so"; + }; + in + trace' "mkBurrito" { + inherit + BURRITO_TARGET + env + esbuild + pname + sass + SECRET_KEY_BASE + src + version + ; + } + beam.mixRelease ( + { + inherit + env + mixFodDeps + pname + src + version + ; + dontFixup = true; + + buildInputs = mkBurritoBuildInputs pkgs ++ buildInputs; + + buildPhase = '' + runHook preBuild + + export HOME=$(mktemp -d) + mix release --no-deps-check + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + cp ./burrito_out/* $out/bin + + runHook postInstall + ''; + } + // optionalAttrs (preConfigure != null) { + inherit preConfigure; + } + // optionalAttrs (preBuild != null) { + inherit preBuild; + } + ); + + mkBurritoPackages = { + buildInputs ? [], + esbuild ? null, + mixFodDeps, + pkgs, + pname, + preBuild ? null, + preConfigure ? null, + sass ? null, + SECRET_KEY_BASE ? "", + src, + version, + } @ args: { + "${pname}-burrito" = args; + "${pname}-burrito-aarch64-darwin" = mkBurrito (args + // { + BURRITO_TARGET = "aarch64_darwin"; + }); + "${pname}-burrito-aarch64-linux-gnu" = mkBurrito (args + // { + BURRITO_TARGET = "aarch64_linux_gnu"; + }); + "${pname}-burrito-aarch64-linux-musl" = mkBurrito (args + // { + BURRITO_TARGET = "aarch64_linux_musl"; + }); + "${pname}-burrito-x86_64-darwin" = mkBurrito (args + // { + BURRITO_TARGET = "x86_64_darwin"; + }); + "${pname}-burrito-x86_64-linux-gnu" = mkBurrito (args + // { + BURRITO_TARGET = "x86_64_linux_gnu"; + }); + "${pname}-burrito-x86_64-linux-musl" = mkBurrito (args + // { + BURRITO_TARGET = "x86_64_linux_musl"; + }); + "${pname}-burrito-x86_64-windows" = mkBurrito (args + // { + BURRITO_TARGET = "x86_64_windows"; + }); + }; + in + mkFlake { + overlays = [ + fenix.overlays.default + ]; + + withPackages = { + packages, + pkgs, + ... + }: let + interpreters.aarch64-unknown-linux-gnu = "/lib/ld-linux-aarch64.so.1"; + interpreters.aarch64-unknown-linux-musl = "/lib/ld-musl-aarch64.so.1"; + interpreters.x86_64-unknown-linux-gnu = "/lib64/ld-linux-x86-64.so.2"; + interpreters.x86_64-unknown-linux-musl = "/lib/ld-musl-x86_64.so.1"; + + nifAttrs = mkNifAttrs pkgs; + + mkFHS = { + name, + src, + interpreter, + }: + pkgs.stdenv.mkDerivation { + inherit + name + src + ; + + buildInputs = [ + pkgs.patchelf + ]; + + dontBuild = true; + dontFixup = true; + + installPhase = '' + runHook preInstall + + for p in $(find . -type f); do + # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + if head -c 4 $p | grep $'\x7FELF' > /dev/null; then + patchelf --set-rpath /lib $p || true + patchelf --set-interpreter ${interpreter} $p || true + fi + done + + mkdir -p $out + cp -R * $out + + runHook postInstall + ''; + }; + + mkMac = { + name, + src, + }: + pkgs.stdenv.mkDerivation { + inherit + name + src + ; + + buildInputs = [ + pkgs.darwin.cctools + ]; + + dontBuild = true; + dontFixup = true; + + installPhase = '' + runHook preInstall + + for p in $(find . -type f); do + # https://en.wikipedia.org/wiki/Mach-O#Mach-O_header + case $(head -c 4 $p | od -An -t x1 | tr -d ' ') in + feedface | cefaedfe | feedfacf | cffaedfe) + for lib in $(otool -L $p | tail -n +1 | grep -v $p | grep ${storeDir} | cut -f 2 | cut -d ' ' -f 1); do + name=$(cut -d '/' -f 2- <<< ''${lib#${storeDir}/}) + name=''${name#lib/} + case $name in + libiconv.dylib) + install_name_tool -change "$lib" '/usr/lib/libiconv.2.dylib' "$p" >&2 + ;; + *) + install_name_tool -change "$lib" "/usr/lib/$name" "$p" >&2 + ;; + esac + done + ;; + esac + done + + mkdir -p $out + cp -R * $out + + runHook postInstall + ''; + + # By some reason, binaries with paths changed via `install_name_tool` fail to load on aarch64 Macs + # TODO: Fix it or at least figure out the reason for the runtime failure + meta.broken = true; + meta.platforms = platforms.darwin; + }; + + # Partially-applied `mkErlang`, which takes an optional cross package set as argument + # mkErlang' :: AttributeSet -> Package + mkErlang' = {pkgsCross ? pkgs}: + mkErlang { + inherit + pkgs + pkgsCross + ; + }; + erlang = mkErlang' {}; + erlang-aarch64-unknown-linux-gnu = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != aarch64-unknown-linux-gnu) { + pkgsCross = pkgs.pkgsCross.aarch64-multiplatform; + }); + erlang-aarch64-unknown-linux-musl = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != aarch64-unknown-linux-musl) { + pkgsCross = pkgs.pkgsCross.aarch64-multiplatform-musl; + }); + erlang-x86_64-unknown-linux-gnu = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != x86_64-unknown-linux-gnu) { + pkgsCross = pkgs.pkgsCross.gnu64; + }); + erlang-x86_64-unknown-linux-musl = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != x86_64-unknown-linux-musl) { + pkgsCross = pkgs.pkgsCross.musl64; + }); + + erlang-aarch64-unknown-linux-gnu-fhs = mkFHS { + name = "erlang-aarch64-unknown-linux-gnu-fhs"; + src = erlang-aarch64-unknown-linux-gnu; + interpreter = interpreters.aarch64-unknown-linux-gnu; + }; + erlang-aarch64-unknown-linux-musl-fhs = mkFHS { + name = "erlang-aarch64-unknown-linux-musl-fhs"; + src = erlang-aarch64-unknown-linux-musl; + interpreter = interpreters.aarch64-unknown-linux-musl; + }; + erlang-x86_64-unknown-linux-gnu-fhs = mkFHS { + name = "erlang-x86_64-unknown-linux-gnu-fhs"; + src = erlang-x86_64-unknown-linux-gnu; + interpreter = interpreters.x86_64-unknown-linux-gnu; + }; + erlang-x86_64-unknown-linux-musl-fhs = mkFHS { + name = "erlang-x86_64-unknown-linux-musl-fhs"; + src = erlang-x86_64-unknown-linux-musl; + interpreter = interpreters.x86_64-unknown-linux-musl; + }; + + mkErts = { + name, + erlang, + }: + pkgs.stdenv.mkDerivation { + name = "${name}.tar.gz"; + src = erlang; + + buildInputs = [ + pkgs.gnutar + ]; + + dontBuild = true; + + installPhase = '' + tar czf $out --transform 's,^\.,./otp-${otpVersion},' -C $src . + ''; + }; + + erts-aarch64-apple-darwin-mac = mkErts { + name = "erts-aarch64-apple-darwin-mac"; + erlang = erlang-aarch64-apple-darwin-mac; + }; + erts-aarch64-unknown-linux-gnu = mkErts { + name = "erts-aarch64-unknown-linux-gnu"; + erlang = erlang-aarch64-unknown-linux-gnu; + }; + erts-aarch64-unknown-linux-gnu-fhs = mkErts { + name = "erts-aarch64-unknown-linux-gnu-fhs"; + erlang = erlang-aarch64-unknown-linux-gnu-fhs; + }; + erts-aarch64-unknown-linux-musl = mkErts { + name = "erts-aarch64-unknown-linux-musl"; + erlang = erlang-aarch64-unknown-linux-musl; + }; + erts-aarch64-unknown-linux-musl-fhs = mkErts { + name = "erts-aarch64-unknown-linux-musl-fhs"; + erlang = erlang-aarch64-unknown-linux-musl-fhs; + }; + + erts-x86_64-apple-darwin-mac = mkErts { + name = "erts-x86_64-apple-darwin-mac"; + erlang = erlang-x86_64-apple-darwin-mac; + }; + erts-x86_64-unknown-linux-gnu = mkErts { + name = "erts-x86_64-unknown-linux-gnu"; + erlang = erlang-x86_64-unknown-linux-gnu; + }; + erts-x86_64-unknown-linux-gnu-fhs = mkErts { + name = "erts-x86_64-unknown-linux-gnu-fhs"; + erlang = erlang-x86_64-unknown-linux-gnu-fhs; + }; + erts-x86_64-unknown-linux-musl = mkErts { + name = "erts-x86_64-unknown-linux-musl"; + erlang = erlang-x86_64-unknown-linux-musl; + }; + erts-x86_64-unknown-linux-musl-fhs = mkErts { + name = "erts-x86_64-unknown-linux-musl-fhs"; + erlang = erlang-x86_64-unknown-linux-musl-fhs; + }; + + hostcorePkgs = let + src = ./host_core; + pname = "host_core"; + mixFodDeps = mkMixDeps { + inherit + pkgs + pname + src + version + ; + sha256 = "sha256-JsYht6LlamYLvlIkDfHo35a4BWQd62iPO3NwtUF1gUo="; + }; + in + mkBurritoPackages { + inherit + mixFodDeps + pkgs + pname + src + version + ; + }; + + wasmcloudPkgs = let + src = ./.; + pname = "wasmcloud_host"; + SECRET_KEY_BASE = readFile secret-key-base; + mixFodDeps = mkMixDeps { + inherit + pkgs + pname + SECRET_KEY_BASE + src + version + ; + sha256 = "sha256-b8Fg0IhAzsjT1kvp7yzg0kPZM7sO/HBlcuFalDITjqA="; + installPhase = '' + runHook preInstall + + cd ./host_core + mix deps.get ''${MIX_ENV:+--only $MIX_ENV} + + cd ../wasmcloud_host + mix deps.get ''${MIX_ENV:+--only $MIX_ENV} + + find "$TEMPDIR/deps" -path '*/.git/*' -a ! -name HEAD -exec rm -rf {} + + cp -r --no-preserve=mode,ownership,timestamps $TEMPDIR/deps $out + + runHook postInstall + ''; + }; + in + mkBurritoPackages { + inherit + mixFodDeps + pkgs + pname + SECRET_KEY_BASE + src + version + ; + inherit + (pkgs) + sass + esbuild + ; + + buildInputs = [ + pkgs.esbuild + pkgs.sass + ]; + + preConfigure = '' + cd ./wasmcloud_host + ln -s $MIX_DEPS_PATH ./deps + ''; + + preBuild = '' + mkdir -p ./priv/static/assets + + mix do deps.loadpaths --no-deps-check, sass default assets/css/app.scss ./priv/static/assets/app.css + mix do deps.loadpaths --no-deps-check, assets.deploy + + cp -r assets/static/* ./priv/static/ + cp -r assets/css/coreui ./priv/static/assets/coreui + ''; + }; + in + fix ( + self': + packages + // { + inherit + erlang + erlang-aarch64-unknown-linux-gnu + erlang-aarch64-unknown-linux-gnu-fhs + erlang-aarch64-unknown-linux-musl + erlang-aarch64-unknown-linux-musl-fhs + erlang-x86_64-unknown-linux-gnu + erlang-x86_64-unknown-linux-gnu-fhs + erlang-x86_64-unknown-linux-musl + erlang-x86_64-unknown-linux-musl-fhs + erts-aarch64-apple-darwin-mac + erts-aarch64-unknown-linux-gnu + erts-aarch64-unknown-linux-gnu-fhs + erts-aarch64-unknown-linux-musl + erts-aarch64-unknown-linux-musl-fhs + erts-x86_64-apple-darwin-mac + erts-x86_64-unknown-linux-gnu + erts-x86_64-unknown-linux-gnu-fhs + erts-x86_64-unknown-linux-musl + erts-x86_64-unknown-linux-musl-fhs + ; + + erts-x86_64-pc-windows-msvc = pkgs.stdenv.mkDerivation { + name = "erts-x86_64-pc-windows-msvc.exe"; + src = erts-x86_64-pc-windows-msvc; + + dontUnpack = true; + dontBuild = true; + + installPhase = '' + install $src $out + ''; + }; + + hostcore_wasmcloud_native-aarch64-unknown-linux-gnu-fhs = mkFHS { + name = "hostcore_wasmcloud_native-aarch64-unknown-linux-gnu-fhs"; + src = self'.hostcore_wasmcloud_native-aarch64-unknown-linux-gnu; + interpreter = interpreters.aarch64-unknown-linux-gnu; + }; + hostcore_wasmcloud_native-aarch64-unknown-linux-musl-fhs = mkFHS { + name = "hostcore_wasmcloud_native-aarch64-unknown-linux-musl-fhs"; + src = self'.hostcore_wasmcloud_native-aarch64-unknown-linux-musl; + interpreter = interpreters.aarch64-unknown-linux-musl; + }; + hostcore_wasmcloud_native-x86_64-unknown-linux-gnu-fhs = mkFHS { + name = "hostcore_wasmcloud_native-x86_64-unknown-linux-gnu-fhs"; + src = self'.hostcore_wasmcloud_native-x86_64-unknown-linux-gnu; + interpreter = interpreters.x86_64-unknown-linux-gnu; + }; + hostcore_wasmcloud_native-x86_64-unknown-linux-musl-fhs = mkFHS { + name = "hostcore_wasmcloud_native-x86_64-unknown-linux-musl-fhs"; + src = self'.hostcore_wasmcloud_native-x86_64-unknown-linux-musl; + interpreter = interpreters.x86_64-unknown-linux-musl; + }; + } + // optionalAttrs pkgs.stdenv.buildPlatform.isDarwin { + erlang-aarch64-apple-darwin-mac = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != aarch64-apple-darwin) { + pkgsCross = pkgs.pkgsCross.aarch64-darwin; + }); + erts-aarch64-apple-darwin-mac = mkErts { + name = "erts-aarch64-apple-darwin-mac"; + erlang = self'.erlang-aarch64-apple-darwin-mac; + }; + erlang-x86_64-apple-darwin-mac = mkErlang' (optionalAttrs (pkgs.stdenv.hostPlatform.config != x86_64-apple-darwin) { + pkgsCross = pkgs.pkgsCross.x86_64-darwin; + }); + erts-x86_64-apple-darwin-mac = mkErts { + name = "erts-x86_64-apple-darwin-mac"; + erlang = self'.erlang-x86_64-apple-darwin-mac; + }; + hostcore_wasmcloud_native-aarch64-apple-darwin-mac = mkMac { + name = "hostcore_wasmcloud_native-aarch64-apple-darwin-mac"; + src = self'.hostcore_wasmcloud_native-aarch64-apple-darwin; + }; + hostcore_wasmcloud_native-x86_64-apple-darwin-mac = mkMac { + name = "hostcore_wasmcloud_native-x86_64-apple-darwin-mac"; + src = self'.hostcore_wasmcloud_native-x86_64-apple-darwin; + }; + } + // nifAttrs.packages + // hostcorePkgs + // wasmcloudPkgs + ); + + withDevShells = { + devShells, + pkgs, + ... + }: let + nifAttrs = mkNifAttrs pkgs; + + burritoBuildInputs = mkBurritoBuildInputs pkgs; + + build-mac-erts = pkgs.writeShellScriptBin "build-mac-erts" '' + set -xe + + out=''${1:-_nix/out} + mkdir -p $out + + nix build -L \ + ${self}\#packages.aarch64-darwin.erts-aarch64-apple-darwin-mac \ + ${self}\#packages.x86_64-darwin.erts-x86_64-apple-darwin-mac \ + -o $out/result + + nix build -L \ + ${self}\#packages.aarch64-darwin.erts-aarch64-apple-darwin-mac \ + -o $out/erts-aarch64-apple-darwin-mac.tar.gz + + nix build -L \ + ${self}\#packages.x86_64-darwin.erts-x86_64-apple-darwin-mac \ + -o $out/erts-x86_64-apple-darwin-mac.tar.gz + ''; + + release = pkgs.writeShellScriptBin "release" ('' + if [ -z $HOSTCORE_WASMCLOUD_NATIVE_AARCH64_APPLE_DARWIN ]; then + echo "HOSTCORE_WASMCLOUD_NATIVE_AARCH64_APPLE_DARWIN must be set to a path containing 'lib/libhostcore_wasmcloud_native.dylib'" + exit 1 + fi + if [ -z $HOSTCORE_WASMCLOUD_NATIVE_X86_64_APPLE_DARWIN ]; then + echo "HOSTCORE_WASMCLOUD_NATIVE_X86_64_APPLE_DARWIN must be set to a path containing 'lib/libhostcore_wasmcloud_native.dylib'" + exit 1 + fi + if [ -z $HOSTCORE_WASMCLOUD_NATIVE_X86_64_PC_WINDOWS_MSVC ]; then + echo "HOSTCORE_WASMCLOUD_NATIVE_X86_64_PC_WINDOWS_MSVC must be set to a path containing 'lib/hostcore_wasmcloud_native.dll'" + exit 1 + fi + + set -xe + + out=''${1:-_nix/out} + mkdir -p $out + + printf '%s' "''${SECRET_KEY_BASE}" | nix build -L \ + --override-input secret-key-base "file:///dev/stdin" \ + --override-input hostcore_wasmcloud_native-aarch64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_AARCH64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-pc-windows-msvc "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_PC_WINDOWS_MSVC" \ + ${self}\#host_core-burrito-aarch64-darwin \ + ${self}\#host_core-burrito-aarch64-linux-gnu \ + ${self}\#host_core-burrito-aarch64-linux-musl \ + ${self}\#host_core-burrito-x86_64-darwin \ + ${self}\#host_core-burrito-x86_64-linux-gnu \ + ${self}\#host_core-burrito-x86_64-linux-musl \ + ${self}\#host_core-burrito-x86_64-windows \ + ${self}\#wasmcloud_host-burrito-aarch64-darwin \ + ${self}\#wasmcloud_host-burrito-aarch64-linux-gnu \ + ${self}\#wasmcloud_host-burrito-aarch64-linux-musl \ + ${self}\#wasmcloud_host-burrito-x86_64-darwin \ + ${self}\#wasmcloud_host-burrito-x86_64-linux-gnu \ + ${self}\#wasmcloud_host-burrito-x86_64-linux-musl \ + ${self}\#wasmcloud_host-burrito-x86_64-windows \ + -o $out/result + + '' + + concatMapStringsSep "\n" (target: '' + printf '%s' "''${SECRET_KEY_BASE}" | nix build -L \ + --override-input secret-key-base "file:///dev/stdin" \ + --override-input hostcore_wasmcloud_native-aarch64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_AARCH64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-pc-windows-msvc "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_PC_WINDOWS_MSVC" \ + ${self}\#host_core-burrito-${target} -o $out/host_core-burrito-${target} + + printf '%s' "''${SECRET_KEY_BASE}" | nix build -L \ + --override-input secret-key-base "file:///dev/stdin" \ + --override-input hostcore_wasmcloud_native-aarch64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_AARCH64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-apple-darwin-mac "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_APPLE_DARWIN" \ + --override-input hostcore_wasmcloud_native-x86_64-pc-windows-msvc "$HOSTCORE_WASMCLOUD_NATIVE_X86_64_PC_WINDOWS_MSVC" \ + ${self}\#wasmcloud_host-burrito-${target} -o $out/wasmcloud_host-burrito-${target} + '') [ + "aarch64-darwin" + "aarch64-linux-gnu" + "aarch64-linux-musl" + "x86_64-darwin" + "x86_64-linux-gnu" + "x86_64-linux-musl" + "x86_64-windows" + ]); + in + extendDerivations { + env.MIX_ESBUILD_PATH = "${pkgs.esbuild}/bin/esbuild"; + env.MIX_SASS_PATH = "${pkgs.sass}/bin/sass"; + + buildInputs = + burritoBuildInputs + ++ [ + nifAttrs.hostRustToolchain + + pkgs.beamPackages.hex + pkgs.beamPackages.rebar3 + pkgs.elixir_1_14 + pkgs.esbuild + pkgs.nats-server + pkgs.sass + + build-mac-erts + release + ]; + } + devShells; + }; +} diff --git a/host_core/Dockerfile b/host_core/Dockerfile index 856f8963..29241034 100644 --- a/host_core/Dockerfile +++ b/host_core/Dockerfile @@ -1,61 +1,24 @@ -## -# STEP 1: Retrieve dependencies (must happen before NIF compilation) -## -ARG BUILDER_IMAGE -ARG RELEASE_IMAGE -FROM ${BUILDER_IMAGE} AS builder +FROM debian:bullseye-slim -ARG MIX_ENV=prod -ENV MIX_ENV=${MIX_ENV} \ - WASMEX_BUILD=true +# Install runtime dependencies +RUN apt update && \ + apt install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* -WORKDIR /opt/app -COPY ./ ./ +ARG BIN +COPY ${BIN} /usr/local/bin/host_core -# Install necessary system dependencies -RUN apk add bash git curl build-base - -# Grab platform-specific NIF if building with release_prod, otherwise build NIF manually -RUN if [ "$MIX_ENV" = "release_prod" ] ; then cp ./priv/built/`uname -m`/libhostcore_wasmcloud_native.so ./priv/built/libhostcore_wasmcloud_native.so ; else echo "not grabbing" ; fi -# Ensure intermediate artifacts don't get bundled into the final release -RUN rm -rf priv/built/aarch64 priv/built/x86_64 priv/native - -# Get Rust for building NIFs (wasmex) -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y -# Set PATH to include Rust toolchain, enable static compilation -ENV PATH="/root/.cargo/bin:${PATH}" \ - RUSTFLAGS="-C target-feature=-crt-static" - -# This step installs all the build tools we'll need -RUN mix local.rebar --force && \ - mix local.hex --force && \ - mix deps.get && \ - mix release - -RUN mkdir -p /opt/built && \ - mv _build/${MIX_ENV}/rel/host_core/* /opt/built - -## -# STEP 2: Build optimized final release image -## - -# Release image should be the same as the _base container image_ used for the builder. -# E.g. `elixir:1.13.3-alpine`'s base container image is `alpine:3.15.4' -FROM ${RELEASE_IMAGE} - -WORKDIR /opt/app -# Required for the erlang VM -RUN apk add bash ncurses libgcc libstdc++ - -COPY --from=builder /opt/built . +RUN host_core maintenance install # NATS connection is required and can be overridden # Default configuration assumes a NATS container is running named `nats` and available over port 4222 ARG WASMCLOUD_RPC_HOST=nats ARG WASMCLOUD_PROV_RPC_HOST=nats ARG WASMCLOUD_CTL_HOST=nats -ENV WASMCLOUD_RPC_HOST=${WASMCLOUD_RPC_HOST} \ +ENV LANG="C.UTF-8" \ + LC_TYPE="C.UTF-8" \ + WASMCLOUD_RPC_HOST=${WASMCLOUD_RPC_HOST} \ WASMCLOUD_CTL_HOST=${WASMCLOUD_CTL_HOST} \ WASMCLOUD_PROV_RPC_HOST=${WASMCLOUD_PROV_RPC_HOST} -CMD ["/opt/app/bin/host_core", "start"] \ No newline at end of file +CMD ["/usr/local/bin/host_core"] diff --git a/host_core/Dockerfile.static b/host_core/Dockerfile.static deleted file mode 100644 index 68169949..00000000 --- a/host_core/Dockerfile.static +++ /dev/null @@ -1,27 +0,0 @@ -# we use a common Dockerfile with statically compiled erlang+openssl -# the Makefile handles this for you -# ../Dockerfile.localstatic -ARG BASEIMAGE=localstatic -FROM ${BASEIMAGE} -COPY ./host_core /opt/app/host_core - -WORKDIR /opt/app/host_core - -RUN . /root/.cargo/env \ - && mix deps.get \ - && MIX_ENV=prod mix release host_core - -FROM ubuntu:20.04 -COPY --from=0 /opt/app/host_core/_build/prod/rel/bakeware/host_core /usr/local/bin/host_core - -ENV LANGUAGE en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -RUN mkdir -p /root/.cache \ - && apt-get update \ - && echo "en_US UTF-8" > /etc/locale.gen \ - && apt-get install -y locales \ - && rm -rf /var/lib/apt/* - -CMD ["/usr/local/bin/host_core"] diff --git a/host_core/Makefile b/host_core/Makefile index aa488628..446dd940 100644 --- a/host_core/Makefile +++ b/host_core/Makefile @@ -1,11 +1,7 @@ -.PHONY: build build-image buildx-cross-image run run-interactive test release help deps +.PHONY: build run run-interactive credo test help deps .DEFAULT: help -CARGO ?= cargo --color always EXTRA_TEST_ARGS ?= -DOCKERFILESTATIC ?= host_core/Dockerfile.static -TAG ?= latest -STATIC_NAME ?= wasmcloud/host_core_static help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_\-.*]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -16,29 +12,6 @@ deps: ## Fetch mix dependencies build: deps ## Compile host_core mix compile -build-image: ## Compile host_core docker image - docker build \ - --build-arg BUILDER_IMAGE=elixir:1.14.3-alpine \ - --build-arg RELEASE_IMAGE=alpine:3.17.2 \ - --build-arg MIX_ENV=prod \ - -t host_core:alpine \ - . - -build-image-static: ## build statically compiled docker image - cd ../ \ - && docker build -f Dockerfile.localstatic -t localstatic . \ - && docker build -f $(DOCKERFILESTATIC) . -t $(STATIC_NAME):$(TAG) - -# erlang 24 until 25 stops segfaulting under qemu -buildx-cross-image: ## Compile host_core docker image using buildx for amd64 and arm64 - docker buildx build \ - --build-arg BUILDER_IMAGE=elixir:1.14.3-otp-24-alpine \ - --build-arg RELEASE_IMAGE=alpine:3.17.2 \ - --build-arg MIX_ENV=prod \ - -t host_core:alpine \ - --platform linux/amd64,linux/arm64 \ - . - run: build ## Run host_core mix @@ -52,6 +25,3 @@ test: build ## Run test suite, launch NATS with docker-compose docker compose -f ./test/docker-compose.yml up --detach MIX_ENV=test mix test $(EXTRA_TEST_ARGS) docker compose -f ./test/docker-compose.yml down - -release: deps ## Creates a mix release for host_core - MIX_ENV=prod mix release diff --git a/host_core/config/dev.exs b/host_core/config/dev.exs index 6cd421e0..c3fe798e 100644 --- a/host_core/config/dev.exs +++ b/host_core/config/dev.exs @@ -2,6 +2,8 @@ import Config config :logger, :console, level: :debug +config :host_core, HostCore.WasmCloud.Native, mode: :debug + # Uncomment one of the below items ONLY if you're not using environment variables to # configure the otel exporter AND you're not using wasmcloud_host (e.g. you're running # headless from host_core). diff --git a/host_core/config/prod.exs b/host_core/config/prod.exs index becde769..3d0d6f87 100644 --- a/host_core/config/prod.exs +++ b/host_core/config/prod.exs @@ -1 +1,5 @@ import Config + +config :host_core, HostCore.WasmCloud.Native, + mode: :release, + skip_compilation?: true diff --git a/host_core/config/release_prod.exs b/host_core/config/release_prod.exs deleted file mode 100644 index 4a842fd1..00000000 --- a/host_core/config/release_prod.exs +++ /dev/null @@ -1,7 +0,0 @@ -# This file is only used for assembling final releases and primarily -# is for specifying custom parameters for the Native NIF -import Config - -config :host_core, HostCore.WasmCloud.Native, - crate: :hostcore_wasmcloud_native, - mode: if(Mix.env() == :dev, do: :debug, else: :release) diff --git a/host_core/lib/copy_nif.ex b/host_core/lib/copy_nif.ex new file mode 100644 index 00000000..6e65ad78 --- /dev/null +++ b/host_core/lib/copy_nif.ex @@ -0,0 +1,66 @@ +defmodule HostCore.CopyNIF do + alias Burrito.Builder.Step + @behaviour Step + + @impl Step + def execute(%Burrito.Builder.Context{} = context) do + dir = + Path.join(context.work_dir, [ + "lib", + "/host_core-#{context.mix_release.version}", + "/priv", + "/native" + ]) + + File.mkdir_p!(dir) + + case context.target.alias do + :aarch64_darwin -> + File.copy!( + System.get_env("NIF_AARCH64_DARWIN"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :aarch64_linux_gnu -> + File.copy!( + System.get_env("NIF_AARCH64_LINUX_GNU"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :aarch64_linux_musl -> + File.copy!( + System.get_env("NIF_AARCH64_LINUX_MUSL"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :x86_64_darwin -> + File.copy!( + System.get_env("NIF_X86_64_DARWIN"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :x86_64_linux_gnu -> + File.copy!( + System.get_env("NIF_X86_64_LINUX_GNU"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :x86_64_linux_musl -> + File.copy!( + System.get_env("NIF_X86_64_LINUX_MUSL"), + Path.join(dir, "libhostcore_wasmcloud_native.so") + ) + + :x86_64_windows -> + File.copy!( + System.get_env("NIF_X86_64_WINDOWS"), + Path.join(dir, "libhostcore_wasmcloud_native.dll") + ) + + alias -> + raise "unknown target alias #{inspect(alias)}" + end + + context + end +end diff --git a/host_core/mix.exs b/host_core/mix.exs index a8789eb1..38264823 100644 --- a/host_core/mix.exs +++ b/host_core/mix.exs @@ -10,28 +10,68 @@ defmodule HostCore.MixProject do elixir: "~> 1.14", start_permanent: Mix.env() == :prod, deps: deps(), - rustler_crates: [ - hostcore_wasmcloud_native: [ - mode: if(Mix.env() == :dev, do: :debug, else: :release) - ] - ], - releases: [ - host_core: [ - steps: conditional_steps() - ] - ], - dialyzer: [plt_add_deps: :apps_direct] + dialyzer: [plt_add_deps: :apps_direct], + releases: + if Mix.env() == :prod do + [ + host_core: [ + steps: [:assemble, &Burrito.wrap/1], + burrito: [ + targets: [ + aarch64_darwin: [ + os: :darwin, + cpu: :aarch64, + custom_erts: System.get_env("ERTS_AARCH64_DARWIN") + ], + aarch64_linux_gnu: [ + os: :linux, + cpu: :aarch64, + libc: :gnu, + custom_erts: System.get_env("ERTS_AARCH64_LINUX_GNU") + ], + aarch64_linux_musl: [ + os: :linux, + cpu: :aarch64, + libc: :musl, + custom_erts: System.get_env("ERTS_AARCH64_LINUX_MUSL") + ], + x86_64_darwin: [ + os: :darwin, + cpu: :x86_64, + custom_erts: System.get_env("ERTS_X86_64_DARWIN") + ], + x86_64_linux_gnu: [ + os: :linux, + cpu: :x86_64, + libc: :gnu, + custom_erts: System.get_env("ERTS_X86_64_LINUX_GNU") + ], + x86_64_linux_musl: [ + os: :linux, + cpu: :x86_64, + libc: :musl, + custom_erts: System.get_env("ERTS_X86_64_LINUX_MUSL") + ], + x86_64_windows: [ + os: :windows, + cpu: :x86_64, + custom_erts: System.get_env("ERTS_X86_64_WINDOWS") + ] + ], + extra_steps: [ + patch: [ + pre: [HostCore.CopyNIF] + ] + ] + ] + ] + ] + else + [] + end ] end - # TODO https://github.com/wasmCloud/wasmcloud-otp/issues/570 - defp conditional_steps do - case :os.type() do - {:unix, _} -> [:assemble, &Bakeware.assemble/1] - _ -> [:assemble] - end - end - # In order to ensure that TLS cert check starts before the otel applications, # we disable auto-start from dependencies and start them in explicit order in the # application function @@ -50,7 +90,7 @@ defmodule HostCore.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do - list = [ + [ {:msgpax, "~> 2.3"}, {:rustler, "~> 0.27.0"}, {:timex, "~> 3.7"}, @@ -76,13 +116,8 @@ defmodule HostCore.MixProject do {:yaml_elixir, "~> 2.9.0"}, {:toml, "~> 0.7"}, {:benchee, "~> 1.0", only: :test}, - {:mock, "~> 0.3.0", only: :test} + {:mock, "~> 0.3.0", only: :test}, + {:burrito, github: "burrito-elixir/burrito"} ] - - # TODO https://github.com/wasmCloud/wasmcloud-otp/issues/570 - case :os.type() do - {:unix, _} -> [{:bakeware, "~> 0.2.4"} | list] - _ -> list - end end end diff --git a/host_core/mix.lock b/host_core/mix.lock index 8696c9e7..bf052293 100644 --- a/host_core/mix.lock +++ b/host_core/mix.lock @@ -3,6 +3,7 @@ "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "dfb3e39d478944c6a17fc37c24ed3ad4a8cc2af0", []}, "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, diff --git a/host_core/native/hostcore_wasmcloud_native/Cross.toml b/host_core/native/hostcore_wasmcloud_native/Cross.toml deleted file mode 100644 index 4e51588e..00000000 --- a/host_core/native/hostcore_wasmcloud_native/Cross.toml +++ /dev/null @@ -1,11 +0,0 @@ -[target.aarch64-unknown-linux-gnu] -image = "wasmcloud/cross:aarch64-unknown-linux-gnu" - -[target.x86_64-apple-darwin] -image = "wasmcloud/cross:x86_64-apple-darwin" - -[target.aarch64-apple-darwin] -image = "wasmcloud/cross:aarch64-apple-darwin" - -[target.x86_64-unknown-linux-gnu] -image = "wasmcloud/cross:x86_64-unknown-linux-gnu" diff --git a/host_core/native/hostcore_wasmcloud_native/README.md b/host_core/native/hostcore_wasmcloud_native/README.md index 12127e86..ce3d145f 100644 --- a/host_core/native/hostcore_wasmcloud_native/README.md +++ b/host_core/native/hostcore_wasmcloud_native/README.md @@ -2,9 +2,3 @@ This Native Implemented Function ([NIF](https://www.erlang.org/doc/tutorial/nif.html)) serves two purposes for wasmCloud: 1. Implement functionality that is better suited for the memory safety or static typing of Rust 2. Reuse common functionality in the various crates published in the Rust ecosystem. - -## To build the NIF module: - -- Make sure your projects `mix.exs` has the `:rustler` compiler listed in the `project` function: `compilers: [:rustler] ++ Mix.compilers()` If there already is a `:compilers` list, you should append `:rustler` to it. -- Add your crate to the `rustler_crates` attribute in the `project function. [See here](https://hexdocs.pm/rustler/basics.html#crate-configuration). -- Your NIF will now build along with your project. diff --git a/wasmcloud_host/Dockerfile b/wasmcloud_host/Dockerfile index 8d18a883..96b92175 100644 --- a/wasmcloud_host/Dockerfile +++ b/wasmcloud_host/Dockerfile @@ -1,146 +1,24 @@ -# NOTE: This docker image must be built from the root of this repository in order to copy `host_core`. -# Use the Makefile target `build-image` for best results. +FROM debian:bullseye-slim -ARG BUILDER_IMAGE -ARG RELEASE_IMAGE -ARG MIX_ENV=prod -ARG SECRET_KEY_BASE - -## -# STEP 1: Retrieve dependencies (must happen before NIF compilation) -## -FROM ${BUILDER_IMAGE} AS deps-builder - -ENV MIX_ENV=${MIX_ENV} \ - SECRET_KEY_BASE=${SECRET_KEY_BASE} - -WORKDIR /opt/app -# This copies our app source code into the build container -COPY ./host_core ./host_core -COPY ./wasmcloud_host ./wasmcloud_host - -# Install necessary system dependencies -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ - git \ - ca-certificates && \ - update-ca-certificates - -# This step installs all the build tools we'll need -RUN mix local.rebar --force && \ - mix local.hex --force - -WORKDIR /opt/app/host_core -RUN mix deps.get -WORKDIR /opt/app/wasmcloud_host -RUN mix deps.get - -## -# STEP 2: Build distillery release -## -FROM ${BUILDER_IMAGE} AS builder -# The name of your application/release (required) -ARG APP_NAME -# The version of the application we are building (required) -ARG APP_VSN -# The environment to build with -ARG MIX_ENV=release_prod -# Set this to true if this release is not a Phoenix app -ARG SKIP_PHOENIX=false -# Secret key is required -ARG SECRET_KEY_BASE -# Flag to include elixir runtime assets -ARG INCLUDE_ERTS=true - -ENV SKIP_PHOENIX=${SKIP_PHOENIX} \ - APP_NAME=${APP_NAME} \ - APP_VSN=${APP_VSN} \ - MIX_ENV=${MIX_ENV} \ - SECRET_KEY_BASE=${SECRET_KEY_BASE} \ - INCLUDE_ERTS=${INCLUDE_ERTS} - -# By convention, /opt is typically used for applications -WORKDIR /opt/app - -# This copies our app source code into the build container (including compiled NIFs) -COPY --from=deps-builder /opt/app /opt/app - -# Install dependencies for build container. This may be packages like `curl`, `bash`, -# or even elixir and erlang depending on the base container +# Install runtime dependencies RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ - curl \ - git \ - ca-certificates \ - libssl-dev \ - pkg-config \ - inotify-tools \ - build-essential + apt install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* -# This step installs all the build tools we'll need -RUN mix local.rebar --force && \ - mix local.hex --force - -################# -# Install: Rust # -################# - -# Get Rust for building NIFs (wasmex) -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y - -# Set PATH to include Rust toolchain -ENV PATH="/root/.cargo/bin:${PATH}" - -######################### -# Build: wasmcloud_host # -######################### - -WORKDIR ./wasmcloud_host -RUN mix do deps.compile, compile - -RUN mkdir -p /opt/built && \ - mix release && \ - cp -r _build/${MIX_ENV}/rel/${APP_NAME}/* /opt/built - -## -# STEP 3: Build optimized final release image -## - -# Release image should be the same as the _base container image_ used for the builder. -# E.g. `elixir:1.13.3-alpine`'s base container image is `alpine:3.15.4' -FROM ${RELEASE_IMAGE} - -ARG APP_NAME -ENV REPLACE_OS_VARS=true - -WORKDIR /opt/app -COPY --from=builder /opt/built . - -# Install release image dependencies (e.g. `bash` is required to run the script and a `libc` installation is required for the NIFs) -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ - ca-certificates \ - curl \ - locales \ - libssl-dev \ - inotify-tools \ - procps && \ - export LANG=en_US.UTF-8 && \ - echo $LANG UTF-8 > /etc/locale.gen && \ - locale-gen && \ - update-locale LANG=$LANG && \ - rm -rf /var/lib/apt/lists/* +ARG BIN +COPY ${BIN} /usr/local/bin/wasmcloud_host -# Prevents unnecessary warning messages about language encoding -ENV LC_ALL=en_US.UTF-8 +RUN wasmcloud_host maintenance install # NATS connection is required and can be overridden # Default configuration assumes a NATS container is running named `nats` and available over port 4222 ARG WASMCLOUD_RPC_HOST=nats ARG WASMCLOUD_PROV_RPC_HOST=nats ARG WASMCLOUD_CTL_HOST=nats -ENV WASMCLOUD_RPC_HOST=${WASMCLOUD_RPC_HOST} \ +ENV LANG="C.UTF-8" \ + LC_TYPE="C.UTF-8" \ + WASMCLOUD_RPC_HOST=${WASMCLOUD_RPC_HOST} \ WASMCLOUD_CTL_HOST=${WASMCLOUD_CTL_HOST} \ WASMCLOUD_PROV_RPC_HOST=${WASMCLOUD_PROV_RPC_HOST} -CMD ["/opt/app/bin/wasmcloud_host", "start"] +CMD ["/usr/local/bin/wasmcloud_host"] diff --git a/wasmcloud_host/Dockerfile.static b/wasmcloud_host/Dockerfile.static deleted file mode 100644 index 83a9a9d2..00000000 --- a/wasmcloud_host/Dockerfile.static +++ /dev/null @@ -1,35 +0,0 @@ -# we use a common Dockerfile with statically compiled erlang+openssl -# the Makefile handles this for you -# ../Dockerfile.localstatic -ARG BASEIMAGE=localstatic -FROM ${BASEIMAGE} - -COPY ./host_core /opt/app/host_core -COPY ./wasmcloud_host /opt/app/wasmcloud_host - -ARG SECRET_KEY_BASE -ARG APP_VSN -ENV SECRET_KEY_BASE=${SECRET_KEY_BASE} \ - APP_VSN=${APP_VSN} - -RUN . /root/.cargo/env \ - && cd /opt/app/host_core \ - && mix deps.get \ - && cd /opt/app/wasmcloud_host \ - && mix deps.get \ - && MIX_ENV=prod mix release wasmcloud_host - -FROM ubuntu:20.04 -COPY --from=0 /opt/app/wasmcloud_host/_build/prod/rel/bakeware/wasmcloud_host /usr/local/bin/wasmcloud_host - -ENV LANGUAGE en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -RUN mkdir -p /root/.cache \ - && apt-get update \ - && echo "en_US UTF-8" > /etc/locale.gen \ - && apt-get install -y locales ca-certificates \ - && rm -rf /var/lib/apt/* - -CMD ["/usr/local/bin/wasmcloud_host"] diff --git a/wasmcloud_host/Makefile b/wasmcloud_host/Makefile index 703ece9d..8cb540b4 100644 --- a/wasmcloud_host/Makefile +++ b/wasmcloud_host/Makefile @@ -1,17 +1,8 @@ -.PHONY: help deps build esbuild release build-image buildx-cross-image run run-interactive +.PHONY: help deps build esbuild run run-interactive -NAME ?= `grep 'app:' ./wasmcloud_host/mix.exs | sed -e 's/\[//g' -e 's/ //g' -e 's/app://' -e 's/[:,]//g'` -VERSION ?= `grep '@app_vsn ' ./wasmcloud_host/mix.exs | cut -d '"' -f2` -BUILD ?= `git rev-parse --short HEAD` -DOCKERFILE ?= ./wasmcloud_host/Dockerfile -DOCKERFILESTATIC ?= ./wasmcloud_host/Dockerfile.static SKIP_PHOENIX ?= false -TAG ?= latest SECRET_KEY_BASE ?= $(shell mix phx.gen.secret) - -BASE_ARGS ?= --build-arg APP_NAME=$(NAME) --build-arg APP_VSN=$(VERSION) --build-arg SECRET_KEY_BASE=$(SECRET_KEY_BASE) --build-arg SKIP_PHOENIX=$(SKIP_PHOENIX) -BASE_TAGS ?= -t $(NAME):$(VERSION)-$(BUILD) -t $(NAME):$(TAG) -BASE_TAGS_STATIC ?= -t $(NAME):$(VERSION)-$(BUILD)-static -t $(NAME):$(TAG)-static +WASMCLOUD_HOST_IMAGE ?= wasmcloud_host:latest help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_\-.*]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -29,50 +20,12 @@ esbuild: ## Build frontend code that relies on esbuild build: deps ## Build wasmcloud_host for development mix compile -build-image: build esbuild ## Compile wasmcloud_host docker image, requires local elixir toolchain since dart-sass doesn't work in arm64 environments - cd ../ && \ - docker build $(BASE_ARGS) \ - --build-arg BUILDER_IMAGE=elixir:1.14.3-slim \ - --build-arg RELEASE_IMAGE=debian:bullseye-slim \ - --build-arg MIX_ENV=prod \ - $(BASE_TAGS) \ - -f $(DOCKERFILE) \ - . - -build-image-static: build esbuild ## build statically compiled docker image - cd ../ \ - && docker build -f Dockerfile.localstatic -t localstatic . \ - && docker build $(BASE_ARGS) \ - --build-arg MIX_ENV=prod \ - $(BASE_TAGS_STATIC) \ - -f $(DOCKERFILESTATIC) \ - . - -# erlang 24 until 25 stops segfaulting under qemu -buildx-cross-image: build esbuild ## Compile wasmcloud_host docker image using buildx for amd64 and arm64 - cd ../ && \ - docker buildx build $(BASE_ARGS) \ - --build-arg BUILDER_IMAGE=elixir:1.14.3-otp-24-slim \ - --build-arg RELEASE_IMAGE=debian:bullseye-slim \ - --build-arg MIX_ENV=prod \ - -t wasmcloud_host:slim \ - --platform linux/amd64,linux/arm64 \ - --load \ - -f $(DOCKERFILE) \ - . - -release: build ## Creates a mix release with a generated secret key base - @SECRET_KEY_BASE=$(shell mix phx.gen.secret) \ - MIX_ENV=prod \ - mix release - run: build ## Run development wasmcloud_host mix phx.server run-image: ## Run the docker compose with specified image tag - WASMCLOUD_HOST_IMAGE=wasmcloud_host:latest \ - docker-compose -f docker-compose.yml \ - up + WASMCLOUD_HOST_IMAGE=$(WASMCLOUD_HOST_IMAGE) \ + docker compose up run-interactive: build ## Run development wasmcloud_host with iex iex -S mix phx.server diff --git a/wasmcloud_host/config/config.exs b/wasmcloud_host/config/config.exs index dfd8918f..82915465 100644 --- a/wasmcloud_host/config/config.exs +++ b/wasmcloud_host/config/config.exs @@ -40,6 +40,7 @@ config :mime, :types, %{ # Configure esbuild (the version is required) config :esbuild, version: "0.12.18", + path: System.get_env("MIX_ESBUILD_PATH"), default: [ args: [ "js/app.js", @@ -59,7 +60,8 @@ config :esbuild, # Configure Dart for scss support config :dart_sass, - version: "1.52.1" + version: "1.52.1", + path: System.get_env("MIX_SASS_PATH") # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/wasmcloud_host/config/prod.exs b/wasmcloud_host/config/prod.exs index 9ec4c14f..0fc0b9a8 100644 --- a/wasmcloud_host/config/prod.exs +++ b/wasmcloud_host/config/prod.exs @@ -22,6 +22,10 @@ config :wasmcloud_host, WasmcloudHostWeb.Endpoint, # Do not print debug messages in production config :logger, :console, level: :info +config :host_core, HostCore.WasmCloud.Native, + mode: :release, + skip_compilation?: true + # ## SSL Support # # To get SSL working, you will need to add the `https` key diff --git a/wasmcloud_host/config/release_prod.exs b/wasmcloud_host/config/release_prod.exs deleted file mode 100644 index c58f946a..00000000 --- a/wasmcloud_host/config/release_prod.exs +++ /dev/null @@ -1,9 +0,0 @@ -# This file is only used for assembling final releases and primarily -# is for specifying custom parameters for the Native host_core NIF -import Config - -import_config "prod.exs" - -config :host_core, HostCore.WasmCloud.Native, - crate: :hostcore_wasmcloud_native, - mode: if(Mix.env() == :dev, do: :debug, else: :release) diff --git a/wasmcloud_host/mix.exs b/wasmcloud_host/mix.exs index 46f743b9..46a4dd86 100644 --- a/wasmcloud_host/mix.exs +++ b/wasmcloud_host/mix.exs @@ -13,22 +13,67 @@ defmodule WasmcloudHost.MixProject do start_permanent: Mix.env() == :prod, aliases: aliases(), releases: [ - wasmcloud_host: [ - steps: conditional_steps() - ] + wasmcloud_host: + if Mix.env() == :prod do + [ + steps: [:assemble, &Burrito.wrap/1], + burrito: [ + targets: [ + aarch64_darwin: [ + os: :darwin, + cpu: :aarch64, + custom_erts: System.get_env("ERTS_AARCH64_DARWIN") + ], + aarch64_linux_gnu: [ + os: :linux, + cpu: :aarch64, + libc: :gnu, + custom_erts: System.get_env("ERTS_AARCH64_LINUX_GNU") + ], + aarch64_linux_musl: [ + os: :linux, + cpu: :aarch64, + libc: :musl, + custom_erts: System.get_env("ERTS_AARCH64_LINUX_MUSL") + ], + x86_64_darwin: [ + os: :darwin, + cpu: :x86_64, + custom_erts: System.get_env("ERTS_X86_64_DARWIN") + ], + x86_64_linux_gnu: [ + os: :linux, + cpu: :x86_64, + libc: :gnu, + custom_erts: System.get_env("ERTS_X86_64_LINUX_GNU") + ], + x86_64_linux_musl: [ + os: :linux, + cpu: :x86_64, + libc: :musl, + custom_erts: System.get_env("ERTS_X86_64_LINUX_MUSL") + ], + x86_64_windows: [ + os: :windows, + cpu: :x86_64, + custom_erts: System.get_env("ERTS_X86_64_WINDOWS") + ] + ], + extra_steps: [ + patch: [ + pre: [HostCore.CopyNIF] + ] + ] + ] + ] + else + [] + end ], deps: deps() ] end - # TODO https://github.com/wasmCloud/wasmcloud-otp/issues/570 - defp conditional_steps do - case :os.type() do - {:unix, _} -> [:assemble, &Bakeware.assemble/1] - _ -> [:assemble] - end - end - # Configuration for the OTP application. # # Type `mix help compile.app` for more information. @@ -48,7 +93,7 @@ defmodule WasmcloudHost.MixProject do # # Type `mix help deps` for examples and options. defp deps do - list = [ + [ {:phoenix, "~> 1.6.0"}, {:phoenix_html, "~> 3.0.4"}, {:phoenix_live_view, "~> 0.16.4"}, @@ -64,17 +109,9 @@ defmodule WasmcloudHost.MixProject do {:host_core, path: "../host_core"}, {:file_system, "~> 0.2"}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false} + {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:burrito, github: "burrito-elixir/burrito"} ] - - # TODO https://github.com/wasmCloud/wasmcloud-otp/issues/570 - case :os.type() do - {:unix, _} -> - [{:bakeware, "~> 0.2.4"} | list] - - _ -> - list - end end # Aliases are shortcuts or tasks specific to the current project. diff --git a/wasmcloud_host/mix.lock b/wasmcloud_host/mix.lock index 57099b2c..41b33ea2 100644 --- a/wasmcloud_host/mix.lock +++ b/wasmcloud_host/mix.lock @@ -2,6 +2,7 @@ "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "burrito": {:git, "https://github.com/burrito-elixir/burrito.git", "dfb3e39d478944c6a17fc37c24ed3ad4a8cc2af0", []}, "castore": {:hex, :castore, "0.1.17", "ba672681de4e51ed8ec1f74ed624d104c0db72742ea1a5e74edbc770c815182f", [:mix], [], "hexpm", "d9844227ed52d26e7519224525cb6868650c272d4a3d327ce3ca5570c12163f9"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, @@ -19,6 +20,7 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"}, "floki": {:hex, :floki, "0.33.1", "f20f1eb471e726342b45ccb68edb9486729e7df94da403936ea94a794f072781", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "461035fd125f13fdf30f243c85a0b1e50afbec876cbf1ceefe6fddd2e6d712c6"}, "gettext": {:hex, :gettext, "0.20.0", "75ad71de05f2ef56991dbae224d35c68b098dd0e26918def5bb45591d5c8d429", [:mix], [], "hexpm", "1c03b177435e93a47441d7f681a7040bd2a816ece9e2666d1c9001035121eb3d"}, "gnat": {:hex, :gnat, "1.6.0", "a6a586bb8d4d94ddd19f4f4776ea61550ce596d2604b120bc4bd30a20202ac3b", [:mix], [{:cowlib, "~> 2.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:nkeys, "~> 0.2", [hex: :nkeys, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d62c5353262290ee0d2b1b1b495e274e4c70126eb8b3ed94ecdc9f8c2784605b"}, @@ -27,14 +29,18 @@ "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "hashids": {:hex, :hashids, "2.0.5", "d9839924c8221b954da8b110eda3e59c2c03df0389bac6e7d0e535f937033df1", [:mix], [], "hexpm", "ef47d8679f20d7bea59d0d49c202258c89f61b9b741bd3dceef2c1985cf95554"}, "hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"}, + "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, + "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, "msgpax": {:hex, :msgpax, "2.3.0", "14f52ad249a3f77b5e2d59f6143e6c18a6e74f34666989e22bac0a465f9835cc", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "65c36846a62ed5615baf7d7d47babb6541313a6c0b6d2ff19354bd518f52df7e"}, + "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, + "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, "nkeys": {:hex, :nkeys, "0.2.1", "c183105135de9d0c07d649f6a13e39792b095772cdbeaacf37dea4e35b647100", [:mix], [{:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}], "hexpm", "946551188bd103559e5129835f782f32ea26027d0ee7063eb6910ffd707fc1aa"}, "norm": {:hex, :norm, "0.13.0", "2c562113f3205e3f195ee288d3bd1ab903743e7e9f3282562c56c61c4d95dec4", [:mix], [{:stream_data, "~> 0.5", [hex: :stream_data, repo: "hexpm", optional: true]}], "hexpm", "447cc96dd2d0e19dcb37c84b5fc0d6842aad69386e846af048046f95561d46d7"}, "opentelemetry": {:hex, :opentelemetry, "1.2.1", "6b374a0b4ebbd206d0c7c13ea34a8b46c63bfefa86d813ed07613876f1811575", [:rebar3], [{:opentelemetry_api, "~> 1.2.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "9a627f3bffcb972d404590ef686309863d0fe09ddf1a4612430b99d781ecf940"}, @@ -55,6 +61,7 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"}, "plug_crypto": {:hex, :plug_crypto, "1.2.3", "8f77d13aeb32bfd9e654cb68f0af517b371fb34c56c9f2b58fe3df1235c1251a", [:mix], [], "hexpm", "b5672099c6ad5c202c45f5a403f21a3411247f164e4a8fab056e5cd8a290f4a2"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "req": {:hex, :req, "0.3.6", "541350d2cc359a8ad17059f2629c18be56d0c85ce0e4ddb27694b6ba482fe923", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "9181047f32b05f8737f6b5917af5ee5385219158bbe4e507f4ec57791a0a78c3"}, "rustler": {:hex, :rustler, "0.27.0", "53ffe86586fd1a2ea60ad07f1506962914eb669dba26c23010cf672662ec8d64", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "d7f5ccaec6e7a96f700330898ff2e9d48818e40789fd2951ba41ecf457986e92"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.5.1", "93df423bd7b14b67dcacf994443d132d300623f80756974cac4febeab40af74a", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3f8cbc8e92eef4e1a71bf441b568b868b16a3730f63f5b803c68073017e30b13"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, From e5a9ab094d85f17685f18d752638a825667c50f1 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Mon, 8 May 2023 14:53:46 +0200 Subject: [PATCH 3/8] ci: only build Docker images on tag pushes We should be testing this always, not only on tag pushes Signed-off-by: Roman Volosatovs --- .github/workflows/release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57132ec1..b2b15cd5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -259,6 +259,13 @@ jobs: Taskkill /im wasmcloud_host_x86_64_windows.exe /f release: + # TODO: Build on PRs and pushes to main (but do not push) + # This is currently blocked on the `app-version` computation, which only supports semver tags + # E.g.: + # ``` + # Error: buildx failed with: ERROR: invalid tag "wasmcloud.azurecr.io/wasmcloud_host:616/merge": invalid reference format + # ``` + if: startswith(github.ref, 'refs/tags/') # Only run on tag push strategy: matrix: config: From 9dec3382beecfc772256d3215b3edfa8f8ebcc67 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Mon, 8 May 2023 17:19:01 +0200 Subject: [PATCH 4/8] ci: pin OS versions Signed-off-by: Roman Volosatovs --- .github/workflows/release.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b2b15cd5..11710ca1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - x86_64-apple-darwin name: nif-${{ matrix.target }} - runs-on: macos-latest + runs-on: macos-11 steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal @@ -41,7 +41,7 @@ jobs: nif-windows: name: nif-x86_64-pc-windows-msvc - runs-on: windows-latest + runs-on: windows-2022 steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal @@ -75,7 +75,7 @@ jobs: needs: - nif-darwin - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/install-nix @@ -132,7 +132,7 @@ jobs: arch: x86_64 libc: musl - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/install-nix @@ -162,7 +162,7 @@ jobs: needs: - nif-windows - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/install-nix @@ -190,7 +190,7 @@ jobs: if-no-files-found: error test-darwin: - runs-on: macos-latest + runs-on: macos-11 needs: - burrito-darwin steps: @@ -212,7 +212,7 @@ jobs: ./wasmcloud_host_x86_64_darwin & sleep 30; kill $! test-linux: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - burrito-linux services: @@ -234,7 +234,7 @@ jobs: - run: ./wasmcloud_host_x86_64_linux_gnu & sleep 30; kill $! test-windows: - runs-on: windows-latest + runs-on: windows-2022 needs: - burrito-windows steps: @@ -285,7 +285,7 @@ jobs: target: x86_64_linux_gnu platform: linux/amd64 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - test-darwin - test-linux @@ -342,7 +342,7 @@ jobs: if: startswith(github.ref, 'refs/tags/') # Only run on tag push needs: - release - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/download-artifact@v3 with: @@ -402,7 +402,7 @@ jobs: release-charts: if: ${{ (startswith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch') && startswith(github.repository.name, 'wasmcloud') }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v3 From 7f0b74282ec0d059ef71b2090742984b892b6fff Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Mon, 8 May 2023 17:21:23 +0200 Subject: [PATCH 5/8] build: default to Docker Hub `wasmcloud_host` image Signed-off-by: Roman Volosatovs --- wasmcloud_host/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasmcloud_host/Makefile b/wasmcloud_host/Makefile index 8cb540b4..49301871 100644 --- a/wasmcloud_host/Makefile +++ b/wasmcloud_host/Makefile @@ -2,7 +2,7 @@ SKIP_PHOENIX ?= false SECRET_KEY_BASE ?= $(shell mix phx.gen.secret) -WASMCLOUD_HOST_IMAGE ?= wasmcloud_host:latest +WASMCLOUD_HOST_IMAGE ?= wasmcloud/wasmcloud_host:latest help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_\-.*]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) From 45c0195897ba4389ba7567908ce1d8dc5e87b7a6 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Mon, 8 May 2023 18:03:17 +0200 Subject: [PATCH 6/8] build: downgrade aarch64 glibc Signed-off-by: Roman Volosatovs --- flake.nix | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 5fc6a90d..ed204bc4 100644 --- a/flake.nix +++ b/flake.nix @@ -73,12 +73,12 @@ pkgs, pkgsCross ? pkgs, }: let - pkgsOld = nixpkgs-old.legacyPackages.${pkgs.stdenv.hostPlatform.system}; + pkgsOld = nixpkgs-old.legacyPackages.${pkgs.stdenv.buildPlatform.system}; in if pkgsCross.stdenv.hostPlatform.isGnu && pkgsCross.stdenv.hostPlatform.isAarch64 then let cc = - if pkgs.stdenv.hostPlatform.isGnu && pkgs.stdenv.hostPlatform.isAarch64 + if pkgs.stdenv.buildPlatform.isGnu && pkgs.stdenv.buildPlatform.isAarch64 # Pull in newer gcc to enable support for aarch64 atomics (e.g. `__aarch64_ldadd4_relax`) then pkgsOld.gcc10Stdenv.cc else pkgsOld.pkgsCross.aarch64-multiplatform.gcc10Stdenv.cc; @@ -87,7 +87,7 @@ else if pkgsCross.stdenv.hostPlatform.isGnu && pkgsCross.stdenv.hostPlatform.isx86_64 then let cc = - if pkgs.stdenv.hostPlatform.isGnu && pkgs.stdenv.hostPlatform.isx86_64 + if pkgs.stdenv.buildPlatform.isGnu && pkgs.stdenv.buildPlatform.isx86_64 then pkgsOld.stdenv.cc else pkgsOld.pkgsCross.gnu64.stdenv.cc; in @@ -112,9 +112,17 @@ } @ args: { depsBuildBuild ? [], buildInputs ? [], + CARGO_TARGET ? pkgsCross.stdenv.hostPlatform.config, ... }: let crossPlatform = pkgsCross.stdenv.hostPlatform; + + crossStdenv = mkStdenv { + inherit + pkgs + pkgsCross + ; + }; in { buildInputs = @@ -131,16 +139,11 @@ pkgsCross.libiconv ]; } - // optionalAttrs (crossPlatform.isGnu && crossPlatform.isx86_64) { - # TODO: Downgrade aarch64 libc - stdenv = mkStdenv { - inherit - pkgs - pkgsCross - ; - }; + // optionalAttrs crossPlatform.isGnu { + "CC_${CARGO_TARGET}" = "${crossStdenv.cc}/bin/${crossStdenv.cc.targetPrefix}cc"; + "CARGO_TARGET_${toUpper (replaceStrings ["-"] ["_"] CARGO_TARGET)}_LINKER" = "${crossStdenv.cc}/bin/${crossStdenv.cc.targetPrefix}cc"; - meta.broken = crossPlatform.isGnu && pkgs.stdenv.hostPlatform.isDarwin; # downgrading glibc breaks Darwin support here + meta.broken = crossPlatform.isGnu && pkgs.stdenv.buildPlatform.isDarwin; # downgrading glibc breaks Darwin support here }; }; From 0f97bf46c9bf7e19cac29f84192763789ea58ad8 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Mon, 8 May 2023 18:03:37 +0200 Subject: [PATCH 7/8] ci: test on various OS versions Signed-off-by: Roman Volosatovs --- .github/workflows/release.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 11710ca1..09241bbb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -190,7 +190,14 @@ jobs: if-no-files-found: error test-darwin: - runs-on: macos-11 + strategy: + matrix: + os: + - macos-11 + - macos-12 + - macos-13 + + runs-on: ${{ matrix.os }} needs: - burrito-darwin steps: @@ -212,7 +219,13 @@ jobs: ./wasmcloud_host_x86_64_darwin & sleep 30; kill $! test-linux: - runs-on: ubuntu-22.04 + strategy: + matrix: + os: + - ubuntu-20.04 + - ubuntu-22.04 + + runs-on: ${{ matrix.os }} needs: - burrito-linux services: @@ -234,7 +247,13 @@ jobs: - run: ./wasmcloud_host_x86_64_linux_gnu & sleep 30; kill $! test-windows: - runs-on: windows-2022 + strategy: + matrix: + os: + - windows-2019 + - windows-2022 + + runs-on: ${{ matrix.os }} needs: - burrito-windows steps: From 441c0517d4ffc19d5e3d3ac7ae007f3f759cb8a5 Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Wed, 10 May 2023 15:28:51 +0200 Subject: [PATCH 8/8] ci: update to `macos-11` Signed-off-by: Roman Volosatovs --- .github/workflows/host_core.yml | 2 +- .github/workflows/wasmcloud_host.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/host_core.yml b/.github/workflows/host_core.yml index 3273b20f..6a3e19ef 100644 --- a/.github/workflows/host_core.yml +++ b/.github/workflows/host_core.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + os: [ubuntu-20.04, windows-2019, macos-11] elixir: [1.14.3] otp: [25] diff --git a/.github/workflows/wasmcloud_host.yml b/.github/workflows/wasmcloud_host.yml index de5be783..47056fb3 100644 --- a/.github/workflows/wasmcloud_host.yml +++ b/.github/workflows/wasmcloud_host.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + os: [ubuntu-20.04, windows-2019, macos-11] elixir: [1.14.3] otp: [25]