Consolidate release artefacts (#329)

Brings in the previously wasm.yml into python.yml and the new file is
renamed as build.yml.

python.yml already had a version and pre-release jobs. These jobs
downloaded artefacts from prior ran jobs (python wheel builds). The
newly attached emscripten build now uploads artefacts of a WebAssembly
binary and javascript file which are fed into the release and
pre-release jobs in addition to the existing python builds.

.github/workflows/python.yml → .github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: "Python Bindings"
+name: "Build"
 'on':
   push:
     branches:
@@ -11,6 +11,7 @@ name: "Python Bindings"
       - '**'
 env:
   qt_version: "6.2.1" # only used by build-macos
+  emsdk_version: 2.0.9 # For use in emscripten build
   ccache_basedir: ${{ github.workspace }}
   ccache_dir: "${{ github.workspace }}/.ccache"
   ccache_compilercheck: content
@@ -230,16 +231,161 @@ jobs:
         with:
             path: ${{github.workspace}}/dist/bergamot-*.whl
 
+    build-wasm:
+      name: "emscripten"
+      runs-on: ubuntu-latest
+      steps:
+
+        - name: Checkout
+          uses: actions/checkout@v2
+          with:
+            submodules: recursive
+
+        - name: Set ccache environment for emcc
+          run: |
+            # We are hardcoding this to mtime instead of env pickup. Rest use content.
+            echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV
+
+            echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
+            echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
+            echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
+            echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
+            echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
+            # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper
+            echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV
+            
+        # This need to be run before setup, so ccache build caching doesn't complain.
+        - name: Obtain emsdk sources
+          run: |
+              git clone --depth 1 https://github.com/emscripten-core/emsdk.git
+
+        - name: Cache-op for build-cache through ccache
+          uses: actions/cache@v2
+          with:
+            path: |
+                ${{ env.ccache_dir }}
+                ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/
+            key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
+            restore-keys: |-
+              ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
+              ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}
+              ccache-${{ github.job }}-${{ env.emsdk_version }}
+
+        - name: Setup Emscripten toolchain
+          run: |
+              (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
+              (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
+              # mtime of this file is checked by ccache, we set it to avoid cache misses.
+              touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten
+
+              # These needs to be done in the activated shell.
+              eval $(./emsdk/emsdk construct_env \
+                  | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \
+                  | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' );
+
+              # This looks more permanent than version pinned, so keeping temporarily to avoid failures.
+              echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH
+
+        - name: Generate ccache_vars for ccache based on machine
+          shell: bash
+          id: ccache_vars
+          run: |-
+            echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
+            echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
+
+        - name: Verify Emscripten setup
+          run: |
+              emcc --version
+              emcmake cmake --version
+              emmake make --version
+
+        - name: ccache prolog
+          run: |-
+            ccache -s # Print current cache stats
+            ccache -z # Zero cache entry
+
+        # WORMHOLE=off
+        - name: "Configure builds for WORMHOLE=off"
+          run: |
+            mkdir -p build-wasm-without-wormhole
+            cd build-wasm-without-wormhole
+            emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ..
+
+
+        - name: "Compile with WORMHOLE=off"
+          working-directory: build-wasm-without-wormhole
+          run: |
+            emmake make -j2
+
+        - name: ccache epilog
+          run: |
+            ccache -s # Print current cache stats
+
+        - name: Import GEMM library from a separate wasm module
+          working-directory: build-wasm-without-wormhole
+          run: bash ../wasm/patch-artifacts-import-gemm-module.sh
+
+
+        # WORMHOLE=on
+        - name: "Configure builds for WORMHOLE=on"
+          run: |
+            mkdir -p build-wasm-with-wormhole
+            cd build-wasm-with-wormhole
+            emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=on ..
+
+
+        - name: "Compile with WORMHOLE=on"
+          working-directory: build-wasm-with-wormhole
+          run: |
+            emmake make -j2
+
+        - name: ccache epilog
+          run: |
+            ccache -s # Print current cache stats
+
+        - name: Instantiate simd wormhole
+          working-directory: build-wasm-with-wormhole
+          run: bash ../wasm/patch-artifacts-enable-wormhole.sh
+
+        - name: Import GEMM library from a separate wasm module
+          working-directory: build-wasm-with-wormhole
+          run: bash ../wasm/patch-artifacts-import-gemm-module.sh
+
+        # Rename the wormhole on builds
+        - name: Rename artefacts with wormhole
+          working-directory: build-wasm-with-wormhole
+          run: |
+                mv bergamot-translator-worker{,-with-wormhole}.js
+                mv bergamot-translator-worker{,-with-wormhole}.js.bak
+                mv bergamot-translator-worker{,-with-wormhole}.wasm
+
+
+        # Upload both together.
+        - name: Upload wasm artifact
+          uses: actions/upload-artifact@v2
+          with:
+            name: wasm-artefacts
+            if-no-files-found: error
+            path: |
+                # Without wormhole
+                ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js
+                ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.wasm
+                ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js.bak
+
+                ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
+                ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
+                ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js.bak
+
   # Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model
     release-latest:
       name: Release Latest Build
       runs-on: ubuntu-latest
-      needs: [python-ubuntu, python-macos]
+      needs: [python-ubuntu, python-macos, build-wasm]
       if: github.ref == 'refs/heads/main'
       steps:
        - name: Download artifacts
          uses: actions/download-artifact@v2
-
+        
        - name: Update GitHub prerelease
          uses: marvinpinto/action-automatic-releases@latest
          with:
@@ -248,12 +394,16 @@ jobs:
            prerelease: true
            title: "Latest Build"
            files: |
-                  ${{github.workspace}}/artifact/*.whl
+                artifact/*.whl
+                wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js
+                wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm
+                wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
+                wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
   
     release-version:
       name: Release version 
       runs-on: ubuntu-latest
-      needs: [python-ubuntu, python-macos]
+      needs: [python-ubuntu, python-macos, build-wasm]
       permissions:
         contents: "write"
         packages: "write"
@@ -271,7 +421,12 @@ jobs:
            prerelease: false
            title: "${{ github.ref_name }}"
            files: |
-                  ${{github.workspace}}/artifact/*.whl
+                artifact/*.whl
+                wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js
+                wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm
+                wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js
+                wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm
+
   
     python-checks:
       name: "formatting and typechecks"