Allow to build Theia IDE based on local Theia sources #304

Introduce docker image that maybe used to build theia locally and
publish it in a local verdaccio package registry.

Adapt the existing next scripts to build a preview Theia IDE with the
next build results.

Contributed on behalf of STMicroelectronics

Signed-off-by: Johannes Faltermeier <[email protected]>

.github/workflows/publish-theia-builder.yml

@@ -0,0 +1,38 @@
+name: Publish Theia Builder Docker Image
+
+on:
+  push:
+    branches:
+      - jf/local-theia-build
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: The image's tag
+        required: true
+        default: next
+
+jobs:
+  build:
+    name: Build and push theia builder image to Github Packages
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Log in to the Github Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/local-theia-build
+          file: local-theia-build.Dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}/blueprint-theia-builder:latest
+          # ghcr.io/${{ github.repository }}/blueprint-theia-builder:${{ github.event.inputs.tag }}

applications/electron/package.json

@@ -129,7 +129,7 @@
     "package:preview": "yarn clean:dist && yarn rebuild && electron-builder -c.mac.identity=null --dir",
     "update:checksum": "ts-node scripts/update-checksum.ts",
     "update:blockmap": "ts-node scripts/update-blockmap.ts",
-    "update:next": "ts-node ../../scripts/update-theia-to-next.ts",
+    "update:next": "ts-node ../../scripts/update-theia-to-next.ts && ts-node ../../scripts/update-application-to-preview.ts && ts-node ../../scripts/update-builder-config-to-preview.ts",
     "test": "mocha --timeout 60000 \"./test/*.spec.js\"",
     "lint": "eslint --ext js,jsx,ts,tsx scripts && eslint --ext js,jsx,ts,tsx test",
     "lint:fix": "eslint --ext js,jsx,ts,tsx scripts --fix && eslint --ext js,jsx,ts,tsx test -fix"

docker/local-theia-build/adduser

@@ -0,0 +1,16 @@
+#!/usr/bin/expect -f
+set timeout 10
+
+spawn npm adduser --registry http://localhost:4873/
+match_max 100000
+
+expect "Username"
+send "theia-build\r"
+
+expect "Password"
+send "The14-8uiLd\r"
+
+expect "Email: (this IS public)"
+send "[email protected]\r"
+
+expect "Logged in on http://localhost:4873/."

docker/local-theia-build/entrypoint

@@ -0,0 +1,52 @@
+#!/bin/sh
+set -e
+
+BUILD_THEIA=0
+ADD_USER=0
+
+if [ -n "$1" ]; then
+    if [ "$1" = "--buildTheia" ]; then
+        BUILD_THEIA=1
+    elif [ "$1" = "--addUser" ]; then
+        ADD_USER=1
+    fi
+fi
+
+if [ -n "$2" ]; then
+    if [ "$2" = "--buildTheia" ]; then
+        BUILD_THEIA=1
+    elif [ "$2" = "--addUser" ]; then
+        ADD_USER=1
+    fi
+fi
+
+REGISTRY=http://localhost:4873/
+echo "Launching Verdaccio..."
+verdaccio --listen http://0.0.0.0:4873 &
+VERDACCIO_PID=$!
+while ! nc -z localhost 4873; do   
+sleep 1
+done
+
+if [ "$ADD_USER" -eq 1 ]; then
+    echo "Adding User..."
+    expect -f /tmp/adduser
+fi
+
+echo "Log in User..."
+expect -f /tmp/loginuser
+
+if [ "$BUILD_THEIA" -eq 1 ]; then
+    echo "Building Theia..."
+    cd /tmp/theia
+    npm config set registry $REGISTRY
+    yarn config set registry $REGISTRY
+    yarn
+    yarn build
+
+    echo "Publishing Theia..."
+    yarn lerna publish preminor --exact --canary --preid next --dist-tag next --no-git-reset --no-git-tag-version --no-push --yes --registry $REGISTRY || true
+fi
+
+echo "Wait while Verdaccio is running..."
+wait $VERDACCIO_PID

docker/local-theia-build/loginuser

@@ -0,0 +1,13 @@
+#!/usr/bin/expect -f
+set timeout 10
+
+spawn npm login --registry http://localhost:4873/
+match_max 100000
+
+expect "Username"
+send "theia-build\r"
+
+expect "Password"
+send "The14-8uiLd\r"
+
+expect "Logged in on http://localhost:4873/."

local-theia-build.Dockerfile

@@ -0,0 +1,34 @@
+# Prerequisites
+# https://github.com/eclipse-theia/theia/blob/master/doc/Developing.md#prerequisites
+FROM node:18.17.0
+RUN apt-get update && \
+    apt-get install -y \
+    make \
+    gcc \
+    pkg-config \
+    build-essential \
+    libx11-dev \
+    libxkbfile-dev \
+    libsecret-1-dev && \
+    # install local npm registry and helper tools
+    yarn global add verdaccio && \
+    apt-get install -y expect-dev netcat-openbsd && \
+    mkdir /tmp/verdaccio && chmod 777 /tmp/verdaccio && \
+    # create some common default directories/files with write access for any user
+    touch /.yarnrc && chmod 777 /.yarnrc && \
+    touch /.npmrc && chmod 777 /.npmrc && \
+    mkdir /.cache && chmod 777 /.cache && \
+    mkdir /.yarn && chmod 777 /.yarn && \
+    mkdir /.npm && chmod 777 /.npm
+
+# Set storage location for verdaccio
+ENV VERDACCIO_STORAGE_PATH /tmp/verdaccio
+
+# Switch to expected workdir
+WORKDIR /tmp
+
+COPY adduser /tmp/adduser
+COPY loginuser /tmp/loginuser
+COPY entrypoint /tmp/entrypoint
+
+ENTRYPOINT ["/tmp/entrypoint"]

local-theia-build.md

@@ -0,0 +1,53 @@
+# Local Theia Build
+
+## Prerequisites
+
+You can create an image with all required tools to build theia and to host your temporary next builds with this command:
+
+```sh
+docker build -t local-theia-builder -f local-theia-build.Dockerfile ./docker/local-theia-build
+```
+
+## Example Local Build
+
+```sh
+# switch to your checked out theia code
+cd ~/git/theia
+# Cleaning any locals build results is advied. Please be sure to commit any changes before cleaning.
+git clean -xfd
+
+# Export location where to save the built results
+# Please make sure that this location exists
+export VERDACCIO_STORAGE_PATH="/home/user/tmp/verdaccio"
+
+# build Theia Next with our builder image
+#
+# This command mounts your theia directory (${PWD}) inside the container
+#
+# The built results will be available at $VERDACCIO_STORAGE_PATH
+# You may omit this volume mount, however the build results will the be local to the docker container
+# 
+# --addUser adds a user in the verdaccio registry. This can be omitted if a VERDACCIO_STORAGE_PATH with a preexisting user is mounted
+#
+# --buildTheia can be omitted if you don't want to build theia but just want to serve the existing results at VERDACCIO_STORAGE_PATH
+#
+docker run --rm \
+    -v ${PWD}:/tmp/theia \
+    -v ${VERDACCIO_STORAGE_PATH}:/tmp/verdaccio  \
+    -u $(id -u ${USER}):$(id -g ${USER}) \
+    -p=4873:4873 \
+    local-theia-builder --addUser --buildTheia
+
+# in a new shell, open theia blueprint
+cd ~/git/theia-blueprint
+
+# update theia version to @next
+yarn && yarn update:next
+
+# run yarn with verdaccio registry to update yarn.lock
+yarn --registry http://localhost:4873/
+
+# build theia blueprint
+yarn build && yarn electron package
+
+```

scripts/update-application-to-preview.ts

@@ -0,0 +1,38 @@
+/********************************************************************************
+ * Copyright (C) 2023 EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ *
+ * SPDX-License-Identifier: MIT
+ ********************************************************************************/
+import * as fs from 'fs';
+import * as path from 'path';
+
+execute();
+
+async function execute(): Promise<void> {
+    const packageJsonPath = path.resolve(
+        './',
+        'package.json'
+    );
+
+    console.log(`Updating ${packageJsonPath}...`);
+
+    const packageJsonContents: string = fs.readFileSync(packageJsonPath, { encoding: 'utf8' });
+    const packageJson = JSON.parse(packageJsonContents);
+
+    console.log('...application name...');
+    packageJson.theia.frontend.config.applicationName = 'Theia IDE Preview';
+    console.log('...done...');
+
+    console.log('...version...');
+    // in order to work well with electron updater we keep a constant prerelease tag (-preview) and keep on increasing the patch segment
+    const curVersion: string = packageJson.version;
+    packageJson.version = curVersion.substring(0, curVersion.lastIndexOf('.') + 1) + new Date().valueOf() + '-preview';
+    console.log('...done...');
+
+    // note: "null" is valid as per `stringify()` signature
+    // eslint-disable-next-line no-null/no-null
+    fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
+}

scripts/update-builder-config-to-preview.ts

@@ -0,0 +1,45 @@
+/********************************************************************************
+ * Copyright (C) 2023 EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ *
+ * SPDX-License-Identifier: MIT
+ ********************************************************************************/
+import * as fs from 'fs';
+import * as path from 'path';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import * as jsyaml from 'js-yaml';
+
+execute();
+
+async function execute(): Promise<void> {
+    const electronBuilderYamlPath = path.resolve(
+        './',
+        'electron-builder.yml'
+    );
+
+    console.log(`Updating ${electronBuilderYamlPath}...`);
+
+    const electronBuilderYamlContents: string = fs.readFileSync(electronBuilderYamlPath, { encoding: 'utf8' });
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const electronBuilderYaml: any = jsyaml.load(electronBuilderYamlContents);
+
+    console.log('...appId...');
+    electronBuilderYaml.appId = 'eclipse.theia.preview';
+    console.log('...done...');
+
+    console.log('...productName...');
+    electronBuilderYaml.productName = 'TheiaIDEPreview';
+    console.log('...done...');
+
+    console.log('...publish urls...');
+    electronBuilderYaml.win.publish.url = 'https://download.eclipse.org/theia/next/windows';
+    electronBuilderYaml.mac.publish.url = 'https://download.eclipse.org/theia/next/macos';
+    electronBuilderYaml.linux.publish.url = 'https://download.eclipse.org/theia/next/linux';
+    console.log('...done...');
+
+    // line width -1 to avoid adding >- on long strings like a hash
+    const newYamlContents = jsyaml.dump(electronBuilderYaml, { lineWidth: -1 });
+    fs.writeFileSync(electronBuilderYamlPath, newYamlContents);
+}