refactor: new dev/prod containers

- Use a single Dockerfile for all Zeppelin services
- Add a Dockerfile in project root that can be used by
  app hosting services
- Provide a standalone and lightweight prod setup
  - Standalone is the same as the old setup, with mysql+nginx
  - Lightweight only runs bot+backend+dash, no mysql/nginx
- Remove mounted mysql data folders for dev and prod
  - This resolves permission issues caused by the mount
  - The mysql service uses a regular named volume now
- Simplify .env options and clearly separate different prod setups
- Remove update.sh
  - Different setups require different update procedures, so a common
    update.sh no longer works

.devcontainer/devcontainer.json

@@ -5,5 +5,5 @@
 
   "service": "devenv",
   "remoteUser": "ubuntu",
-  "workspaceFolder": "/home/ubuntu/zeppelin"
+  "workspaceFolder": "/workspace/zeppelin"
 }

.dockerignore

@@ -8,4 +8,97 @@
 
 node_modules
 /backend/dist
+/shared/dist
 /dashboard/dist
+
+# FROM GITIGNORE:
+
+# Created by .ignore support plugin (hsz.mobi)
+### Node template
+# Logs
+/logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.clinic
+.clinic-bot
+.clinic-api
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+*.env
+.env
+
+# windows folder options
+desktop.ini
+
+# PHPStorm
+.idea/
+
+# Misc
+/convert.js
+/startscript.js
+.cache
+npm-ls.txt
+npm-audit.txt
+.vscode/launch.json
+
+# Debug files
+*.debug.ts
+*.debug.js
+
+.vscode/
+
+config-errors.txt
+/config-schema.json
+
+*.tsbuildinfo

.env.example

@@ -1,3 +1,7 @@
+# ==========================
+# GENERAL OPTIONS
+# ==========================
+
 # 32 character encryption key
 KEY=
 
@@ -17,58 +21,63 @@ STAFF=
 # A comma-separated list of server IDs that should be allowed by default
 DEFAULT_ALLOWED_SERVERS=
 
-# When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api.
-API_PORT=3000
-
 # Only required if relevant feature is used
 #PHISHERMAN_API_KEY=
 
-# The user ID and group ID that should be used within the Docker containers
-# This should match your own user ID and group ID. Run `id -u` and `id -g` to find them.
-DOCKER_USER_UID=
-DOCKER_USER_GID=
 
-#
-# DOCKER (DEVELOPMENT)
-# NOTE: You only need to fill in these values for running the development environment. See production config further below.
-#
+# ==========================
+# DEVELOPMENT
+# NOTE: You only need to fill in these values for running the development environment
+# ==========================
 
-DOCKER_DEV_WEB_PORT=3300
+DEVELOPMENT_WEB_PORT=3300
 
 # The MySQL database running in the container is exposed to the host on this port,
 # allowing access with database tools such as DBeaver
-DOCKER_DEV_MYSQL_PORT=3001
+DEVELOPMENT_MYSQL_PORT=3356
 # Password for the Zeppelin database user
-DOCKER_DEV_MYSQL_PASSWORD=
+DEVELOPMENT_MYSQL_PASSWORD=password
 # Password for the MySQL root user
-DOCKER_DEV_MYSQL_ROOT_PASSWORD=
+DEVELOPMENT_MYSQL_ROOT_PASSWORD=password
 
 # The development environment container has an SSH server that you can connect to.
 # This is the port that server is exposed to the host on.
-DOCKER_DEV_SSH_PORT=3002
-DOCKER_DEV_SSH_PASSWORD=password
+DEVELOPMENT_SSH_PORT=3022
+DEVELOPMENT_SSH_PASSWORD=password
 
 # If your user has a different UID than 1000, you might have to fill that in here to avoid permission issues
-#DOCKER_DEV_UID=1000
+#DEVELOPMENT_UID=1000
+
+
+# ==========================
+# PRODUCTION - STANDALONE
+# NOTE: You only need to fill in these values for running the standalone production environment
+# ==========================
+
+STANDALONE_DOMAIN=
 
-#
-# DOCKER (PRODUCTION)
-# NOTE: You only need to fill in these values for running the production environment. See development config above.
-#
+STANDALONE_WEB_PORT=443
 
-DOCKER_PROD_DOMAIN=
-DOCKER_PROD_WEB_PORT=443
 # The MySQL database running in the container is exposed to the host on this port,
 # allowing access with database tools such as DBeaver
-DOCKER_PROD_MYSQL_PORT=3001
+STANDALONE_MYSQL_PORT=3356
 # Password for the Zeppelin database user
-DOCKER_PROD_MYSQL_PASSWORD=
+STANDALONE_MYSQL_PASSWORD=
 # Password for the MySQL root user
-DOCKER_PROD_MYSQL_ROOT_PASSWORD=
-
-# You only need to set these if you're running an external database.
-# In a standard setup, the database is run in a docker container.
-#DB_HOST=
-#DB_USER=
-#DB_PASSWORD=
-#DB_DATABASE=
+STANDALONE_MYSQL_ROOT_PASSWORD=
+
+
+# ==========================
+# PRODUCTION - LIGHTWEIGHT
+# NOTE: You only need to fill in these values for running the lightweight production environment
+# ==========================
+
+# Ports where the API/dashboard are exposed on the host
+LIGHTWEIGHT_API_PORT=3001
+LIGHTWEIGHT_DASHBOARD_PORT=3002
+
+LIGHTWEIGHT_DB_HOST=
+LIGHTWEIGHT_DB_PORT=
+LIGHTWEIGHT_DB_USER=
+LIGHTWEIGHT_DB_PASSWORD=
+LIGHTWEIGHT_DB_DATABASE=

.gitignore

@@ -87,3 +87,7 @@ config-errors.txt
 /config-schema.json
 
 *.tsbuildinfo
+
+# Legacy data folders
+/docker/development/data
+/docker/production/data

Dockerfile

@@ -0,0 +1,20 @@
+FROM node:20
+USER node
+
+COPY --chown=node:node . /zeppelin
+
+# Install dependencies for all packages
+WORKDIR /zeppelin
+RUN npm ci
+
+# Build backend
+WORKDIR /zeppelin/backend
+RUN npm run build
+
+# Build dashboard
+WORKDIR /zeppelin/dashboard
+RUN npm run build
+
+# Prune dev dependencies
+WORKDIR /zeppelin
+RUN npm prune --production

PRODUCTION.md

@@ -1,34 +1,67 @@
 # Zeppelin production environment
-Zeppelin's production environment - that is, the **bot, API, and dashboard** - uses Docker.
+Zeppelin's production environment uses Docker. There are a few different ways to run Zeppelin in production:
 
-## Starting the production environment
+1. **Standalone**
+  * The easiest way to get Zeppelin up and running. This setup comes with a built-in database and web server.
+2. **Lightweight**
+  * In case you don't want to use the built-in database and web server. This setup only runs the bot, API, and dashboard themselves. You'll have to provide your own database connection options and set up a proxy server for the API and dashboard.
+3. **Manual**
+  * If you only want to run a specific service, you can use Zeppelin's Dockerfile directly.
+
+## Standalone
+
+### Setup
 1. Install Docker on the machine running the bot
 2. Make a copy of `.env.example` called `.env`
-3. Fill in the missing values in `.env`
-4. Run `docker compose -f docker-compose.production.yml build`
-5. Run `docker compose -f docker-compose.production.yml up -d`
+3. Fill in the missing values in `.env` (including the "PRODUCTION - STANDALONE" section)
+
+**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. A popular option for this is [Cloudflare Tunnel](https://www.cloudflare.com/products/tunnel/).
+
+### Running the bot
+`docker compose -f docker-compose.standalone.yml up -d`
+
+### Shutting the bot down
+`docker compose -f docker-compose.standalone.yml down`
+
+### Updating the bot
+1. Shut the bot down
+2. Update the files (e.g. `git pull`)
+3. Rebuild: `docker compose -f docker-compose.standalone.yml build`
+4. Run the bot again
+
+### Viewing logs
+`docker compose -f docker-compose.standalone.yml logs -t -f`
+
+## Lightweight
 
-**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. Cloudflare is a popular choice here.
+### Setup
+1. Install Docker on the machine running the bot
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env` (including the "PRODUCTION - LIGHTWEIGHT" section)
 
-## Updating the bot
+### Running the bot
+`docker compose -f docker-compose.lightweight.yml up -d`
 
-### One-click script
-If you've downloaded the bot's files by cloning the git repository, you can use `update.sh` to update the bot.
+### Shutting the bot down
+`docker compose -f docker-compose.lightweight.yml down`
 
-### Manual instructions
-1. Shut the bot down: `docker compose -f docker-compose.production.yml down`
+### Updating the bot
+1. Shut the bot down
 2. Update the files (e.g. `git pull`)
-3. Build new images: `docker compose -f docker-compose.production.yml build`
-3. Start the bot again: `docker compose -f docker-compose.production.yml up -d`
+3. Rebuild: `docker compose -f docker-compose.lightweight.yml build`
+4. Run the bot again
+
+### Viewing logs
+`docker compose -f docker-compose.lightweight.yml logs -t -f`
 
-### Ephemeral hotfixes
-If you need to make a hotfix to the bot's source files directly on the server:
-1. Shut the bot down: `docker compose -f docker-compose.production.yml down`
-2. Make your edits
-3. Build new images: `docker compose -f docker-compose.production.yml build`
-4. Start the bot again: `docker compose -f docker-compose.production.yml up -d`
+## Manual
+1. Build the Zeppelin image: `docker build --tag 'zeppelin' .`
+2. Run the service:
+  * Bot: `docker run zeppelin npm run start-bot`
+  * API: `docker run zeppelin npm run start-api`
+  * Dashboard: `docker run zeppelin npm run start-dashboard`
 
-Make sure to revert any hotfixes before updating the bot normally.
+If you're using an application platform such as Railway, you can simply point it to Zeppelin's repository and it should pick up the Dockerfile from there.
+For the start command, you can use the same commands as above: `npm run start-bot`, `npm run start-api`, `npm run start-dashboard`.
 
-## View logs
-To view real-time logs, run `docker compose -f docker-compose.production.yml logs -t -f`
+**Note:** You'll need to provide the necessary env variables. For example, `docker run --env-file .env zeppelin`

backend/package.json

@@ -4,28 +4,28 @@
   "description": "",
   "private": true,
   "scripts": {
-    "watch": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node start-dev.js\"",
-    "watch-yaml-parse-test": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node dist/backend/src/yamlParseTest.js\"",
+    "watch": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"node start-dev.js\"",
+    "watch-yaml-parse-test": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"node dist/backend/src/yamlParseTest.js\"",
     "build": "tsc --build",
-    "start-bot-dev": "cross-env NODE_ENV=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
+    "start-bot-dev": "NODE_ENV=development HOST_MODE=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
     "start-bot-dev-debug": "NODE_ENV=development DEBUG=true clinic heapprofiler --collect-only --dest .clinic-bot -- node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
     "start-bot-prod": "cross-env NODE_ENV=production node --enable-source-maps --stack-trace-limit=30 dist/backend/src/index.js",
     "start-bot-prod-debug": "NODE_ENV=production DEBUG=true clinic heapprofiler --collect-only --dest .clinic-bot -- node --enable-source-maps --stack-trace-limit=30 dist/backend/src/index.js",
-    "watch-bot": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-bot-dev\"",
-    "start-api-dev": "cross-env NODE_ENV=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
+    "watch-bot": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"npm run start-bot-dev\"",
+    "start-api-dev": "NODE_ENV=development HOST_MODE=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
     "start-api-dev-debug": "NODE_ENV=development DEBUG=true clinic heapprofiler --collect-only --dest .clinic-api -- node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
     "start-api-prod": "cross-env NODE_ENV=production node --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
     "start-api-prod-debug": "NODE_ENV=production DEBUG=true clinic heapprofiler --collect-only --dest .clinic-api -- node --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
-    "watch-api": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-api-dev\"",
+    "watch-api": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"npm run start-api-dev\"",
     "typeorm": "node ./node_modules/typeorm/cli.js",
     "migrate": "npm run typeorm -- migration:run -d dist/backend/src/data/dataSource.js",
     "migrate-prod": "cross-env NODE_ENV=production npm run migrate",
-    "migrate-dev": "cross-env NODE_ENV=development npm run build && npm run migrate",
+    "migrate-dev": "NODE_ENV=development HOST_MODE=development npm run build && npm run migrate",
     "migrate-rollback": "npm run typeorm -- migration:revert -d dist/backend/src/data/dataSource.js",
     "migrate-rollback-prod": "cross-env NODE_ENV=production npm run migrate",
-    "migrate-rollback-dev": "cross-env NODE_ENV=development npm run build && npm run migrate",
-    "validate-active-configs": "cross-env NODE_ENV=development node --enable-source-maps dist/backend/src/validateActiveConfigs.js > ../config-errors.txt",
-    "export-config-json-schema": "cross-env NODE_ENV=development node --enable-source-maps dist/backend/src/exportSchemas.js > ../config-schema.json",
+    "migrate-rollback-dev": "NODE_ENV=development HOST_MODE=development npm run build && npm run migrate",
+    "validate-active-configs": "NODE_ENV=development HOST_MODE=development node --enable-source-maps dist/backend/src/validateActiveConfigs.js > ../config-errors.txt",
+    "export-config-json-schema": "NODE_ENV=development HOST_MODE=development node --enable-source-maps dist/backend/src/exportSchemas.js > ../config-schema.json",
     "test": "npm run build && npm run run-tests",
     "run-tests": "ava",
     "test-watch": "tsc-watch --onSuccess \"npx ava\""

backend/src/api/start.ts

@@ -51,7 +51,7 @@ app.use((req, res, next) => {
   return notFound(res);
 });
 
-const port = env.API_PORT;
+const port = 3001;
 app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line
 
 startBackgroundTasks();