V2 Migration (game-ci#47)

* Fix issues discovered during deployment

* Fix compile issues

* Fix predeploy

* Add test function deployment

* Fix linting

* Fix linting

* Pin versions, add module to package type

* Fix linting

* Revert to commonjs

* Revert module resolution

* Fix missing token field

* Remove token

* Fix lint

* Revert node fetch version

* Cleanup test function

* Use force delete

* Finish migration to v2 functions. Converts to new secrets management. Update packages

* Fix formatting

* Remove require

* Read file contents

* Downgrade node-fetch

* Downgrade octokit

* Verbose secret paths

* Improve service account credential read

* Force service account credential

* Revert secret names

* Add fallback path to service account credentials

* Fixes from code review. Docs on database backup/restore

Author: AndrewKahr

.github/workflows/main.yml

@@ -1,61 +1,86 @@
 name: 🚀
 
 on:
-  pull_request: { }
-  push: { branches: [ main ] }
+  pull_request: {}
+  push: { branches: [main] }
 
 jobs:
   test:
     name: 🧪 Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
       - name: install dependencies
         run: yarn && yarn --cwd ./functions
       - name: run linter
         run: yarn lint && yarn --cwd ./functions lint
       - name: run tests
         run: yarn test && yarn --cwd ./functions test
-#      - name: Upload test results
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: Test results
-#          path: "**/artifacs"
+  #      - name: Upload test results
+  #        uses: actions/upload-artifact@v1
+  #        with:
+  #          name: Test results
+  #          path: "**/artifacs"
 
   build:
     name: 🛠 Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
       - name: install dependencies
         run: yarn && yarn --cwd ./functions
       - name: build
         run: yarn --cwd ./functions build
 
+  testDeploy:
+    name: Test Deploy
+    needs: [test, build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+      - name: install dependencies
+        run: yarn && yarn --cwd ./functions
+      - name: Deploy test to Firebase
+        uses: w9jds/[email protected]
+        with:
+          args: deploy --only functions:testFunction
+        env:
+          GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'
+      - name: Cleanup Firebase Test
+        uses: w9jds/[email protected]
+        with:
+          args: functions:delete testFunction --force
+        env:
+          GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'
+
   deploy:
     name: ✨ Deploy
-    needs: [ test, build ]
+    needs: [test, build, testDeploy]
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 16
-          yarn: 'yarn'
+          node-version: 20
+          cache: 'yarn'
       - name: install dependencies
         run: yarn && yarn --cwd ./functions
       - name: Deploy to Firebase
-        uses: w9jds/firebase-action@v11.24.1
+        uses: w9jds/firebase-action@v13.10.2
         with:
-          args: deploy
+          args: deploy --except functions:testFunction
         env:
-          FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
+          GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'

.gitignore

@@ -68,4 +68,5 @@ node_modules/
 # IDE
 .idea
 .vs
-.vscode
+.vscode
+functions/.yarn/

README.md

@@ -34,3 +34,14 @@ Completed CiJobs are reported to Discord.
 ## Ingeminator
 
 TODO - Describe how it works
+
+## Database Backup
+
+The firestore database can be backed up with the following command:
+`yarn run backfire export ./export/versioningBackendBackup --project unity-ci-versions --keyFile <PATH_TO_GOOGLE_CLOUD_SERVICE_ACCOUNT_KEYFILE.json>`
+
+Similarly, it can be used to restore a backup with:
+`yarn run backfire import ./export/versioningBackendBackup --project unity-ci-versions --keyFile <PATH_TO_GOOGLE_CLOUD_SERVICE_ACCOUNT_KEYFILE.json>`
+
+You likely would want to empty the database before restoring but you can also use flags like overwrite, merge, etc to control the restoration
+rules.

firebase.json

@@ -5,9 +5,9 @@
   },
   "functions": {
     "predeploy": [
-      "npm --prefix \"$RESOURCE_DIR\" run lint",
-      "npm --prefix \"$RESOURCE_DIR\" run test",
-      "npm --prefix \"$RESOURCE_DIR\" run build"
+      "yarn --cwd \"$RESOURCE_DIR\" lint",
+      "yarn --cwd \"$RESOURCE_DIR\" test",
+      "yarn --cwd \"$RESOURCE_DIR\" build"
     ]
   },
   "hosting": {

functions/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

functions/package.json

@@ -12,35 +12,40 @@
     "test": "echo \"No tests yet, feel free to add\" && exit 0"
   },
   "engines": {
-    "node": "16"
+    "node": "20"
   },
   "main": "lib/index.js",
   "dependencies": {
-    "@octokit/auth-app": "^4.0.9",
-    "@octokit/rest": "^19.0.7",
+    "@octokit/auth-app": "^6.1.1",
+    "@octokit/rest": "^20.1.1",
     "eris": "^0.17.2",
-    "firebase-admin": "^11.7.0",
-    "firebase-functions": "^4.3.0",
+    "firebase-admin": "^12.1.1",
+    "firebase-functions": "^5.0.1",
     "httpie": "^1.1.2",
-    "jsdom": "^21.1.1",
-    "lodash": "^4.17.20",
-    "node-fetch": "^3.3.1",
-    "semver": "^7.3.2"
+    "jsdom": "^24.1.0",
+    "lodash": "^4.17.21",
+    "node-fetch": "^2.7.0",
+    "semver": "^7.6.2"
   },
   "devDependencies": {
-    "@types/jsdom": "^21.1.1",
-    "@types/lodash": "^4.14.162",
-    "@types/node": "^18.15.11",
-    "@types/node-fetch": "^2.5.7",
-    "@types/semver": "^7.3.4",
-    "@types/ws": "^8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.59.0",
-    "@typescript-eslint/parser": "^5.59.0",
-    "eslint": "^8.38.0",
-    "eslint-plugin-import": "^2.22.0",
-    "firebase-functions-test": "^3.1.0",
-    "jest": "^29.5.0",
-    "typescript": "^5.0.4"
+    "@octokit/types": "^13.5.0",
+    "@types/jsdom": "^21.1.7",
+    "@types/lodash": "^4.17.4",
+    "@types/node": "^20.14.0",
+    "@types/node-fetch": "^2.6.11",
+    "@types/semver": "^7.5.8",
+    "@types/ws": "^8.5.10",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
+    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
+    "firebase-functions-test": "^3.3.0",
+    "jest": "^29.7.0",
+    "typescript": "^5.4.5"
   },
-  "private": true
+  "private": true,
+  "volta": {
+    "node": "20.14.0",
+    "yarn": "1.22.22"
+  }
 }

functions/src/api/index.ts

@@ -4,3 +4,4 @@ export { reportNewBuild } from './reportNewBuild';
 export { reportPublication } from './reportPublication';
 export { unityVersions } from './unityVersions';
 export { retryBuild } from './retryBuild';
+export { testFunction } from './testFunction';

functions/src/api/queueStatus.ts

@@ -1,10 +1,9 @@
-import { functions } from '../service/firebase';
-import { Request } from 'firebase-functions/v2/https';
+import { onRequest, Request } from 'firebase-functions/v2/https';
 import { Response } from 'express-serve-static-core';
 import { CiJobs } from '../model/ciJobs';
 import { CiBuilds } from '../model/ciBuilds';
 
-export const queueStatus = functions.https.onRequest(async (req: Request, res: Response) => {
+export const queueStatus = onRequest(async (req: Request, res: Response) => {
   const jobs = await CiJobs.getAll();
   const builds = await CiBuilds.getAll();
 

functions/src/api/repoVerions.ts

@@ -0,0 +1,15 @@
+import { onRequest, Request } from 'firebase-functions/v2/https';
+import { logger } from 'firebase-functions/v2';
+import { Response } from 'express-serve-static-core';
+import { RepoVersionInfo } from '../model/repoVersionInfo';
+
+export const repoVersions = onRequest(async (request: Request, response: Response) => {
+  try {
+    const versions = await RepoVersionInfo.getAllIds();
+
+    response.send(versions);
+  } catch (err) {
+    logger.error(err);
+    response.send('Oops.');
+  }
+});

functions/src/api/repoVersions.ts

@@ -1,43 +1,56 @@
-import { firebase, functions } from '../service/firebase';
-import { Request } from 'firebase-functions/v2/https';
+import { onRequest, Request } from 'firebase-functions/v2/https';
+import { logger } from 'firebase-functions/v2';
 import { Response } from 'express-serve-static-core';
 import { Token } from '../config/token';
 import { BuildFailure, CiBuilds } from '../model/ciBuilds';
 import { CiJobs } from '../model/ciJobs';
 import { Discord } from '../service/discord';
+import { defineSecret } from 'firebase-functions/params';
 
-export const reportBuildFailure = functions.https.onRequest(async (req: Request, res: Response) => {
-  try {
-    if (!Token.isValid(req.header('authorization'))) {
-      firebase.logger.warn('unauthorised request', req.headers);
-      res.status(403).send('Unauthorized');
-      return;
-    }
+const discordToken = defineSecret('DISCORD_TOKEN');
+const internalToken = defineSecret('INTERNAL_TOKEN');
+
+export const reportBuildFailure = onRequest(
+  { secrets: [discordToken, internalToken] },
+  async (req: Request, res: Response) => {
+    const discordClient = new Discord();
+    await discordClient.init(discordToken.value());
+
+    try {
+      if (!Token.isValid(req.header('authorization'), internalToken.value())) {
+        logger.warn('unauthorised request', req.headers);
+        res.status(403).send('Unauthorized');
+        return;
+      }
 
-    const { body } = req;
-    firebase.logger.debug('Build failure report incoming.', body);
+      const { body } = req;
+      logger.debug('Build failure report incoming.', body);
 
-    const { jobId, buildId, reason } = body;
-    const failure: BuildFailure = { reason };
+      const { jobId, buildId, reason } = body;
+      const failure: BuildFailure = { reason };
 
-    await CiJobs.markFailureForJob(jobId);
-    await CiBuilds.markBuildAsFailed(buildId, failure);
+      await CiJobs.markFailureForJob(jobId);
+      await CiBuilds.markBuildAsFailed(buildId, failure);
 
-    firebase.logger.info('Build failure reported.', body);
-    res.status(200).send('OK');
-  } catch (err: any) {
-    const message = `
+      logger.info('Build failure reported.', body);
+      res.status(200).send('OK');
+    } catch (err: any) {
+      const message = `
       Something went wrong while reporting a build failure
       ${err.message}
     `;
-    firebase.logger.error(message, err);
-    await Discord.sendAlert(message);
+      logger.error(message, err);
+
+      await discordClient.sendAlert(message);
+
+      if (req.body?.jobId?.toString().startsWith('dryRun')) {
+        await CiBuilds.removeDryRunBuild(req.body.buildId);
+        await CiJobs.removeDryRunJob(req.body.jobId);
+      }
 
-    if (req.body?.jobId?.toString().startsWith('dryRun')) {
-      await CiBuilds.removeDryRunBuild(req.body.buildId);
-      await CiJobs.removeDryRunJob(req.body.jobId);
+      res.status(500).send('Something went wrong');
     }
 
-    res.status(500).send('Something went wrong');
-  }
-});
+    await discordClient.disconnect();
+  },
+);