Refactor: streamline workflow sorting and enhance workflow management (#187)

* Refactor: streamline workflow sorting and enhance workflow management

- Updated workflow sorting to be alphabetical for improved organization.
- Refactored workflow_page_controller.dart to ensure workflows are ordered by name.
- Added deletion alert feature for workflows to enhance user experience.

* Refactor: enhance Firestore transaction handling in VM Runner

- Introduced a new function `runFirestoreTransaction` to encapsulate Firestore transaction logic.
- Updated Firestore client variable naming for clarity.
- Improved error handling and logging for fetching build jobs and updating document statuses.
- Streamlined the process of updating build job statuses from "queued" to "inProgress".

* Refactor: improve Firestore transaction handling and remove redundant build status update

- Removed unnecessary build status update call in RunnerCommand.
- Enhanced runFirestoreTransaction to return the fetched build job document snapshot.
- Improved error handling and logging for Firestore transactions in VM Runner.
- Streamlined the process of handling build job documents.

* Enhance workflow fetching in VM Runner

- Added functionality to fetch all workflow documents from Firestore.
- Improved error handling and logging for workflow retrieval.
- Updated logging to include details of found workflow documents.

* feat: add GitHub installation token retrieval and update dependencies

- Introduced functionality to retrieve GitHub installation tokens using JWT authentication.
- Added a new command-line flag for specifying the GitHub App's PEM file.
- Updated dependencies in go.mod to include `cloud.google.com/go/firestore` and `github.com/dgrijalva/jwt-go`.
- Improved logging for workflow and build job processing in the VM Runner.

* add: execute ssh command

* refactor

* Refactor: streamline logging in VM Runner

- Removed logger parameters from RunApp function and initialized loggers within the function.
- Deleted the unused binary file 'vm-runner' to clean up the project structure.

* Refactor: enhance VM process handling and SSH command execution

- Improved error handling in RunApp to log VM process failures without stopping execution.
- Added cleanup logic for stopping and deleting VMs after execution.
- Refactored ExecuteSSHCommand to return structured results, including stdout, stderr, and exit code.
- Updated logging for SSH command outputs for better clarity.

* Refactor: simplify VM cleanup logic in handleVMProcess

- Moved VM cleanup logic into a separate function `cleanupVM` for better readability and maintainability.
- Updated the deferred function to call `cleanupVM`, ensuring VMs are stopped and deleted after execution.
- Improved logging for VM stop and delete actions.

* add: firebase storage for storing log

* feat: get installation token

* refactor

* fix: correct SSH command in handleVMProcess

- Updated the SSH command from 'lsa' to 'ls' in the handleVMProcess function to ensure proper execution and output retrieval.
- This change improves the accuracy of the command being executed during VM process handling.

* refactor: update BuildJob structure and enhance VM command execution

- Renamed fields in BuildJob from `repoUrl` to `repositoryUrl` and `branch` to `buildBranch` for clarity.
- Introduced a new `cloneCommand` function to streamline the repository cloning process using the specified build branch and GitHub token.
- Enhanced command execution in `handleVMProcess` to iterate over workflow steps, improving flexibility in command handling.
- Added a new `Step` struct to represent individual commands in the workflow, allowing for better organization and management of execution steps.

* feat: secrets

* feat: cli with golang

Author: mafreud

apps/firebase_functions/functions/src/index.ts

@@ -4,10 +4,83 @@ import { getFirestore } from "firebase-admin/firestore";
 import { githubApp } from "./github/github_app.js";
 import { updateGitHubCheckStatus } from "./github/update_github_checks_status.js";
 import { updateGitHubChecksLog } from "./github/update_github_checks_log.js";
+import { onObjectFinalized } from "firebase-functions/v2/storage";
+import type { OpenCIGithub } from "./models/BuildJob.js";
+import { getGitHubInstallationToken } from "./github/get_github_installation_token.js";
+import { Octokit } from "@octokit/rest";
+import { getStorage } from "firebase-admin/storage";
 
 const firebaseApp = initializeApp();
 export const firestore = getFirestore(firebaseApp);
+const storage = getStorage(firebaseApp);
 
 export const githubAppFunction = githubApp;
 export const updateGitHubCheckStatusFunction = updateGitHubCheckStatus;
 export const updateGitHubChecksLogFunction = updateGitHubChecksLog;
+
+export const generateThumbnail = onObjectFinalized(
+	{
+		region: "asia-northeast1",
+		secrets: ["APP_ID", "PRIVATE_KEY", "GITHUB_WEBHOOK_SECRET"],
+	},
+	async (event) => {
+		console.log(`event.source: ${event.source}`);
+		console.log(`event.data: ${event.data}`);
+		console.log(`event.data.bucket: ${event.data.bucket}`);
+		console.log(`event.data.name: ${event.data.name}`);
+		console.log(`event.data.contentType: ${event.data.contentType}`);
+
+		const bucket = storage.bucket(event.data.bucket);
+		const file = bucket.file(event.data.name);
+
+		const [content] = await file.download();
+		const logs = content.toString("utf-8");
+
+		const jobId = event.data.name.split("/")[1];
+		const docs = await firestore.collection("build_jobs_v3").doc(jobId).get();
+		const data = docs.data();
+		const openciGitHub = data?.github as OpenCIGithub | null;
+		console.log(`openciGitHub: ${openciGitHub}`);
+
+		if (openciGitHub == null) {
+			console.log("No openciGitHub found");
+			return;
+		}
+
+		const installationId = openciGitHub.installationId;
+
+		const appId = process.env.APP_ID;
+		const privateKey = process.env.PRIVATE_KEY;
+
+		if (!appId || !privateKey) {
+			console.error("Missing appId or privateKey");
+			return;
+		}
+
+		const token = await getGitHubInstallationToken(
+			installationId,
+			appId,
+			privateKey,
+		);
+		const octokit = new Octokit({ auth: token });
+
+		await setGitHubCheckStatusToInProgress(octokit, openciGitHub, logs);
+	},
+);
+
+async function setGitHubCheckStatusToInProgress(
+	octokit: Octokit,
+	github: OpenCIGithub,
+	logs: string,
+) {
+	await octokit.checks.update({
+		owner: github.owner,
+		repo: github.repositoryName,
+		check_run_id: github.checkRunId,
+		output: {
+			title: "Updated Build Results",
+			summary: "Build progress update",
+			text: logs,
+		},
+	});
+}

apps/openci_runner/lib/src/commands/runner_command.dart

@@ -131,10 +131,6 @@ class RunnerCommand extends Command<int> {
           final vmName = generateUUID;
           final logId = generateUUID;
           try {
-            await updateBuildStatus(
-              jobId: buildJob.id,
-            );
-
             final workflow =
                 await getWorkflowModel(firestore, buildJob.workflowId);
             if (workflow == null) {

apps/vm-runner/.gitignore

@@ -0,0 +1 @@
+run.sh

apps/vm-runner/build_job.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	"cloud.google.com/go/firestore"
+	"github.com/getsentry/sentry-go"
+)
+
+type BuildJob struct {
+	ID     string `json:"id"`
+	GitHub struct {
+		RepoURL        string `json:"repositoryUrl"`
+		InstallationId int    `json:"installationId"`
+		AppId          int    `json:"appId"`
+		BuildBranch    string `json:"buildBranch"`
+	} `json:"github"`
+	WorkflowId string `json:"workflowId"`
+	Status     string `json:"status"`
+}
+
+func ParseBuildJob(jsonData []byte) (*BuildJob, error) {
+	var job BuildJob
+
+	err := json.Unmarshal(jsonData, &job)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal build job: %w", err)
+	}
+
+	fmt.Printf("Successfully parsed BuildJob: %v\n", job)
+
+	return &job, nil
+}
+
+func GetBuildJob(ctx context.Context, firestoreClient *firestore.Client, infoLogger *log.Logger) (*BuildJob, error) {
+	var snap *firestore.DocumentSnapshot
+
+	err := firestoreClient.RunTransaction(ctx, func(ctx context.Context, tx *firestore.Transaction) error {
+		ref := firestoreClient.Collection("build_jobs_v3").
+			Where("buildStatus", "==", "queued").
+			OrderBy("createdAt", firestore.Asc).
+			Limit(1)
+
+		docs, err := ref.Documents(ctx).GetAll()
+		if err != nil {
+			return fmt.Errorf("failed to fetch build jobs: %v", err)
+		}
+
+		if len(docs) == 0 {
+			return fmt.Errorf("no queued build jobs found")
+		}
+
+		docRef := docs[0].Ref
+		freshDocSnap, err := tx.Get(docRef)
+		if err != nil {
+			return fmt.Errorf("failed to get document snapshot: %v", err)
+		}
+
+		if !freshDocSnap.Exists() {
+			return fmt.Errorf("document does not exist")
+		}
+
+		data := freshDocSnap.Data()
+		freshStatus, ok := data["buildStatus"].(string)
+		if !ok {
+			return fmt.Errorf("invalid buildStatus field")
+		}
+
+		if freshStatus == "queued" {
+			err = tx.Update(docRef, []firestore.Update{
+				{
+					Path:  "buildStatus",
+					Value: "inProgress",
+				},
+				{
+					Path:  "updatedAt",
+					Value: time.Now(),
+				},
+			})
+			if err != nil {
+				return fmt.Errorf("failed to update document: %v", err)
+			}
+			infoLogger.Printf("Document %s updated to inProgress.", docRef.ID)
+		} else {
+			return fmt.Errorf("build job status is not queued: %s", freshStatus)
+		}
+
+		snap = freshDocSnap
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if snap == nil {
+		return nil, fmt.Errorf("no valid build job found")
+	}
+
+	data := snap.Data()
+
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal workflow data: %v", err)
+	}
+
+	job, err := ParseBuildJob(jsonData)
+	if err == nil {
+		infoLogger.Printf("Successfully parsed BuildJob: %v\n", job)
+	} else {
+		sentry.CaptureMessage(fmt.Sprintf("Failed to parse BuildJob: %v", err))
+		return nil, fmt.Errorf("failed to parse BuildJob: %v", err)
+	}
+
+	return job, nil
+}

apps/vm-runner/firebase.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"regexp"
+
+	firebase "firebase.google.com/go"
+)
+
+func UploadLogToFirebaseStorage(
+	ctx context.Context,
+	app *firebase.App,
+	jobId string,
+	cmdRes SSHCommandResult,
+) error {
+	fmt.Printf("UploadLogToFirebaseStorage: %+v", cmdRes)
+	fileName := logPath(jobId)
+	fmt.Printf("fileName: %s", fileName)
+	newContent := cmdLog(cmdRes)
+
+	storageClient, err := app.Storage(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create storage client: %v", err)
+	}
+
+	bucket, err := storageClient.DefaultBucket()
+	if err != nil {
+		return fmt.Errorf("failed to get default storage bucket: %v", err)
+	}
+
+	obj := bucket.Object(fileName)
+	reader, err := obj.NewReader(ctx)
+
+	var content string
+	if err == nil {
+		existingContent, err := io.ReadAll(reader)
+		reader.Close()
+		if err != nil {
+			return fmt.Errorf("failed to read existing log: %v", err)
+		}
+		content = string(existingContent) + "\n" + newContent
+	} else {
+		content = newContent
+	}
+
+	writer := obj.NewWriter(ctx)
+	_, err = writer.Write([]byte(content))
+	if err != nil {
+		writer.Close()
+		return fmt.Errorf("failed to write log to storage: %v", err)
+	}
+
+	if err = writer.Close(); err != nil {
+		return fmt.Errorf("failed to close writer: %v", err)
+	}
+
+	return nil
+}
+
+func logPath(buildId string) string {
+	return fmt.Sprintf("logs_v1/%s/openci_log.log", buildId)
+}
+
+func cmdLog(output SSHCommandResult) string {
+	logContent := fmt.Sprintf(
+		"=== Command Execution Result ===\n"+
+			"Command: %s\n"+
+			"Exit Code: %d\n"+
+			"=== Stdout ===\n%s\n"+
+			"=== Stderr ===\n%s\n"+
+			"=========================",
+		output.Command,
+		output.ExitCode,
+		output.Stdout,
+		output.Stderr,
+	)
+
+	return redactSensitiveInfo(logContent)
+}
+
+func redactSensitiveInfo(input string) string {
+	patterns := []struct {
+		regex       string
+		replacement string
+	}{
+		{`[A-Za-z0-9+/]{24,}={0,3}`, "REDACTED_BASE64"},
+		{`(ghs_|github_)[0-9a-f]{40}`, "REDACTED_GITHUB_TOKEN"},
+		{`[A-Za-z0-9_\-]{40,}`, "REDACTED_ACCESS_TOKEN"},
+		{`(?i)password=([^\s&]+)`, "password=REDACTED_PASSWORD"},
+	}
+
+	result := input
+	for _, pattern := range patterns {
+		re := regexp.MustCompile(pattern.regex)
+		result = re.ReplaceAllString(result, pattern.replacement)
+	}
+	return result
+}

apps/vm-runner/github.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+)
+
+func GetGitHubInstallationToken(pemPath string, installationID int64, appID int64, infoLogger *log.Logger) (string, error) {
+	infoLogger.Printf("Getting GitHub installation token for installation ID: %d, app ID: %d, pemPath: %v\n", installationID, appID, pemPath)
+	pemBytes, err := os.ReadFile(pemPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read pem file: %v", err)
+	}
+
+	privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(pemBytes)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse private key: %v", err)
+	}
+
+	now := time.Now()
+	claims := jwt.StandardClaims{
+		IssuedAt:  now.Unix(),
+		ExpiresAt: now.Add(9 * time.Minute).Unix(),
+		Issuer:    fmt.Sprintf("%d", appID),
+	}
+
+	jwtToken, err := jwt.NewWithClaims(jwt.SigningMethodRS256, claims).SignedString(privateKey)
+	if err != nil {
+		return "", fmt.Errorf("failed to sign jwt: %v", err)
+	}
+
+	url := fmt.Sprintf("https://api.github.com/app/installations/%d/access_tokens", installationID)
+	req, err := http.NewRequest("POST", url, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request for installation token: %v", err)
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", jwtToken))
+	req.Header.Set("Accept", "application/vnd.github.v3+json")
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to request GitHub installation token: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusCreated {
+		return "", fmt.Errorf("failed to get installation token, status code: %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %v", err)
+	}
+
+	var tokenResponse struct {
+		Token string `json:"token"`
+	}
+	if err := json.Unmarshal(body, &tokenResponse); err != nil {
+		return "", fmt.Errorf("failed to unmarshal token response: %v", err)
+	}
+
+	if len(tokenResponse.Token) == 0 {
+		return "", fmt.Errorf("installation token not found in response")
+	}
+
+	return tokenResponse.Token, nil
+}