From 970e76484bd70f6efe533a2389e9cc14a4c156c6 Mon Sep 17 00:00:00 2001 From: Lorenz Hohmann Date: Wed, 15 Jan 2025 17:31:59 +0100 Subject: [PATCH] DockerProvider: Add mounting a kubeconfig file to environments --- backend/src/Configuration.ts | 48 +++++++++++++++++++++++++ backend/src/Environment.ts | 2 ++ backend/src/KubernetesManager.ts | 42 ++++++++++++++++++++++ backend/src/providers/DockerProvider.ts | 48 +++++++++++++++++++------ backend/src/providers/Provider.ts | 1 + backend/src/routes/k8s.ts | 5 ++- 6 files changed, 134 insertions(+), 12 deletions(-) diff --git a/backend/src/Configuration.ts b/backend/src/Configuration.ts index c71bb9b..cd72c47 100644 --- a/backend/src/Configuration.ts +++ b/backend/src/Configuration.ts @@ -708,6 +708,54 @@ environments.set("KommProt-Ue1-Test", { "/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md", }); +environments.set("CC-Test", { + providerImage: "cc-container", + // providerImage: "ssh:ubuntu", + providerDockerCmd: "", + mountKubeconfig: true, + terminals: [ + [ + { + type: "Shell", + name: "host1", + cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer", + executable: "", + params: [], + provideTty: true, + }, + ], + ], + editableFiles: [ + { + absFilePath: + "/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md", + alias: "README", + }, + ], + stopCommands: [ + { + type: "Shell", + name: "host1", + cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer", + executable: "sudo clab destroy", + params: [], + provideTty: false, + }, + { + type: "Shell", + name: "host2", + cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer", + executable: "", + params: [], + provideTty: false, + }, + ], + description: "CC-Test description", + assignmentLabSheetLocation: "instance", + assignmentLabSheet: + "/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md", +}); + export default environments; export function updateEnvironments( diff --git a/backend/src/Environment.ts b/backend/src/Environment.ts index 7053dea..9efda20 100644 --- a/backend/src/Environment.ts +++ b/backend/src/Environment.ts @@ -146,6 +146,7 @@ export interface EnvironmentDescription { useCollaboration?: boolean; useLanguageClient?: boolean; maxBonusPoints?: number; + mountKubeconfig?: boolean; } const DenyStartOfMissingInstanceErrorMessage = @@ -571,6 +572,7 @@ export default class Environment { rootDrive: this.configuration.providerRootDrive, proxmoxTemplateTag: this.configuration.providerProxmoxTemplateTag, + mountKubeconfig: this.configuration.mountKubeconfig, }, ), ); diff --git a/backend/src/KubernetesManager.ts b/backend/src/KubernetesManager.ts index 429d1eb..c7eced0 100644 --- a/backend/src/KubernetesManager.ts +++ b/backend/src/KubernetesManager.ts @@ -1,4 +1,5 @@ import { Client } from "ssh2"; +import fs from "fs"; export interface KubernetesCert { key: string; @@ -318,4 +319,45 @@ users: client-key-data: ${cert.key} `; } + + /** + * This function stores a kubeconfig file on the local filesystem. It overwrites any existing file. + * @param kubeconfig - The kubeconfig file to be stored. + * @param groupNumber - The group number of the user. + */ + storeLocalKubeconfig(kubeconfig: string, groupNumber: number): void { + const localPath = process.env.KUBECTL_STORE_PATH || "/tmp"; + + fs.writeFileSync( + `${localPath}/kubeconfig-group-${groupNumber}`, + kubeconfig, + ); + } + + /** + * This function deletes a kubeconfig file from the local filesystem. + * @param groupNumber - The group number of the user. + */ + deleteLocalKubeconfig(groupNumber: number): void { + const localPath = process.env.KUBECTL_STORE_PATH || "/tmp"; + + if (fs.existsSync(`${localPath}/kubeconfig-group-${groupNumber}`)) { + fs.unlinkSync(`${localPath}/kubeconfig-group-${groupNumber}`); + } + } + + /** + * Get the path to the kubeconfig file on the local filesystem. + * @param groupNumber - The group number of the user. + * @returns The path to the kubeconfig file as a string. + */ + getLocalKubeconfigPath(groupNumber: number): string { + const localPath = process.env.KUBECTL_STORE_PATH || "/tmp"; + + if (!fs.existsSync(`${localPath}/kubeconfig-group-${groupNumber}`)) { + throw new Error("KubernetesManager: Kubeconfig file not found."); + } + + return `${localPath}/kubeconfig-group-${groupNumber}`; + } } diff --git a/backend/src/providers/DockerProvider.ts b/backend/src/providers/DockerProvider.ts index 6d149cc..ae29f6f 100644 --- a/backend/src/providers/DockerProvider.ts +++ b/backend/src/providers/DockerProvider.ts @@ -11,6 +11,7 @@ import Dockerode, { import { ToadScheduler, SimpleIntervalJob, AsyncTask } from "toad-scheduler"; import { Client } from "ssh2"; import Environment from "../Environment"; +import KubernetesManager from "../KubernetesManager"; const schedulerIntervalSeconds = 5 * 60; @@ -165,6 +166,7 @@ export default class DockerProvider implements InstanceProvider { image?: string; dockerCmd?: string; dockerSupplementalPorts?: string[]; + mountKubeconfig?: boolean; }, ): Promise { const containerImage = options.image ?? this.image; @@ -192,6 +194,28 @@ export default class DockerProvider implements InstanceProvider { portBindings[port] = [{}]; } + const envs: string[] = []; + const hostConfigBinds: string[] = []; + + // Add kubeconfig to container if requested + if (options.mountKubeconfig) { + const containerKubeconfigPath: string = "/home/p4/.kube/config"; + let kubeconfigPath: string; + + const k8s: KubernetesManager = new KubernetesManager(); + try { + kubeconfigPath = k8s.getLocalKubeconfigPath(groupNumber); + } catch (err) { + throw new Error( + "DockerProvider: Could not get kubeconfig path.\n" + + (err as Error).message, + ); + } + + envs.push(`KUBECONFIG=${containerKubeconfigPath}`); + hostConfigBinds.push(`${kubeconfigPath}:${containerKubeconfigPath}`); + } + const containerOptions: ContainerCreateOptions = { Image: containerImage, name: `${username}-${groupNumber}-${environment}`, @@ -207,7 +231,9 @@ export default class DockerProvider implements InstanceProvider { PortBindings: portBindings, Privileged: true, AutoRemove: true, + Binds: hostConfigBinds, }, + Env: envs, }; // TODO: handle container already exists? @@ -347,22 +373,22 @@ export default class DockerProvider implements InstanceProvider { `${container.Names[0]} was created at ${createdAt.toISOString()} and should be deleted`, ); - const deleted = await Environment.deleteInstanceEnvironments(container.Id).catch( - (reason) => { - const originalMessage = - reason instanceof Error ? reason.message : "Unknown error"; - console.log( - `DockerProvider: Error while deleting environment after pruning container (${container.Names[0]}).\n` + - originalMessage, - ); - }, - ); + const deleted = await Environment.deleteInstanceEnvironments( + container.Id, + ).catch((reason) => { + const originalMessage = + reason instanceof Error ? reason.message : "Unknown error"; + console.log( + `DockerProvider: Error while deleting environment after pruning container (${container.Names[0]}).\n` + + originalMessage, + ); + }); if (!deleted) { console.log( `DockerProvider: Could not delete environment during pruning, environment seams to be gone already, deleting leftover container: (${container.Names[0]}).`, ); await this.deleteServer(container.Id).catch((reason) => { - const originalMessage = + const originalMessage = reason instanceof Error ? reason.message : "Unknown error"; throw new Error( `DockerProvider: Failed to delete container (${container.Names[0]}) to be pruned.\n` + diff --git a/backend/src/providers/Provider.ts b/backend/src/providers/Provider.ts index e34b25c..112d917 100644 --- a/backend/src/providers/Provider.ts +++ b/backend/src/providers/Provider.ts @@ -27,6 +27,7 @@ export interface InstanceProvider { kernelBootARGs?: string; rootDrive?: string; proxmoxTemplateTag?: string; + mountKubeconfig?: boolean; }, ): Promise; getServer(instance: string): Promise; diff --git a/backend/src/routes/k8s.ts b/backend/src/routes/k8s.ts index 1e2bc47..24dfd66 100644 --- a/backend/src/routes/k8s.ts +++ b/backend/src/routes/k8s.ts @@ -18,7 +18,8 @@ export default (): Router => { .createUserCert(namespaceName) .then((cert: KubernetesCert) => { const kubeconfig: string = k8s.getKubeconfig(cert, namespaceName); - console.log(kubeconfig); + + k8s.storeLocalKubeconfig(kubeconfig, reqWithUser.user.groupNumber); k8s .setupNamespace(namespaceName) @@ -46,6 +47,8 @@ export default (): Router => { k8s .undeployNamespace(namespaceName) .then(() => { + k8s.deleteLocalKubeconfig(reqWithUser.user.groupNumber); + res .status(200) .json({ status: "success", message: "K8S undeploy successful" });