Skip to content

Commit

Permalink
DockerProvider: Add mounting a kubeconfig file to environments
Browse files Browse the repository at this point in the history
  • Loading branch information
lorenzhohmann committed Jan 15, 2025
1 parent 6de10b6 commit 970e764
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 12 deletions.
48 changes: 48 additions & 0 deletions backend/src/Configuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,54 @@ environments.set("KommProt-Ue1-Test", {
"/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md",
});

environments.set("CC-Test", {
providerImage: "cc-container",
// providerImage: "ssh:ubuntu",
providerDockerCmd: "",
mountKubeconfig: true,
terminals: [
[
{
type: "Shell",
name: "host1",
cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer",
executable: "",
params: [],
provideTty: true,
},
],
],
editableFiles: [
{
absFilePath:
"/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md",
alias: "README",
},
],
stopCommands: [
{
type: "Shell",
name: "host1",
cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer",
executable: "sudo clab destroy",
params: [],
provideTty: false,
},
{
type: "Shell",
name: "host2",
cwd: "/home/p4/kommprot-labs/kommprot-lab-application-layer",
executable: "",
params: [],
provideTty: false,
},
],
description: "CC-Test description",
assignmentLabSheetLocation: "instance",
assignmentLabSheet:
"/home/p4/kommprot-labs/kommprot-lab-application-layer/README.md",
});

export default environments;

export function updateEnvironments(
Expand Down
2 changes: 2 additions & 0 deletions backend/src/Environment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ export interface EnvironmentDescription {
useCollaboration?: boolean;
useLanguageClient?: boolean;
maxBonusPoints?: number;
mountKubeconfig?: boolean;
}

const DenyStartOfMissingInstanceErrorMessage =
Expand Down Expand Up @@ -571,6 +572,7 @@ export default class Environment {
rootDrive: this.configuration.providerRootDrive,
proxmoxTemplateTag:
this.configuration.providerProxmoxTemplateTag,
mountKubeconfig: this.configuration.mountKubeconfig,
},
),
);
Expand Down
42 changes: 42 additions & 0 deletions backend/src/KubernetesManager.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Client } from "ssh2";
import fs from "fs";

export interface KubernetesCert {
key: string;
Expand Down Expand Up @@ -318,4 +319,45 @@ users:
client-key-data: ${cert.key}
`;
}

/**
* This function stores a kubeconfig file on the local filesystem. It overwrites any existing file.
* @param kubeconfig - The kubeconfig file to be stored.
* @param groupNumber - The group number of the user.
*/
storeLocalKubeconfig(kubeconfig: string, groupNumber: number): void {
const localPath = process.env.KUBECTL_STORE_PATH || "/tmp";

fs.writeFileSync(
`${localPath}/kubeconfig-group-${groupNumber}`,
kubeconfig,
);
}

/**
* This function deletes a kubeconfig file from the local filesystem.
* @param groupNumber - The group number of the user.
*/
deleteLocalKubeconfig(groupNumber: number): void {
const localPath = process.env.KUBECTL_STORE_PATH || "/tmp";

if (fs.existsSync(`${localPath}/kubeconfig-group-${groupNumber}`)) {
fs.unlinkSync(`${localPath}/kubeconfig-group-${groupNumber}`);
}
}

/**
* Get the path to the kubeconfig file on the local filesystem.
* @param groupNumber - The group number of the user.
* @returns The path to the kubeconfig file as a string.
*/
getLocalKubeconfigPath(groupNumber: number): string {
const localPath = process.env.KUBECTL_STORE_PATH || "/tmp";

if (!fs.existsSync(`${localPath}/kubeconfig-group-${groupNumber}`)) {
throw new Error("KubernetesManager: Kubeconfig file not found.");
}

return `${localPath}/kubeconfig-group-${groupNumber}`;
}
}
48 changes: 37 additions & 11 deletions backend/src/providers/DockerProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import Dockerode, {
import { ToadScheduler, SimpleIntervalJob, AsyncTask } from "toad-scheduler";
import { Client } from "ssh2";
import Environment from "../Environment";
import KubernetesManager from "../KubernetesManager";

const schedulerIntervalSeconds = 5 * 60;

Expand Down Expand Up @@ -165,6 +166,7 @@ export default class DockerProvider implements InstanceProvider {
image?: string;
dockerCmd?: string;
dockerSupplementalPorts?: string[];
mountKubeconfig?: boolean;
},
): Promise<VMEndpoint> {
const containerImage = options.image ?? this.image;
Expand Down Expand Up @@ -192,6 +194,28 @@ export default class DockerProvider implements InstanceProvider {
portBindings[port] = [{}];
}

const envs: string[] = [];
const hostConfigBinds: string[] = [];

// Add kubeconfig to container if requested
if (options.mountKubeconfig) {
const containerKubeconfigPath: string = "/home/p4/.kube/config";
let kubeconfigPath: string;

const k8s: KubernetesManager = new KubernetesManager();
try {
kubeconfigPath = k8s.getLocalKubeconfigPath(groupNumber);
} catch (err) {
throw new Error(
"DockerProvider: Could not get kubeconfig path.\n" +
(err as Error).message,
);
}

envs.push(`KUBECONFIG=${containerKubeconfigPath}`);
hostConfigBinds.push(`${kubeconfigPath}:${containerKubeconfigPath}`);
}

const containerOptions: ContainerCreateOptions = {
Image: containerImage,
name: `${username}-${groupNumber}-${environment}`,
Expand All @@ -207,7 +231,9 @@ export default class DockerProvider implements InstanceProvider {
PortBindings: portBindings,
Privileged: true,
AutoRemove: true,
Binds: hostConfigBinds,
},
Env: envs,
};

// TODO: handle container already exists?
Expand Down Expand Up @@ -347,22 +373,22 @@ export default class DockerProvider implements InstanceProvider {
`${container.Names[0]} was created at ${createdAt.toISOString()} and should be deleted`,
);

const deleted = await Environment.deleteInstanceEnvironments(container.Id).catch(
(reason) => {
const originalMessage =
reason instanceof Error ? reason.message : "Unknown error";
console.log(
`DockerProvider: Error while deleting environment after pruning container (${container.Names[0]}).\n` +
originalMessage,
);
},
);
const deleted = await Environment.deleteInstanceEnvironments(
container.Id,
).catch((reason) => {
const originalMessage =
reason instanceof Error ? reason.message : "Unknown error";
console.log(
`DockerProvider: Error while deleting environment after pruning container (${container.Names[0]}).\n` +
originalMessage,
);
});
if (!deleted) {
console.log(
`DockerProvider: Could not delete environment during pruning, environment seams to be gone already, deleting leftover container: (${container.Names[0]}).`,
);
await this.deleteServer(container.Id).catch((reason) => {
const originalMessage =
const originalMessage =
reason instanceof Error ? reason.message : "Unknown error";
throw new Error(
`DockerProvider: Failed to delete container (${container.Names[0]}) to be pruned.\n` +
Expand Down
1 change: 1 addition & 0 deletions backend/src/providers/Provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export interface InstanceProvider {
kernelBootARGs?: string;
rootDrive?: string;
proxmoxTemplateTag?: string;
mountKubeconfig?: boolean;
},
): Promise<VMEndpoint>;
getServer(instance: string): Promise<VMEndpoint>;
Expand Down
5 changes: 4 additions & 1 deletion backend/src/routes/k8s.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ export default (): Router => {
.createUserCert(namespaceName)
.then((cert: KubernetesCert) => {
const kubeconfig: string = k8s.getKubeconfig(cert, namespaceName);
console.log(kubeconfig);

k8s.storeLocalKubeconfig(kubeconfig, reqWithUser.user.groupNumber);

k8s
.setupNamespace(namespaceName)
Expand Down Expand Up @@ -46,6 +47,8 @@ export default (): Router => {
k8s
.undeployNamespace(namespaceName)
.then(() => {
k8s.deleteLocalKubeconfig(reqWithUser.user.groupNumber);

res
.status(200)
.json({ status: "success", message: "K8S undeploy successful" });
Expand Down

0 comments on commit 970e764

Please sign in to comment.