Merge pull request #1040 from polyseam/ow-no-unstage

[Feature]: `kubeseal` validation before shellout; `cndi ow` will destory any files that aren't regenerated;

src/actions/overwrite.worker.ts

@@ -107,24 +107,6 @@ self.onmessage = async (message: OverwriteWorkerMessage) => {
   if (message.data.type === "begin-overwrite") {
     const options = message.data.options as OverwriteActionOptions;
 
-    const pathToKubernetesManifests = path.join(
-      options.output,
-      "cndi",
-      "cluster_manifests",
-    );
-
-    const pathToTerraformResources = path.join(
-      options.output,
-      "cndi",
-      "terraform",
-    );
-
-    const pathToFunctionsOutput = path.join(
-      options.output,
-      "cndi",
-      "functions",
-    );
-
     const envPath = path.join(options.output, ".env");
 
     const [errorLoadingConfig, result] = await loadCndiConfig(options.output);
@@ -217,14 +199,6 @@ self.onmessage = async (message: OverwriteWorkerMessage) => {
       ".ts",
     );
 
-    try {
-      await Deno.remove(pathToFunctionsOutput, {
-        recursive: true,
-      });
-    } catch {
-      // folder did not exist
-    }
-
     if (shouldBuildFunctions) {
       const fnsWorkflowPath = path.join(
         ".github",
@@ -358,15 +332,6 @@ self.onmessage = async (message: OverwriteWorkerMessage) => {
       );
     }
 
-    try {
-      // remove all files in cndi/terraform
-      await Deno.remove(pathToTerraformResources, {
-        recursive: true,
-      });
-    } catch {
-      // folder did not exist
-    }
-
     const terraformStatePassphrase = Deno.env.get("TERRAFORM_STATE_PASSPHRASE");
 
     if (!terraformStatePassphrase) {
@@ -488,15 +453,6 @@ self.onmessage = async (message: OverwriteWorkerMessage) => {
         }
       }
 
-      try {
-        // remove all files in cndi/cluster
-        await Deno.remove(pathToKubernetesManifests, {
-          recursive: true,
-        });
-      } catch {
-        // folder did not exist
-      }
-
       await Deno.writeTextFile(
         tempPublicKeyFilePath,
         sealedSecretsKeys.sealed_secrets_public_key,
@@ -823,7 +779,9 @@ self.onmessage = async (message: OverwriteWorkerMessage) => {
 
     console.log(ccolors.success("staged terraform stack"));
 
-    const errPersistingStagedFiles = await persistStagedFiles(options.output);
+    const errPersistingStagedFiles = await persistStagedFiles(options.output, {
+      purge: ["cndi"], // effectively says "always empty contents of cndi/ before staging"
+    });
 
     if (errPersistingStagedFiles) {
       await self.postMessage(errPersistingStagedFiles.owWorkerErrorMessage);

src/outputs/application-manifest.ts

@@ -173,6 +173,18 @@ const getApplicationManifest = (
     applicationSpec?.syncPolicy || {},
   );
 
+  const finalSyncOptions: Record<string, string> = {};
+
+  for (const syncOption of syncPolicy.syncOptions) {
+    const [name, status] = syncOption.split("=");
+    finalSyncOptions[name] = status;
+  }
+
+  // syncOptions are deduped and the user-supplied values are preferred
+  syncPolicy.syncOptions = Object.entries(finalSyncOptions).map(([k, v]) =>
+    `${k}=${v}`
+  );
+
   const {
     repoURL,
     path,

src/tests/commands/ow.test.ts

@@ -5,6 +5,7 @@ import { runCndi } from "src/tests/helpers/run-cndi.ts";
 import {
   listAllFilePaths,
   listChangedFilePaths,
+  listFilepathsBeforeAndAfter,
   setsAreEquivalent,
 } from "src/tests/helpers/util.ts";
 
@@ -30,11 +31,11 @@ Deno.test(
     });
 
     await t.step("test", async () => {
-      const changedFilePaths = await listChangedFilePaths(async () => {
+      const { before, after } = await listFilepathsBeforeAndAfter(async () => {
         const { status } = await runCndi("ow");
         assert(status.success);
       });
-      assert(changedFilePaths.length === 0);
+      assert(setsAreEquivalent(new Set(before), new Set(after)));
     });
 
     await t.step("cleanup", cleanup);
@@ -65,6 +66,112 @@ Deno.test(
   },
 );
 
+Deno.test(`'cndi ow' should successfully convert secrets if correctly defined`, async (t) => {
+  let dir = "";
+  await t.step("setup", async () => {
+    dir = await Deno.makeTempDir();
+    Deno.chdir(dir);
+    await runCndi("init", "-t", "basic", "-l", "aws/eks");
+  });
+
+  await t.step("test", async () => {
+    const { before, after } = await listFilepathsBeforeAndAfter(async () => {
+      const [errorLoadingConfig, result] = await loadCndiConfig(dir);
+
+      if (errorLoadingConfig) {
+        assert(false, errorLoadingConfig.message);
+      }
+
+      Deno.env.set("FOOSECRET", "realstring");
+
+      const { config } = result;
+      config.cluster_manifests["new-secret"] = {
+        kind: "Secret",
+        metadata: {
+          name: "my-secret",
+        },
+        stringData: {
+          password: "$cndi_on_ow.seal_secret_from_env_var(FOOSECRET)",
+        },
+      };
+      config.cluster_manifests["new-secret-yet-to-be-defined"] = {
+        kind: "Secret",
+        metadata: {
+          name: "my-secret-2",
+        },
+        stringData: {
+          // BAR_SECRET will not be defined and should be included in .env with a placeholder
+          magic: "$cndi_on_ow.seal_secret_from_env_var(BAR_SECRET)",
+        },
+      };
+
+      await Deno.writeTextFile(
+        `${dir}/cndi_config.yaml`,
+        YAML.stringify(config),
+      );
+      await runCndi("ow", "--loud");
+    });
+
+    const beforeSet = new Set(before);
+    const afterSet = new Set(after);
+
+    const added = afterSet.difference(beforeSet);
+    const _removed = beforeSet.difference(afterSet);
+
+    const expectedToAdd = new Set([
+      path.join("cndi", "cluster_manifests", "new-secret.yaml"),
+      // new-secret-yet-to-be-defined will update .env with a placeholder, but not yield a secret
+    ]);
+
+    assert(setsAreEquivalent(added, expectedToAdd));
+    const dotenv = await Deno.readTextFile(path.join(dir, ".env"));
+    assert(dotenv.includes("BAR_SECRET"));
+  });
+
+  await t.step("cleanup", cleanup);
+});
+
+Deno.test(
+  `'cndi ow' should fail if secrets use plaintext in their definition`,
+  async (t) => {
+    let dir = "";
+    await t.step("setup", async () => {
+      dir = await Deno.makeTempDir();
+      Deno.chdir(dir);
+      await runCndi("init", "-t", "basic", "-l", "aws/eks");
+    });
+
+    await t.step("test", async () => {
+      const { before, after } = await listFilepathsBeforeAndAfter(async () => {
+        const [errorLoadingConfig, result] = await loadCndiConfig(dir);
+
+        if (errorLoadingConfig) {
+          assert(false, errorLoadingConfig.message);
+        }
+
+        const { config } = result;
+        config.cluster_manifests["new-secret"] = {
+          kind: "Secret",
+          metadata: {
+            name: "my-secret",
+          },
+          stringData: {
+            password: "plaintext",
+          },
+        };
+        await Deno.writeTextFile(
+          `${dir}/cndi_config.yaml`,
+          YAML.stringify(config),
+        );
+        await runCndi("ow");
+      });
+      assert(setsAreEquivalent(new Set(before), new Set(after)));
+    });
+
+    await t.step("cleanup", cleanup);
+  },
+);
+
 Deno.test(
   "'cndi ow' should create a manifest file if one has been added to config",
   async (t) => {

src/tests/helpers/util.ts

@@ -50,6 +50,21 @@ export async function listAllFilePaths(directory: string): Promise<string[]> {
   return filePaths;
 }
 
+type FilePathsBeforeAndAfter = {
+  before: string[];
+  after: string[];
+};
+
+export async function listFilepathsBeforeAndAfter(
+  operationFn: () => Promise<void>,
+  dir = ".",
+): Promise<FilePathsBeforeAndAfter> {
+  const before = await listAllFilePaths(dir);
+  await operationFn();
+  const after = await listAllFilePaths(dir);
+  return { before, after };
+}
+
 export async function listChangedFilePaths(
   operationFn: () => Promise<void>,
   dir = ".",