(Feat): Interacting with projects programmatically

.github/workflows/check-api-for-breaking-changes.yml

@@ -72,4 +72,4 @@ jobs:
         run: |
           docker-compose -f "docker-compose.dev.yml" down
           docker stop infisical-api
-          docker remove infisical-api
+          docker remove infisical-api

.github/workflows/run-backend-tests.yml

@@ -44,4 +44,4 @@ jobs:
           ENCRYPTION_KEY: 4bnfe4e407b8921c104518903515b218
       - name: cleanup
         run: |
-          docker-compose -f "docker-compose.dev.yml" down
+          docker-compose -f "docker-compose.dev.yml" down

backend/e2e-test/vitest-environment-knex.ts

@@ -56,6 +56,7 @@ export default {
         { expiresIn: cfg.JWT_AUTH_LIFETIME }
       );
     } catch (error) {
+      console.log("[TEST] Error setting up environment", error);
       await db.destroy();
       throw error;
     }

backend/src/db/migrations/20240216154123_ghost_users.ts

@@ -0,0 +1,39 @@
+import { Knex } from "knex";
+
+import { ProjectVersion, TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (!hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.boolean("isGhost").defaultTo(false).notNullable();
+    });
+  }
+
+  if (!hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.integer("version").defaultTo(ProjectVersion.V1).notNullable();
+      t.string("upgradeStatus").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "isGhost");
+  const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version");
+
+  if (hasGhostUserColumn) {
+    await knex.schema.alterTable(TableName.Users, (t) => {
+      t.dropColumn("isGhost");
+    });
+  }
+
+  if (hasProjectVersionColumn) {
+    await knex.schema.alterTable(TableName.Project, (t) => {
+      t.dropColumn("version");
+      t.dropColumn("upgradeStatus");
+    });
+  }
+}

backend/src/db/schemas/models.ts

@@ -112,6 +112,17 @@ export enum SecretType {
   Personal = "personal"
 }
 
+export enum ProjectVersion {
+  V1 = 1,
+  V2 = 2
+}
+
+export enum ProjectUpgradeStatus {
+  InProgress = "IN_PROGRESS",
+  // Completed -> Will be null if completed. So a completed status is not needed
+  Failed = "FAILED"
+}
+
 export enum IdentityAuthMethod {
   Univeral = "universal-auth"
 }

backend/src/db/schemas/projects.ts

@@ -14,7 +14,9 @@ export const ProjectsSchema = z.object({
   autoCapitalization: z.boolean().default(true).nullable().optional(),
   orgId: z.string().uuid(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  version: z.number().default(1),
+  upgradeStatus: z.string().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;

backend/src/db/schemas/users.ts

@@ -19,7 +19,8 @@ export const UsersSchema = z.object({
   mfaMethods: z.string().array().nullable().optional(),
   devices: z.unknown().nullable().optional(),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  isGhost: z.boolean().default(false)
 });
 
 export type TUsers = z.infer<typeof UsersSchema>;

backend/src/db/seed-data.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-mutable-exports */
 import crypto from "node:crypto";
 
 import argon2, { argon2id } from "argon2";
@@ -15,9 +16,12 @@ import {
 
 import { TSecrets, TUserEncryptionKeys } from "./schemas";
 
+export let userPrivateKey: string | undefined;
+export let userPublicKey: string | undefined;
+
 export const seedData1 = {
   id: "3dafd81d-4388-432b-a4c5-f735616868c1",
-  email: "[email protected]",
+  email: process.env.TEST_USER_EMAIL || "[email protected]",
   password: process.env.TEST_USER_PASSWORD || "testInfisical@1",
   organization: {
     id: "180870b7-f464-4740-8ffe-9d11c9245ea7",
@@ -42,6 +46,12 @@ export const seedData1 = {
   },
   token: {
     id: "a9dfafba-a3b7-42e3-8618-91abb702fd36"
+  },
+
+  // We set these values during user creation, and later re-use them during project seeding.
+  encryptionKeys: {
+    publicKey: "",
+    privateKey: ""
   }
 };
 

backend/src/ee/services/audit-log/audit-log-service.ts

@@ -58,6 +58,7 @@ export const auditLogServiceFactory = ({
     if (data.event.type !== EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH) {
       if (!data.projectId && !data.orgId) throw new BadRequestError({ message: "Must either project id or org id" });
     }
+
     return auditLogQueue.pushToLog(data);
   };
 

backend/src/ee/services/saml-config/saml-config-service.ts

@@ -338,7 +338,8 @@ export const samlConfigServiceFactory = ({
             email,
             firstName,
             lastName,
-            authMethods: [AuthMethod.EMAIL]
+            authMethods: [AuthMethod.EMAIL],
+            isGhost: false
           },
           tx
         );

backend/src/ee/services/scim/scim-service.ts

@@ -251,7 +251,8 @@ export const scimServiceFactory = ({
             email,
             firstName,
             lastName,
-            authMethods: [AuthMethod.EMAIL]
+            authMethods: [AuthMethod.EMAIL],
+            isGhost: false
           },
           tx
         );

backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts

@@ -1,8 +1,13 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { SecretApprovalRequestsSecretsSchema, TableName, TSecretTags } from "@app/db/schemas";
-import { DatabaseError } from "@app/lib/errors";
+import {
+  SecretApprovalRequestsSecretsSchema,
+  TableName,
+  TSecretApprovalRequestsSecrets,
+  TSecretTags
+} from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
 
 export type TSecretApprovalRequestSecretDALFactory = ReturnType<typeof secretApprovalRequestSecretDALFactory>;
@@ -11,6 +16,35 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
   const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret);
   const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag);
 
+  const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => {
+    try {
+      const existingApprovalSecrets = await secretApprovalRequestSecretOrm.find(
+        {
+          $in: {
+            id: data.map((el) => el.id)
+          }
+        },
+        { tx }
+      );
+
+      if (existingApprovalSecrets.length !== data.length) {
+        throw new BadRequestError({ message: "Some of the secret approvals do not exist" });
+      }
+
+      if (data.length === 0) return [];
+
+      const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret)
+        .insert(data)
+        .onConflict("id") // this will cause a conflict then merge the data
+        .merge() // Merge the data with the existing data
+        .returning("*");
+
+      return updatedApprovalSecrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
   const findByRequestId = async (requestId: string, tx?: Knex) => {
     try {
       const doc = await (tx || db)({
@@ -190,6 +224,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
   return {
     ...secretApprovalRequestSecretOrm,
     findByRequestId,
+    bulkUpdateNoVersionIncrement,
     insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany
   };
 };

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -11,6 +11,7 @@ import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
 import { groupBy, pick, unique } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { ActorType } from "@app/services/auth/auth-type";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -47,6 +48,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany">;
+  projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   secretService: Pick<
     TSecretServiceFactory,
     | "fnSecretBulkInsert"
@@ -67,6 +69,7 @@ export const secretApprovalRequestServiceFactory = ({
   secretApprovalRequestReviewerDAL,
   secretApprovalRequestSecretDAL,
   secretBlindIndexDAL,
+  projectDAL,
   permissionService,
   snapshotService,
   secretService,
@@ -434,6 +437,8 @@ export const secretApprovalRequestServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
+    await projectDAL.checkProjectUpgradeStatus(projectId);
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "GenSecretApproval" });
     const folderId = folder.id;

backend/src/lib/crypto/encryption.ts

@@ -8,6 +8,9 @@ import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
 
 import { getConfig } from "../config/env";
 
+export const decodeBase64 = (s: string) => naclUtils.decodeBase64(s);
+export const encodeBase64 = (u: Uint8Array) => naclUtils.encodeBase64(u);
+
 export type TDecryptSymmetricInput = {
   ciphertext: string;
   iv: string;

backend/src/lib/crypto/index.ts

@@ -1,12 +1,20 @@
 export {
   buildSecretBlindIndexFromName,
   createSecretBlindIndex,
+  decodeBase64,
   decryptAsymmetric,
   decryptSymmetric,
   decryptSymmetric128BitHexKeyUTF8,
+  encodeBase64,
   encryptAsymmetric,
   encryptSymmetric,
   encryptSymmetric128BitHexKeyUTF8,
   generateAsymmetricKeyPair
 } from "./encryption";
+export {
+  decryptIntegrationAuths,
+  decryptSecretApprovals,
+  decryptSecrets,
+  decryptSecretVersions
+} from "./secret-encryption";
 export { generateSrpServerKey, srpCheckClientProof } from "./srp";