Feature/export import stage 3

packages/components/src/utils.ts

@@ -1,13 +1,13 @@
+import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'
 import axios from 'axios'
 import { load } from 'cheerio'
+import { AES, enc } from 'crypto-js'
 import * as fs from 'fs'
-import * as path from 'path'
 import { JSDOM } from 'jsdom'
-import { z } from 'zod'
+import * as path from 'path'
 import { DataSource } from 'typeorm'
+import { z } from 'zod'
 import { ICommonObject, IDatabaseEntity, IMessage, INodeData, IVariable } from './Interface'
-import { AES, enc } from 'crypto-js'
-import { AIMessage, HumanMessage, BaseMessage } from '@langchain/core/messages'
 
 export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
 export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank

packages/server/src/Interface.ts

@@ -1,5 +1,4 @@
-import { IAction } from 'flowise-components'
-import { ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
+import { IAction, ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
@@ -90,6 +89,23 @@ export interface ICredential {
     encryptedData: string
     updatedDate: Date
     createdDate: Date
+    isEncryptionKeyLost: boolean
+}
+
+export interface IEncryption {
+    id: string
+    name: string
+    encryptionKey: string
+    updatedDate: Date
+    createdDate: Date
+}
+
+export interface IEncryptionCredential {
+    id: string
+    encryptionId: string
+    credentialId: string
+    updatedDate: Date
+    createdDate: Date
 }
 
 export interface IVariable {

packages/server/src/controllers/export-import/index.ts

@@ -0,0 +1,15 @@
+import { NextFunction, Request, Response } from 'express'
+import exportImportService from '../../services/export-import'
+
+const getAll = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const apiResponse = await exportImportService.getAll()
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
+export default {
+    getAll
+}

packages/server/src/database/entities/Credential.ts

@@ -1,5 +1,5 @@
 /* eslint-disable */
-import { Entity, Column, PrimaryGeneratedColumn, Index, CreateDateColumn, UpdateDateColumn } from 'typeorm'
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from 'typeorm'
 import { ICredential } from '../../Interface'
 
 @Entity()
@@ -23,4 +23,7 @@ export class Credential implements ICredential {
     @Column({ type: 'timestamp' })
     @UpdateDateColumn()
     updatedDate: Date
+
+    @Column({ type: 'boolean', default: false })
+    isEncryptionKeyLost: boolean
 }

packages/server/src/database/entities/Encryption.ts

@@ -0,0 +1,23 @@
+/* eslint-disable */
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from 'typeorm'
+import { IEncryption } from '../../Interface'
+
+@Entity()
+export class Encryption implements IEncryption {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Column()
+    name: string
+
+    @Column()
+    encryptionKey: string
+
+    @Column({ type: 'timestamp' })
+    @CreateDateColumn()
+    createdDate: Date
+
+    @Column({ type: 'timestamp' })
+    @UpdateDateColumn()
+    updatedDate: Date
+}

packages/server/src/database/entities/EncryptionCredential.ts

@@ -0,0 +1,23 @@
+/* eslint-disable */
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from 'typeorm'
+import { IEncryptionCredential } from '../../Interface'
+
+@Entity()
+export class EncryptionCredential implements IEncryptionCredential {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Column()
+    encryptionId: string
+
+    @Column()
+    credentialId: string
+
+    @Column({ type: 'timestamp' })
+    @CreateDateColumn()
+    createdDate: Date
+
+    @Column({ type: 'timestamp' })
+    @UpdateDateColumn()
+    updatedDate: Date
+}

packages/server/src/database/entities/index.ts

@@ -1,21 +1,25 @@
+import { Assistant } from './Assistant'
 import { ChatFlow } from './ChatFlow'
 import { ChatMessage } from './ChatMessage'
 import { ChatMessageFeedback } from './ChatMessageFeedback'
 import { Credential } from './Credential'
-import { Tool } from './Tool'
-import { Assistant } from './Assistant'
-import { Variable } from './Variable'
 import { DocumentStore } from './DocumentStore'
 import { DocumentStoreFileChunk } from './DocumentStoreFileChunk'
+import { Encryption } from './Encryption'
+import { EncryptionCredential } from './EncryptionCredential'
 import { Lead } from './Lead'
+import { Tool } from './Tool'
 import { UpsertHistory } from './UpsertHistory'
+import { Variable } from './Variable'
 import { ApiKey } from './ApiKey'
 
 export const entities = {
     ChatFlow,
     ChatMessage,
     ChatMessageFeedback,
     Credential,
+    Encryption,
+    EncryptionCredential,
     Tool,
     Assistant,
     Variable,

packages/server/src/database/migrations/mariadb/1721309779046-AddEncryption.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddEncryption1721308320215 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`encryption\` (
+            \`id\` VARCHAR(36) NOT NULL,
+            \`name\` VARCHAR(255) NOT NULL,
+            \`encryptionKey\` VARCHAR(255) NOT NULL,
+            \`createdDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+            \`updatedDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+            PRIMARY KEY (\`id\`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`encryption_credential\` (
+            \`id\` VARCHAR(255) NOT NULL, 
+            \`encryptionId\` VARCHAR(36) NOT NULL, 
+            \`credentialId\` VARCHAR(36) NOT NULL, 
+            \`createdDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), 
+            \`updatedDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), 
+            FOREIGN KEY (\`encryptionId\`) REFERENCES \`encryption\`(\`id\`), 
+            FOREIGN KEY (\`credentialId\`) REFERENCES \`credential\`(\`id\`), 
+            PRIMARY KEY (\`id\`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS \`encryption_credential\`;`)
+        await queryRunner.query(`DROP TABLE IF EXISTS \`encryption\`;`)
+    }
+}

packages/server/src/database/migrations/mariadb/1722473901281-AddIsEncryptionKeyLostToCredential.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddIsEncryptionKeyLostToCredential1722473901281 implements MigrationInterface {
+    name = 'AddIsEncryptionKeyLostToCredential1722473901281'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` ADD COLUMN \`isEncryptionKeyLost\` TINYINT(1) NOT NULL DEFAULT 0;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` DROP COLUMN \`isEncryptionKeyLost\`;`)
+    }
+}

packages/server/src/database/migrations/mariadb/index.ts

@@ -22,7 +22,9 @@ import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-Add
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
 import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
+import { AddEncryption1721308320215 } from './1721309779046-AddEncryption'
 import { LongTextColumn1722301395521 } from './1722301395521-LongTextColumn'
+import { AddIsEncryptionKeyLostToCredential1722473901281 } from './1722473901281-AddIsEncryptionKeyLostToCredential'
 
 export const mariadbMigrations = [
     Init1693840429259,
@@ -46,8 +48,10 @@ export const mariadbMigrations = [
     AddLead1710832127079,
     AddLeadToChatMessage1711538023578,
     AddAgentReasoningToChatMessage1714679514451,
+    AddEncryption1721308320215,
     AddTypeToChatFlow1716300000000,
     AddApiKey1720230151480,
     AddActionToChatMessage1721078251523,
+    AddIsEncryptionKeyLostToCredential1722473901281,
     LongTextColumn1722301395521
 ]

packages/server/src/database/migrations/mysql/1721309779046-AddEncryption.ts

@@ -0,0 +1,30 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddEncryption1721308320215 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`encryption\` (
+            \`id\` VARCHAR(36) PRIMARY KEY NOT NULL, 
+            \`name\` VARCHAR(255) NOT NULL, \`encryptionKey\` VARCHAR(255) NOT NULL, 
+            \`createdDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), 
+            \`updatedDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`encryption_credential\` (
+            \`id\` VARCHAR(255) PRIMARY KEY NOT NULL, 
+            \`encryptionId\` VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, 
+            \`credentialId\` VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, 
+            \`createdDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), 
+            \`updatedDate\` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), 
+            FOREIGN KEY (\`encryptionId\`) REFERENCES \`encryption\`(\`id\`), 
+            FOREIGN KEY (\`credentialId\`) REFERENCES \`credential\`(\`id\`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS \`encryption_credential\`;`)
+        await queryRunner.query(`DROP TABLE IF EXISTS \`encryption\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/1722473901281-AddIsEncryptionKeyLostToCredential.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddIsEncryptionKeyLostToCredential1722473901281 implements MigrationInterface {
+    name = 'AddIsEncryptionKeyLostToCredential1722473901281'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` ADD COLUMN \`isEncryptionKeyLost\` TINYINT(1) NOT NULL DEFAULT 0;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`credential\` DROP COLUMN \`isEncryptionKeyLost\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -17,13 +17,15 @@ import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
 import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
 import { AddLead1710832127079 } from './1710832127079-AddLead'
 import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
-import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
+import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
 import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
+import { AddEncryption1721308320215 } from './1721309779046-AddEncryption'
 import { LongTextColumn1722301395521 } from './1722301395521-LongTextColumn'
+import { AddIsEncryptionKeyLostToCredential1722473901281 } from './1722473901281-AddIsEncryptionKeyLostToCredential'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -47,9 +49,11 @@ export const mysqlMigrations = [
     AddLead1710832127079,
     AddLeadToChatMessage1711538023578,
     AddAgentReasoningToChatMessage1714679514451,
+    AddEncryption1721308320215,
     AddTypeToChatFlow1716300000000,
     AddVectorStoreConfigToDocStore1715861032479,
     AddApiKey1720230151480,
     AddActionToChatMessage1721078251523,
+    AddIsEncryptionKeyLostToCredential1722473901281,
     LongTextColumn1722301395521
 ]

packages/server/src/database/migrations/postgres/1721309779046-AddEncryption.ts

@@ -0,0 +1,32 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddEncryption1721308320215 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "encryption" (
+            "id" UUID PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(), 
+            "name" VARCHAR NOT NULL, 
+            "encryptionKey" VARCHAR NOT NULL, 
+            "createdDate" TIMESTAMP NOT NULL DEFAULT now(), 
+            "updatedDate" TIMESTAMP NOT NULL DEFAULT now()
+            );`
+        )
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS "encryption_credential" (
+            "id" UUID PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(), 
+            "encryptionId" UUID NOT NULL, 
+            "credentialId" UUID NOT NULL, 
+            "createdDate" TIMESTAMP NOT NULL DEFAULT now(), 
+            "updatedDate" TIMESTAMP NOT NULL DEFAULT now(), 
+            CONSTRAINT "fk_encryption" FOREIGN KEY ("encryptionId") REFERENCES "encryption"("id"), 
+            CONSTRAINT "fk_credential" FOREIGN KEY ("credentialId") REFERENCES "credential"("id"));`
+        )
+        await queryRunner.query(`CREATE INDEX IF NOT EXISTS "idx_encryptionId" ON"encryption_credential"("encryptionId");`)
+        await queryRunner.query(`CREATE INDEX IF NOT EXISTS "idx_credentialId" ON"encryption_credential"("credentialId");`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS encryption_credential;`)
+        await queryRunner.query(`DROP TABLE IF EXISTS encryption;`)
+    }
+}

packages/server/src/database/migrations/postgres/1722473901281-AddIsEncryptionKeyLostToCredential.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddIsEncryptionKeyLostToCredential1722473901281 implements MigrationInterface {
+    name = 'AddIsEncryptionKeyLostToCredential1722473901281'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "credential" ADD COLUMN "isEncryptionKeyLost" BOOLEAN NOT NULL DEFAULT FALSE;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "credential" DROP COLUMN "isEncryptionKeyLost";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -18,12 +18,14 @@ import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHi
 import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
 import { AddLead1710832137905 } from './1710832137905-AddLead'
 import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'
-import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
 import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
+import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
 import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
 import { AddApiKey1720230151480 } from './1720230151480-AddApiKey'
 import { AddActionToChatMessage1721078251523 } from './1721078251523-AddActionToChatMessage'
+import { AddEncryption1721308320215 } from './1721309779046-AddEncryption'
+import { AddIsEncryptionKeyLostToCredential1722473901281 } from './1722473901281-AddIsEncryptionKeyLostToCredential'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -48,8 +50,10 @@ export const postgresMigrations = [
     AddLead1710832137905,
     AddLeadToChatMessage1711538016098,
     AddAgentReasoningToChatMessage1714679514451,
+    AddEncryption1721308320215,
     AddTypeToChatFlow1716300000000,
     AddVectorStoreConfigToDocStore1715861032479,
     AddApiKey1720230151480,
-    AddActionToChatMessage1721078251523
+    AddActionToChatMessage1721078251523,
+    AddIsEncryptionKeyLostToCredential1722473901281
 ]