diff --git a/packages/insomnia/src/common/database-provider/conditional.ts b/packages/insomnia/src/common/database-provider/conditional.ts new file mode 100644 index 000000000..710ae3a87 --- /dev/null +++ b/packages/insomnia/src/common/database-provider/conditional.ts @@ -0,0 +1,73 @@ +import { BaseModel } from "../../models" +import { BaseImplementation, Database, Query, Sort } from "./index" +import { MultipleDatabase } from "./multiple" + +export class ConditionalWriter extends MultipleDatabase { + constructor(primary: Database, secondary: Database, rootId: string) { + super(primary, [new RejectWriter(secondary, rootId)]) + } +} + +class RejectWriter extends BaseImplementation { + constructor(private readonly database: Database, private rootId: string) { + super() + } + + private async getIds(doc: T): Promise { + return [doc._id, ...(await this.database.withAncestors(doc)).map(doc => doc._id)] + } + + private async inPath(doc: T): Promise { + if (doc._id === this.rootId) { + return true; + } + if (doc.parentId === this.rootId) { + return true; + } + return (await this.getIds(doc)).includes(this.rootId); + } + + count(type: string, query?: Query | undefined): Promise { + return this.database.count(type, query) + } + find(type: string, query?: string | Query | undefined, sort?: Sort | undefined): Promise { + return this.database.find(type, query, sort) + } + findMostRecentlyModified(type: string, query?: Query | undefined, limit?: number | null | undefined): Promise { + return this.database.findMostRecentlyModified(type, query, limit); + } + init(types: string[], config: object, forceReset?: boolean | undefined, consoleLog?: { (...data: any[]): void; (message?: any, ...optionalParams: any[]): void } | undefined): Promise { + return this.database.init(types, config, forceReset, consoleLog) + } + initClient(): Promise { + return this.database.initClient() + } + async insert(doc: T, fromSync?: boolean | undefined, initializeModel?: boolean | undefined): Promise { + if (!await this.inPath(doc)) { + throw new Error() + } + return this.insert(doc, fromSync, initializeModel) + } + async remove(doc: T, fromSync?: boolean | undefined): Promise { + if (!await this.inPath(doc)) { + throw new Error() + } + return this.remove(doc, fromSync) + } + async removeWhere(type: string, query: Query): Promise { + const docs = await this.find(type, query) + await Promise.allSettled(docs.map(doc => this.remove(doc, false))) + } + async unsafeRemove(doc: T, fromSync?: boolean | undefined): Promise { + if (!await this.inPath(doc)) { + throw new Error() + } + return this.unsafeRemove(doc, fromSync) + } + async update(doc: T, fromSync?: boolean | undefined): Promise { + if (!await this.inPath(doc)) { + throw new Error() + } + return this.update(doc, fromSync) + } +} diff --git a/packages/insomnia/src/common/database-provider/fs.ts b/packages/insomnia/src/common/database-provider/fs.ts new file mode 100644 index 000000000..2957a5a2e --- /dev/null +++ b/packages/insomnia/src/common/database-provider/fs.ts @@ -0,0 +1,191 @@ +import * as models from "../../models"; +import { BaseModel } from "../../models"; +import { + BaseImplementation, + ChangeBufferEvent, + Database, + Query, + Sort, +} from "./index"; +import { + access, + mkdir, + readdir, + readFile, + unlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import electron from "electron"; + +class FSDatabase extends BaseImplementation implements Database { + private rootPath: string = ""; + private getDBFilePath(modelType: string) { + // NOTE: Do not EVER change this. EVER! + return join(this.rootPath, "insomnia.DB", modelType); + } + + private getDocFilePath( + doc: T | { _id: string; type: string } + ): string { + // NOTE: Do not EVER change this. EVER! + return join(this.getDBFilePath(doc.type), `${doc._id}.json`); + } + + count( + type: string, + query?: Query | undefined + ): Promise { + if (query !== undefined) { + throw new Error("Method not implemented."); + } + return readdir(this.getDBFilePath(type)).then( + (paths) => paths.filter((path) => path.endsWith(".json")).length + ); + } + + async find( + type: string, + query?: Query, + sort?: Sort + ): Promise { + if ( + query === undefined || + (typeof query === "object" && Object.keys(query).length === 0) + ) { + let filesPath: string[] = []; + try { + filesPath = (await readdir(this.getDBFilePath(type))).filter((path) => + path.endsWith(".json") + ); + } catch (e) { + console.error(e); + filesPath = []; + } + const rawFiles = await Promise.all( + filesPath.map((path) => + readFile(join(this.getDBFilePath(type), path), { + encoding: "utf8", + }).then((body) => JSON.parse(body) as T) + ) + ); + const files = await Promise.all( + rawFiles.map((file) => models.initModel(type, file)) + ); + return BaseImplementation.sortList(files, sort); + } + + return BaseImplementation.filterList(await this.find(type), query); + } + + get( + type: string, + id?: string | undefined + ): Promise { + if (!id && id === "n/a") return Promise.resolve(null); + return readFile(this.getDocFilePath({ _id: id!, type }), { + encoding: "utf8", + }).then((body) => JSON.parse(body) as T); + } + + async init( + types: string[], + config = {}, + forceReset: boolean = false, + consoleLog: typeof console.log = console.log + ): Promise { + this.rootPath ||= + process.env["INSOMNIA_DATA_PATH"] || electron.app?.getPath("userData"); + await Promise.all( + types.map(async (type) => { + const path = this.getDBFilePath(type); + try { + await access(path); + } catch (e) { + await mkdir(path, { mode: 0o755, recursive: true }); + } + }) + ); + } + + async initClient() { + this.rootPath ||= + process.env["INSOMNIA_DATA_PATH"] || window?.app?.getPath("userData"); + console.log("[db] Initialized DB client"); + } + + async insert( + doc: T, + fromSync: boolean = false, + initializeModel: boolean = true + ): Promise { + const docWithDefaults = initializeModel + ? await models.initModel(doc.type, doc) + : doc; + + return writeFile( + this.getDocFilePath(docWithDefaults), + JSON.stringify(docWithDefaults), + { encoding: "utf8" } + ).then((_) => { + // NOTE: This needs to be after we resolve + this.notifyOfChange("insert", docWithDefaults, fromSync); + return docWithDefaults; + }); + } + + async remove( + doc: T, + fromSync: boolean = false + ): Promise { + const flushId = await this.bufferChanges(); + + const docs = await this.withDescendants(doc); + + await Promise.all(docs.map((doc) => unlink(this.getDocFilePath(doc)))); + docs.map((d) => this.notifyOfChange("remove", d, fromSync)); + await this.flushChanges(flushId); + } + + async removeWhere(type: string, query: Query): Promise { + const flushId = await this.bufferChanges(); + + const foundDocs = await this.find(type, query); + await Promise.all( + foundDocs.map(async (doc) => { + const docs = await this.withDescendants(doc); + + await Promise.all(docs.map((doc) => unlink(this.getDocFilePath(doc)))); + docs.map((d) => this.notifyOfChange("remove", d, false)); + }) + ); + await this.flushChanges(flushId); + } + + async unsafeRemove( + doc: T, + fromSync: boolean = false + ): Promise { + await unlink(this.getDocFilePath(doc)); + this.notifyOfChange("remove", doc, fromSync); + } + + async update( + doc: T, + fromSync: boolean = false + ): Promise { + const docWithDefaults = await models.initModel(doc.type, doc); + + return writeFile( + this.getDocFilePath(docWithDefaults), + JSON.stringify(docWithDefaults), + { encoding: "utf8" } + ).then((_) => { + // NOTE: This needs to be after we resolve + this.notifyOfChange("update", docWithDefaults, fromSync); + return docWithDefaults; + }); + } +} + +export const database = new FSDatabase(); diff --git a/packages/insomnia/src/common/database-provider/git.ts b/packages/insomnia/src/common/database-provider/git.ts new file mode 100644 index 000000000..bc9b49fe2 --- /dev/null +++ b/packages/insomnia/src/common/database-provider/git.ts @@ -0,0 +1,262 @@ +import { BaseModel } from "../../models"; +import { BaseImplementation, Query, Sort } from "./index" +import * as models from "../../models" +import { join } from "node:path" + +export const FileMode = { + symlink(): number { + return 120000 + }, + tree(): number { + return 40000 + }, + normal(permissions: Record<"owner" | "group" | "other", {write?: boolean; read?:boolean; execute?:boolean}> | string | number): number { + function bool2number({ write, read, execute }: {write?: boolean; read?:boolean; execute?:boolean}) { + let mode = 0; + if (execute) mode += 1; + if (write) mode += 2; + if (read) mode += 4; + return mode + } + if (typeof permissions === "number") { + return 100_000 + permissions; + } + if (typeof permissions === "string") { + const groups = Array.from(permissions.matchAll(/(?...)/g)).map(i => i.groups?.mode ?? '---') + return FileMode.normal({ + owner: { + read: groups[0].charAt(0) === 'r', + write: groups[0].charAt(1) === 'w', + execute: groups[0].charAt(2) === 'x', + }, + group: { + read: groups[1].charAt(0) === 'r', + write: groups[1].charAt(1) === 'w', + execute: groups[1].charAt(2) === 'x', + }, + other: { + read: groups[2].charAt(0) === 'r', + write: groups[2].charAt(1) === 'w', + execute: groups[2].charAt(2) === 'x', + }, + }) + } + return 100_000 + bool2number(permissions.owner) * 100 + bool2number(permissions.group) * 10 + bool2number(permissions.owner) + }, +} + +interface GitRunner { + /** + * `echo '<>' | git hash-object -w --stdin` + * @example `hashObject('hello world');` + * @param content The content of the blob + * @return the blob hash + */ + hashObject(content: string): Promise; + + /** + * `git update-index --add --cacheinfo <>,<>,<>` + * @example `updateIndexAdd(100664, '3b18e512dba79e4c8300dd08aeb37f8e728b8dad', 'foobar.txt');` + * @param mode + * @param objectHash + * @param path + */ + updateIndexAdd(mode: number, objectHash: string, path: string): Promise; + + /** + * `git update-index --remove -- <>` + * @example `updateIndexRemove('foobar.txt');` + * @param path + */ + updateIndexRemove(path: string): Promise; + + /** + * `git write-tree` + */ + writeTree(): Promise; + + /** + * `echo '<>' | git commit-tree <> -p <>` + * `echo '<>' | git commit-tree <>` + * @example `commitTree('update foobar', '3f0fa67e5448c7bc3248333938a69f1aaf0c299f', '7d88fc945a64642e6248aeae05ea4ae1d1d3cb5b');` + * @param message + * @param treeHash + * @param previousCommitHash + */ + commitTree(message: string, treeHash: string, previousCommitHash?: string): Promise; + + /** + * `git update-ref <> <>` + * @example `updateRef('refs/heads/main', '4be0f30b8c5fb8e5740d3dcd1034c274fa94ba07');` + * @param ref + * @param commitHash + */ + updateRef(ref: string, commitHash: string): Promise; + + /** + * `git rev-parse <>` + * @example `revParse('refs/heads/main');` + * @param ref + */ + revParse(ref: string): Promise; + + /** + * `git ls-tree <> <>` + * `git ls-tree <>` + * @example `lsTree('refs/heads/main', 'src/');` + * @param ref + * @param path + */ + lsTree(ref: string, path?: string): Promise; + + /** + * `git cat-file <> <>` + * @example `catFile('blob', '3b18e512dba79e4c8300dd08aeb37f8e728b8dad');` + * @param type + * @param hash + */ + catFile(type: "blob" | "tree" | "commit" | "tag", hash: string): Promise; + + /** + * `git show <>:<>` + * @example `show('refs/heads/main', 'foobar.txt');` + * @param ref + * @param path + */ + show(ref: string, path: string): Promise; + + /** + * `git push <> <>:<>` + * `git push <> <>:<>` + * @param remote + * @param localRef + * @param remoteRef + */ + push(remote: string, localRef: string, remoteRef?: string): Promise; + + /** + * `git fetch <>` + * @param remote + */ + fetch(remote: string): Promise; + + /** + * `git rebase <>` + * @param ref + */ + rebase(ref: string): Promise; +} + +interface GitRunnerLsTree { + mode: string; + type: "blob" | "tree"; + hash: string; + path: string; +} + +export class GitDatabase extends BaseImplementation { + + constructor(private readonly git: GitRunner, private readonly branch: string) { + super() + } + + private getDBFilePath(modelType: string) { + return join(".insomnium", modelType); + } + + private getDocFilePath( + doc: T | { _id: string; type: string } + ): string { + return join(this.getDBFilePath(doc.type), `${doc._id}.json`); + } + + count(type: string, query?: Query | undefined): Promise { + return this.git.lsTree(this.branch, this.getDBFilePath(type)) + .then(list => list.filter(item => item.type === "blob")) + .then(list => list.length); + } + async find(type: string, query?: Query | undefined, sort?: Sort | undefined): Promise { + if ( + query === undefined || + (typeof query === "object" && Object.keys(query).length === 0) + ) { + const lines = await this.git.lsTree(this.branch, this.getDBFilePath(type)); + const contents = await Promise.all(lines.filter(line => line.type === 'blob').map(line => this.git.catFile('blob', line.hash).then(content => JSON.parse(content) as T))) + return BaseImplementation.sortList(contents, sort); + } + + return BaseImplementation.filterList(await this.find(type, undefined, sort), query); + } + + get( + type: string, + id?: string | undefined + ): Promise { + if (!id && id === "n/a") return Promise.resolve(null); + return this.git.show(this.branch, this.getDocFilePath({ _id: id!, type })).then(content => JSON.parse(content) as T) + } + + init(types: string[], config: object, forceReset?: boolean | undefined, consoleLog?: { (...data: any[]): void; (message?: any, ...optionalParams: any[]): void } | undefined): Promise { + return Promise.resolve() + } + initClient(): Promise { + return Promise.resolve() + } + async insert(doc: T, fromSync: boolean = false, initializeModel: boolean = true): Promise { + const docWithDefaults = initializeModel + ? await models.initModel(doc.type, doc) + : doc; + + const objectPath = this.getDocFilePath(doc); + const objectHash = await this.git.hashObject(JSON.stringify(docWithDefaults)) + await this.git.updateIndexAdd(FileMode.normal('644'), objectHash, objectPath) + const treeHash = await this.git.writeTree() + const commitHash = await this.git.commitTree(`update ${objectPath}`, treeHash, await this.git.revParse(this.branch)) + await this.git.updateRef(this.branch, commitHash) + + this.notifyOfChange("insert", docWithDefaults, fromSync); + + return docWithDefaults; + } + async remove(doc: T, fromSync: boolean = false): Promise { + const flushId = await this.bufferChanges(); + + const docs = await this.withDescendants(doc); + + await Promise.all(docs.map((doc) => this.git.updateIndexRemove(this.getDocFilePath(doc)))); + docs.map((d) => this.notifyOfChange("remove", d, fromSync)); + const treeHash = await this.git.writeTree() + const commitHash = await this.git.commitTree(`remove ${this.getDocFilePath(doc)} and children`, treeHash, await this.git.revParse(this.branch)) + await this.git.updateRef(this.branch, commitHash) + await this.flushChanges(flushId); + } + async removeWhere(type: string, query: Query): Promise { + const flushId = await this.bufferChanges(); + + const foundDocs = await this.find(type, query); + await Promise.all( + foundDocs.map(async (doc) => { + const docs = await this.withDescendants(doc); + + await Promise.all(docs.map((doc) => this.git.updateIndexRemove(this.getDocFilePath(doc)))); + docs.map((d) => this.notifyOfChange("remove", d, false)); + }) + ); + const treeHash = await this.git.writeTree() + const commitHash = await this.git.commitTree(`remove ${foundDocs.length} documents and children`, treeHash, await this.git.revParse(this.branch)) + await this.git.updateRef(this.branch, commitHash) + await this.flushChanges(flushId); + } + async unsafeRemove(doc: T, fromSync?: boolean | undefined): Promise { + const objectPath = this.getDocFilePath(doc); + await this.git.updateIndexRemove(this.getDocFilePath(doc)); + const treeHash = await this.git.writeTree() + const commitHash = await this.git.commitTree(`remove ${objectPath}`, treeHash, await this.git.revParse(this.branch)) + await this.git.updateRef(this.branch, commitHash) + } + update(doc: T, fromSync?: boolean | undefined): Promise { + return this.insert(doc, fromSync, false); + } + + +} diff --git a/packages/insomnia/src/common/database-provider/index.ts b/packages/insomnia/src/common/database-provider/index.ts new file mode 100644 index 000000000..2cc910b43 --- /dev/null +++ b/packages/insomnia/src/common/database-provider/index.ts @@ -0,0 +1,521 @@ +import { BaseModel, mustGetModel } from "../../models/index"; +import * as models from "../../models"; +import { generateId } from "../misc"; +import electron from "electron"; + +export interface Query { + _id?: string | SpecificQuery; + parentId?: string | SpecificQuery | null; + remoteId?: string | null; + plugin?: string; + key?: string; + environmentId?: string | null; + protoFileId?: string; +} + +export type Sort = Record; + +export interface Operation { + upsert?: BaseModel[]; + remove?: BaseModel[]; +} + +export interface SpecificQuery { + $gt?: number; + $in?: string[]; + $nin?: string[]; +} + +export type ModelQuery = Partial< + Record +>; +export type ChangeType = "insert" | "update" | "remove"; + +export interface Database { + all(type: string): Promise; + + batchModifyDocs(operation: Operation): Promise; + + /** buffers database changes and returns a buffer id */ + bufferChanges(millis?: number): Promise; + + /** buffers database changes and returns a buffer id */ + bufferChangesIndefinitely(): Promise; + + count(type: string, query?: Query): Promise; + + docCreate( + type: string, + ...patches: Patch[] + ): Promise; + + docUpdate( + originalDoc: T, + ...patches: Patch[] + ): Promise; + + duplicate(originalDoc: T, patch?: Patch): Promise; + + find( + type: string, + query?: Query, + sort?: Sort + ): Promise; + + findMostRecentlyModified( + type: string, + query?: Query, + limit?: number | null + ): Promise; + + flushChanges(id?: number, fake?: boolean): Promise; + + get(type: string, id?: string): Promise; + + getMostRecentlyModified( + type: string, + query?: Query + ): Promise; + + getWhere( + type: string, + query: ModelQuery | Query + ): Promise; + + init( + types: string[], + config: object, + forceReset?: boolean, + consoleLog?: typeof console.log + ): Promise; + + initClient(): Promise; + + insert( + doc: T, + fromSync?: boolean, + initializeModel?: boolean + ): Promise; + + onChange(callback: ChangeListener): void; + + offChange(callback: ChangeListener): void; + + remove(doc: T, fromSync?: boolean): Promise; + + removeWhere(type: string, query: Query): Promise; + + /** Removes entries without removing their children */ + unsafeRemove(doc: T, fromSync?: boolean): Promise; + + update(doc: T, fromSync?: boolean): Promise; + + upsert(doc: T, fromSync?: boolean): Promise; + + withAncestors( + doc: T | null, + types?: string[] + ): Promise; + + withDescendants( + doc: T | null, + stopType?: string | null + ): Promise; +} + +export type ChangeBufferEvent = [ + event: ChangeType, + doc: T, + fromSync: boolean +]; + +export type ChangeListener = (changes: ChangeBufferEvent[]) => void; + +export type Patch = Partial; + +export abstract class BaseImplementation implements Database { + private bufferingChanges = false; + private bufferChangesId = 0; + protected changeListeners: ChangeListener[] = []; + protected changeBuffer: ChangeBufferEvent[] = []; + + async all(type: string): Promise { + return this.find(type); + } + + async batchModifyDocs({ + upsert = [], + remove = [], + }: Operation): Promise { + const flushId = await this.bufferChanges(); + + // Perform from least to most dangerous + await Promise.all([ + ...upsert.map((doc) => this.upsert(doc, true)), + ...remove.map((doc) => this.unsafeRemove(doc, true)), + ]); + + await this.flushChanges(flushId); + } + + /** buffers database changes and returns a buffer id */ + async bufferChanges(millis = 1000) { + this.bufferingChanges = true; + setTimeout(this.flushChanges, millis); + return ++this.bufferChangesId; + } + + /** buffers database changes and returns a buffer id */ + async bufferChangesIndefinitely() { + this.bufferingChanges = true; + return ++this.bufferChangesId; + } + + abstract count( + type: string, + query?: Query + ): Promise; + + async docCreate( + type: string, + ...patches: Patch[] + ): Promise { + const doc = await models.initModel( + type, + ...patches, + // Fields that the user can't touch + { + type: type, + } + ); + return this.insert(doc); + } + + async docUpdate( + originalDoc: T, + ...patches: Patch[] + ): Promise { + // No need to re-initialize the model during update; originalDoc will be in a valid state by virtue of loading + const doc = await models.initModel( + originalDoc.type, + originalDoc, + + // NOTE: This is before `patches` because we want `patch.modified` to win if it has it + { + modified: Date.now(), + }, + ...patches + ); + return this.update(doc); + } + + async duplicate(originalDoc: T, patch: Patch = {}) { + const flushId = await this.bufferChanges(); + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + async function next(docToCopy: T, patch: Patch) { + const model = mustGetModel(docToCopy.type); + const overrides = { + _id: generateId(model.prefix), + modified: Date.now(), + created: Date.now(), + type: docToCopy.type, // Ensure this is not overwritten by the patch + }; + + // 1. Copy the doc + const newDoc = Object.assign({}, docToCopy, patch, overrides); + + // Don't initialize the model during insert, and simply duplicate + const createdDoc = await self.insert(newDoc, false, false); + + // 2. Get all the children + for (const type of models.types()) { + // Note: We never want to duplicate a response + if (!models.canDuplicate(type)) { + continue; + } + + const parentId = docToCopy._id; + const children = await self.find(type, { parentId }); + + for (const doc of children) { + await next(doc, { parentId: createdDoc._id }); + } + } + + return createdDoc; + } + + const createdDoc = await next(originalDoc, patch); + await this.flushChanges(flushId); + return createdDoc; + } + + abstract find( + type: string, + query?: Query | string, + sort?: Sort + ): Promise; + + async findMostRecentlyModified(type: string, query?: Query | undefined, limit?: number | null | undefined): Promise { + const result = await this.find(type, query, { modified: -1 } as Sort); + if (limit) { + return result.slice(0, limit); + } + return result; + } + + async flushChanges(id = 0, fake = false) { + // Only flush if ID is 0 or the current flush ID is the same as passed + if (id !== 0 && this.bufferChangesId !== id) { + return; + } + + this.bufferingChanges = false; + const changes = [...this.changeBuffer]; + this.changeBuffer = []; + + if (changes.length === 0) { + // No work to do + return; + } + + if (fake) { + console.log(`[db] Dropped ${changes.length} changes.`); + return; + } + // Notify local listeners too + for (const fn of this.changeListeners) { + await fn(changes); + } + // Notify remote listeners + const isMainContext = process.type === "browser"; + if (isMainContext) { + const windows = electron.BrowserWindow.getAllWindows(); + + for (const window of windows) { + window.webContents.send("db.changes", changes); + } + } + } + + async get(type: string, id?: string) { + // Short circuit IDs used to represent nothing + if (!id || id === "n/a") { + return null; + } else { + return this.getWhere(type, { _id: id }); + } + } + + async getMostRecentlyModified( + type: string, + query?: Query + ): Promise { + const docs = await this.findMostRecentlyModified(type, query, 1); + return docs.length ? docs[0] : null; + } + + async getWhere( + type: string, + query: Query | Partial> + ): Promise { + const result = await this.find(type, query); + return result.length > 0 ? result[0] : null; + } + + abstract init( + types: string[], + config: object, + forceReset?: boolean, + consoleLog?: typeof console.log + ): Promise; + + abstract initClient(): Promise; + + abstract insert( + doc: T, + fromSync?: boolean, + initializeModel?: boolean + ): Promise; + + onChange(callback: ChangeListener) { + this.changeListeners.push(callback); + } + + offChange(callback: ChangeListener) { + this.changeListeners = this.changeListeners.filter((l) => l !== callback); + } + + abstract remove( + doc: T, + fromSync?: boolean + ): Promise; + + abstract removeWhere(type: string, query: Query): Promise; + + /** Removes entries without removing their children */ + abstract unsafeRemove( + doc: T, + fromSync?: boolean + ): Promise; + + abstract update(doc: T, fromSync?: boolean): Promise; + + async upsert(doc: T, fromSync = false): Promise { + const existingDoc = await this.get(doc.type, doc._id); + + if (existingDoc) { + return this.update(doc, fromSync); + } else { + return this.insert(doc, fromSync); + } + } + + async withAncestors( + doc: T | null, + types: string[] = models.types() + ) { + if (!doc) { + return []; + } + + let docsToReturn: T[] = doc ? [doc] : []; + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + async function next(docs: T[]): Promise { + const foundDocs: T[] = []; + + for (const d of docs) { + for (const type of types) { + // If the doc is null, we want to search for parentId === null + const another = await self.get(type, d.parentId); + another && foundDocs.push(another); + } + } + + if (foundDocs.length === 0) { + // Didn't find anything. We're done + return docsToReturn; + } + + // Continue searching for children + docsToReturn = [...docsToReturn, ...foundDocs]; + return next(foundDocs); + } + + return next([doc]); + } + + async withDescendants( + doc: T | null, + stopType: string | null = null + ): Promise { + let docsToReturn: BaseModel[] = doc ? [doc] : []; + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + async function next(docs: (BaseModel | null)[]): Promise { + let foundDocs: BaseModel[] = []; + + for (const doc of docs) { + if (stopType && doc && doc.type === stopType) { + continue; + } + + const promises: Promise[] = []; + + for (const type of models.types()) { + // If the doc is null, we want to search for parentId === null + const parentId = doc ? doc._id : null; + const promise = self.find(type, { parentId }); + promises.push(promise); + } + + for (const more of await Promise.all(promises)) { + foundDocs = [...foundDocs, ...more]; + } + } + + if (foundDocs.length === 0) { + // Didn't find anything. We're done + return docsToReturn; + } + + // Continue searching for children + docsToReturn = [...docsToReturn, ...foundDocs]; + return next(foundDocs); + } + + return next([doc]); + } + + protected async notifyOfChange( + event: ChangeType, + doc: T, + fromSync: boolean + ) { + this.changeBuffer.push([event, doc, fromSync]); + + // Flush right away if we're not buffering + if (!this.bufferingChanges) { + await this.flushChanges(); + } + } + + static sortList(list: T[], sort?: Sort): T[] { + list.sort((itemA, itemB) => { + return Object.entries(sort ?? {}).reduce((result, [key, direction]) => { + if (direction !== 1 && direction !== -1) { + throw new Error("Method not implemented."); + } + if (result !== 0) { + return result; + } + if (typeof itemA[key as keyof T] === "string") + return ( + (itemA[key as keyof T] as string).localeCompare( + itemB[key as keyof T] as string + ) * direction + ); + return ( + (itemA[key as keyof T] as number) - + (itemB[key as keyof T] as number) * direction + ); + }, 0); + }); + return list; + } + + static filterList(list: T[], query: Query): T[] { + return list.filter(item => { + return Object.entries(query ?? {}).every(([key, value]) => { + if (typeof value === "object") { + let valid = true; + const operators = Object.keys(value); + operators.forEach((operator) => { + switch (operator) { + case "$gt": + valid &&= item[key as keyof T] > value[operator]; + break; + case "$in": + valid &&= value[operator].includes(item[key as keyof T]); + break; + case "$nin": { + valid &&= !value[operator].includes(item[key as keyof T]); + break; + } + default: + throw new Error("Method not implemented."); + } + }); + + return valid; + } + return item[key as keyof T] === value; + }); + }) + } +} diff --git a/packages/insomnia/src/common/database-provider/migration.ts b/packages/insomnia/src/common/database-provider/migration.ts new file mode 100644 index 000000000..1eb5a6cd6 --- /dev/null +++ b/packages/insomnia/src/common/database-provider/migration.ts @@ -0,0 +1,13 @@ +import { Database } from "./index" + +export async function migrate(from: Database, to: Database, types: string[]): Promise { + const missing = [] + for (const type of types) { + const [allFrom, allTo] = await Promise.all([from.all(type), to.all(type)]) + const toIds = allTo.map(item => item._id); + missing.push(...allFrom.filter(item => !toIds.includes(item._id))) + } + await to.batchModifyDocs({ + upsert: missing, + }) +} diff --git a/packages/insomnia/src/common/database-provider/multiple.ts b/packages/insomnia/src/common/database-provider/multiple.ts new file mode 100644 index 000000000..23365fe1f --- /dev/null +++ b/packages/insomnia/src/common/database-provider/multiple.ts @@ -0,0 +1,108 @@ +import { BaseModel } from "../../models" +import { BaseImplementation, Database, Query, Sort, SpecificQuery } from "./index" + +export class MultipleDatabase extends BaseImplementation { + constructor(private primary: Database, private readonly secondaries: Database[]) { + super() + } + + private deduplicate(docs: T[]): T[] { + return Array.from(docs.reduce((grouped, doc) => { + if (!grouped.has(doc._id)) { + grouped.set(doc._id, doc) + } + return grouped; + }, new Map() as Map).values()) + } + + count(type: string, query?: Query | undefined): Promise { + return Promise.allSettled([this.primary.count(type, query), ...this.secondaries.map(database => database.count(type, query))]) + .then(_ => _.filter(settled => settled.status === "fulfilled") as PromiseFulfilledResult[]) + .then(_ => _.map(settled => settled.value)) + .then(_ => Math.max(..._.flat())) + } + + find(type: string, query?: string | Query | undefined, sort?: Sort | undefined): Promise { + return Promise.allSettled([this.primary.find(type, query, sort), ...this.secondaries.map(database => database.find(type, query, sort))]) + .then(_ => _.filter(settled => settled.status === "fulfilled") as PromiseFulfilledResult[]) + .then(_ => _.map(settled => settled.value)) + .then(_ => _.flat()) + .then(this.deduplicate) + } + + findMostRecentlyModified(type: string, query?: Query | undefined, limit?: number | null | undefined): Promise { + return Promise.allSettled([this.primary.findMostRecentlyModified(type, query, limit), ...this.secondaries.map(database => database.findMostRecentlyModified(type, query, limit))]) + .then(_ => _.filter(settled => settled.status === "fulfilled") as PromiseFulfilledResult[]) + .then(_ => _.map(settled => settled.value)) + .then(_ => _.flat()) + .then(this.deduplicate) + } + + get(type: string, id?: string | undefined): Promise { + return Promise.any([this.primary.get(type, id), ...this.secondaries.map(database => database.get(type, id))]) + } + + getMostRecentlyModified(type: string, query?: Query | undefined): Promise { + return Promise.any([this.primary.getMostRecentlyModified(type, query), ...this.secondaries.map(database => database.getMostRecentlyModified(type, query))]) + } + + getWhere(type: string, query: Query | Partial>): Promise { + return Promise.any([this.primary.getWhere(type, query), ...this.secondaries.map(database => database.getWhere(type, query))]) + } + + init(types: string[], config: object, forceReset?: boolean | undefined, consoleLog?: { + (...data: any[]): void; + (message?: any, ...optionalParams: any[]): void; + } | undefined): Promise { + return Promise.allSettled([this.primary.init(types, config, forceReset, consoleLog), ...this.secondaries.map(database => database.init(types, config, forceReset, consoleLog))]).then(_ => {}) + } + + initClient(): Promise { + return Promise.allSettled([this.primary.initClient(), ...this.secondaries.map(database => database.initClient())]).then(_ => {}) + } + + async insert(doc: T, fromSync?: boolean | undefined, initializeModel?: boolean | undefined): Promise { + const document = await this.primary.insert(doc, fromSync, initializeModel); + await Promise.allSettled(this.secondaries.map(database => database.upsert(document, fromSync))) + return document + } + + remove(doc: T, fromSync?: boolean | undefined): Promise { + return Promise.allSettled([this.primary.remove(doc, fromSync), ...this.secondaries.map(database => database.remove(doc, fromSync))]).then(_ => {}) + } + + removeWhere(type: string, query: Query): Promise { + return Promise.allSettled([this.primary.removeWhere(type, query), ...this.secondaries.map(database => database.removeWhere(type, query))]).then(_ => {}) + } + + unsafeRemove(doc: T, fromSync?: boolean | undefined): Promise { + return Promise.allSettled([this.primary.unsafeRemove(doc, fromSync), ...this.secondaries.map(database => database.unsafeRemove(doc, fromSync))]).then(_ => {}) + } + + update(doc: T, fromSync?: boolean | undefined): Promise { + return Promise.any([this.primary.update(doc, fromSync), ...this.secondaries.map(database => database.update(doc, fromSync))]) + } + + async upsert(doc: T, fromSync?: boolean | undefined): Promise { + const document = await this.primary.upsert(doc, fromSync); + await Promise.allSettled(this.secondaries.map(database => database.upsert(document, fromSync))) + return document + } + + withAncestors(doc: T | null, types?: string[] | undefined): Promise { + return Promise.allSettled([this.primary.withAncestors(doc, types), ...this.secondaries.map(database => database.withAncestors(doc, types))]) + .then(_ => _.filter(settled => settled.status === "fulfilled") as PromiseFulfilledResult[]) + .then(_ => _.map(settled => settled.value)) + .then(_ => _.flat()) + .then(this.deduplicate) + } + + withDescendants(doc: T | null, stopType?: string | null | undefined): Promise { + return Promise.allSettled([this.primary.withDescendants(doc, stopType), ...this.secondaries.map(database => database.withDescendants(doc, stopType))]) + .then(_ => _.filter(settled => settled.status === "fulfilled") as PromiseFulfilledResult[]) + .then(_ => _.map(settled => settled.value)) + .then(_ => _.flat()) + .then(this.deduplicate) + } + +} diff --git a/packages/insomnia/src/common/database-provider/nedb.ts b/packages/insomnia/src/common/database-provider/nedb.ts new file mode 100644 index 000000000..f663f7b97 --- /dev/null +++ b/packages/insomnia/src/common/database-provider/nedb.ts @@ -0,0 +1,656 @@ +/* eslint-disable prefer-rest-params -- don't want to change ...arguments usage for these sensitive functions without more testing */ +import electron from "electron"; +// to-do > "@seald-io/nedb": "^2.0.0", +// import NeDB from '@seald-io/nedb'; +import NeDB from "nedb"; +import fsPath from "path"; +import { v4 as uuidv4 } from "uuid"; + +import { CookieJar } from "../../models/cookie-jar"; +import { Environment } from "../../models/environment"; +import { GitRepository } from "../../models/git-repository"; +import type { BaseModel } from "../../models"; +import * as models from "../../models/index"; +import type { Workspace } from "../../models/workspace"; +import { DB_PERSIST_INTERVAL } from "../constants"; +import { + BaseImplementation, + Database, + Operation, + Query, + Sort, + ChangeBufferEvent, + ModelQuery, + Patch, +} from "./index"; + +class NeDBImplementation extends BaseImplementation implements Database { + async all(type: string): Promise { + if (db._empty) { + return _send("all", ...arguments); + } + return super.all(type); + } + + async batchModifyDocs({ upsert = [], remove = [] }: Operation) { + if (db._empty) { + return _send("batchModifyDocs", ...arguments); + } + return super.batchModifyDocs({ upsert, remove }); + } + + /** buffers database changes and returns a buffer id */ + async bufferChanges(millis = 1000) { + if (db._empty) { + return _send("bufferChanges", ...arguments); + } + return super.bufferChanges(millis); + } + + /** buffers database changes and returns a buffer id */ + async bufferChangesIndefinitely() { + if (db._empty) { + return _send("bufferChangesIndefinitely", ...arguments); + } + return super.bufferChangesIndefinitely(); + } + + async count(type: string, query: Query = {}) { + if (db._empty) { + return _send("count", ...arguments); + } + return new Promise((resolve, reject) => { + (db[type] as NeDB).count(query, (err, count) => { + if (err) { + return reject(err); + } + + resolve(count); + }); + }); + } + + async duplicate(originalDoc: T, patch: Patch = {}) { + if (db._empty) { + return _send("duplicate", ...arguments); + } + return super.duplicate(originalDoc, patch); + } + + async find( + type: string, + query: Query | string = {}, + sort: Sort = { created: 1 } + ) { + if (db._empty) { + return _send("find", ...arguments); + } + return new Promise((resolve, reject) => { + (db[type] as NeDB) + .find(query) + .sort(sort) + .exec(async (err, rawDocs) => { + if (err) { + reject(err); + return; + } + + const docs: T[] = []; + + for (const rawDoc of rawDocs) { + docs.push(await models.initModel(type, rawDoc)); + } + + resolve(docs); + }); + }); + } + + async findMostRecentlyModified( + type: string, + query: Query = {}, + limit: number | null = null + ) { + if (db._empty) { + return _send("findMostRecentlyModified", ...arguments); + } + return new Promise((resolve) => { + (db[type] as NeDB) + .find(query) + .sort({ + modified: -1, + }) + // @ts-expect-error -- TSCONVERSION limit shouldn't be applied if it's null, or default to something that means no-limit + .limit(limit) + .exec(async (err, rawDocs) => { + if (err) { + console.warn("[db] Failed to find docs", err); + resolve([]); + return; + } + + const docs: T[] = []; + + for (const rawDoc of rawDocs) { + docs.push(await models.initModel(type, rawDoc)); + } + + resolve(docs); + }); + }); + } + + async flushChanges(id = 0, fake = false) { + if (db._empty) { + return _send("flushChanges", ...arguments); + } + return super.flushChanges(id, fake); + } + + async get(type: string, id?: string) { + if (db._empty) { + return _send("get", ...arguments); + } + + return super.get(type, id) + } + + getMostRecentlyModified( + type: string, + query: Query = {} + ) { + if (db._empty) { + return _send("getMostRecentlyModified", ...arguments); + } + return super.getMostRecentlyModified(type, query); + } + + async getWhere( + type: string, + query: ModelQuery | Query + ) { + if (db._empty) { + return _send("getWhere", ...arguments); + } + return super.getWhere(type, query); + } + + async init( + types: string[], + config: NeDB.DataStoreOptions = {}, + forceReset = false, + consoleLog: typeof console.log = console.log + ) { + if (forceReset) { + this.changeListeners = []; + + for (const attr of Object.keys(db)) { + if (attr === "_empty") { + continue; + } + + delete db[attr]; + } + } + + // Fill in the defaults + for (const modelType of types) { + if (db[modelType]) { + consoleLog(`[db] Already initialized DB.${modelType}`); + continue; + } + + const filePath = getDBFilePath(modelType); + const collection = new NeDB( + Object.assign( + { + autoload: true, + filename: filePath, + corruptAlertThreshold: 0.9, + }, + config + ) + ); + if (!config.inMemoryOnly) { + collection.persistence.setAutocompactionInterval(DB_PERSIST_INTERVAL); + } + db[modelType] = collection; + } + + delete db._empty; + electron.ipcMain.on("db.fn", async (e, fnName, replyChannel, ...args) => { + try { + // @ts-expect-error -- mapping unsoundness + const result = await this[fnName](...args); + e.sender.send(replyChannel, null, result); + } catch (err) { + e.sender.send(replyChannel, { + message: err.message, + stack: err.stack, + }); + } + }); + + // NOTE: Only repair the DB if we're not running in memory. Repairing here causes tests to hang indefinitely for some reason. + // TODO: Figure out why this makes tests hang + if (!config.inMemoryOnly) { + await _fixDBShape(); + consoleLog(`[db] Initialized DB at ${getDBFilePath("$TYPE")}`); + } + + // This isn't the best place for this but w/e + // Listen for response deletions and delete corresponding response body files + this.onChange(async (changes: ChangeBufferEvent[]) => { + for (const [type, doc] of changes) { + // TODO(TSCONVERSION) what's returned here is the entire model implementation, not just a model + // The type definition will be a little confusing + const m: Record | null = models.getModel(doc.type); + + if (!m) { + continue; + } + + if (type === "remove" && typeof m.hookRemove === "function") { + try { + await m.hookRemove(doc, consoleLog); + } catch (err) { + consoleLog( + `[db] Delete hook failed for ${type} ${doc._id}: ${err.message}` + ); + } + } + + if (type === "insert" && typeof m.hookInsert === "function") { + try { + await m.hookInsert(doc, consoleLog); + } catch (err) { + consoleLog( + `[db] Insert hook failed for ${type} ${doc._id}: ${err.message}` + ); + } + } + + if (type === "update" && typeof m.hookUpdate === "function") { + try { + await m.hookUpdate(doc, consoleLog); + } catch (err) { + consoleLog( + `[db] Update hook failed for ${type} ${doc._id}: ${err.message}` + ); + } + } + } + }); + + for (const model of models.all()) { + // @ts-expect-error -- TSCONVERSION optional type on response + if (typeof model.hookDatabaseInit === "function") { + // @ts-expect-error -- TSCONVERSION optional type on response + await model.hookDatabaseInit?.(consoleLog); + } + } + } + + async initClient() { + electron.ipcRenderer.on("db.changes", async (_e, changes) => { + for (const fn of this.changeListeners) { + await fn(changes); + } + }); + console.log("[db] Initialized DB client"); + } + + async insert( + doc: T, + fromSync = false, + initializeModel = true + ) { + if (db._empty) { + return _send("insert", ...arguments); + } + return new Promise(async (resolve, reject) => { + let docWithDefaults: T | null = null; + + try { + if (initializeModel) { + docWithDefaults = await models.initModel(doc.type, doc); + } else { + docWithDefaults = doc; + } + } catch (err) { + return reject(err); + } + + (db[doc.type] as NeDB).insert(docWithDefaults, (err, newDoc: T) => { + if (err) { + return reject(err); + } + + resolve(newDoc); + // NOTE: This needs to be after we resolve + this.notifyOfChange("insert", newDoc, fromSync); + }); + }); + } + + async remove(doc: T, fromSync = false) { + if (db._empty) { + return _send("remove", ...arguments); + } + + const flushId = await this.bufferChanges(); + + const docs = await this.withDescendants(doc); + const docIds = docs.map((d) => d._id); + const types = [...new Set(docs.map((d) => d.type))]; + + // Don't really need to wait for this to be over; + types.map((t) => + db[t].remove( + { + _id: { + $in: docIds, + }, + }, + { + multi: true, + } + ) + ); + + docs.map((d) => this.notifyOfChange("remove", d, fromSync)); + await this.flushChanges(flushId); + } + + async removeWhere(type: string, query: Query) { + if (db._empty) { + return _send("removeWhere", ...arguments); + } + const flushId = await this.bufferChanges(); + + for (const doc of await this.find(type, query)) { + const docs = await this.withDescendants(doc); + const docIds = docs.map((d) => d._id); + const types = [...new Set(docs.map((d) => d.type))]; + + // Don't really need to wait for this to be over; + types.map((t) => + db[t].remove( + { + _id: { + $in: docIds, + }, + }, + { + multi: true, + } + ) + ); + docs.map((d) => this.notifyOfChange("remove", d, false)); + } + + await this.flushChanges(flushId); + } + + /** Removes entries without removing their children */ + async unsafeRemove(doc: T, fromSync = false) { + if (db._empty) { + return _send("unsafeRemove", ...arguments); + } + + (db[doc.type] as NeDB).remove({ _id: doc._id }); + this.notifyOfChange("remove", doc, fromSync); + } + + async update(doc: T, fromSync = false) { + if (db._empty) { + return _send("update", ...arguments); + } + + return new Promise(async (resolve, reject) => { + let docWithDefaults: T; + + try { + docWithDefaults = await models.initModel(doc.type, doc); + } catch (err) { + return reject(err); + } + + (db[doc.type] as NeDB).update( + { _id: docWithDefaults._id }, + docWithDefaults, + // TODO(TSCONVERSION) see comment below, upsert can happen automatically as part of the update + // @ts-expect-error -- TSCONVERSION expects 4 args but only sent 3. Need to validate what UpdateOptions should be. + (err) => { + if (err) { + return reject(err); + } + + resolve(docWithDefaults); + // NOTE: This needs to be after we resolve + this.notifyOfChange("update", docWithDefaults, fromSync); + } + ); + }); + } + + // TODO(TSCONVERSION) the update method above can now take an upsert property + async upsert(doc: T, fromSync = false) { + if (db._empty) { + return _send("upsert", ...arguments); + } + return super.upsert(doc, fromSync); + } + + async withAncestors( + doc: T | null, + types: string[] = models.types() + ) { + if (db._empty) { + return _send("withAncestors", ...arguments); + } + return super.withAncestors(doc, types); + } + + async withDescendants( + doc: T | null, + stopType: string | null = null + ): Promise { + if (db._empty) { + return _send("withDescendants", ...arguments); + } + return super.withDescendants(doc, stopType); + } +} + +export const database = new NeDBImplementation(); + +interface DB { + [index: string]: NeDB; +} + +// @ts-expect-error -- TSCONVERSION _empty doesn't match the index signature, use something other than _empty in future +const db: DB = { + _empty: true, +} as DB; + +// ~~~~~~~ // +// HELPERS // +// ~~~~~~~ // + +function getDBFilePath(modelType: string) { + // NOTE: Do not EVER change this. EVER! + return fsPath.join( + process.env["INSOMNIA_DATA_PATH"] || electron.app.getPath("userData"), + `insomnia.${modelType}.db` + ); +} + +// ~~~~~~~~~~~~~~~~~~~ // +// DEFAULT MODEL STUFF // +// ~~~~~~~~~~~~~~~~~~~ // + +// ~~~~~~~ // +// Helpers // +// ~~~~~~~ // +async function _send(fnName: string, ...args: any[]) { + return new Promise((resolve, reject) => { + const replyChannel = `db.fn.reply:${uuidv4()}`; + electron.ipcRenderer.send("db.fn", fnName, replyChannel, ...args); + electron.ipcRenderer.once(replyChannel, (_e, err, result: T) => { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + }); +} + +/** + * Run various database repair scripts + */ +export async function _fixDBShape() { + console.log("[fix] Running database repairs"); + const workspaces = await database.find(models.workspace.type); + for (const workspace of workspaces) { + await _repairBaseEnvironments(workspace); + await _fixMultipleCookieJars(workspace); + await _applyApiSpecName(workspace); + } + + console.log(["workspaces"], workspaces); + + for (const gitRepository of await database.find( + models.gitRepository.type + )) { + await _fixOldGitURIs(gitRepository); + } +} + +/** + * This function ensures that apiSpec exists for each workspace + * If the filename on the apiSpec is not set or is the default initialized name + * It will apply the workspace name to it + */ +async function _applyApiSpecName(workspace: Workspace) { + const apiSpec = await models.apiSpec.getByParentId(workspace._id); + if (apiSpec === null) { + return; + } + + if ( + !apiSpec.fileName || + apiSpec.fileName === models.apiSpec.init().fileName + ) { + await models.apiSpec.update(apiSpec, { + fileName: workspace.name, + }); + } +} + +/** + * This function repairs workspaces that have multiple base environments. Since a workspace + * can only have one, this function walks over all base environments, merges the data, and + * moves all children as well. + */ +async function _repairBaseEnvironments(workspace: Workspace) { + const baseEnvironments = await database.find( + models.environment.type, + { + parentId: workspace._id, + } + ); + + // Nothing to do here + if (baseEnvironments.length <= 1) { + return; + } + + const chosenBase = baseEnvironments[0]; + + for (const baseEnvironment of baseEnvironments) { + if (baseEnvironment._id === chosenBase._id) { + continue; + } + + chosenBase.data = Object.assign(baseEnvironment.data, chosenBase.data); + const subEnvironments = await database.find( + models.environment.type, + { + parentId: baseEnvironment._id, + } + ); + + for (const subEnvironment of subEnvironments) { + await database.docUpdate(subEnvironment, { + parentId: chosenBase._id, + }); + } + + // Remove unnecessary base env + await database.remove(baseEnvironment); + } + + // Update remaining base env + await database.update(chosenBase); + console.log( + `[fix] Merged ${baseEnvironments.length} base environments under ${workspace.name}` + ); +} + +/** + * This function repairs workspaces that have multiple cookie jars. Since a workspace + * can only have one, this function walks over all jars and merges them and their cookies + * together. + */ +async function _fixMultipleCookieJars(workspace: Workspace) { + const cookieJars = await database.find(models.cookieJar.type, { + parentId: workspace._id, + }); + + // Nothing to do here + if (cookieJars.length <= 1) { + return; + } + + const chosenJar = cookieJars[0]; + + for (const cookieJar of cookieJars) { + if (cookieJar._id === chosenJar._id) { + continue; + } + + for (const cookie of cookieJar.cookies) { + if (chosenJar.cookies.find((c) => c.id === cookie.id)) { + continue; + } + + chosenJar.cookies.push(cookie); + } + + // Remove unnecessary jar + await database.remove(cookieJar); + } + + // Update remaining jar + await database.update(chosenJar); + console.log( + `[fix] Merged ${cookieJars.length} cookie jars under ${workspace.name}` + ); +} + +// Append .git to old git URIs to mimic previous isomorphic-git behaviour +async function _fixOldGitURIs(doc: GitRepository) { + if (!doc.uriNeedsMigration) { + return; + } + + if (!doc.uri.endsWith(".git")) { + doc.uri += ".git"; + } + + doc.uriNeedsMigration = false; + await database.update(doc); + console.log(`[fix] Fixed git URI for ${doc._id}`); +} diff --git a/packages/insomnia/src/common/database.ts b/packages/insomnia/src/common/database.ts index abff46ecf..286afbbbd 100644 --- a/packages/insomnia/src/common/database.ts +++ b/packages/insomnia/src/common/database.ts @@ -1,858 +1,7 @@ -/* eslint-disable prefer-rest-params -- don't want to change ...arguments usage for these sensitive functions without more testing */ -import electron from 'electron'; -// to-do > "@seald-io/nedb": "^2.0.0", -// import NeDB from '@seald-io/nedb'; -import NeDB from 'nedb'; -import fsPath from 'path'; -import { v4 as uuidv4 } from 'uuid'; +export type { Query, Operation, SpecificQuery, ModelQuery, ChangeType, ChangeBufferEvent } from "./database-provider" +import { database as fsDB } from "./database-provider/fs" +import { database as neDB } from "./database-provider/nedb" -import { mustGetModel } from '../models'; -import { CookieJar } from '../models/cookie-jar'; -import { Environment } from '../models/environment'; -import { GitRepository } from '../models/git-repository'; -import type { BaseModel } from '../models/index'; -import * as models from '../models/index'; -import type { Workspace } from '../models/workspace'; -import { DB_PERSIST_INTERVAL } from './constants'; -import { generateId } from './misc'; -import { dummyStartingWorkspace, importToWorkspaceFromJSON } from './import'; +export async function _fixDBShape() {} -export interface Query { - _id?: string | SpecificQuery; - parentId?: string | SpecificQuery | null; - remoteId?: string | null; - plugin?: string; - key?: string; - environmentId?: string | null; - protoFileId?: string; -} - -type Sort = Record; - -export interface Operation { - upsert?: BaseModel[]; - remove?: BaseModel[]; -} - -export interface SpecificQuery { - $gt?: number; - $in?: string[]; - $nin?: string[]; -} - -export type ModelQuery = Partial>; -export type ChangeType = 'insert' | 'update' | 'remove'; -export const database = { - all: async function(type: string) { - if (db._empty) { - return _send('all', ...arguments); - } - return database.find(type); - }, - - batchModifyDocs: async function({ upsert = [], remove = [] }: Operation) { - if (db._empty) { - return _send('batchModifyDocs', ...arguments); - } - const flushId = await database.bufferChanges(); - - // Perform from least to most dangerous - await Promise.all(upsert.map(doc => database.upsert(doc, true))); - await Promise.all(remove.map(doc => database.unsafeRemove(doc, true))); - - await database.flushChanges(flushId); - }, - - /** buffers database changes and returns a buffer id */ - bufferChanges: async function(millis = 1000) { - if (db._empty) { - return _send('bufferChanges', ...arguments); - } - bufferingChanges = true; - setTimeout(database.flushChanges, millis); - return ++bufferChangesId; - }, - - /** buffers database changes and returns a buffer id */ - bufferChangesIndefinitely: async function() { - if (db._empty) { - return _send('bufferChangesIndefinitely', ...arguments); - } - bufferingChanges = true; - return ++bufferChangesId; - }, - - count: async function(type: string, query: Query = {}) { - if (db._empty) { - return _send('count', ...arguments); - } - return new Promise((resolve, reject) => { - (db[type] as NeDB).count(query, (err, count) => { - if (err) { - return reject(err); - } - - resolve(count); - }); - }); - }, - - docCreate: async (type: string, ...patches: Patch[]) => { - const doc = await models.initModel( - type, - ...patches, - // Fields that the user can't touch - { - type: type, - }, - ); - return database.insert(doc); - }, - - docUpdate: async (originalDoc: T, ...patches: Patch[]) => { - // No need to re-initialize the model during update; originalDoc will be in a valid state by virtue of loading - const doc = await models.initModel( - originalDoc.type, - originalDoc, - - // NOTE: This is before `patches` because we want `patch.modified` to win if it has it - { - modified: Date.now(), - }, - ...patches, - ); - return database.update(doc); - }, - - duplicate: async function(originalDoc: T, patch: Patch = {}) { - if (db._empty) { - return _send('duplicate', ...arguments); - } - const flushId = await database.bufferChanges(); - - async function next(docToCopy: T, patch: Patch) { - const model = mustGetModel(docToCopy.type); - const overrides = { - _id: generateId(model.prefix), - modified: Date.now(), - created: Date.now(), - type: docToCopy.type, // Ensure this is not overwritten by the patch - }; - - // 1. Copy the doc - const newDoc = Object.assign({}, docToCopy, patch, overrides); - - // Don't initialize the model during insert, and simply duplicate - const createdDoc = await database.insert(newDoc, false, false); - - // 2. Get all the children - for (const type of allTypes()) { - // Note: We never want to duplicate a response - if (!models.canDuplicate(type)) { - continue; - } - - const parentId = docToCopy._id; - const children = await database.find(type, { parentId }); - - for (const doc of children) { - await next(doc, { parentId: createdDoc._id }); - } - } - - return createdDoc; - } - - const createdDoc = await next(originalDoc, patch); - await database.flushChanges(flushId); - return createdDoc; - }, - - find: async function( - type: string, - query: Query | string = {}, - sort: Sort = { created: 1 }, - ) { - if (db._empty) { - return _send('find', ...arguments); - } - return new Promise((resolve, reject) => { - (db[type] as NeDB) - .find(query) - .sort(sort) - .exec(async (err, rawDocs) => { - if (err) { - reject(err); - return; - } - - const docs: T[] = []; - - for (const rawDoc of rawDocs) { - docs.push(await models.initModel(type, rawDoc)); - } - - resolve(docs); - }); - }); - }, - - findMostRecentlyModified: async function( - type: string, - query: Query = {}, - limit: number | null = null, - ) { - if (db._empty) { - return _send('findMostRecentlyModified', ...arguments); - } - return new Promise(resolve => { - (db[type] as NeDB) - .find(query) - .sort({ - modified: -1, - }) - // @ts-expect-error -- TSCONVERSION limit shouldn't be applied if it's null, or default to something that means no-limit - .limit(limit) - .exec(async (err, rawDocs) => { - if (err) { - console.warn('[db] Failed to find docs', err); - resolve([]); - return; - } - - const docs: T[] = []; - - for (const rawDoc of rawDocs) { - docs.push(await models.initModel(type, rawDoc)); - } - - resolve(docs); - }); - }); - }, - - flushChanges: async function(id = 0, fake = false) { - if (db._empty) { - return _send('flushChanges', ...arguments); - } - - // Only flush if ID is 0 or the current flush ID is the same as passed - if (id !== 0 && bufferChangesId !== id) { - return; - } - - bufferingChanges = false; - const changes = [...changeBuffer]; - changeBuffer = []; - - if (changes.length === 0) { - // No work to do - return; - } - - if (fake) { - console.log(`[db] Dropped ${changes.length} changes.`); - return; - } - // Notify local listeners too - for (const fn of changeListeners) { - await fn(changes); - } - // Notify remote listeners - const isMainContext = process.type === 'browser'; - if (isMainContext) { - const windows = electron.BrowserWindow.getAllWindows(); - - for (const window of windows) { - window.webContents.send('db.changes', changes); - } - } - }, - - get: async function(type: string, id?: string) { - if (db._empty) { - return _send('get', ...arguments); - } - - // Short circuit IDs used to represent nothing - if (!id || id === 'n/a') { - return null; - } else { - return database.getWhere(type, { _id: id }); - } - }, - - getMostRecentlyModified: async function(type: string, query: Query = {}) { - if (db._empty) { - return _send('getMostRecentlyModified', ...arguments); - } - const docs = await database.findMostRecentlyModified(type, query, 1); - return docs.length ? docs[0] : null; - }, - - getWhere: async function(type: string, query: ModelQuery | Query) { - if (db._empty) { - return _send('getWhere', ...arguments); - } - const docs = await database.find(type, query); - return docs.length ? docs[0] : null; - }, - - init: async ( - types: string[], - config: NeDB.DataStoreOptions = {}, - forceReset = false, - consoleLog: typeof console.log = console.log, - ) => { - if (forceReset) { - changeListeners = []; - - for (const attr of Object.keys(db)) { - if (attr === '_empty') { - continue; - } - - delete db[attr]; - } - } - - // Fill in the defaults - for (const modelType of types) { - if (db[modelType]) { - consoleLog(`[db] Already initialized DB.${modelType}`); - continue; - } - - const filePath = getDBFilePath(modelType); - const collection = new NeDB( - Object.assign( - { - autoload: true, - filename: filePath, - corruptAlertThreshold: 0.9, - }, - config, - ), - ); - if (!config.inMemoryOnly) { - collection.persistence.setAutocompactionInterval(DB_PERSIST_INTERVAL); - } - db[modelType] = collection; - } - - delete db._empty; - electron.ipcMain.on('db.fn', async (e, fnName, replyChannel, ...args) => { - try { - // @ts-expect-error -- mapping unsoundness - const result = await database[fnName](...args); - e.sender.send(replyChannel, null, result); - } catch (err) { - e.sender.send(replyChannel, { - message: err.message, - stack: err.stack, - }); - } - }); - - // NOTE: Only repair the DB if we're not running in memory. Repairing here causes tests to hang indefinitely for some reason. - // TODO: Figure out why this makes tests hang - if (!config.inMemoryOnly) { - await _fixDBShape(); - consoleLog(`[db] Initialized DB at ${getDBFilePath('$TYPE')}`); - - } - - // This isn't the best place for this but w/e - // Listen for response deletions and delete corresponding response body files - database.onChange(async (changes: ChangeBufferEvent[]) => { - for (const [type, doc] of changes) { - // TODO(TSCONVERSION) what's returned here is the entire model implementation, not just a model - // The type definition will be a little confusing - const m: Record | null = models.getModel(doc.type); - - if (!m) { - continue; - } - - if (type === 'remove' && typeof m.hookRemove === 'function') { - try { - await m.hookRemove(doc, consoleLog); - } catch (err) { - consoleLog(`[db] Delete hook failed for ${type} ${doc._id}: ${err.message}`); - } - } - - if (type === 'insert' && typeof m.hookInsert === 'function') { - try { - await m.hookInsert(doc, consoleLog); - } catch (err) { - consoleLog(`[db] Insert hook failed for ${type} ${doc._id}: ${err.message}`); - } - } - - if (type === 'update' && typeof m.hookUpdate === 'function') { - try { - await m.hookUpdate(doc, consoleLog); - } catch (err) { - consoleLog(`[db] Update hook failed for ${type} ${doc._id}: ${err.message}`); - } - } - } - }); - - for (const model of models.all()) { - // @ts-expect-error -- TSCONVERSION optional type on response - if (typeof model.hookDatabaseInit === 'function') { - // @ts-expect-error -- TSCONVERSION optional type on response - await model.hookDatabaseInit?.(consoleLog); - } - } - }, - - initClient: async () => { - electron.ipcRenderer.on('db.changes', async (_e, changes) => { - for (const fn of changeListeners) { - await fn(changes); - } - }); - console.log('[db] Initialized DB client'); - }, - - insert: async function(doc: T, fromSync = false, initializeModel = true) { - if (db._empty) { - return _send('insert', ...arguments); - } - return new Promise(async (resolve, reject) => { - let docWithDefaults: T | null = null; - - try { - if (initializeModel) { - docWithDefaults = await models.initModel(doc.type, doc); - } else { - docWithDefaults = doc; - } - } catch (err) { - return reject(err); - } - - (db[doc.type] as NeDB).insert(docWithDefaults, (err, newDoc: T) => { - if (err) { - return reject(err); - } - - resolve(newDoc); - // NOTE: This needs to be after we resolve - notifyOfChange('insert', newDoc, fromSync); - }); - }); - }, - - onChange: (callback: ChangeListener) => { - changeListeners.push(callback); - }, - - offChange: (callback: ChangeListener) => { - changeListeners = changeListeners.filter(l => l !== callback); - }, - - remove: async function(doc: T, fromSync = false) { - if (db._empty) { - return _send('remove', ...arguments); - } - - const flushId = await database.bufferChanges(); - - const docs = await database.withDescendants(doc); - const docIds = docs.map(d => d._id); - const types = [...new Set(docs.map(d => d.type))]; - - // Don't really need to wait for this to be over; - types.map(t => - db[t].remove( - { - _id: { - $in: docIds, - }, - }, - { - multi: true, - }, - ), - ); - - docs.map(d => notifyOfChange('remove', d, fromSync)); - await database.flushChanges(flushId); - }, - - removeWhere: async function(type: string, query: Query) { - if (db._empty) { - return _send('removeWhere', ...arguments); - } - const flushId = await database.bufferChanges(); - - for (const doc of await database.find(type, query)) { - const docs = await database.withDescendants(doc); - const docIds = docs.map(d => d._id); - const types = [...new Set(docs.map(d => d.type))]; - - // Don't really need to wait for this to be over; - types.map(t => - db[t].remove( - { - _id: { - $in: docIds, - }, - }, - { - multi: true, - }, - ), - ); - docs.map(d => notifyOfChange('remove', d, false)); - } - - await database.flushChanges(flushId); - }, - - /** Removes entries without removing their children */ - unsafeRemove: async function(doc: T, fromSync = false) { - if (db._empty) { - return _send('unsafeRemove', ...arguments); - } - - (db[doc.type] as NeDB).remove({ _id: doc._id }); - notifyOfChange('remove', doc, fromSync); - }, - - update: async function(doc: T, fromSync = false) { - if (db._empty) { - return _send('update', ...arguments); - } - - return new Promise(async (resolve, reject) => { - let docWithDefaults: T; - - try { - docWithDefaults = await models.initModel(doc.type, doc); - } catch (err) { - return reject(err); - } - - (db[doc.type] as NeDB).update( - { _id: docWithDefaults._id }, - docWithDefaults, - // TODO(TSCONVERSION) see comment below, upsert can happen automatically as part of the update - // @ts-expect-error -- TSCONVERSION expects 4 args but only sent 3. Need to validate what UpdateOptions should be. - err => { - if (err) { - return reject(err); - } - - resolve(docWithDefaults); - // NOTE: This needs to be after we resolve - notifyOfChange('update', docWithDefaults, fromSync); - }, - ); - }); - }, - - // TODO(TSCONVERSION) the update method above can now take an upsert property - upsert: async function(doc: T, fromSync = false) { - if (db._empty) { - return _send('upsert', ...arguments); - } - const existingDoc = await database.get(doc.type, doc._id); - - if (existingDoc) { - return database.update(doc, fromSync); - } else { - return database.insert(doc, fromSync); - } - }, - - withAncestors: async function(doc: T | null, types: string[] = allTypes()) { - if (db._empty) { - return _send('withAncestors', ...arguments); - } - - if (!doc) { - return []; - } - - let docsToReturn: T[] = doc ? [doc] : []; - - async function next(docs: T[]): Promise { - const foundDocs: T[] = []; - - for (const d of docs) { - for (const type of types) { - // If the doc is null, we want to search for parentId === null - const another = await database.get(type, d.parentId); - another && foundDocs.push(another); - } - } - - if (foundDocs.length === 0) { - // Didn't find anything. We're done - return docsToReturn; - } - - // Continue searching for children - docsToReturn = [ - ...docsToReturn, - ...foundDocs, - ]; - return next(foundDocs); - } - - return next([doc]); - }, - - withDescendants: async function(doc: T | null, stopType: string | null = null): Promise { - if (db._empty) { - return _send('withDescendants', ...arguments); - } - let docsToReturn: BaseModel[] = doc ? [doc] : []; - - async function next(docs: (BaseModel | null)[]): Promise { - let foundDocs: BaseModel[] = []; - - for (const doc of docs) { - if (stopType && doc && doc.type === stopType) { - continue; - } - - const promises: Promise[] = []; - - for (const type of allTypes()) { - // If the doc is null, we want to search for parentId === null - const parentId = doc ? doc._id : null; - const promise = database.find(type, { parentId }); - promises.push(promise); - } - - for (const more of await Promise.all(promises)) { - foundDocs = [ - ...foundDocs, - ...more, - ]; - } - } - - if (foundDocs.length === 0) { - // Didn't find anything. We're done - return docsToReturn; - } - - // Continue searching for children - docsToReturn = [...docsToReturn, ...foundDocs]; - return next(foundDocs); - } - - return next([doc]); - }, -}; - -interface DB { - [index: string]: NeDB; -} - -// @ts-expect-error -- TSCONVERSION _empty doesn't match the index signature, use something other than _empty in future -const db: DB = { - _empty: true, -} as DB; - -// ~~~~~~~ // -// HELPERS // -// ~~~~~~~ // -const allTypes = () => Object.keys(db); - -function getDBFilePath(modelType: string) { - // NOTE: Do not EVER change this. EVER! - return fsPath.join(process.env['INSOMNIA_DATA_PATH'] || electron.app.getPath('userData'), `insomnia.${modelType}.db`); -} - -// ~~~~~~~~~~~~~~~~ // -// Change Listeners // -// ~~~~~~~~~~~~~~~~ // -let bufferingChanges = false; -let bufferChangesId = 1; - -export type ChangeBufferEvent = [ - event: ChangeType, - doc: T, - fromSync: boolean -]; - -let changeBuffer: ChangeBufferEvent[] = []; - -type ChangeListener = (changes: ChangeBufferEvent[]) => void; - -let changeListeners: ChangeListener[] = []; - -async function notifyOfChange(event: ChangeType, doc: T, fromSync: boolean) { - const updatedDoc = doc; - - changeBuffer.push([event, updatedDoc, fromSync]); - - // Flush right away if we're not buffering - if (!bufferingChanges) { - await database.flushChanges(); - } -} - -// ~~~~~~~~~~~~~~~~~~~ // -// DEFAULT MODEL STUFF // -// ~~~~~~~~~~~~~~~~~~~ // - -type Patch = Partial; - -// ~~~~~~~ // -// Helpers // -// ~~~~~~~ // -async function _send(fnName: string, ...args: any[]) { - return new Promise((resolve, reject) => { - const replyChannel = `db.fn.reply:${uuidv4()}`; - electron.ipcRenderer.send('db.fn', fnName, replyChannel, ...args); - electron.ipcRenderer.once(replyChannel, (_e, err, result: T) => { - if (err) { - reject(err); - } else { - resolve(result); - } - }); - }); -} - -/** - * Run various database repair scripts - */ -export async function _fixDBShape() { - console.log('[fix] Running database repairs'); - const workspaces = await database.find(models.workspace.type); - for (const workspace of workspaces) { - await _repairBaseEnvironments(workspace); - await _fixMultipleCookieJars(workspace); - await _applyApiSpecName(workspace); - } - - console.log(['workspaces'], workspaces); - - for (const gitRepository of await database.find(models.gitRepository.type)) { - await _fixOldGitURIs(gitRepository); - } -} - -/** - * This function ensures that apiSpec exists for each workspace - * If the filename on the apiSpec is not set or is the default initialized name - * It will apply the workspace name to it - */ -async function _applyApiSpecName(workspace: Workspace) { - const apiSpec = await models.apiSpec.getByParentId(workspace._id); - if (apiSpec === null) { - return; - } - - if (!apiSpec.fileName || apiSpec.fileName === models.apiSpec.init().fileName) { - await models.apiSpec.update(apiSpec, { - fileName: workspace.name, - }); - } -} - -/** - * This function repairs workspaces that have multiple base environments. Since a workspace - * can only have one, this function walks over all base environments, merges the data, and - * moves all children as well. - */ -async function _repairBaseEnvironments(workspace: Workspace) { - const baseEnvironments = await database.find(models.environment.type, { - parentId: workspace._id, - }); - - // Nothing to do here - if (baseEnvironments.length <= 1) { - return; - } - - const chosenBase = baseEnvironments[0]; - - for (const baseEnvironment of baseEnvironments) { - if (baseEnvironment._id === chosenBase._id) { - continue; - } - - chosenBase.data = Object.assign(baseEnvironment.data, chosenBase.data); - const subEnvironments = await database.find(models.environment.type, { - parentId: baseEnvironment._id, - }); - - for (const subEnvironment of subEnvironments) { - await database.docUpdate(subEnvironment, { - parentId: chosenBase._id, - }); - } - - // Remove unnecessary base env - await database.remove(baseEnvironment); - } - - // Update remaining base env - await database.update(chosenBase); - console.log(`[fix] Merged ${baseEnvironments.length} base environments under ${workspace.name}`); -} - -/** - * This function repairs workspaces that have multiple cookie jars. Since a workspace - * can only have one, this function walks over all jars and merges them and their cookies - * together. - */ -async function _fixMultipleCookieJars(workspace: Workspace) { - const cookieJars = await database.find(models.cookieJar.type, { - parentId: workspace._id, - }); - - // Nothing to do here - if (cookieJars.length <= 1) { - return; - } - - const chosenJar = cookieJars[0]; - - for (const cookieJar of cookieJars) { - if (cookieJar._id === chosenJar._id) { - continue; - } - - for (const cookie of cookieJar.cookies) { - if (chosenJar.cookies.find(c => c.id === cookie.id)) { - continue; - } - - chosenJar.cookies.push(cookie); - } - - // Remove unnecessary jar - await database.remove(cookieJar); - } - - // Update remaining jar - await database.update(chosenJar); - console.log(`[fix] Merged ${cookieJars.length} cookie jars under ${workspace.name}`); -} - -// Append .git to old git URIs to mimic previous isomorphic-git behaviour -async function _fixOldGitURIs(doc: GitRepository) { - if (!doc.uriNeedsMigration) { - return; - } - - if (!doc.uri.endsWith('.git')) { - doc.uri += '.git'; - } - - doc.uriNeedsMigration = false; - await database.update(doc); - console.log(`[fix] Fixed git URI for ${doc._id}`); -} +export { fsDB as database } diff --git a/packages/insomnia/tsconfig.build.json b/packages/insomnia/tsconfig.build.json index 95274e951..29309c567 100644 --- a/packages/insomnia/tsconfig.build.json +++ b/packages/insomnia/tsconfig.build.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "lib": ["DOM.Iterable", "DOM"], + "lib": ["DOM.Iterable", "DOM", "ES2021.Promise"], "isolatedModules": true, "jsx": "react", "outDir": "./build",