From ea5bd343398dde2251e0d0c8023ae2ce380da691 Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Sat, 18 Jan 2025 12:26:53 -0800 Subject: [PATCH] docs: minor comments added --- packages/core/src/source/Document.ts | 9 ++-- packages/core/src/source/DocumentConverter.ts | 3 ++ .../browser/indexeddb/IndexedDbJobQueue.ts | 5 +- .../indexeddb/IndexedDbTaskGraphRepository.ts | 4 ++ .../IndexedDbTaskOutputRepository.ts | 4 ++ .../inmemory/InMemoryModelRepository.ts | 4 ++ .../browser/inmemory/InMemoryRateLimiter.ts | 4 ++ .../inmemory/InMemoryTaskGraphRepository.ts | 4 ++ .../inmemory/InMemoryTaskOutputRepository.ts | 4 ++ .../storage/src/bun/sqlite/SqliteJobQueue.ts | 49 +++++++++++++++++++ .../src/bun/sqlite/SqliteModelRepository.ts | 4 ++ .../src/bun/sqlite/SqliteRateLimiter.ts | 4 ++ .../bun/sqlite/SqliteTaskGraphRepository.ts | 4 ++ .../bun/sqlite/SqliteTaskOutputRepository.ts | 4 ++ .../filesystem/FileTaskGraphRepository.ts | 4 ++ .../filesystem/FileTaskOutputRepository.ts | 4 ++ .../src/node/postgres/PostgresJobQueue.ts | 49 +++++++++++++++++++ .../src/node/postgres/PostgresRateLimiter.ts | 20 ++++++++ .../postgres/PostgresTaskGraphRepository.ts | 4 ++ .../postgres/PostgresTaskOutputRepository.ts | 4 ++ 20 files changed, 185 insertions(+), 6 deletions(-) diff --git a/packages/core/src/source/Document.ts b/packages/core/src/source/Document.ts index c3e64bf..b5f1293 100644 --- a/packages/core/src/source/Document.ts +++ b/packages/core/src/source/Document.ts @@ -21,6 +21,9 @@ export interface DocumentSectionMetadata { title: string; } +/** + * Represents a document with its content and metadata. + */ export class Document { public metadata: DocumentMetadata; @@ -81,11 +84,7 @@ export class Document { } export class DocumentSection extends Document { - constructor( - public parent: Document, - content?: ContentType, - metadata?: DocumentSectionMetadata - ) { + constructor(public parent: Document, content?: ContentType, metadata?: DocumentSectionMetadata) { super(content, metadata); this.parent = parent; } diff --git a/packages/core/src/source/DocumentConverter.ts b/packages/core/src/source/DocumentConverter.ts index 3eb7d1b..5bae40b 100644 --- a/packages/core/src/source/DocumentConverter.ts +++ b/packages/core/src/source/DocumentConverter.ts @@ -7,6 +7,9 @@ import { Document, DocumentMetadata } from "./Document"; +/** + * Abstract class for converting different types of content into a Document. + */ export abstract class DocumentConverter { public metadata: DocumentMetadata; constructor(metadata: DocumentMetadata) { diff --git a/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts b/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts index 75c5834..c515a23 100644 --- a/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts +++ b/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts @@ -10,7 +10,10 @@ import { Job, JobQueue, ILimiter } from "ellmers-core"; import { makeFingerprint } from "../../util/Misc"; import { ensureIndexedDbTable } from "./base/IndexedDbTable"; -// The IndexedDbQueue class +/** + * IndexedDB implementation of a job queue. + * Provides storage and retrieval for job execution states using IndexedDB. + */ export class IndexedDbQueue extends JobQueue { private dbPromise: Promise; diff --git a/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts b/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts index a0932f5..3201748 100644 --- a/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts +++ b/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts @@ -8,6 +8,10 @@ import { TaskGraphRepository } from "ellmers-core"; import { IndexedDbKVRepository } from "./base/IndexedDbKVRepository"; +/** + * IndexedDB implementation of a task graph repository. + * Provides storage and retrieval for task graphs using IndexedDB. + */ export class IndexedDbTaskGraphRepository extends TaskGraphRepository { kvRepository: IndexedDbKVRepository; public type = "IndexedDbTaskGraphRepository" as const; diff --git a/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts b/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts index 809a625..798737d 100644 --- a/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts +++ b/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts @@ -13,6 +13,10 @@ import { } from "ellmers-core"; import { IndexedDbKVRepository } from "./base/IndexedDbKVRepository"; +/** + * IndexedDB implementation of a task output repository. + * Provides storage and retrieval for task outputs using IndexedDB. + */ export class IndexedDbTaskOutputRepository extends TaskOutputRepository { kvRepository: IndexedDbKVRepository< TaskOutputPrimaryKey, diff --git a/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts b/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts index a4fd0d0..50d276c 100644 --- a/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts +++ b/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts @@ -19,6 +19,10 @@ import { } from "ellmers-ai"; import { DefaultValueType } from "ellmers-core"; +/** + * In-memory implementation of a model repository. + * Provides storage and retrieval for models and task-to-model mappings. + */ export class InMemoryModelRepository extends ModelRepository { modelKvRepository: InMemoryKVRepository< ModelPrimaryKey, diff --git a/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts b/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts index 298116e..f26db0f 100644 --- a/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts +++ b/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts @@ -7,6 +7,10 @@ import { ILimiter } from "ellmers-core"; +/** + * In-memory implementation of a rate limiter. + * Manages request counts and delays to control job execution. + */ export class InMemoryRateLimiter implements ILimiter { private requests: Date[] = []; private nextAvailableTime: Date = new Date(); // New property to track externally set delay diff --git a/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts b/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts index 84eb836..4db0f75 100644 --- a/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts +++ b/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts @@ -8,6 +8,10 @@ import { TaskGraphRepository } from "ellmers-core"; import { InMemoryKVRepository } from "./base/InMemoryKVRepository"; +/** + * In-memory implementation of a task graph repository. + * Provides storage and retrieval for task graphs. + */ export class InMemoryTaskGraphRepository extends TaskGraphRepository { kvRepository: InMemoryKVRepository; public type = "InMemoryTaskGraphRepository" as const; diff --git a/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts b/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts index 66b294a..d61cb62 100644 --- a/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts +++ b/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts @@ -15,6 +15,10 @@ import { } from "ellmers-core"; import { InMemoryKVRepository } from "./base/InMemoryKVRepository"; +/** + * In-memory implementation of a task output repository. + * Provides storage and retrieval for task outputs. + */ export class InMemoryTaskOutputRepository extends TaskOutputRepository { kvRepository: InMemoryKVRepository< TaskOutputPrimaryKey, diff --git a/packages/storage/src/bun/sqlite/SqliteJobQueue.ts b/packages/storage/src/bun/sqlite/SqliteJobQueue.ts index 6df5ed5..bbe31b3 100644 --- a/packages/storage/src/bun/sqlite/SqliteJobQueue.ts +++ b/packages/storage/src/bun/sqlite/SqliteJobQueue.ts @@ -11,6 +11,10 @@ import { type Database } from "bun:sqlite"; // TODO: reuse prepared statements +/** + * SQLite implementation of a job queue. + * Provides storage and retrieval for job execution states using SQLite. + */ export class SqliteJobQueue extends JobQueue { constructor( protected db: Database, @@ -49,6 +53,11 @@ export class SqliteJobQueue extends JobQueue { return this; } + /** + * Creates a new job instance from the provided database results. + * @param results - The job data from the database + * @returns A new Job instance with populated properties + */ public createNewJob(results: any): Job { return new this.jobClass({ ...results, @@ -61,6 +70,11 @@ export class SqliteJobQueue extends JobQueue { }); } + /** + * Adds a new job to the queue. + * @param job - The job to add + * @returns The ID of the added job + */ public async add(job: Job) { job.queueName = this.queue; const fingerprint = await makeFingerprint(job.input); @@ -95,6 +109,11 @@ export class SqliteJobQueue extends JobQueue { return result?.id; } + /** + * Retrieves a job by its ID. + * @param id - The ID of the job to retrieve + * @returns The job if found, undefined otherwise + */ public async get(id: string) { const JobQuery = ` SELECT * @@ -106,6 +125,11 @@ export class SqliteJobQueue extends JobQueue { return result ? this.createNewJob(result) : undefined; } + /** + * Retrieves a slice of jobs from the queue. + * @param num - Maximum number of jobs to return + * @returns An array of jobs + */ public async peek(num: number = 100) { num = Number(num) || 100; // TS does not validate, so ensure it is a number since we put directly in SQL string const FutureJobQuery = ` @@ -123,6 +147,10 @@ export class SqliteJobQueue extends JobQueue { return ret; } + /** + * Retrieves all jobs currently being processed. + * @returns An array of jobs + */ public async processing() { const ProcessingQuery = ` SELECT * @@ -136,6 +164,10 @@ export class SqliteJobQueue extends JobQueue { return ret; } + /** + * Retrieves the next available job that is ready to be processed. + * @returns The next job or undefined if no job is available + */ public async next() { let id: string | undefined; { @@ -164,6 +196,11 @@ export class SqliteJobQueue extends JobQueue { } } + /** + * Retrieves the number of jobs in the queue with a specific status. + * @param status - The status of the jobs to count + * @returns The count of jobs with the specified status + */ public async size(status = JobStatus.PENDING) { const sizeQuery = ` SELECT COUNT(*) as count @@ -175,6 +212,13 @@ export class SqliteJobQueue extends JobQueue { return result.count; } + /** + * Marks a job as complete with its output or error. + * Handles retries for failed jobs and triggers completion callbacks. + * @param id - ID of the job to complete + * @param output - Result of the job execution + * @param error - Optional error message if job failed + */ public async complete(id: string, output: Output | null = null, error: string | null = null) { const job = await this.get(id); if (!job) throw new Error(`Job ${id} not found`); @@ -206,6 +250,11 @@ export class SqliteJobQueue extends JobQueue { await this.limiter.clear(); } + /** + * Looks up cached output for a given task type and input + * Uses input fingerprinting for efficient matching + * @returns The cached output or null if not found + */ public async outputForInput(taskType: string, input: Input) { const fingerprint = await makeFingerprint(input); const OutputQuery = ` diff --git a/packages/storage/src/bun/sqlite/SqliteModelRepository.ts b/packages/storage/src/bun/sqlite/SqliteModelRepository.ts index 673f08d..abe4aa5 100644 --- a/packages/storage/src/bun/sqlite/SqliteModelRepository.ts +++ b/packages/storage/src/bun/sqlite/SqliteModelRepository.ts @@ -17,6 +17,10 @@ import { import { SqliteKVRepository } from "./base/SqliteKVRepository"; import { DefaultValueType } from "ellmers-core"; +/** + * SQLite implementation of a model repository. + * Provides storage and retrieval for models and task-to-model mappings using SQLite. + */ export class SqliteModelRepository extends ModelRepository { public type = "SqliteModelRepository" as const; modelKvRepository: SqliteKVRepository< diff --git a/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts b/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts index 731dc2d..152a69a 100644 --- a/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts +++ b/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts @@ -9,6 +9,10 @@ import { type Database } from "better-sqlite3"; import { ILimiter } from "ellmers-core"; import { toSQLiteTimestamp } from "../../util/Misc"; +/** + * SQLite implementation of a rate limiter. + * Manages request counts and delays to control job execution. + */ export class SqliteRateLimiter implements ILimiter { private readonly db: Database; private readonly queueName: string; diff --git a/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts b/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts index 941880a..006ddc9 100644 --- a/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts +++ b/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts @@ -8,6 +8,10 @@ import { TaskGraphRepository } from "ellmers-core"; import { SqliteKVRepository } from "./base/SqliteKVRepository"; +/** + * SQLite implementation of a task graph repository. + * Provides storage and retrieval for task graphs using SQLite. + */ export class SqliteTaskGraphRepository extends TaskGraphRepository { kvRepository: SqliteKVRepository; public type = "SqliteTaskGraphRepository" as const; diff --git a/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts b/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts index 2ea177e..7a239ca 100644 --- a/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts +++ b/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts @@ -13,6 +13,10 @@ import { } from "ellmers-core"; import { SqliteKVRepository } from "./base/SqliteKVRepository"; +/** + * SQLite implementation of a task output repository. + * Provides storage and retrieval for task outputs using SQLite. + */ export class SqliteTaskOutputRepository extends TaskOutputRepository { kvRepository: SqliteKVRepository< TaskOutputPrimaryKey, diff --git a/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts b/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts index 378974d..a1c10ca 100644 --- a/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts +++ b/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts @@ -8,6 +8,10 @@ import { TaskGraphRepository } from "ellmers-core"; import { FileKVRepository } from "./base/FileKVRepository"; +/** + * File-based implementation of a task graph repository. + * Provides storage and retrieval for task graphs using a file system. + */ export class FileTaskGraphRepository extends TaskGraphRepository { kvRepository: FileKVRepository; public type = "FileTaskGraphRepository" as const; diff --git a/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts b/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts index af762ae..31d1545 100644 --- a/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts +++ b/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts @@ -14,6 +14,10 @@ import { } from "ellmers-core"; import { FileKVRepository } from "./base/FileKVRepository"; +/** + * File-based implementation of a task output repository. + * Provides storage and retrieval for task outputs using a file system. + */ export class FileTaskOutputRepository extends TaskOutputRepository { kvRepository: FileKVRepository< TaskOutputPrimaryKey, diff --git a/packages/storage/src/node/postgres/PostgresJobQueue.ts b/packages/storage/src/node/postgres/PostgresJobQueue.ts index 8acf0e8..c3bba93 100644 --- a/packages/storage/src/node/postgres/PostgresJobQueue.ts +++ b/packages/storage/src/node/postgres/PostgresJobQueue.ts @@ -11,6 +11,10 @@ import { makeFingerprint } from "../../util/Misc"; // TODO: prepared statements +/** + * PostgreSQL implementation of a job queue. + * Provides storage and retrieval for job execution states using PostgreSQL. + */ export class PostgresJobQueue extends JobQueue { constructor( protected readonly sql: Sql, @@ -49,6 +53,11 @@ export class PostgresJobQueue extends JobQueue { return this; } + /** + * Creates a new job instance from the provided database results. + * @param results - The job data from the database + * @returns A new Job instance with populated properties + */ public createNewJob(results: any): Job { return new this.jobClass({ ...results, @@ -61,6 +70,11 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Adds a new job to the queue. + * @param job - The job to add + * @returns The ID of the added job + */ public async add(job: Job) { job.queueName = this.queue; const fingerprint = await makeFingerprint(job.input); @@ -74,6 +88,11 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Retrieves a job by its ID. + * @param id - The ID of the job to retrieve + * @returns The job if found, undefined otherwise + */ public async get(id: number) { return await this.sql.begin(async (sql) => { const result = await sql` @@ -86,6 +105,11 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Retrieves a slice of jobs from the queue. + * @param num - Maximum number of jobs to return + * @returns An array of jobs + */ public async peek(num: number = 100) { num = Number(num) || 100; // TS does not validate, so ensure it is a number return await this.sql.begin(async (sql) => { @@ -104,6 +128,10 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Retrieves all jobs currently being processed. + * @returns An array of jobs + */ public async processing() { return await this.sql.begin(async (sql) => { const result = await sql` @@ -117,6 +145,10 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Retrieves the next available job that is ready to be processed. + * @returns The next job or undefined if no job is available + */ public async next() { return await this.sql.begin(async (sql) => { const result = await sql` @@ -140,6 +172,11 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Retrieves the number of jobs in the queue with a specific status. + * @param status - The status of the jobs to count + * @returns The count of jobs with the specified status + */ public async size(status = JobStatus.PENDING) { return await this.sql.begin(async (sql) => { const result = await sql` @@ -152,6 +189,13 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Marks a job as complete with its output or error. + * Handles retries for failed jobs and triggers completion callbacks. + * @param id - ID of the job to complete + * @param output - Result of the job execution + * @param error - Optional error message if job failed + */ public async complete(id: number, output: Output | null = null, error: string | null = null) { const job = await this.get(id); if (!job) throw new Error(`Job ${id} not found`); @@ -183,6 +227,11 @@ export class PostgresJobQueue extends JobQueue { }); } + /** + * Looks up cached output for a given task type and input + * Uses input fingerprinting for efficient matching + * @returns The cached output or null if not found + */ public async outputForInput(taskType: string, input: Input) { const fingerprint = await makeFingerprint(input); return await this.sql.begin(async (sql) => { diff --git a/packages/storage/src/node/postgres/PostgresRateLimiter.ts b/packages/storage/src/node/postgres/PostgresRateLimiter.ts index a4d3a81..07941c8 100644 --- a/packages/storage/src/node/postgres/PostgresRateLimiter.ts +++ b/packages/storage/src/node/postgres/PostgresRateLimiter.ts @@ -8,6 +8,10 @@ import { ILimiter } from "ellmers-core"; import { Sql } from "postgres"; +/** + * PostgreSQL implementation of a rate limiter. + * Manages request counts and delays to control job execution. + */ export class PostgresRateLimiter implements ILimiter { private readonly windowSizeInMilliseconds: number; @@ -36,6 +40,10 @@ export class PostgresRateLimiter implements ILimiter { await this.sql`DELETE FROM job_rate_limit`; } + /** + * Checks if a job can proceed based on rate limiting rules. + * @returns True if the job can proceed, false otherwise + */ async canProceed(): Promise { const now = new Date(); const attemptedAtThreshold = new Date(now.getTime() - this.windowSizeInMilliseconds); @@ -66,6 +74,10 @@ export class PostgresRateLimiter implements ILimiter { return true; } + /** + * Records a new job attempt. + * @returns The ID of the added job + */ async recordJobStart(): Promise { // Record a new job attempt await this.sql` @@ -78,6 +90,10 @@ export class PostgresRateLimiter implements ILimiter { // Optional for rate limiting: Cleanup or track completions if needed } + /** + * Retrieves the next available time for the specific queue. + * @returns The next available time + */ async getNextAvailableTime(): Promise { // Query for the earliest job attempt within the window that reaches the limit const result = await this.sql` @@ -96,6 +112,10 @@ export class PostgresRateLimiter implements ILimiter { } } + /** + * Sets the next available time for the specific queue. + * @param date - The new next available time + */ async setNextAvailableTime(date: Date): Promise { // Update the next available time for the specific queue. If no entry exists, insert a new one. await this.sql` diff --git a/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts b/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts index 855e44a..f8c43b8 100644 --- a/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts +++ b/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts @@ -8,6 +8,10 @@ import { TaskGraphRepository } from "ellmers-core"; import { PostgresKVRepository } from "./base/PostgresKVRepository"; +/** + * PostgreSQL implementation of a task graph repository. + * Provides storage and retrieval for task graphs using PostgreSQL. + */ export class PostgresTaskGraphRepository extends TaskGraphRepository { kvRepository: PostgresKVRepository; public type = "PostgresTaskGraphRepository" as const; diff --git a/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts b/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts index e69e24b..311a540 100644 --- a/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts +++ b/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts @@ -13,6 +13,10 @@ import { } from "ellmers-core"; import { PostgresKVRepository } from "./base/PostgresKVRepository"; +/** + * PostgreSQL implementation of a task output repository. + * Provides storage and retrieval for task outputs using PostgreSQL. + */ export class PostgresTaskOutputRepository extends TaskOutputRepository { kvRepository: PostgresKVRepository< TaskOutputPrimaryKey,