diff --git a/packages/core/src/source/Document.ts b/packages/core/src/source/Document.ts
index c3e64bf..b5f1293 100644
--- a/packages/core/src/source/Document.ts
+++ b/packages/core/src/source/Document.ts
@@ -21,6 +21,9 @@ export interface DocumentSectionMetadata {
title: string;
}
+/**
+ * Represents a document with its content and metadata.
+ */
export class Document {
public metadata: DocumentMetadata;
@@ -81,11 +84,7 @@ export class Document {
}
export class DocumentSection extends Document {
- constructor(
- public parent: Document,
- content?: ContentType,
- metadata?: DocumentSectionMetadata
- ) {
+ constructor(public parent: Document, content?: ContentType, metadata?: DocumentSectionMetadata) {
super(content, metadata);
this.parent = parent;
}
diff --git a/packages/core/src/source/DocumentConverter.ts b/packages/core/src/source/DocumentConverter.ts
index 3eb7d1b..5bae40b 100644
--- a/packages/core/src/source/DocumentConverter.ts
+++ b/packages/core/src/source/DocumentConverter.ts
@@ -7,6 +7,9 @@
import { Document, DocumentMetadata } from "./Document";
+/**
+ * Abstract class for converting different types of content into a Document.
+ */
export abstract class DocumentConverter {
public metadata: DocumentMetadata;
constructor(metadata: DocumentMetadata) {
diff --git a/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts b/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts
index 75c5834..c515a23 100644
--- a/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts
+++ b/packages/storage/src/browser/indexeddb/IndexedDbJobQueue.ts
@@ -10,7 +10,10 @@ import { Job, JobQueue, ILimiter } from "ellmers-core";
import { makeFingerprint } from "../../util/Misc";
import { ensureIndexedDbTable } from "./base/IndexedDbTable";
-// The IndexedDbQueue class
+/**
+ * IndexedDB implementation of a job queue.
+ * Provides storage and retrieval for job execution states using IndexedDB.
+ */
export class IndexedDbQueue extends JobQueue {
private dbPromise: Promise;
diff --git a/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts b/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts
index a0932f5..3201748 100644
--- a/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts
+++ b/packages/storage/src/browser/indexeddb/IndexedDbTaskGraphRepository.ts
@@ -8,6 +8,10 @@
import { TaskGraphRepository } from "ellmers-core";
import { IndexedDbKVRepository } from "./base/IndexedDbKVRepository";
+/**
+ * IndexedDB implementation of a task graph repository.
+ * Provides storage and retrieval for task graphs using IndexedDB.
+ */
export class IndexedDbTaskGraphRepository extends TaskGraphRepository {
kvRepository: IndexedDbKVRepository;
public type = "IndexedDbTaskGraphRepository" as const;
diff --git a/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts b/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts
index 809a625..798737d 100644
--- a/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts
+++ b/packages/storage/src/browser/indexeddb/IndexedDbTaskOutputRepository.ts
@@ -13,6 +13,10 @@ import {
} from "ellmers-core";
import { IndexedDbKVRepository } from "./base/IndexedDbKVRepository";
+/**
+ * IndexedDB implementation of a task output repository.
+ * Provides storage and retrieval for task outputs using IndexedDB.
+ */
export class IndexedDbTaskOutputRepository extends TaskOutputRepository {
kvRepository: IndexedDbKVRepository<
TaskOutputPrimaryKey,
diff --git a/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts b/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts
index a4fd0d0..50d276c 100644
--- a/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts
+++ b/packages/storage/src/browser/inmemory/InMemoryModelRepository.ts
@@ -19,6 +19,10 @@ import {
} from "ellmers-ai";
import { DefaultValueType } from "ellmers-core";
+/**
+ * In-memory implementation of a model repository.
+ * Provides storage and retrieval for models and task-to-model mappings.
+ */
export class InMemoryModelRepository extends ModelRepository {
modelKvRepository: InMemoryKVRepository<
ModelPrimaryKey,
diff --git a/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts b/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts
index 298116e..f26db0f 100644
--- a/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts
+++ b/packages/storage/src/browser/inmemory/InMemoryRateLimiter.ts
@@ -7,6 +7,10 @@
import { ILimiter } from "ellmers-core";
+/**
+ * In-memory implementation of a rate limiter.
+ * Manages request counts and delays to control job execution.
+ */
export class InMemoryRateLimiter implements ILimiter {
private requests: Date[] = [];
private nextAvailableTime: Date = new Date(); // New property to track externally set delay
diff --git a/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts b/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts
index 84eb836..4db0f75 100644
--- a/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts
+++ b/packages/storage/src/browser/inmemory/InMemoryTaskGraphRepository.ts
@@ -8,6 +8,10 @@
import { TaskGraphRepository } from "ellmers-core";
import { InMemoryKVRepository } from "./base/InMemoryKVRepository";
+/**
+ * In-memory implementation of a task graph repository.
+ * Provides storage and retrieval for task graphs.
+ */
export class InMemoryTaskGraphRepository extends TaskGraphRepository {
kvRepository: InMemoryKVRepository;
public type = "InMemoryTaskGraphRepository" as const;
diff --git a/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts b/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts
index 66b294a..d61cb62 100644
--- a/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts
+++ b/packages/storage/src/browser/inmemory/InMemoryTaskOutputRepository.ts
@@ -15,6 +15,10 @@ import {
} from "ellmers-core";
import { InMemoryKVRepository } from "./base/InMemoryKVRepository";
+/**
+ * In-memory implementation of a task output repository.
+ * Provides storage and retrieval for task outputs.
+ */
export class InMemoryTaskOutputRepository extends TaskOutputRepository {
kvRepository: InMemoryKVRepository<
TaskOutputPrimaryKey,
diff --git a/packages/storage/src/bun/sqlite/SqliteJobQueue.ts b/packages/storage/src/bun/sqlite/SqliteJobQueue.ts
index 6df5ed5..bbe31b3 100644
--- a/packages/storage/src/bun/sqlite/SqliteJobQueue.ts
+++ b/packages/storage/src/bun/sqlite/SqliteJobQueue.ts
@@ -11,6 +11,10 @@ import { type Database } from "bun:sqlite";
// TODO: reuse prepared statements
+/**
+ * SQLite implementation of a job queue.
+ * Provides storage and retrieval for job execution states using SQLite.
+ */
export class SqliteJobQueue extends JobQueue {
constructor(
protected db: Database,
@@ -49,6 +53,11 @@ export class SqliteJobQueue extends JobQueue {
return this;
}
+ /**
+ * Creates a new job instance from the provided database results.
+ * @param results - The job data from the database
+ * @returns A new Job instance with populated properties
+ */
public createNewJob(results: any): Job {
return new this.jobClass({
...results,
@@ -61,6 +70,11 @@ export class SqliteJobQueue extends JobQueue {
});
}
+ /**
+ * Adds a new job to the queue.
+ * @param job - The job to add
+ * @returns The ID of the added job
+ */
public async add(job: Job) {
job.queueName = this.queue;
const fingerprint = await makeFingerprint(job.input);
@@ -95,6 +109,11 @@ export class SqliteJobQueue extends JobQueue {
return result?.id;
}
+ /**
+ * Retrieves a job by its ID.
+ * @param id - The ID of the job to retrieve
+ * @returns The job if found, undefined otherwise
+ */
public async get(id: string) {
const JobQuery = `
SELECT *
@@ -106,6 +125,11 @@ export class SqliteJobQueue extends JobQueue {
return result ? this.createNewJob(result) : undefined;
}
+ /**
+ * Retrieves a slice of jobs from the queue.
+ * @param num - Maximum number of jobs to return
+ * @returns An array of jobs
+ */
public async peek(num: number = 100) {
num = Number(num) || 100; // TS does not validate, so ensure it is a number since we put directly in SQL string
const FutureJobQuery = `
@@ -123,6 +147,10 @@ export class SqliteJobQueue extends JobQueue {
return ret;
}
+ /**
+ * Retrieves all jobs currently being processed.
+ * @returns An array of jobs
+ */
public async processing() {
const ProcessingQuery = `
SELECT *
@@ -136,6 +164,10 @@ export class SqliteJobQueue extends JobQueue {
return ret;
}
+ /**
+ * Retrieves the next available job that is ready to be processed.
+ * @returns The next job or undefined if no job is available
+ */
public async next() {
let id: string | undefined;
{
@@ -164,6 +196,11 @@ export class SqliteJobQueue extends JobQueue {
}
}
+ /**
+ * Retrieves the number of jobs in the queue with a specific status.
+ * @param status - The status of the jobs to count
+ * @returns The count of jobs with the specified status
+ */
public async size(status = JobStatus.PENDING) {
const sizeQuery = `
SELECT COUNT(*) as count
@@ -175,6 +212,13 @@ export class SqliteJobQueue extends JobQueue {
return result.count;
}
+ /**
+ * Marks a job as complete with its output or error.
+ * Handles retries for failed jobs and triggers completion callbacks.
+ * @param id - ID of the job to complete
+ * @param output - Result of the job execution
+ * @param error - Optional error message if job failed
+ */
public async complete(id: string, output: Output | null = null, error: string | null = null) {
const job = await this.get(id);
if (!job) throw new Error(`Job ${id} not found`);
@@ -206,6 +250,11 @@ export class SqliteJobQueue extends JobQueue {
await this.limiter.clear();
}
+ /**
+ * Looks up cached output for a given task type and input
+ * Uses input fingerprinting for efficient matching
+ * @returns The cached output or null if not found
+ */
public async outputForInput(taskType: string, input: Input) {
const fingerprint = await makeFingerprint(input);
const OutputQuery = `
diff --git a/packages/storage/src/bun/sqlite/SqliteModelRepository.ts b/packages/storage/src/bun/sqlite/SqliteModelRepository.ts
index 673f08d..abe4aa5 100644
--- a/packages/storage/src/bun/sqlite/SqliteModelRepository.ts
+++ b/packages/storage/src/bun/sqlite/SqliteModelRepository.ts
@@ -17,6 +17,10 @@ import {
import { SqliteKVRepository } from "./base/SqliteKVRepository";
import { DefaultValueType } from "ellmers-core";
+/**
+ * SQLite implementation of a model repository.
+ * Provides storage and retrieval for models and task-to-model mappings using SQLite.
+ */
export class SqliteModelRepository extends ModelRepository {
public type = "SqliteModelRepository" as const;
modelKvRepository: SqliteKVRepository<
diff --git a/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts b/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts
index 731dc2d..152a69a 100644
--- a/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts
+++ b/packages/storage/src/bun/sqlite/SqliteRateLimiter.ts
@@ -9,6 +9,10 @@ import { type Database } from "better-sqlite3";
import { ILimiter } from "ellmers-core";
import { toSQLiteTimestamp } from "../../util/Misc";
+/**
+ * SQLite implementation of a rate limiter.
+ * Manages request counts and delays to control job execution.
+ */
export class SqliteRateLimiter implements ILimiter {
private readonly db: Database;
private readonly queueName: string;
diff --git a/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts b/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts
index 941880a..006ddc9 100644
--- a/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts
+++ b/packages/storage/src/bun/sqlite/SqliteTaskGraphRepository.ts
@@ -8,6 +8,10 @@
import { TaskGraphRepository } from "ellmers-core";
import { SqliteKVRepository } from "./base/SqliteKVRepository";
+/**
+ * SQLite implementation of a task graph repository.
+ * Provides storage and retrieval for task graphs using SQLite.
+ */
export class SqliteTaskGraphRepository extends TaskGraphRepository {
kvRepository: SqliteKVRepository;
public type = "SqliteTaskGraphRepository" as const;
diff --git a/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts b/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts
index 2ea177e..7a239ca 100644
--- a/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts
+++ b/packages/storage/src/bun/sqlite/SqliteTaskOutputRepository.ts
@@ -13,6 +13,10 @@ import {
} from "ellmers-core";
import { SqliteKVRepository } from "./base/SqliteKVRepository";
+/**
+ * SQLite implementation of a task output repository.
+ * Provides storage and retrieval for task outputs using SQLite.
+ */
export class SqliteTaskOutputRepository extends TaskOutputRepository {
kvRepository: SqliteKVRepository<
TaskOutputPrimaryKey,
diff --git a/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts b/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts
index 378974d..a1c10ca 100644
--- a/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts
+++ b/packages/storage/src/node/filesystem/FileTaskGraphRepository.ts
@@ -8,6 +8,10 @@
import { TaskGraphRepository } from "ellmers-core";
import { FileKVRepository } from "./base/FileKVRepository";
+/**
+ * File-based implementation of a task graph repository.
+ * Provides storage and retrieval for task graphs using a file system.
+ */
export class FileTaskGraphRepository extends TaskGraphRepository {
kvRepository: FileKVRepository;
public type = "FileTaskGraphRepository" as const;
diff --git a/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts b/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts
index af762ae..31d1545 100644
--- a/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts
+++ b/packages/storage/src/node/filesystem/FileTaskOutputRepository.ts
@@ -14,6 +14,10 @@ import {
} from "ellmers-core";
import { FileKVRepository } from "./base/FileKVRepository";
+/**
+ * File-based implementation of a task output repository.
+ * Provides storage and retrieval for task outputs using a file system.
+ */
export class FileTaskOutputRepository extends TaskOutputRepository {
kvRepository: FileKVRepository<
TaskOutputPrimaryKey,
diff --git a/packages/storage/src/node/postgres/PostgresJobQueue.ts b/packages/storage/src/node/postgres/PostgresJobQueue.ts
index 8acf0e8..c3bba93 100644
--- a/packages/storage/src/node/postgres/PostgresJobQueue.ts
+++ b/packages/storage/src/node/postgres/PostgresJobQueue.ts
@@ -11,6 +11,10 @@ import { makeFingerprint } from "../../util/Misc";
// TODO: prepared statements
+/**
+ * PostgreSQL implementation of a job queue.
+ * Provides storage and retrieval for job execution states using PostgreSQL.
+ */
export class PostgresJobQueue extends JobQueue {
constructor(
protected readonly sql: Sql,
@@ -49,6 +53,11 @@ export class PostgresJobQueue extends JobQueue {
return this;
}
+ /**
+ * Creates a new job instance from the provided database results.
+ * @param results - The job data from the database
+ * @returns A new Job instance with populated properties
+ */
public createNewJob(results: any): Job {
return new this.jobClass({
...results,
@@ -61,6 +70,11 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Adds a new job to the queue.
+ * @param job - The job to add
+ * @returns The ID of the added job
+ */
public async add(job: Job) {
job.queueName = this.queue;
const fingerprint = await makeFingerprint(job.input);
@@ -74,6 +88,11 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Retrieves a job by its ID.
+ * @param id - The ID of the job to retrieve
+ * @returns The job if found, undefined otherwise
+ */
public async get(id: number) {
return await this.sql.begin(async (sql) => {
const result = await sql`
@@ -86,6 +105,11 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Retrieves a slice of jobs from the queue.
+ * @param num - Maximum number of jobs to return
+ * @returns An array of jobs
+ */
public async peek(num: number = 100) {
num = Number(num) || 100; // TS does not validate, so ensure it is a number
return await this.sql.begin(async (sql) => {
@@ -104,6 +128,10 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Retrieves all jobs currently being processed.
+ * @returns An array of jobs
+ */
public async processing() {
return await this.sql.begin(async (sql) => {
const result = await sql`
@@ -117,6 +145,10 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Retrieves the next available job that is ready to be processed.
+ * @returns The next job or undefined if no job is available
+ */
public async next() {
return await this.sql.begin(async (sql) => {
const result = await sql`
@@ -140,6 +172,11 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Retrieves the number of jobs in the queue with a specific status.
+ * @param status - The status of the jobs to count
+ * @returns The count of jobs with the specified status
+ */
public async size(status = JobStatus.PENDING) {
return await this.sql.begin(async (sql) => {
const result = await sql`
@@ -152,6 +189,13 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Marks a job as complete with its output or error.
+ * Handles retries for failed jobs and triggers completion callbacks.
+ * @param id - ID of the job to complete
+ * @param output - Result of the job execution
+ * @param error - Optional error message if job failed
+ */
public async complete(id: number, output: Output | null = null, error: string | null = null) {
const job = await this.get(id);
if (!job) throw new Error(`Job ${id} not found`);
@@ -183,6 +227,11 @@ export class PostgresJobQueue extends JobQueue {
});
}
+ /**
+ * Looks up cached output for a given task type and input
+ * Uses input fingerprinting for efficient matching
+ * @returns The cached output or null if not found
+ */
public async outputForInput(taskType: string, input: Input) {
const fingerprint = await makeFingerprint(input);
return await this.sql.begin(async (sql) => {
diff --git a/packages/storage/src/node/postgres/PostgresRateLimiter.ts b/packages/storage/src/node/postgres/PostgresRateLimiter.ts
index a4d3a81..07941c8 100644
--- a/packages/storage/src/node/postgres/PostgresRateLimiter.ts
+++ b/packages/storage/src/node/postgres/PostgresRateLimiter.ts
@@ -8,6 +8,10 @@
import { ILimiter } from "ellmers-core";
import { Sql } from "postgres";
+/**
+ * PostgreSQL implementation of a rate limiter.
+ * Manages request counts and delays to control job execution.
+ */
export class PostgresRateLimiter implements ILimiter {
private readonly windowSizeInMilliseconds: number;
@@ -36,6 +40,10 @@ export class PostgresRateLimiter implements ILimiter {
await this.sql`DELETE FROM job_rate_limit`;
}
+ /**
+ * Checks if a job can proceed based on rate limiting rules.
+ * @returns True if the job can proceed, false otherwise
+ */
async canProceed(): Promise {
const now = new Date();
const attemptedAtThreshold = new Date(now.getTime() - this.windowSizeInMilliseconds);
@@ -66,6 +74,10 @@ export class PostgresRateLimiter implements ILimiter {
return true;
}
+ /**
+ * Records a new job attempt.
+ * @returns The ID of the added job
+ */
async recordJobStart(): Promise {
// Record a new job attempt
await this.sql`
@@ -78,6 +90,10 @@ export class PostgresRateLimiter implements ILimiter {
// Optional for rate limiting: Cleanup or track completions if needed
}
+ /**
+ * Retrieves the next available time for the specific queue.
+ * @returns The next available time
+ */
async getNextAvailableTime(): Promise {
// Query for the earliest job attempt within the window that reaches the limit
const result = await this.sql`
@@ -96,6 +112,10 @@ export class PostgresRateLimiter implements ILimiter {
}
}
+ /**
+ * Sets the next available time for the specific queue.
+ * @param date - The new next available time
+ */
async setNextAvailableTime(date: Date): Promise {
// Update the next available time for the specific queue. If no entry exists, insert a new one.
await this.sql`
diff --git a/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts b/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts
index 855e44a..f8c43b8 100644
--- a/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts
+++ b/packages/storage/src/node/postgres/PostgresTaskGraphRepository.ts
@@ -8,6 +8,10 @@
import { TaskGraphRepository } from "ellmers-core";
import { PostgresKVRepository } from "./base/PostgresKVRepository";
+/**
+ * PostgreSQL implementation of a task graph repository.
+ * Provides storage and retrieval for task graphs using PostgreSQL.
+ */
export class PostgresTaskGraphRepository extends TaskGraphRepository {
kvRepository: PostgresKVRepository;
public type = "PostgresTaskGraphRepository" as const;
diff --git a/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts b/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts
index e69e24b..311a540 100644
--- a/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts
+++ b/packages/storage/src/node/postgres/PostgresTaskOutputRepository.ts
@@ -13,6 +13,10 @@ import {
} from "ellmers-core";
import { PostgresKVRepository } from "./base/PostgresKVRepository";
+/**
+ * PostgreSQL implementation of a task output repository.
+ * Provides storage and retrieval for task outputs using PostgreSQL.
+ */
export class PostgresTaskOutputRepository extends TaskOutputRepository {
kvRepository: PostgresKVRepository<
TaskOutputPrimaryKey,