docs: Add various doc comments

examples/cli/src/TaskHelper.ts

@@ -8,6 +8,18 @@
 import chalk from "chalk";
 import { ListrTaskWrapper } from "listr2";
 
+/**
+ * TaskHelper provides CLI progress visualization utilities.
+ *
+ * Features:
+ * - Unicode-based progress bar generation
+ * - Customizable bar length and progress indication
+ * - Color-coded output using chalk
+ *
+ * Used to create visual feedback for long-running tasks in the CLI interface,
+ * with smooth progress transitions and clear visual indicators.
+ */
+
 export function createBar(progress: number, length: number): string {
   let distance = progress * length;
   let bar = "";
@@ -47,11 +59,7 @@ export function createBar(progress: number, length: number): string {
   // Extend empty bar
   bar += "\u258F".repeat(length > bar.length ? length - bar.length : 0);
 
-  return chalk.rgb(
-    70,
-    70,
-    240
-  )("\u2595" + chalk.bgRgb(20, 20, 70)(bar) + "\u258F");
+  return chalk.rgb(70, 70, 240)("\u2595" + chalk.bgRgb(20, 20, 70)(bar) + "\u258F");
 }
 
 export class TaskHelper<T = any> {

packages/ai/src/provider/ProviderRegistry.ts

@@ -14,11 +14,18 @@ import {
   JobQueueTask,
 } from "ellmers-core";
 
+/**
+ * Enum to define the types of job queue execution
+ */
 export enum JobQueueRunType {
   local = "local",
   api = "api",
 }
 
+/**
+ * Extends the base Job class to provide custom execution functionality
+ * through a provided function.
+ */
 class ProviderJob<Input, Output> extends Job<Input, Output> {
   constructor(
     details: JobConstructorDetails<Input, Output> & {
@@ -35,9 +42,22 @@ class ProviderJob<Input, Output> extends Job<Input, Output> {
   }
 }
 
+/**
+ * Registry that manages provider-specific task execution functions and job queues.
+ * Handles the registration, retrieval, and execution of task processing functions
+ * for different model providers and task types.
+ */
 export class ProviderRegistry<Input, Output> {
+  // Registry of task execution functions organized by task type and model provider
   runFnRegistry: Record<string, Record<string, (task: any, runInputData: any) => Promise<Output>>> =
     {};
+
+  /**
+   * Registers a task execution function for a specific task type and model provider
+   * @param taskType - The type of task (e.g., 'text-generation', 'embedding')
+   * @param modelProvider - The provider of the model (e.g., 'hf-transformers', 'tf-mediapipe', 'openai', etc)
+   * @param runFn - The function that executes the task
+   */
   registerRunFn(
     taskType: string,
     modelProvider: string,
@@ -47,6 +67,10 @@ export class ProviderRegistry<Input, Output> {
     this.runFnRegistry[taskType][modelProvider] = runFn;
   }
 
+  /**
+   * Creates a job wrapper around a task execution function
+   * This allows the task to be queued and executed asynchronously
+   */
   jobAsRunFn(runtype: string, modelType: string) {
     const fn = this.runFnRegistry[runtype]?.[modelType];
     return async (task: JobQueueTask, input: Input) => {
@@ -68,11 +92,21 @@ export class ProviderRegistry<Input, Output> {
     };
   }
 
+  /**
+   * Retrieves the direct execution function for a task type and model
+   * Bypasses the job queue system for immediate execution
+   */
   getDirectRunFn(taskType: string, modelType: string) {
     return this.runFnRegistry[taskType]?.[modelType];
   }
 
+  // Map of model types to their corresponding job queues
   queues: Map<string, JobQueue<Input, Output>> = new Map();
+
+  /**
+   * Queue management methods for starting, stopping, and clearing job queues
+   * These methods help control the execution flow of tasks across all providers
+   */
   registerQueue(modelType: string, jobQueue: JobQueue<Input, Output>) {
     this.queues.set(modelType, jobQueue);
   }
@@ -100,6 +134,7 @@ export class ProviderRegistry<Input, Output> {
   }
 }
 
+// Singleton instance management for the ProviderRegistry
 let providerRegistry: ProviderRegistry<TaskInput, TaskOutput>;
 export function getProviderRegistry() {
   if (!providerRegistry) providerRegistry = new ProviderRegistry();

packages/ai/src/task/TextEmbeddingTask.ts

@@ -22,7 +22,11 @@ export type TextEmbeddingTaskInput = CreateMappedType<typeof TextEmbeddingTask.i
 export type TextEmbeddingTaskOutput = CreateMappedType<typeof TextEmbeddingTask.outputs>;
 
 /**
- * This is a task that generates an embedding for a single piece of text
+ * A task that generates vector embeddings for text using a specified embedding model.
+ * Embeddings are numerical representations of text that capture semantic meaning,
+ * useful for similarity comparisons and semantic search.
+ *
+ * @extends JobQueueLlmTask
  */
 export class TextEmbeddingTask extends JobQueueLlmTask {
   public static inputs = [
@@ -41,6 +45,7 @@ export class TextEmbeddingTask extends JobQueueLlmTask {
   constructor(config: JobQueueTaskConfig & { input?: TextEmbeddingTaskInput } = {}) {
     super(config);
   }
+
   declare runInputData: TextEmbeddingTaskInput;
   declare runOutputData: TextEmbeddingTaskOutput;
   declare defaults: Partial<TextEmbeddingTaskInput>;
@@ -52,11 +57,20 @@ TaskRegistry.registerTask(TextEmbeddingTask);
 type TextEmbeddingCompoundTaskOutput = ConvertAllToArrays<TextEmbeddingTaskOutput>;
 type TextEmbeddingCompoundTaskInput = ConvertSomeToOptionalArray<TextEmbeddingTaskInput, "model">;
 
+/**
+ * A compound task factory that creates a task capable of processing multiple texts
+ * and generating embeddings in parallel
+ */
 export const TextEmbeddingCompoundTask = arrayTaskFactory<
   TextEmbeddingCompoundTaskInput,
   TextEmbeddingCompoundTaskOutput
 >(TextEmbeddingTask, ["model", "text"]);
 
+/**
+ * Convenience function to create and run a TextEmbeddingCompoundTask
+ * @param {TextEmbeddingCompoundTaskInput} input - Input containing text(s) and model(s) for embedding
+ * @returns {Promise<TextEmbeddingCompoundTaskOutput>} Promise resolving to the generated embeddings
+ */
 export const TextEmbedding = (input: TextEmbeddingCompoundTaskInput) => {
   return new TextEmbeddingCompoundTask({ input }).run();
 };

packages/ai/src/task/TextGenerationTask.ts

@@ -55,11 +55,22 @@ type TextGenerationCompoundTaskInput = ConvertSomeToOptionalArray<
   TextGenerationTaskInput,
   "model" | "prompt"
 >;
+
+/**
+ * Factory-generated task class for handling batch text generation operations.
+ * Created using arrayTaskFactory to support processing multiple prompts/models simultaneously.
+ */
 export const TextGenerationCompoundTask = arrayTaskFactory<
   TextGenerationCompoundTaskInput,
   TextGenerationCompoundOutput
 >(TextGenerationTask, ["model", "prompt"]);
 
+/**
+ * Convenience function to run text generation tasks.
+ * Creates and executes a TextGenerationCompoundTask with the provided input.
+ * @param input The input parameters for text generation (prompts and models)
+ * @returns Promise resolving to the generated text output(s)
+ */
 export const TextGeneration = (input: TextGenerationCompoundTaskInput) => {
   return new TextGenerationCompoundTask({ input }).run();
 };

packages/core/src/source/MasterDocument.ts

@@ -8,6 +8,19 @@
 import { Document, DocumentMetadata, TextFragment } from "./Document";
 import { DocumentConverter } from "./DocumentConverter";
 
+/**
+ * MasterDocument represents a container for managing multiple versions/variants of a document.
+ * It maintains the original document and its transformed variants for different use cases.
+ *
+ * Key features:
+ * - Stores original document and metadata
+ * - Maintains a master version and variants
+ * - Automatically creates paragraph-split variant
+ *
+ * The paragraph variant splits text fragments by newlines while preserving other fragment types,
+ * which is useful for more granular text processing.
+ */
+
 export class MasterDocument {
   public metadata: DocumentMetadata;
   public original: DocumentConverter;

packages/core/src/storage/taskgraph/TaskGraphRepository.ts

@@ -11,22 +11,53 @@ import { KVRepository } from "../base/KVRepository";
 import { CompoundTask } from "../../task/base/Task";
 import { TaskRegistry } from "../../task/base/TaskRegistry";
 
+/**
+ * Events that can be emitted by the TaskGraphRepository
+ */
 export type TaskGraphEvents = "graph_saved" | "graph_retrieved" | "graph_cleared";
 
+/**
+ * Abstract repository class for managing task graphs persistence and retrieval.
+ * Provides functionality to save, load, and manipulate task graphs with their associated tasks and data flows.
+ */
 export abstract class TaskGraphRepository {
   public type = "TaskGraphRepository";
   abstract kvRepository: KVRepository;
   private events = new EventEmitter<TaskGraphEvents>();
+
+  /**
+   * Registers an event listener for the specified event
+   * @param name The event name to listen for
+   * @param fn The callback function to execute when the event occurs
+   */
   on(name: TaskGraphEvents, fn: (...args: any[]) => void) {
     this.events.on.call(this.events, name, fn);
   }
+
+  /**
+   * Removes an event listener for the specified event
+   * @param name The event name to stop listening for
+   * @param fn The callback function to remove
+   */
   off(name: TaskGraphEvents, fn: (...args: any[]) => void) {
     this.events.off.call(this.events, name, fn);
   }
+
+  /**
+   * Emits an event with the given arguments
+   * @param name The event name to emit
+   * @param args Additional arguments to pass to the event listeners
+   */
   emit(name: TaskGraphEvents, ...args: any[]) {
     this.events.emit.call(this.events, name, ...args);
   }
 
+  /**
+   * Creates a task instance from a task graph item JSON representation
+   * @param item The JSON representation of the task
+   * @returns A new task instance
+   * @throws Error if required fields are missing or invalid
+   */
   private createTask(item: TaskGraphItemJson) {
     if (!item.id) throw new Error("Task id required");
     if (!item.type) throw new Error("Task type required");
@@ -51,6 +82,11 @@ export abstract class TaskGraphRepository {
     return task;
   }
 
+  /**
+   * Creates a TaskGraph instance from its JSON representation
+   * @param graphJsonObj The JSON representation of the task graph
+   * @returns A new TaskGraph instance with all tasks and data flows
+   */
   public createSubGraph(graphJsonObj: TaskGraphJson) {
     const subGraph = new TaskGraph();
     for (const subitem of graphJsonObj.nodes) {
@@ -69,12 +105,24 @@ export abstract class TaskGraphRepository {
     return subGraph;
   }
 
+  /**
+   * Saves a task graph to persistent storage
+   * @param key The unique identifier for the task graph
+   * @param output The task graph to save
+   * @emits graph_saved when the operation completes
+   */
   async saveTaskGraph(key: string, output: TaskGraph): Promise<void> {
     const value = JSON.stringify(output.toJSON());
     await this.kvRepository.put(key, value);
     this.emit("graph_saved", key);
   }
 
+  /**
+   * Retrieves a task graph from persistent storage
+   * @param key The unique identifier of the task graph to retrieve
+   * @returns The retrieved task graph, or undefined if not found
+   * @emits graph_retrieved when the operation completes successfully
+   */
   async getTaskGraph(key: string): Promise<TaskGraph | undefined> {
     const jsonStr = (await this.kvRepository.get(key)) as string;
     if (!jsonStr) {
@@ -88,11 +136,19 @@ export abstract class TaskGraphRepository {
     return graph;
   }
 
+  /**
+   * Clears all task graphs from the repository
+   * @emits graph_cleared when the operation completes
+   */
   async clear(): Promise<void> {
     await this.kvRepository.deleteAll();
     this.emit("graph_cleared");
   }
 
+  /**
+   * Returns the number of task graphs stored in the repository
+   * @returns The count of stored task graphs
+   */
   async size(): Promise<number> {
     return await this.kvRepository.size();
   }

packages/core/src/storage/taskoutput/TaskOutputRepository.ts

@@ -21,6 +21,10 @@ export const TaskOutputPrimaryKeySchema = {
   taskType: "string",
 } as const;
 
+/**
+ * Abstract class for managing task outputs in a repository
+ * Provides methods for saving, retrieving, and clearing task outputs
+ */
 export abstract class TaskOutputRepository {
   public type = "TaskOutputRepository";
   abstract kvRepository: KVRepository<
@@ -29,35 +33,73 @@ export abstract class TaskOutputRepository {
     typeof TaskOutputPrimaryKeySchema
   >;
   private events = new EventEmitter<TaskOutputEvents>();
+
+  /**
+   * Registers an event listener for a specific event
+   * @param name The event name to listen for
+   * @param fn The callback function to execute when the event occurs
+   */
   on(name: TaskOutputEvents, fn: (...args: any[]) => void) {
     this.events.on.call(this.events, name, fn);
   }
+
+  /**
+   * Removes an event listener for a specific event
+   * @param name The event name to stop listening for
+   * @param fn The callback function to remove
+   */
   off(name: TaskOutputEvents, fn: (...args: any[]) => void) {
     this.events.off.call(this.events, name, fn);
   }
+
+  /**
+   * Emits an event with the given arguments
+   * @param name The event name to emit
+   * @param args Additional arguments to pass to the event listeners
+   */
   emit(name: TaskOutputEvents, ...args: any[]) {
     this.events.emit.call(this.events, name, ...args);
   }
 
+  /**
+   * Saves a task output to the repository
+   * @param taskType The type of task to save the output for
+   * @param inputs The input parameters for the task
+   * @param output The task output to save
+   */
   async saveOutput(taskType: string, inputs: TaskInput, output: TaskOutput): Promise<void> {
     const key = await makeFingerprint(inputs);
     const value = JSON.stringify(output);
     await this.kvRepository.putKeyValue({ key, taskType }, { "kv-value": value });
     this.emit("output_saved", taskType);
   }
 
+  /**
+   * Retrieves a task output from the repository
+   * @param taskType The type of task to retrieve the output for
+   * @param inputs The input parameters for the task
+   * @returns The retrieved task output, or undefined if not found
+   */
   async getOutput(taskType: string, inputs: TaskInput): Promise<TaskOutput | undefined> {
     const key = await makeFingerprint(inputs);
     const output = await this.kvRepository.getKeyValue({ key, taskType });
     this.emit("output_retrieved", taskType);
     return output ? (JSON.parse(output["kv-value"]) as TaskOutput) : undefined;
   }
 
+  /**
+   * Clears all task outputs from the repository
+   * @emits output_cleared when the operation completes
+   */
   async clear(): Promise<void> {
     await this.kvRepository.deleteAll();
     this.emit("output_cleared");
   }
 
+  /**
+   * Returns the number of task outputs stored in the repository
+   * @returns The count of stored task outputs
+   */
   async size(): Promise<number> {
     return await this.kvRepository.size();
   }