Merge pull request #23 from sroussey/remove-type-lookups

refactor: Unroll the auto type system and make explicit

docs/developers/01_getting_started.md

@@ -131,22 +131,24 @@ And unrolling the config helpers, we get the following equivalent code:
 
 ```ts
 import {
-  DownloadModelTask,
-  TextRewriterCompoundTask,
   DebugLog,
   DataFlow,
   TaskGraph,
   TaskGraphRunner,
-  getProviderRegistry,
-  HuggingFaceLocal_DownloadRun,
-  HuggingFaceLocal_TextRewriterRun,
-  InMemoryJobQueue,
-  ModelProcessorEnum,
   ConcurrencyLimiter,
   TaskInput,
   TaskOutput,
 } from "ellmers-core";
 
+import { DownloadModelTask, TextRewriterCompoundTask, getAiProviderRegistry } from "ellmers-ai";
+
+import {
+  HuggingFaceLocal_DownloadRun,
+  HuggingFaceLocal_TextRewriterRun,
+} from "ellmers-ai-provider/hf-transformers";
+
+import { InMemoryJobQueue } from "ellmers-storage/inmemory";
+
 // config and start up
 getGlobalModelRepository(new InMemoryModelRepository());
 await getGlobalModelRepository().addModel({
@@ -166,7 +168,7 @@ await getGlobalModelRepository().connectTaskToModel(
   "ONNX Xenova/LaMini-Flan-T5-783M q8"
 );
 
-const ProviderRegistry = getProviderRegistry();
+const ProviderRegistry = getAiProviderRegistry();
 ProviderRegistry.registerRunFn(
   DownloadModelTask.type,
   ModelProcessorEnum.LOCAL_ONNX_TRANSFORMERJS,
@@ -187,7 +189,9 @@ jobQueue.start();
 
 // build and run graph
 const graph = new TaskGraph();
-graph.addTask(new DownloadModel({ id: "1", input: { model: "Xenova/LaMini-Flan-T5-783M" } }));
+graph.addTask(
+  new DownloadModel({ id: "1", input: { model: "ONNX Xenova/LaMini-Flan-T5-783M q8" } })
+);
 graph.addTask(
   new TextRewriterCompoundTask({
     id: "2",
@@ -256,7 +260,7 @@ An example is TextEmbeddingTask and TextEmbeddingCompoundTask. The first takes a
 import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
-  model: "Xenova/LaMini-Flan-T5-783M",
+  model: "ONNX Xenova/LaMini-Flan-T5-783M q8",
   text: "The quick brown fox jumps over the lazy dog.",
 });
 await builder.run();
@@ -268,7 +272,7 @@ OR
 import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder.TextEmbedding({
-  model: ["Xenova/LaMini-Flan-T5-783M", "Universal Sentence Encoder"],
+  model: ["ONNX Xenova/LaMini-Flan-T5-783M q8", "Universal Sentence Encoder"],
   text: "The quick brown fox jumps over the lazy dog.",
 });
 await builder.run();
@@ -281,7 +285,7 @@ import { TaskGraphBuilder } from "ellmers-core";
 const builder = new TaskGraphBuilder();
 builder
   .DownloadModel({
-    model: ["Xenova/LaMini-Flan-T5-783M", "Universal Sentence Encoder"],
+    model: ["ONNX Xenova/LaMini-Flan-T5-783M q8", "Universal Sentence Encoder"],
   })
   .TextEmbedding({
     text: "The quick brown fox jumps over the lazy dog.",
@@ -316,7 +320,7 @@ There is a JSONTask that can be used to build a graph. This is useful for saving
     "dependencies": {
       "model": {
         "id": "1",
-        "output": "text_generation_model"
+        "output": "generation_model"
       }
     }
   },
@@ -376,7 +380,7 @@ To use a task, instantiate it with some input and call `run()`:
 const task = new TextEmbeddingTask({
   id: "1",
   input: {
-    model: "Xenova/LaMini-Flan-T5-783M",
+    model: "ONNX Xenova/LaMini-Flan-T5-783M q8",
     text: "The quick brown fox jumps over the lazy dog.",
   },
 });
@@ -397,7 +401,7 @@ const graph = new TaskGraph();
 graph.addTask(
   new TextRewriterCompoundTask({
     input: {
-      model: "Xenova/LaMini-Flan-T5-783M",
+      model: "ONNX Xenova/LaMini-Flan-T5-783M q8",
       text: "The quick brown fox jumps over the lazy dog.",
       prompt: ["Rewrite the following text in reverse:", "Rewrite this to sound like a pirate:"],
     },
@@ -417,7 +421,7 @@ graph.addTask(
   new TextRewriterCompoundTask({
     id: "1",
     input: {
-      model: "Xenova/LaMini-Flan-T5-783M",
+      model: "ONNX Xenova/LaMini-Flan-T5-783M q8",
       text: "The quick brown fox jumps over the lazy dog.",
       prompt: ["Rewrite the following text in reverse:", "Rewrite this to sound like a pirate:"],
     },

examples/cli/src/ellmers.ts

@@ -3,7 +3,7 @@
 import { program } from "commander";
 import { argv } from "process";
 import { AddBaseCommands } from "./TaskCLI";
-import { getProviderRegistry } from "ellmers-ai";
+import { getAiProviderRegistry } from "ellmers-ai";
 import {
   registerHuggingfaceLocalModels,
   registerHuggingfaceLocalTasksInMemory,
@@ -24,4 +24,4 @@ registerMediaPipeTfJsLocalInMemory();
 
 await program.parseAsync(argv);
 
-getProviderRegistry().stopQueues();
+getAiProviderRegistry().stopQueues();

examples/web/src/App.tsx

@@ -20,7 +20,7 @@ import { QueuesStatus } from "./QueueSatus";
 import { OutputRepositoryStatus } from "./OutputRepositoryStatus";
 import { GraphStoreStatus } from "./GraphStoreStatus";
 import { InMemoryJobQueue } from "ellmers-storage/inmemory";
-import { getProviderRegistry } from "ellmers-ai";
+import { getAiProviderRegistry } from "ellmers-ai";
 import {
   LOCAL_ONNX_TRANSFORMERJS,
   registerHuggingfaceLocalTasks,
@@ -38,7 +38,7 @@ import { env } from "@huggingface/transformers";
 env.backends.onnx.wasm.proxy = true;
 env.allowLocalModels = true;
 
-const ProviderRegistry = getProviderRegistry();
+const ProviderRegistry = getAiProviderRegistry();
 
 registerHuggingfaceLocalTasks();
 ProviderRegistry.registerQueue(
@@ -77,8 +77,8 @@ const resetGraph = () => {
     })
     .TextTranslation({
       model: "ONNX Xenova/m2m100_418M q8",
-      source: "en",
-      target: "es",
+      source_lang: "en",
+      target_lang: "es",
     })
     .rename("text", "message")
     .rename("text", "message", -2)

examples/web/src/QueueSatus.tsx

@@ -1,9 +1,9 @@
 import { JobStatus } from "ellmers-core";
-import { getProviderRegistry } from "ellmers-ai";
+import { getAiProviderRegistry } from "ellmers-ai";
 import { useCallback, useEffect, useState } from "react";
 
 export function QueueStatus({ queueType }: { queueType: string }) {
-  const queue = getProviderRegistry().getQueue(queueType);
+  const queue = getAiProviderRegistry().getQueue(queueType);
   const [pending, setPending] = useState<number>(0);
   const [processing, setProcessing] = useState<number>(0);
   const [completed, setCompleted] = useState<number>(0);
@@ -51,7 +51,7 @@ export function QueueStatus({ queueType }: { queueType: string }) {
 }
 
 export function QueuesStatus() {
-  const queues = getProviderRegistry().queues;
+  const queues = getAiProviderRegistry().queues;
   const queueKeys = Array.from(queues.keys());
 
   return (

packages/ai-provider/src/hf-transformers/bindings/registerTasks.ts

@@ -1,5 +1,5 @@
 import {
-  getProviderRegistry,
+  getAiProviderRegistry,
   DownloadModelTask,
   TextEmbeddingTask,
   TextGenerationTask,
@@ -20,7 +20,7 @@ import {
 import { LOCAL_ONNX_TRANSFORMERJS } from "../model/ONNXTransformerJsModel";
 
 export async function registerHuggingfaceLocalTasks() {
-  const ProviderRegistry = getProviderRegistry();
+  const ProviderRegistry = getAiProviderRegistry();
 
   ProviderRegistry.registerRunFn(
     DownloadModelTask.type,

packages/ai-provider/src/hf-transformers/provider/HuggingFaceLocal_TaskRun.ts

@@ -20,8 +20,7 @@ import {
   type TranslationSingle,
   TextStreamer,
 } from "@huggingface/transformers";
-import { ElVector } from "ellmers-core";
-import { getGlobalModelRepository } from "ellmers-ai";
+import { ElVector, getGlobalModelRepository } from "ellmers-ai";
 import type {
   JobQueueLlmTask,
   DownloadModelTask,
@@ -229,8 +228,8 @@ export async function HuggingFaceLocal_TextTranslationRun(
   });
 
   let results = await translate(runInputData.text, {
-    src_lang: runInputData.source,
-    tgt_lang: runInputData.target,
+    src_lang: runInputData.source_lang,
+    tgt_lang: runInputData.target_lang,
     streamer,
   } as any);
   if (!Array.isArray(results)) {

packages/ai-provider/src/hf-transformers/test/HFTransformersBinding.test.ts

@@ -8,7 +8,7 @@
 import { describe, expect, it } from "bun:test";
 import { ConcurrencyLimiter, TaskGraphBuilder, TaskInput, TaskOutput } from "ellmers-core";
 import {
-  getProviderRegistry,
+  getAiProviderRegistry,
   getGlobalModelRepository,
   setGlobalModelRepository,
 } from "ellmers-ai";
@@ -23,7 +23,7 @@ const HFQUEUE = "local_hf";
 describe("HFTransformersBinding", () => {
   describe("InMemoryJobQueue", () => {
     it("Should have an item queued", async () => {
-      const providerRegistry = getProviderRegistry();
+      const providerRegistry = getAiProviderRegistry();
       const jobQueue = new InMemoryJobQueue<TaskInput, TaskOutput>(
         HFQUEUE,
         new ConcurrencyLimiter(1, 10),
@@ -85,7 +85,7 @@ describe("HFTransformersBinding", () => {
         "TextRewritingTask",
         "ONNX Xenova/LaMini-Flan-T5-783M q8"
       );
-      const providerRegistry = getProviderRegistry();
+      const providerRegistry = getAiProviderRegistry();
       const jobQueue = new SqliteJobQueue<TaskInput, TaskOutput>(
         getDatabase(":memory:"),
         HFQUEUE,

packages/ai-provider/src/tf-mediapipe/bindings/registerTasks.ts

@@ -1,4 +1,4 @@
-import { getProviderRegistry } from "ellmers-ai";
+import { getAiProviderRegistry } from "ellmers-ai";
 import { DownloadModelTask, TextEmbeddingTask } from "ellmers-ai";
 import {
   MediaPipeTfJsLocal_Download,
@@ -7,7 +7,7 @@ import {
 import { MEDIA_PIPE_TFJS_MODEL } from "..";
 
 export const registerMediaPipeTfJsLocalTasks = () => {
-  const ProviderRegistry = getProviderRegistry();
+  const ProviderRegistry = getAiProviderRegistry();
 
   ProviderRegistry.registerRunFn(
     DownloadModelTask.type,

packages/ai-provider/src/tf-mediapipe/provider/MediaPipeLocalTaskRun.ts

@@ -6,13 +6,13 @@
 //    *******************************************************************************
 
 import { FilesetResolver, TextEmbedder } from "@mediapipe/tasks-text";
-import { ElVector } from "ellmers-core";
 import {
   DownloadModelTask,
   DownloadModelTaskInput,
   TextEmbeddingTask,
   TextEmbeddingTaskInput,
   getGlobalModelRepository,
+  ElVector,
 } from "ellmers-ai";
 
 /**

packages/ai-provider/src/tf-mediapipe/test/TfMediaPipeBinding.test.ts

@@ -7,7 +7,7 @@
 
 // import { describe, expect, it } from "bun:test";
 // import { ConcurrencyLimiter, TaskGraphBuilder, TaskInput, TaskOutput } from "ellmers-core";
-// import { getGlobalModelRepository, getProviderRegistry, Model } from "ellmers-ai";
+// import { getGlobalModelRepository, getAiProviderRegistry, Model } from "ellmers-ai";
 // import { InMemoryJobQueue } from "ellmers-storage/inmemory";
 // import { SqliteJobQueue } from "../../../../storage/dist/bun/sqlite";
 // import { registerMediaPipeTfJsLocalTasks } from "../bindings/registerTasks";
@@ -35,7 +35,7 @@
 //         universal_sentence_encoder.name
 //       );
 //       registerMediaPipeTfJsLocalTasks();
-//       const ProviderRegistry = getProviderRegistry();
+//       const ProviderRegistry = getAiProviderRegistry();
 //       const jobQueue = new InMemoryJobQueue<TaskInput, TaskOutput>(
 //         TFQUEUE,
 //         new ConcurrencyLimiter(1, 10),
@@ -74,7 +74,7 @@
 //         universal_sentence_encoder.name
 //       );
 //       registerMediaPipeTfJsLocalTasks();
-//       const ProviderRegistry = getProviderRegistry();
+//       const ProviderRegistry = getAiProviderRegistry();
 //       const jobQueue = new SqliteJobQueue<TaskInput, TaskOutput>(
 //         getDatabase(":memory:"),
 //         TFQUEUE,

packages/ai/src/index.ts

@@ -2,4 +2,7 @@ export * from "./task";
 export * from "./model/Model";
 export * from "./model/ModelRegistry";
 export * from "./model/ModelRepository";
-export * from "./provider/ProviderRegistry";
+export * from "./source/Document";
+export * from "./source/DocumentConverterText";
+export * from "./source/DocumentConverterMarkdown";
+export * from "./provider/AiProviderRegistry";

packages/ai/src/provider/ProviderRegistry.ts → packages/ai/src/provider/AiProviderRegistry.ts

@@ -47,7 +47,7 @@ class ProviderJob<Input, Output> extends Job<Input, Output> {
  * Handles the registration, retrieval, and execution of task processing functions
  * for different model providers and task types.
  */
-export class ProviderRegistry<Input, Output> {
+export class AiProviderRegistry<Input, Output> {
   // Registry of task execution functions organized by task type and model provider
   runFnRegistry: Record<string, Record<string, (task: any, runInputData: any) => Promise<Output>>> =
     {};
@@ -135,11 +135,11 @@ export class ProviderRegistry<Input, Output> {
 }
 
 // Singleton instance management for the ProviderRegistry
-let providerRegistry: ProviderRegistry<TaskInput, TaskOutput>;
-export function getProviderRegistry() {
-  if (!providerRegistry) providerRegistry = new ProviderRegistry();
+let providerRegistry: AiProviderRegistry<TaskInput, TaskOutput>;
+export function getAiProviderRegistry() {
+  if (!providerRegistry) providerRegistry = new AiProviderRegistry();
   return providerRegistry;
 }
-export function setProviderRegistry(pr: ProviderRegistry<TaskInput, TaskOutput>) {
+export function setAiProviderRegistry(pr: AiProviderRegistry<TaskInput, TaskOutput>) {
   providerRegistry = pr;
 }

packages/core/src/source/Document.ts → packages/ai/src/source/Document.ts

@@ -13,6 +13,18 @@ enum DocumentType {
   TABLE = "table",
 }
 
+const doc_variants = [
+  "tree",
+  "flat",
+  "tree-paragraphs",
+  "flat-paragraphs",
+  "tree-sentences",
+  "flat-sentences",
+] as const;
+type DocVariant = (typeof doc_variants)[number];
+const doc_parsers = ["txt", "md"] as const; // | "html" | "pdf" | "csv";
+type DocParser = (typeof doc_parsers)[number];
+
 export interface DocumentMetadata {
   title: string;
 }

packages/ai/src/task/DocumentSplitterTask.ts

@@ -5,18 +5,15 @@
 //    *   Licensed under the Apache License, Version 2.0 (the "License");           *
 //    *******************************************************************************
 
-import {
-  Document,
-  DocumentFragment,
-  SingleTask,
-  TaskGraphBuilder,
-  TaskGraphBuilderHelper,
-  CreateMappedType,
-  TaskRegistry,
-} from "ellmers-core";
-
-export type DocumentSplitterTaskInput = CreateMappedType<typeof DocumentSplitterTask.inputs>;
-export type DocumentSplitterTaskOutput = CreateMappedType<typeof DocumentSplitterTask.outputs>;
+import { SingleTask, TaskGraphBuilder, TaskGraphBuilderHelper, TaskRegistry } from "ellmers-core";
+import { Document, DocumentFragment } from "../source/Document";
+export type DocumentSplitterTaskInput = {
+  parser: "txt" | "md";
+  file: Document;
+};
+export type DocumentSplitterTaskOutput = {
+  texts: string[];
+};
 
 export class DocumentSplitterTask extends SingleTask {
   static readonly type: string = "DocumentSplitterTask";