Merge pull request #23 from lalalune/evaluators

Add Evaluators

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bgent",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "private": false,
   "description": "bgent. because agent was taken.",
   "type": "module",

scripts/concat.mjs

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url'
 const instructions = 'The above code was taken from my codebase at https://github.com/lalalune/bgent. You are writing tests and documentation for my codebase. Please use the above code as a reference. Tests should be written with Jest and Typescript. Do not use mocks or stubs. Keep it very simple and straightforward.'
 
 // Patterns to ignore
-const ignorePatterns = ['actionExamples.ts', 'agents', 'goal', 'goals', 'utils', 'logger', 'index', 'data', 'constants', 'templates', 'worker']
+const ignorePatterns = ['actionExamples.ts', 'agents', 'goal', 'cache', 'goals', 'supabase',  'utils', 'logger', 'index', 'data', 'constants', 'templates', 'worker']
 
 // __dirname is not defined in ES module scope, so we need to create it
 const __filename = fileURLToPath(import.meta.url)
@@ -24,7 +24,7 @@ const shouldIgnore = (filePath) => {
 
 // Function to recursively read through directories and concatenate .ts files
 const readDirectory = (dirPath) => {
-    let concatenatedContent = '# START MY CODEBASE'
+    let concatenatedContent = ''
 
     fs.readdirSync(dirPath).forEach(file => {
         const filePath = path.join(dirPath, file)
@@ -51,7 +51,7 @@ const readDirectory = (dirPath) => {
 }
 
 // Start reading from the root TypeScript directory
-const concatenatedContent = readDirectory(directoryPath)
+const concatenatedContent = '# START MY CODEBASE' + readDirectory(directoryPath)
 
 // Write the concatenated content to the output file
 fs.writeFileSync(outputFile, concatenatedContent + '# END MY CODEBASE\n\n' + instructions)

src/agents/cj/actions/__tests__/introduce.test.ts

@@ -23,9 +23,9 @@ const zeroUuid = "00000000-0000-0000-0000-000000000000" as UUID;
 
 describe("Introduce Action", () => {
   test("Introduce the user", async () => {
-    const { user, runtime } = await createRuntime(
-      process.env as Record<string, string>,
-    );
+    const { user, runtime } = await createRuntime({
+      env: process.env as Record<string, string>,
+    });
 
     const data = await getRelationship({
       runtime,

src/agents/cj/evaluators/__tests__/details.test.ts

@@ -21,10 +21,9 @@ const zeroUuid = "00000000-0000-0000-0000-000000000000" as UUID;
 
 describe("User Details", () => {
   test("Get user details", async () => {
-    const { user, runtime } = await createRuntime(
-      process.env as Record<string, string>,
-      24,
-    );
+    const { user, runtime } = await createRuntime({
+      env: process.env as Record<string, string>,
+    });
 
     const data = await getRelationship({
       runtime,

src/agents/cj/evaluators/__tests__/profile.test.ts

@@ -28,7 +28,9 @@ let user: User;
 
 describe("User Profile", () => {
   beforeAll(async () => {
-    const result = await createRuntime();
+    const result = await createRuntime({
+      env: process.env as Record<string, string>,
+    });
     runtime = result.runtime;
     user = result.session.user;
   });
@@ -56,9 +58,9 @@ describe("User Profile", () => {
   });
 
   test("Get user profile", async () => {
-    const { user, runtime } = await createRuntime(
-      process.env as Record<string, string>,
-    );
+    const { user, runtime } = await createRuntime({
+      env: process.env as Record<string, string>,
+    });
 
     const data = await getRelationship({
       runtime,
@@ -164,7 +166,7 @@ describe("User Profile", () => {
 
       expect(
         result.toLowerCase().includes("startup") ||
-          result.toLowerCase().includes("programmer"),
+        result.toLowerCase().includes("programmer"),
       ).toBe(true);
 
       const descriptions = await runtime.descriptionManager.getMemoriesByIds({

src/lib/__tests__/actions.test.ts

@@ -2,8 +2,10 @@ import { type User } from "@supabase/supabase-js";
 import { type UUID } from "crypto";
 import dotenv from "dotenv";
 import { createRuntime } from "../../test/createRuntime";
+import { TEST_ACTION, TEST_ACTION_FAIL } from "../../test/testAction";
 import { getRelationship } from "../relationships";
 import { type BgentRuntime } from "../runtime";
+import { type Message } from "../types";
 
 dotenv.config();
 
@@ -12,24 +14,27 @@ const zeroUuid = "00000000-0000-0000-0000-000000000000" as UUID;
 describe("Actions", () => {
   let user: User;
   let runtime: BgentRuntime;
-  // let room_id: UUID;
+  let room_id: UUID;
 
   afterAll(async () => {
     await cleanup();
   });
 
   beforeAll(async () => {
-    const setup = await createRuntime();
-    user = setup.session.user;
-    runtime = setup.runtime;
+    const { session, runtime: _runtime } = await createRuntime({
+      env: process.env as Record<string, string>,
+      actions: [TEST_ACTION, TEST_ACTION_FAIL],
+    });
+    user = session.user;
+    runtime = _runtime;
 
-    // const data = await getRelationship({
-    //   runtime,
-    //   userA: user?.id as UUID,
-    //   userB: zeroUuid,
-    // });
+    const data = await getRelationship({
+      runtime,
+      userA: user?.id as UUID,
+      userB: zeroUuid,
+    });
 
-    // room_id = data?.room_id;
+    room_id = data?.room_id;
 
     await cleanup();
   });
@@ -45,96 +50,88 @@ describe("Actions", () => {
     ]);
   }
 
-  describe("Actions", () => {
-    let user: User;
-    let runtime: BgentRuntime;
-    let room_id: UUID;
-
-    afterAll(async () => {
-      await cleanup();
-    });
-
-    beforeAll(async () => {
-      const setup = await createRuntime();
-      user = setup.session.user;
-      runtime = setup.runtime;
+  // Test that actions are being loaded into context properly
+  test("Actions are loaded into context", async () => {
+    const actions = runtime.actions;
+    expect(actions).toBeDefined();
+    expect(actions.length).toBeGreaterThan(0);
+    // Ensure the CONTINUE action is part of the loaded actions
+    const continueAction = actions.find((action) => action.name === "CONTINUE");
+    expect(continueAction).toBeDefined();
+  });
 
-      const data = await getRelationship({
-        runtime,
-        userA: user?.id as UUID,
-        userB: zeroUuid,
+  // Test that actions are validated properly
+  test("Continue action is always valid", async () => {
+    const continueAction = runtime.actions.find(
+      (action) => action.name === "CONTINUE",
+    );
+    expect(continueAction).toBeDefined();
+    if (continueAction && continueAction.validate) {
+      const isValid = await continueAction.validate(runtime, {
+        agentId: zeroUuid,
+        senderId: user.id as UUID,
+        userIds: [user.id as UUID, zeroUuid],
+        content: "Test message",
+        room_id: room_id,
       });
+      expect(isValid).toBeTruthy();
+    } else {
+      throw new Error(
+        "Continue action or its validation function is undefined",
+      );
+    }
+  });
 
-      room_id = data?.room_id;
+  test("Validate that TEST_ACTION is in the state", async () => {
+    const message: Message = {
+      senderId: user.id as UUID,
+      agentId: zeroUuid,
+      userIds: [user.id as UUID, zeroUuid],
+      content:
+        "Please respond with the message 'ok' and the action TEST_ACTION",
+      room_id,
+    };
 
-      await cleanup();
-    });
+    const response = await runtime.handleRequest(message);
 
-    async function cleanup() {
-      await runtime.summarizationManager.removeAllMemoriesByUserIds([
-        user?.id as UUID,
-        zeroUuid,
-      ]);
-      await runtime.messageManager.removeAllMemoriesByUserIds([
-        user?.id as UUID,
-        zeroUuid,
-      ]);
-    }
+    expect(response.action).toEqual("TEST_ACTION");
+  });
 
-    // Test that actions are being loaded into context properly
-    test("Actions are loaded into context", async () => {
-      const actions = runtime.getActions();
-      expect(actions).toBeDefined();
-      expect(actions.length).toBeGreaterThan(0);
-      // Ensure the CONTINUE action is part of the loaded actions
-      const continueAction = actions.find(
-        (action) => action.name === "CONTINUE",
-      );
-      expect(continueAction).toBeDefined();
-    });
+  test("Test that actions are properly validated in state", async () => {
+    const message: Message = {
+      senderId: user.id as UUID,
+      agentId: zeroUuid,
+      userIds: [user.id as UUID, zeroUuid],
+      content:
+        "Please respond with the message 'ok' and the action TEST_ACTION",
+      room_id,
+    };
 
-    // Test that actions are validated properly
-    test("Continue action is always valid", async () => {
-      const continueAction = runtime
-        .getActions()
-        .find((action) => action.name === "CONTINUE");
-      expect(continueAction).toBeDefined();
-      if (continueAction && continueAction.validate) {
-        const isValid = await continueAction.validate(runtime, {
-          agentId: zeroUuid,
-          senderId: user.id as UUID,
-          userIds: [user.id as UUID, zeroUuid],
-          content: "Test message",
-          room_id: room_id,
-        });
-        expect(isValid).toBeTruthy();
-      } else {
-        throw new Error(
-          "Continue action or its validation function is undefined",
-        );
-      }
-    });
+    const state = await runtime.composeState(message);
+    expect(state.actionNames).not.toContain("TEST_ACTION_FAIL");
 
-    // Test that action handlers are being called properly
-    test("Continue action handler is called", async () => {
-      const continueAction = runtime
-        .getActions()
-        .find((action) => action.name === "CONTINUE");
-      expect(continueAction).toBeDefined();
-      if (continueAction && continueAction.handler) {
-        const mockMessage = {
-          agentId: zeroUuid,
-          senderId: user.id as UUID,
-          userIds: [user.id as UUID, zeroUuid],
-          content: "Test message for CONTINUE action",
-          room_id: room_id,
-        };
-        const response = await continueAction.handler(runtime, mockMessage);
-        expect(response).toBeDefined();
-        // Further assertions can be made based on the expected outcome of the CONTINUE action handler
-      } else {
-        throw new Error("Continue action or its handler function is undefined");
-      }
-    }, 20000);
+    expect(state.actionNames).toContain("TEST_ACTION");
   });
+
+  // Test that action handlers are being called properly
+  test("Continue action handler is called", async () => {
+    const continueAction = runtime.actions.find(
+      (action) => action.name === "CONTINUE",
+    );
+    expect(continueAction).toBeDefined();
+    if (continueAction && continueAction.handler) {
+      const mockMessage = {
+        agentId: zeroUuid,
+        senderId: user.id as UUID,
+        userIds: [user.id as UUID, zeroUuid],
+        content: "Test message for CONTINUE action",
+        room_id: room_id,
+      };
+      const response = await continueAction.handler(runtime, mockMessage);
+      expect(response).toBeDefined();
+      // Further assertions can be made based on the expected outcome of the CONTINUE action handler
+    } else {
+      throw new Error("Continue action or its handler function is undefined");
+    }
+  }, 20000);
 });