Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions examples/04-langchain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
ZeroEvalCallbackHandler,
setGlobalCallbackHandler,
} from "zeroeval/langchain";
import { init } from "zeroeval"

import { ChatOpenAI } from "@langchain/openai";
import { ChatPromptTemplate } from "@langchain/core/prompts";
Expand Down Expand Up @@ -106,7 +107,7 @@ async function main() {
const structuredReport = await structuredModel.invoke(
`Based on this weather information: "${weatherInfo}", generate a detailed weather report.`
);

console.log("\nStructured Weather Report:");
console.log(JSON.stringify(structuredReport, null, 2));

Expand All @@ -132,17 +133,17 @@ async function main() {
const structuredReport2 = await structuredModel.invoke(
`Based on this weather information: "${weatherInfo2}", generate a detailed weather report.`
);

console.log("\nStructured Weather Report for NY:");
console.log(JSON.stringify(structuredReport2, null, 2));

// Example 3: Direct structured output without tool calling
console.log("\n\nDirect structured output example...\n");

const directStructuredResult = await structuredModel.invoke(
"Generate a weather report for London, UK. Make it rainy and cold, around 45°F."
);

console.log("Direct Structured Output:");
console.log(JSON.stringify(directStructuredResult, null, 2));

Expand Down
32 changes: 23 additions & 9 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "zeroeval",
"version": "0.1.6",
"version": "0.1.8",
"description": "ZeroEval TypeScript SDK",
"keywords": [
"observability",
Expand Down Expand Up @@ -112,14 +112,15 @@
},
"devDependencies": {
"@eslint/js": "^9.30.1",
"@types/node": "^20.19.4",
"@types/node": "^20.19.6",
"@typescript-eslint/eslint-plugin": "^8.36.0",
"@typescript-eslint/parser": "^8.36.0",
"@vitest/coverage-v8": "^3.2.4",
"@vitest/ui": "^3.2.4",
"eslint": "^9.30.1",
"eslint-config-prettier": "^10.1.5",
"eslint-plugin-prettier": "^5.5.1",
"globals": "^16.3.0",
"prettier": "^3.6.2",
"rimraf": "^5.0.0",
"ts-node": "^10.9.2",
Expand Down
2 changes: 1 addition & 1 deletion src/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { tracer } from './observability/Tracer';
import { Span } from './observability/Span';
import type { Span } from './observability/Span';

/** Return the current active Span (or undefined). */
export function getCurrentSpan(): Span | undefined {
Expand Down
36 changes: 36 additions & 0 deletions src/init.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { tracer } from './observability/Tracer';
import { Logger, getLogger } from './observability/logger';

const logger = getLogger('zeroeval');

export interface InitOptions {
apiKey?: string;
Expand All @@ -7,6 +10,7 @@ export interface InitOptions {
maxSpans?: number;
collectCodeDetails?: boolean;
integrations?: Record<string, boolean>;
debug?: boolean;
}

// Track whether init has been called
Expand All @@ -32,8 +36,40 @@ export function init(opts: InitOptions = {}): void {
maxSpans,
collectCodeDetails,
integrations,
debug,
} = opts;

// Check if debug mode is enabled via param or env var
const isDebugMode =
debug || process.env.ZEROEVAL_DEBUG?.toLowerCase() === 'true';

// Enable debug mode
if (isDebugMode) {
process.env.ZEROEVAL_DEBUG = 'true';
Logger.setDebugMode(true);

// Log all configuration values as the first log message
const maskedApiKey = Logger.maskApiKey(
apiKey || process.env.ZEROEVAL_API_KEY
);
const finalApiUrl =
apiUrl || process.env.ZEROEVAL_API_URL || 'https://api.zeroeval.com';

logger.debug('ZeroEval SDK Configuration:');
logger.debug(` API Key: ${maskedApiKey}`);
logger.debug(` API URL: ${finalApiUrl}`);
logger.debug(` Debug Mode: ${isDebugMode}`);
logger.debug(` Flush Interval: ${flushInterval ?? '10s (default)'}`);
logger.debug(` Max Spans: ${maxSpans ?? '100 (default)'}`);
logger.debug(
` Collect Code Details: ${collectCodeDetails ?? 'true (default)'}`
);

logger.info('SDK initialized in debug mode.');
} else {
Logger.setDebugMode(false);
}

if (apiKey) process.env.ZEROEVAL_API_KEY = apiKey;
if (apiUrl) process.env.ZEROEVAL_API_URL = apiUrl;

Expand Down
68 changes: 61 additions & 7 deletions src/observability/Tracer.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import { AsyncLocalStorage } from 'async_hooks';
import { randomUUID } from 'crypto';
import { Span } from './Span';
import { SpanWriter, BackendSpanWriter } from './writer';
import type { SpanWriter } from './writer';
import { BackendSpanWriter } from './writer';
import { setInterval } from 'timers';
import { discoverIntegrations } from './integrations/utils';
import type { Integration } from './integrations/base';
import { getLogger } from './logger';

const logger = getLogger('zeroeval.tracer');

interface ConfigureOptions {
flushInterval?: number;
Expand All @@ -30,6 +34,11 @@
private _shuttingDown = false;

constructor() {
logger.debug('Initializing tracer...');
logger.debug(
`Tracer config: flush_interval=${this._flushIntervalMs}ms, max_spans=${this._maxSpans}`
);

// schedule periodic flush
setInterval(() => {
if (Date.now() - this._lastFlush >= this._flushIntervalMs) {
Expand All @@ -54,10 +63,17 @@

/* CONFIG ----------------------------------------------------------------*/
configure(opts: ConfigureOptions = {}) {
if (opts.flushInterval !== undefined)
if (opts.flushInterval !== undefined) {
this._flushIntervalMs = opts.flushInterval * 1000;
if (opts.maxSpans !== undefined) this._maxSpans = opts.maxSpans;
// Other options ignored for now (collectCodeDetails, integrations)
logger.info(
`Tracer flush_interval configured to ${opts.flushInterval}s.`
);
}
if (opts.maxSpans !== undefined) {
this._maxSpans = opts.maxSpans;
logger.info(`Tracer max_spans configured to ${opts.maxSpans}.`);
}
logger.debug(`Tracer configuration updated:`, opts);
}

/* ACTIVE SPAN -----------------------------------------------------------*/
Expand All @@ -76,6 +92,8 @@
tags?: Record<string, string>;
} = {}
): Span {
logger.debug(`Starting span: ${name}`);

const parent = this.currentSpan();
const span = new Span(name, parent?.traceId);

Expand All @@ -85,10 +103,14 @@
span.sessionName = parent.sessionName;
// inherit tags
span.tags = { ...parent.tags, ...opts.tags };
logger.debug(`Span ${name} inherits from parent ${parent.name}`);
} else {
span.sessionId = opts.sessionId ?? randomUUID();
span.sessionName = opts.sessionName;
span.tags = { ...opts.tags };
logger.debug(
`Span ${name} is a root span with session ${span.sessionId}`
);
}

Object.assign(span.attributes, opts.attributes);
Expand All @@ -107,6 +129,8 @@
endSpan(span: Span): void {
if (!span.endTime) span.end();

logger.debug(`Ending span: ${span.name} (duration: ${span.durationMs}ms)`);

// pop stack
const stack = als.getStore();
if (stack && stack[stack.length - 1] === span) {
Expand All @@ -124,14 +148,25 @@
const ordered = traceBucket.sort((a) => (a.parentId ? 1 : -1));
delete this._traceBuckets[span.traceId];
this._buffer.push(...ordered);

logger.debug(
`Trace ${span.traceId} complete with ${ordered.length} spans`
);
}

// flush if buffer full
if (this._buffer.length >= this._maxSpans) this.flush();
if (this._buffer.length >= this._maxSpans) {
logger.debug(
`Buffer full (${this._buffer.length} spans), triggering flush`
);
this.flush();
}
}

/* TAG HELPERS -----------------------------------------------------------*/
addTraceTags(traceId: string, tags: Record<string, string>): void {
logger.debug(`Adding trace tags to ${traceId}:`, tags);

// update buckets
for (const span of this._traceBuckets[traceId] ?? [])
Object.assign(span.tags, tags);
Expand All @@ -142,6 +177,8 @@
}

addSessionTags(sessionId: string, tags: Record<string, string>): void {
logger.debug(`Adding session tags to ${sessionId}:`, tags);

const all = [...Object.values(this._traceBuckets).flat(), ...this._buffer];
all
.filter((s) => s.sessionId === sessionId)
Expand All @@ -155,31 +192,48 @@
/* FLUSH -----------------------------------------------------------------*/
flush(): void {
if (this._buffer.length === 0) return;

logger.info(`Flushing ${this._buffer.length} spans to backend`);

this._lastFlush = Date.now();
this._writer.write(this._buffer.splice(0));
}

private async _setupAvailableIntegrations(): Promise<void> {
logger.info('Checking for available integrations...');

const available = await discoverIntegrations();

for (const [key, Ctor] of Object.entries(available)) {
try {
const inst = new Ctor();
if ((Ctor as any).isAvailable?.() !== false) {

Check warning on line 210 in src/observability/Tracer.ts

View workflow job for this annotation

GitHub Actions / 🔍 Code Quality

Unexpected any. Specify a different type
logger.info(`Setting up integration: ${key}`);
inst.setup();
this._integrations[key] = inst;
logger.info(`✅ Successfully set up integration: ${key}`);
}
} catch (err) {
// eslint-disable-next-line no-console
console.warn(`[ZeroEval] Failed to setup integration ${key}`, err);
logger.error(`❌ Failed to setup integration ${key}:`, err);
}
}

if (Object.keys(this._integrations).length > 0) {
logger.info(
`Active integrations: ${Object.keys(this._integrations).join(', ')}`
);
} else {
logger.info('No active integrations found.');
}
}

/** Flush remaining spans and teardown integrations */
shutdown(): void {
if (this._shuttingDown) return;
this._shuttingDown = true;

logger.info('Shutting down tracer...');

try {
this.flush();
} catch (_) {}
Expand Down
2 changes: 1 addition & 1 deletion src/observability/integrations/langchain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export class LangChainIntegration extends Integration {
for (const method of methods) {
if (typeof Runnable.prototype[method] !== 'function') continue;
this.patchMethod(
Runnable.prototype as any,
Runnable.prototype,
method as any,
(orig: AnyFn): AnyFn => {
const isAsync = method.toString().startsWith('a');
Expand Down
Loading
Loading