feat: Instrumentation V2

bun.lock

@@ -8,8 +8,10 @@
         "@opentelemetry/exporter-metrics-otlp-grpc": "0.202.0",
         "@opentelemetry/exporter-metrics-otlp-http": "0.202.0",
         "@opentelemetry/exporter-metrics-otlp-proto": "0.202.0",
+        "@opentelemetry/exporter-prometheus": "^0.202.0",
         "@opentelemetry/exporter-trace-otlp-http": "0.202.0",
         "@opentelemetry/resources": "2.0.1",
+        "@opentelemetry/sdk-metrics": "2.0.1",
         "@opentelemetry/sdk-trace-base": "2.0.1",
         "@opentelemetry/sdk-trace-node": "2.0.1",
         "@opentelemetry/semantic-conventions": "^1.34.0",
@@ -250,6 +252,8 @@
 
     "@opentelemetry/exporter-metrics-otlp-proto": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.1", "@opentelemetry/exporter-metrics-otlp-http": "0.202.0", "@opentelemetry/otlp-exporter-base": "0.202.0", "@opentelemetry/otlp-transformer": "0.202.0", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-metrics": "2.0.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-X0RpPpPjyCAmIq9tySZm0Hk3Ltw8KWsqeNq5I7gS9AR9RzbVHb/l+eiMI1CqSRvW9R47HXcUu/epmEzY8ebFAg=="],
 
+    "@opentelemetry/exporter-prometheus": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.1", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-metrics": "2.0.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-6RvQqZHAPFiwL1OKRJe4ta6SgJx/g8or41B+OovVVEie3HeCDhDGL9S1VJNkBozUz6wTY8a47fQwdMrCOUdMhQ=="],
+
     "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.1", "@opentelemetry/otlp-exporter-base": "0.202.0", "@opentelemetry/otlp-transformer": "0.202.0", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-trace-base": "2.0.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-/hKE8DaFCJuaQqE1IxpgkcjOolUIwgi3TgHElPVKGdGRBSmJMTmN/cr6vWa55pCJIXPyhKvcMrbrya7DZ3VmzA=="],
 
     "@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.1", "@opentelemetry/otlp-transformer": "0.202.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-nMEOzel+pUFYuBJg2znGmHJWbmvMbdX5/RhoKNKowguMbURhz0fwik5tUKplLcUtl8wKPL1y9zPnPxeBn65N0Q=="],

docker-compose.jaeger.yml

@@ -0,0 +1,40 @@
+# Temporary - just for testing traces/metrics before switching to mimir collector
+services:
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    ports:
+      # Jaeger UI
+      - "16686:16686"
+      # OTLP gRPC receiver
+      - "4317:4317"
+      # OTLP HTTP receiver
+      - "4318:4318"
+      # Jaeger thrift
+      - "14250:14250"
+      # Jaeger thrift HTTP
+      - "14268:14268"
+      # Zipkin compatible endpoint
+      - "9411:9411"
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+      - LOG_LEVEL=debug
+    command:
+      - "--memory.max-traces=10000"
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--storage.tsdb.retention.time=200h'
+      - '--web.enable-lifecycle'
+
+networks:
+  default:
+    name: seda-otel-network 

package.json

@@ -22,7 +22,48 @@
 		"check-ts": "bunx tsc --noEmit",
 		"fmt": "bunx biome check --write .",
 		"check-fmt": "bunx biome check .",
-		"build-schema": "bun run ./packages/contract/build.ts"
+		"build-schema": "bun run ./packages/contract/build.ts",
+		"node": "bun run check-ts && bun run ./packages/cli run",
+		"node:dev": "bun run ./packages/cli run",
+		"node:inspect": "bun run check-ts && bun run --inspect ./packages/cli run",
+		"identity:info": "bun run check-ts && bun run ./packages/cli identities info",
+		"identity:stake": "bun run check-ts && bun run ./packages/cli identities stake",
+		"identity:unstake": "bun run check-ts && bun run ./packages/cli identities unstake",
+		"identity:withdraw": "bun run check-ts && bun run ./packages/cli identities withdraw",
+		"planet": "bun run check-ts && bun run ./packages/cli run --network planet",
+		"planet:info": "bun run check-ts && bun run ./packages/cli identities info --offline --network planet",
+		"planet:stake": "bun run check-ts && bun run ./packages/cli identities stake --network planet",
+		"planet:unstake": "bun run check-ts && bun run ./packages/cli identities unstake --network planet",
+		"planet:withdraw": "bun run check-ts && bun run ./packages/cli identities withdraw --network planet",
+		"testnet": "bun run check-ts && bun run ./packages/cli run --network testnet",
+		"testnet:info": "bun run check-ts && bun run ./packages/cli identities info --offline --network testnet",
+		"testnet:stake": "bun run check-ts && bun run ./packages/cli identities stake --network testnet",
+		"testnet:unstake": "bun run check-ts && bun run ./packages/cli identities unstake --network testnet",
+		"testnet:withdraw": "bun run check-ts && bun run ./packages/cli identities withdraw --network testnet",
+		"devnet": "bun run check-ts && bun run ./packages/cli run --network devnet",
+		"devnet:info": "bun run check-ts && bun run ./packages/cli identities info --offline --network devnet",
+		"devnet:stake": "bun run check-ts && bun run ./packages/cli identities stake --network devnet",
+		"devnet:unstake": "bun run check-ts && bun run ./packages/cli identities unstake --network devnet",
+		"devnet:withdraw": "bun run check-ts && bun run ./packages/cli identities withdraw --network devnet",
+		"dev": "bun run ./packages/cli",
+		"dev:planet": "bun run ./packages/cli run --network planet",
+		"dev:planet:info": "bun run ./packages/cli identities info --offline --network planet",
+		"dev:planet:stake": "bun run ./packages/cli identities stake --network planet",
+		"dev:planet:unstake": "bun run ./packages/cli identities unstake --network planet",
+		"dev:planet:withdraw": "bun run ./packages/cli identities withdraw --network planet",
+		"dev:testnet": "bun run ./packages/cli run --network testnet",
+		"dev:testnet:info": "bun run ./packages/cli identities info --offline --network testnet",
+		"dev:testnet:stake": "bun run ./packages/cli identities stake --network testnet",
+		"dev:testnet:unstake": "bun run ./packages/cli identities unstake --network testnet",
+		"dev:testnet:withdraw": "bun run ./packages/cli identities withdraw --network testnet",
+		"dev:devnet": "bun run ./packages/cli run --network devnet",
+		"dev:devnet:info": "bun run ./packages/cli identities info --offline --network devnet",
+		"dev:devnet:stake": "bun run ./packages/cli identities stake --network devnet",
+		"dev:devnet:unstake": "bun run ./packages/cli identities unstake --network devnet",
+		"dev:devnet:withdraw": "bun run ./packages/cli identities withdraw --network devnet",
+		"validate": "bun run check-ts && bun run ./packages/cli validate",
+		"tools": "bun run check-ts && bun run ./packages/cli tools",
+		"init": "bun run check-ts && bun run ./packages/cli init"
 	},
 	"devDependencies": {
 		"@types/bun": "latest",
@@ -35,18 +76,20 @@
 		"typescript": "^5.7.3"
 	},
 	"dependencies": {
-		"true-myth": "^8.4.0",
-		"node-fetch": "^3.3.2",
-		"ts-pattern": "^5.7.1",
-		"type-fest": "^4.41.0",
 		"@opentelemetry/api": "^1.9.0",
 		"@opentelemetry/exporter-metrics-otlp-grpc": "0.202.0",
 		"@opentelemetry/exporter-metrics-otlp-http": "0.202.0",
 		"@opentelemetry/exporter-metrics-otlp-proto": "0.202.0",
+		"@opentelemetry/exporter-prometheus": "^0.202.0",
+		"@opentelemetry/exporter-trace-otlp-http": "0.202.0",
 		"@opentelemetry/resources": "2.0.1",
-		"@opentelemetry/semantic-conventions": "^1.34.0",
+		"@opentelemetry/sdk-metrics": "2.0.1",
 		"@opentelemetry/sdk-trace-base": "2.0.1",
 		"@opentelemetry/sdk-trace-node": "2.0.1",
-		"@opentelemetry/exporter-trace-otlp-http": "0.202.0"
+		"@opentelemetry/semantic-conventions": "^1.34.0",
+		"node-fetch": "^3.3.2",
+		"true-myth": "^8.4.0",
+		"ts-pattern": "^5.7.1",
+		"type-fest": "^4.41.0"
 	}
 }

packages/cli/src/commands/run.ts

@@ -20,18 +20,28 @@ export const runCmd = populateWithCommonOptions(new Command("run"))
 		});
 
 		if (config.isOk) {
-			const exitController = new AbortController();
-
 			runNode(config.value, {
-				exitController,
+				skipIdentityInitialization: false,
 			});
 
 			listenForExit(async () => {
-				exitController.abort();
+				// Graceful shutdown handled by telemetry system
 			});
 		} else {
 			logger.error("Error while parsing config:");
 
+			// Record critical boot failure for config errors
+			try {
+				const { metricsHelpers } = await import("@sedaprotocol/overlay-ts-common");
+				const configError = new Error(`Config parsing failed: ${config.error.join(", ")}`);
+				metricsHelpers.recordCriticalError("node_boot", configError, {
+					reason: "config_parsing_failure",
+					boot_phase: "config_validation",
+				});
+			} catch (e) {
+				// Ignore if telemetry is not available
+			}
+
 			for (const error of config.error) {
 				logger.error(error);
 			}

packages/common/src/index.ts

@@ -1,4 +1,15 @@
-import "./telemetry";
+// Enhanced telemetry exports
+export {
+	initializeTelemetry,
+	shutdownTelemetry,
+	telemetryInitialized,
+} from "./telemetry";
+
+export {
+	sedaMetrics,
+	metricsHelpers,
+} from "./telemetry/metrics";
+
 export { isBrowser } from "./services/is-browser";
 export * from "./services/try-async";
 export * from "./services/timer";

packages/common/src/seda/seda-chain.ts

@@ -9,6 +9,7 @@ import type { AppConfig } from "@sedaprotocol/overlay-ts-config";
 import { logger } from "@sedaprotocol/overlay-ts-logger";
 import { EventEmitter } from "eventemitter3";
 import { Maybe, Result } from "true-myth";
+import { metricsHelpers } from "../index";
 import {
 	AlreadyCommitted,
 	AlreadyRevealed,
@@ -134,6 +135,12 @@ export class SedaChain extends EventEmitter<EventMap> {
 					id: txHash,
 				});
 
+				// Record RPC connectivity error
+				metricsHelpers.recordRpcError("general", "getCurrentBlockHeight", currentBlockHeight.error, {
+					tx_hash: txHash,
+					operation: "get_current_block_height",
+				});
+
 				return Result.ok(Maybe.nothing());
 			}
 
@@ -144,6 +151,13 @@ export class SedaChain extends EventEmitter<EventMap> {
 					id: txHash,
 				});
 
+				// Record RPC connectivity error
+				metricsHelpers.recordRpcError("general", "getBlock", block.error, {
+					tx_hash: txHash,
+					block_height: currentBlockHeight.value.toString(),
+					operation: "get_block",
+				});
+
 				// We only want to return an error on transaction level, not on the block level
 				return Result.ok(Maybe.nothing());
 			}
@@ -465,6 +479,15 @@ export class SedaChain extends EventEmitter<EventMap> {
 			logger.error(`Could not find callback for message id: ${txMessage.value.id}: ${txMessage.value}`, {
 				id: txMessage.value.traceId,
 			});
+
+			// HIGH: Callback lookup failure - fishy behavior detected
+			const callbackError = new Error(`Callback not found for message id: ${txMessage.value.id}`);
+			metricsHelpers.recordHighPriorityError("callback_lookup", callbackError, {
+				message_id: txMessage.value.id,
+				trace_id: txMessage.value.traceId ?? "unknown",
+				operation: "callback_lookup",
+			});
+
 			return;
 		}
 
@@ -569,6 +592,13 @@ export class SedaChain extends EventEmitter<EventMap> {
 						id: traceId,
 					});
 
+					// Record RPC connectivity error for transaction fetch failure
+					metricsHelpers.recordRpcError("general", "getTransaction", transactionResult.error, {
+						tx_hash: transactionHash.value,
+						trace_id: traceId ?? "unknown",
+						operation: "get_transaction",
+					});
+
 					const error = narrowDownError(transactionResult.error);
 					clearInterval(checkTransactionInterval);
 					resolve(Result.err(error));

packages/common/src/telemetry/decorators.ts

@@ -0,0 +1,133 @@
+import { type Span, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+
+const tracer = trace.getTracer("seda-overlay-decorators", "1.0.0");
+
+export interface TracedOptions {
+	spanName?: string;
+	spanKind?: SpanKind;
+	attributes?: Record<string, string | number | boolean>;
+}
+
+export function Traced(options: TracedOptions = {}) {
+	return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
+		if (!descriptor.value) {
+			throw new Error("@Traced can only be applied to methods");
+		}
+
+		const originalMethod = descriptor.value;
+		const spanName = options.spanName || `${target.constructor.name}.${propertyKey}`;
+
+		descriptor.value = function (...args: any[]) {
+			return tracer.startActiveSpan(spanName, (span: Span) => {
+				try {
+					span.setAttributes({
+						"method.name": propertyKey,
+						"class.name": target.constructor.name,
+						...options.attributes,
+					});
+
+					const result = originalMethod.apply(this, args);
+
+					if (result && typeof result.then === "function") {
+						return result
+							.then((asyncResult: any) => {
+								span.setStatus({ code: SpanStatusCode.OK });
+								return asyncResult;
+							})
+							.catch((error: Error) => {
+								span.recordException(error);
+								span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+								throw error;
+							})
+							.finally(() => span.end());
+					} else {
+						span.setStatus({ code: SpanStatusCode.OK });
+						span.end();
+						return result;
+					}
+				} catch (error: any) {
+					span.recordException(error);
+					span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+					span.end();
+					throw error;
+				}
+			});
+		};
+
+		return descriptor;
+	};
+}
+
+export function MonitorCritical(options: TracedOptions = {}) {
+	return Traced({
+		...options,
+		spanName: options.spanName || `critical.operation`,
+		attributes: { operation_type: "critical", ...options.attributes },
+	});
+}
+
+export function MonitorRPC(options: TracedOptions & { endpoint?: string } = {}) {
+	const rpcAttributes: Record<string, string | number | boolean> = {
+		...options.attributes,
+	};
+
+	if (options.endpoint) {
+		rpcAttributes.rpc_endpoint = options.endpoint;
+	}
+
+	return Traced({
+		...options,
+		spanName: options.spanName || `rpc.${options.endpoint || "call"}`,
+		spanKind: SpanKind.CLIENT,
+		attributes: rpcAttributes,
+	});
+}
+
+export function TraceClass(options: { prefix?: string } = {}) {
+	return (constructor: any) => {
+		const methodNames = Object.getOwnPropertyNames(constructor.prototype);
+
+		for (const methodName of methodNames) {
+			if (methodName === "constructor") continue;
+
+			const descriptor = Object.getOwnPropertyDescriptor(constructor.prototype, methodName);
+			if (descriptor && typeof descriptor.value === "function") {
+				const spanName = options.prefix ? `${options.prefix}.${methodName}` : `${constructor.name}.${methodName}`;
+				Traced({ spanName })(constructor.prototype, methodName, descriptor);
+				Object.defineProperty(constructor.prototype, methodName, descriptor);
+			}
+		}
+
+		return constructor;
+	};
+}
+
+export async function withSpan<T>(name: string, fn: (span: Span) => Promise<T> | T): Promise<T> {
+	return tracer.startActiveSpan(name, async (span: Span) => {
+		try {
+			const result = await fn(span);
+			span.setStatus({ code: SpanStatusCode.OK });
+			return result;
+		} catch (error: any) {
+			span.recordException(error);
+			span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+			throw error;
+		} finally {
+			span.end();
+		}
+	});
+}
+
+export function setSpanAttributes(attributes: Record<string, string | number | boolean>) {
+	const activeSpan = trace.getActiveSpan();
+	if (activeSpan) {
+		activeSpan.setAttributes(attributes);
+	}
+}
+
+export function addSpanEvent(name: string, attributes?: Record<string, string | number | boolean>) {
+	const activeSpan = trace.getActiveSpan();
+	if (activeSpan) {
+		activeSpan.addEvent(name, attributes);
+	}
+}