Reduce metrics usage (#874)

* Lint.

* Reduce metrics usage.

* set export interval to 60s

.github/workflows/publish.yaml

@@ -102,4 +102,4 @@ jobs:
           if [ -n "$LATEST" ]; then
             echo "::set-output name=image_builder_latest::$REGISTRY/$REPOSITORY:$IMAGE_TAG_LATEST"
           fi
-          echo "::set-output name=image_builder_commit::$REGISTRY/$REPOSITORY:$IMAGE_TAG_COMMIT"
+          echo "::set-output name=image_builder_commit::$REGISTRY/$REPOSITORY:$IMAGE_TAG_COMMIT"

blocks/loader.ts

@@ -9,7 +9,10 @@ import type { DecofileProvider } from "../engine/decofile/provider.ts";
 import { HttpError } from "../engine/errors.ts";
 import type { ResolverMiddlewareContext } from "../engine/middleware.ts";
 import { logger } from "../observability/otel/config.ts";
-import { meter } from "../observability/otel/metrics.ts";
+import {
+  meter,
+  OTEL_ENABLE_EXTRA_METRICS,
+} from "../observability/otel/metrics.ts";
 import { caches, ENABLE_LOADER_CACHE } from "../runtime/caches/mod.ts";
 import type { HttpContext } from "./handler.ts";
 import {
@@ -317,7 +320,9 @@ const wrapLoader = (
         return await flights.do(request.url, staleWhileRevalidate);
       } finally {
         const dimension = { loader, status };
-        stats.latency.record(performance.now() - start, dimension);
+        if (OTEL_ENABLE_EXTRA_METRICS) {
+          stats.latency.record(performance.now() - start, dimension);
+        }
         ctx.monitoring?.currentSpan?.setDesc(status);
       }
     },

observability/http.ts

@@ -4,15 +4,15 @@ import { meter } from "./otel/metrics.ts";
 const httpDuration = meter.createHistogram("http_request_duration", {
   description: "http request duration",
   unit: "ms",
-  valueType: ValueType.DOUBLE,
+  valueType: ValueType.INT,
 });
 /**
  * @returns a end function that when gets called observe the duration of the operation.
  */
 export const startObserve = () => {
   const start = performance.now();
   return (method: string, path: string, status: number) => {
-    httpDuration.record(performance.now() - start, {
+    httpDuration.record(Math.round(performance.now() - start), {
       "http.method": method,
       "http.route": path,
       "http.response.status": `${status}`,

observability/observe.ts

@@ -1,6 +1,6 @@
 import { isWrappedError } from "../blocks/loader.ts";
 import { ValueType } from "../deps.ts";
-import { meter } from "./otel/metrics.ts";
+import { meter, OTEL_ENABLE_EXTRA_METRICS } from "./otel/metrics.ts";
 
 const operationDuration = meter.createHistogram("block_op_duration", {
   description: "operation duration",
@@ -27,9 +27,11 @@ export const observe = async <T>(
     isError = "true";
     throw error;
   } finally {
-    operationDuration.record(performance.now() - start, {
-      "operation.name": op,
-      "operation.is_error": isError,
-    });
+    if (OTEL_ENABLE_EXTRA_METRICS) {
+      operationDuration.record(performance.now() - start, {
+        "operation.name": op,
+        "operation.is_error": isError,
+      });
+    }
   }
 };

observability/otel/metrics.ts

@@ -6,6 +6,17 @@ import {
   View,
 } from "../../deps.ts";
 import { OTEL_IS_ENABLED, resource } from "./config.ts";
+
+export const OTEL_ENABLE_EXTRA_METRICS: boolean = Deno.env.has(
+  "OTEL_ENABLE_EXTRA_METRICS",
+);
+
+// 2 minutes. We don't need frequent updates here.
+export const OTEL_EXPORT_INTERVAL: number = parseInt(
+  Deno.env.get("OTEL_EXPORT_INTERVAL") ?? "60000",
+  10,
+);
+
 // a=b,c=d => {a:b, c:d}
 const headersStringToObject = (headersString: string | undefined | null) => {
   if (!headersString) {
@@ -46,7 +57,7 @@ if (OTEL_IS_ENABLED) {
   meterProvider.addMetricReader(
     new PeriodicExportingMetricReader({
       exporter: metricExporter,
-      exportIntervalMillis: 30_000,
+      exportIntervalMillis: OTEL_EXPORT_INTERVAL,
     }),
   );
 }