Merge pull request #1185 from akhilmhdh/feat/pino-cloudwatch

feat(aws-cloudwatch): added support for aws cloudwatch transport in pino
Infisical · Nov 21, 2023 · d909ff6 · d909ff6
2 parents 9d2a08d + 5301bcc
commit d909ff6
Show file tree

Hide file tree

Showing 6 changed files with 14,534 additions and 12,069 deletions.
diff --git a/backend/package-lock.json b/backend/package-lock.json
diff --git a/backend/package.json b/backend/package.json
@@ -8,6 +8,7 @@
     "@octokit/rest": "^19.0.5",
     "@sentry/node": "^7.77.0",
     "@sentry/tracing": "^7.48.0",
+    "@serdnam/pino-cloudwatch-transport": "^1.0.4",
     "@types/crypto-js": "^4.1.1",
     "@types/libsodium-wrappers": "^0.7.10",
     "@ucast/mongo2js": "^1.3.4",
@@ -71,7 +72,7 @@
   "main": "src/index.js",
   "scripts": {
     "start": "node build/index.js",
-    "dev": "nodemon index.js | pino-pretty --colorize",
+    "dev": "nodemon index.js",
     "swagger-autogen": "node ./swagger/index.ts",
     "build": "rimraf ./build && tsc && cp -R ./src/templates ./build && cp -R ./src/data ./build",
     "lint": "eslint . --ext .ts",

diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts
@@ -89,6 +89,21 @@ export const getClientSecretGitLabLogin = async () =>
 export const getUrlGitLabLogin = async () =>
   (await client.getSecret("URL_GITLAB_LOGIN")).secretValue || GITLAB_URL;
 
+export const getAwsCloudWatchLog = async () => {
+  const logGroupName =
+    (await client.getSecret("AWS_CLOUDWATCH_LOG_GROUP_NAME")).secretValue || "infisical-log-stream";
+  const region = (await client.getSecret("AWS_CLOUDWATCH_LOG_REGION")).secretValue;
+  const accessKeyId = (await client.getSecret("AWS_CLOUDWATCH_LOG_ACCESS_KEY_ID")).secretValue;
+  const accessKeySecret = (await client.getSecret("AWS_CLOUDWATCH_LOG_ACCESS_KEY_SECRET"))
+    .secretValue;
+  const interval = parseInt(
+    (await client.getSecret("AWS_CLOUDWATCH_LOG_INTERVAL")).secretValue || 1000,
+    10
+  );
+  if (!region || !accessKeyId || !accessKeySecret) return;
+  return { logGroupName, region, accessKeySecret, accessKeyId, interval };
+};
+
 export const getPostHogHost = async () =>
   (await client.getSecret("POSTHOG_HOST")).secretValue || "https://app.posthog.com";
 export const getPostHogProjectApiKey = async () =>

diff --git a/backend/src/index.ts b/backend/src/index.ts
@@ -5,7 +5,7 @@ import express from "express";
 require("express-async-errors");
 import helmet from "helmet";
 import cors from "cors";
-import { logger } from "./utils/logging";
+import { initLogger, logger } from "./utils/logging";
 import httpLogger from "pino-http";
 import { DatabaseService } from "./services";
 import { EELicenseService, GithubSecretScanningService } from "./ee/services";
@@ -101,6 +101,7 @@ import { serverConfigInit } from "./config/serverConfig";
 let handler: null | any = null;
 
 const main = async () => {
+  await initLogger();
   const port = await getPort();
 
   // initializing the database connection

diff --git a/backend/src/utils/logging/index.ts b/backend/src/utils/logging/index.ts
@@ -1 +1 @@
-export { logger } from "./logger";
+export { logger, initLogger } from "./logger";
diff --git a/backend/src/utils/logging/logger.ts b/backend/src/utils/logging/logger.ts
@@ -1,15 +1,56 @@
-import pino from "pino";
-
-export const logger = pino({
-  level: process.env.PINO_LOG_LEVEL || "trace",
-  timestamp: pino.stdTimeFunctions.isoTime,
-  formatters: {
-    bindings: (bindings) => {
-        return {
+import pino, { Logger } from "pino";
+import { getAwsCloudWatchLog, getNodeEnv } from "../../config";
+
+export let logger: Logger;
+
+export const initLogger = async () => {
+  const awsCloudWatchLogCfg = await getAwsCloudWatchLog();
+  const nodeEnv = await getNodeEnv();
+  const isProduction = nodeEnv === "production";
+  const targets: pino.TransportMultiOptions["targets"][number][] = [
+    isProduction
+      ? { level: "trace", target: "pino/file", options: {} }
+      : {
+          level: "info",
+          target: "pino-pretty", // must be installed separately
+          options: {
+            colorize: true
+          }
+        }
+  ];
+
+  if (awsCloudWatchLogCfg) {
+    targets.push({
+      target: "@serdnam/pino-cloudwatch-transport",
+      level: "info",
+      options: {
+        logGroupName: awsCloudWatchLogCfg.logGroupName,
+        logStreamName: awsCloudWatchLogCfg.logGroupName,
+        awsRegion: awsCloudWatchLogCfg.region,
+        awsAccessKeyId: awsCloudWatchLogCfg.accessKeyId,
+        awsSecretAccessKey: awsCloudWatchLogCfg.accessKeySecret,
+        interval: awsCloudWatchLogCfg.interval
+      }
+    });
+  }
+
+  const transport = pino.transport({
+    targets
+  });
+
+  logger = pino(
+    {
+      level: process.env.PINO_LOG_LEVEL || "trace",
+      formatters: {
+        bindings: (bindings) => {
+          return {
             pid: bindings.pid,
             hostname: bindings.hostname
             // node_version: process.version
-        };
+          };
+        }
+      }
     },
-  }
-});
+    transport
+  );
+};