Merge pull request #9 from topcoder-platform/fix/wiston-logger-ecs-fix

#8 Winston Logger ECS Deployment Fix

Author: jmgasper

.env.example

@@ -14,6 +14,9 @@ LOG_LEVEL=info
 # The directory to store log files
 LOG_DIR=logs
 
+# leave unset (defaults to false in production)
+ENABLE_FILE_LOGGING=true
+
 # -------------------------------------
 # Kafka Configuration
 # -------------------------------------

README.md

@@ -41,6 +41,9 @@ Open the `.env` file and configure the variables for your environment. It is cru
    PORT=3000
    LOG_LEVEL=debug
    LOG_DIR=logs
+   # Enable file logging in production environments (default: false)
+   # In development environments, file logging is enabled by default
+   ENABLE_FILE_LOGGING=false
 
    # -------------------------------------
    # Kafka Configuration
@@ -206,6 +209,31 @@ tsconfig.json               # TypeScript config
 README.md                   # Documentation
 ```
 
+## Logging Configuration
+
+The autopilot service uses a flexible logging strategy that adapts to different deployment environments:
+
+### Development Environment
+- **File Logging**: Enabled by default for debugging purposes
+- **Console Logging**: Always enabled with colorized output
+- **Log Files**: Created in the `logs/` directory (configurable via `LOG_DIR`)
+
+### Production Environment (ECS)
+- **File Logging**: Disabled by default to support read-only filesystems
+- **Console Logging**: Primary logging method, suitable for container log collection
+- **Override**: Set `ENABLE_FILE_LOGGING=true` to enable file logging if the filesystem is writable
+
+### Environment Variables
+- `LOG_LEVEL`: Controls log verbosity (`error`, `warn`, `info`, `debug`, `verbose`)
+- `LOG_DIR`: Directory for log files (default: `logs`)
+- `ENABLE_FILE_LOGGING`: Boolean flag to control file logging in production
+
+### ECS Deployment Notes
+When deploying to ECS, ensure:
+1. `NODE_ENV=production` is set
+2. `ENABLE_FILE_LOGGING` is either unset or set to `false` for read-only filesystems
+3. Container logging is configured to collect stdout/stderr logs
+
 ## Further Documentation
 
 For more detailed technical information, please see the documents in the `docs/` directory:
@@ -228,3 +256,13 @@ For more detailed technical information, please see the documents in the `docs/`
    ```
 
 - **API Authentication Errors**: Ensure Auth0 credentials (`AUTH0_URL`, `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_AUDIENCE`) in your `.env` file are valid and not expired.
+
+- **ECS Deployment - Read-only Filesystem Error**: If the application fails to start in ECS with errors about creating a 'logs' directory:
+  - Ensure `NODE_ENV=production` is set in your ECS task definition
+  - Verify `ENABLE_FILE_LOGGING` is not set or is set to `false`
+  - Check that your ECS task definition is configured to collect logs from stdout/stderr
+
+- **Logging Issues**:
+  - **No log files in development**: Check that `ENABLE_FILE_LOGGING` is not set to `false`
+  - **File permission errors**: Ensure the application has write permissions to the `LOG_DIR` directory
+  - **Missing logs in production**: Verify your container logging configuration is collecting stdout/stderr output

src/common/constants/config.constants.ts

@@ -3,6 +3,7 @@ export interface AppConfig {
   DEFAULT_NODE_ENV: 'development' | 'production' | 'test';
   DEFAULT_LOG_LEVEL: 'error' | 'warn' | 'info' | 'debug' | 'verbose';
   DEFAULT_LOG_DIR: string;
+  DEFAULT_ENABLE_FILE_LOGGING: boolean;
 }
 
 export interface KafkaConfig {
@@ -51,6 +52,7 @@ export const CONFIG: Config = {
     DEFAULT_NODE_ENV: 'development',
     DEFAULT_LOG_LEVEL: 'info',
     DEFAULT_LOG_DIR: 'logs',
+    DEFAULT_ENABLE_FILE_LOGGING: false,
   },
   KAFKA: {
     DEFAULT_CLIENT_ID: 'autopilot-service',

src/common/services/logger.service.ts

@@ -17,51 +17,86 @@ export class LoggerService implements NestLoggerService {
   }
 
   private createWinstonLogger(): Logger {
+    const baseTransports: any[] = [
+      new transports.Console({
+        level: process.env.LOG_LEVEL || CONFIG.APP.DEFAULT_LOG_LEVEL,
+        format: format.combine(
+          format.colorize(),
+          format.printf(({ timestamp, level, message, context, ...meta }) => {
+            const formattedTimestamp =
+              timestamp instanceof Date
+                ? timestamp.toISOString()
+                : String(timestamp);
+            const formattedContext =
+              typeof context === 'string' ? context : 'App';
+            const formattedMessage =
+              typeof message === 'string'
+                ? message
+                : typeof message === 'object' && message !== null
+                  ? JSON.stringify(message)
+                  : String(message);
+
+            return `${formattedTimestamp} [${formattedContext}] ${String(level)}: ${formattedMessage}${
+              Object.keys(meta).length
+                ? ' ' + JSON.stringify(meta, null, 1)
+                : ''
+            }`;
+          }),
+        ),
+      }),
+    ];
+
+    // Add file transports conditionally based on environment
+    const shouldEnableFileLogging = this.shouldEnableFileLogging();
+    if (shouldEnableFileLogging) {
+      try {
+        baseTransports.push(
+          new transports.File({
+            filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/error.log`,
+            level: 'error',
+          }),
+          new transports.File({
+            filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/combined.log`,
+          }),
+        );
+      } catch (error) {
+        // If file transport creation fails (e.g., read-only filesystem),
+        // log a warning but continue with console-only logging
+        console.warn(
+          'Failed to initialize file transports, falling back to console-only logging:',
+          error instanceof Error ? error.message : String(error),
+        );
+      }
+    }
+
     return createLogger({
       format: format.combine(
         format.timestamp(),
         format.errors({ stack: true }),
         format.json(),
       ),
       defaultMeta: { context: this.context },
-      transports: [
-        new transports.Console({
-          level: process.env.LOG_LEVEL || CONFIG.APP.DEFAULT_LOG_LEVEL,
-          format: format.combine(
-            format.colorize(),
-            format.printf(({ timestamp, level, message, context, ...meta }) => {
-              const formattedTimestamp =
-                timestamp instanceof Date
-                  ? timestamp.toISOString()
-                  : String(timestamp);
-              const formattedContext =
-                typeof context === 'string' ? context : 'App';
-              const formattedMessage =
-                typeof message === 'string'
-                  ? message
-                  : typeof message === 'object' && message !== null
-                    ? JSON.stringify(message)
-                    : String(message);
-
-              return `${formattedTimestamp} [${formattedContext}] ${String(level)}: ${formattedMessage}${
-                Object.keys(meta).length
-                  ? ' ' + JSON.stringify(meta, null, 1)
-                  : ''
-              }`;
-            }),
-          ),
-        }),
-        new transports.File({
-          filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/error.log`,
-          level: 'error',
-        }),
-        new transports.File({
-          filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/combined.log`,
-        }),
-      ],
+      transports: baseTransports,
     });
   }
 
+  /**
+   * Determines whether file logging should be enabled based on environment configuration
+   */
+  private shouldEnableFileLogging(): boolean {
+    const nodeEnv = process.env.NODE_ENV;
+    const enableFileLogging = process.env.ENABLE_FILE_LOGGING;
+
+    // In production, only enable file logging if explicitly requested
+    if (nodeEnv === 'production') {
+      return enableFileLogging === 'true';
+    }
+
+    // In non-production environments (development, test), enable file logging by default
+    // unless explicitly disabled
+    return enableFileLogging !== 'false';
+  }
+
   private formatMessage(message: unknown): string {
     if (typeof message === 'string') {
       return message;

src/config/sections/app.config.ts

@@ -6,5 +6,6 @@ export default registerAs('app', () => ({
   logging: {
     level: process.env.LOG_LEVEL || 'info',
     directory: process.env.LOG_DIR || 'logs',
+    enableFileLogging: process.env.ENABLE_FILE_LOGGING === 'true',
   },
 }));

src/config/validation.ts

@@ -11,6 +11,7 @@ export const validationSchema = Joi.object({
     .valid('error', 'warn', 'info', 'debug', 'verbose')
     .default('info'),
   LOG_DIR: Joi.string().default('logs'),
+  ENABLE_FILE_LOGGING: Joi.boolean().default(false),
 
   // Kafka Configuration
   KAFKA_BROKERS: Joi.string().required(),