CLDSRV-750: add Server Access Logs

Author: leif-scality

config.json

@@ -149,5 +149,9 @@
     },
     "integrityChecks": {
         "objectPutRetention": true
+    },
+    "serverAccessLogs": {
+        "enabled": true,
+        "outputFile": "./logs/server-access.log"
     }
 }

lib/Config.js

@@ -574,6 +574,27 @@ function parseIntegrityChecks(config) {
     return integrityChecks;
 }
 
+function parseServerAccessLogs(config) {
+    const res = {
+        enabled: true,
+        outputFile: './logs/server-access.log',
+    };
+
+    if (config && config.serverAccessLogs) {
+        if ('enabled' in config.serverAccessLogs) {
+            assert(typeof config.serverAccessLogs.enabled === 'boolean');
+            res.enabled = config.serverAccessLogs.enabled;
+        }
+
+        if ('outputFile' in config.serverAccessLogs) {
+            assert(typeof config.serverAccessLogs.outputFile === 'string');
+            res.outputFile = config.serverAccessLogs.outputFile;
+        }
+    }
+
+    return res;
+}
+
 /**
  * Reads from a config file and returns the content as a config object
  */
@@ -1785,7 +1806,7 @@ class Config extends EventEmitter {
             }
         }
         this.integrityChecks = parseIntegrityChecks(config);
-
+        this.serverAccessLogs = parseServerAccessLogs(config);
         /**
          * S3C-10336: PutObject max size of 5GB is new in 9.5.1
          * Provides a way to bypass the new validation if it breaks customer workflows

lib/api/api.js

@@ -264,6 +264,10 @@ const api = {
                 });
 
                 request.on('end', () => {
+                    if (request.serverAccessLog) {
+                        request.serverAccessLog.startTurnAroundTime = process.hrtime.bigint();
+                    }
+
                     if (bodyLength > MAX_BODY_LENGTH) {
                         log.error('body length is too long for request type',
                                   { bodyLength });

lib/metadata/metadataUtils.js

@@ -10,6 +10,23 @@ const { onlyOwnerAllowed } = require('../../constants');
 const { actionNeedQuotaCheck, actionWithDataDeletion } = require('arsenal/build/lib/policyEvaluator/RequestContext');
 const { processBytesToWrite, validateQuotas } = require('../api/apiUtils/quotas/quotaUtils');
 
+function storeServerAccessLogInfo(request, authInfo, bucket) {
+    if (request &&
+        request.serverAccessLog &&
+        authInfo &&
+        bucket &&
+        bucket.getBucketLoggingStatus() &&
+        bucket.getBucketLoggingStatus().getLoggingEnabled()) {
+        /* eslint-disable no-param-reassign */
+        request.serverAccessLog.enabled = true;
+        request.serverAccessLog.bucketOwner = bucket.getOwner();
+        request.serverAccessLog.bucketName = bucket.getName();
+        request.serverAccessLog.authInfo = authInfo;
+        request.serverAccessLog.loggingEnabled = bucket.getBucketLoggingStatus().getLoggingEnabled();
+        /* eslint-enable no-param-reassign */
+    }
+}
+
 /** getNullVersionFromMaster - retrieves the null version
  * metadata via retrieving the master key
  *
@@ -274,6 +291,7 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
                 contentLength, withVersionId, log, err => next(err, bucket, objMD));
         },
     ], (err, bucket, objMD) => {
+        storeServerAccessLogInfo(request, authInfo, bucket);
         if (err) {
             // still return bucket for cors headers
             return callback(err, bucket);
@@ -296,6 +314,7 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
 function standardMetadataValidateBucket(params, actionImplicitDenies, log, callback) {
     const { bucketName } = params;
     return metadata.getBucket(bucketName, log, (err, bucket) => {
+        storeServerAccessLogInfo(params.request, params.authInfo, bucket);
         if (err) {
             // if some implicit actionImplicitDenies, return AccessDenied before
             // leaking any state information

lib/server.js

@@ -26,6 +26,7 @@ const {
 
 const HttpAgent = require('agentkeepalive');
 const QuotaService = require('./utilization/instance');
+const { logServerAccess } = require('./utilities/serverAccesssLogger');
 const { parseLC, MultipleBackendGateway } = arsenal.storage.data;
 const websiteEndpoints = _config.websiteEndpoints;
 let client = dataWrapper.client;
@@ -122,6 +123,20 @@ class S3Server {
         monitoringClient.httpActiveRequests.inc();
         const requestStartTime = process.hrtime.bigint();
 
+        // eslint-disable-next-line no-param-reassign
+        req.serverAccessLog = {
+            enabled: false,
+            startTime: requestStartTime,
+        };
+        // eslint-disable-next-line no-param-reassign
+        res.serverAccessLog = {};
+
+        res.on('finish', () => {
+            // eslint-disable-next-line no-param-reassign
+            req.serverAccessLog.endTime = process.hrtime.bigint();
+            logServerAccess(req.serverAccessLog, req, res);
+        });
+ 
         // disable nagle algorithm
         req.socket.setNoDelay();
         res.on('close', () => {