feat(#8216): propagate api request id to haproxy (#9613)

Adds custom header to pass the id of the original client request from API to all cascading requests to CouchDb.
Changes request ID to being a 12 char long string (uuid slice) instead of the whole uuid.

#8216

api/src/db.js

@@ -6,6 +6,8 @@ PouchDB.plugin(require('pouchdb-adapter-http'));
 PouchDB.plugin(require('pouchdb-session-authentication'));
 PouchDB.plugin(require('pouchdb-find'));
 PouchDB.plugin(require('pouchdb-mapreduce'));
+const asyncLocalStorage = require('./services/async-storage');
+const { REQUEST_ID_HEADER } = require('./server-utils');
 
 const { UNIT_TEST_ENV } = process.env;
 
@@ -74,6 +76,10 @@ if (UNIT_TEST_ENV) {
   const fetch = (url, opts) => {
     // Adding audit flag (haproxy) Service that made the request initially.
     opts.headers.set('X-Medic-Service', 'api');
+    const requestId = asyncLocalStorage.getRequestId();
+    if (requestId) {
+      opts.headers.set(REQUEST_ID_HEADER, requestId);
+    }
     return PouchDB.fetch(url, opts);
   };
 

api/src/routing.js

@@ -77,6 +77,7 @@ const dbDocHandler = require('./controllers/db-doc');
 const extensionLibs = require('./controllers/extension-libs');
 const replication = require('./controllers/replication');
 const app = express.Router({ strict: true });
+const asyncLocalStorage = require('./services/async-storage');
 const moment = require('moment');
 const MAX_REQUEST_SIZE = '32mb';
 
@@ -156,9 +157,15 @@ if (process.argv.slice(2).includes('--allow-cors')) {
   });
 }
 
+const shortUuid = () => {
+  const ID_LENGTH = 12;
+  return uuid.v4().replace(/-/g, '').toLowerCase().slice(0, ID_LENGTH);
+};
+
 app.use((req, res, next) => {
-  req.id = uuid.v4();
-  next();
+  req.id = shortUuid();
+  req.headers[serverUtils.REQUEST_ID_HEADER] = req.id;
+  asyncLocalStorage.set(req, () => next());
 });
 app.use(getLocale);
 

api/src/server-utils.js

@@ -4,6 +4,7 @@ const environment = require('@medic/environment');
 const isClientHuman = require('./is-client-human');
 const logger = require('@medic/logger');
 const MEDIC_BASIC_AUTH = 'Basic realm="Medic Web Services"';
+const REQUEST_ID_HEADER = 'X-Request-Id';
 const cookie = require('./services/cookie');
 const {InvalidArgumentError} = require('@medic/cht-datasource');
 
@@ -49,6 +50,7 @@ const promptForBasicAuth = res => {
 
 module.exports = {
   MEDIC_BASIC_AUTH: MEDIC_BASIC_AUTH,
+  REQUEST_ID_HEADER: REQUEST_ID_HEADER,
 
   /*
    * Attempts to determine the correct response given the error code.

api/src/services/async-storage.js

@@ -0,0 +1,17 @@
+const { AsyncLocalStorage } = require('node:async_hooks');
+const asyncLocalStorage = new AsyncLocalStorage();
+const { REQUEST_ID_HEADER } = require('../server-utils');
+
+const request = require('@medic/couch-request');
+
+module.exports = {
+  set: (req, callback) => {
+    asyncLocalStorage.run({ clientRequest: req }, callback);
+  },
+  getRequestId: () => {
+    const localStorage = asyncLocalStorage.getStore();
+    return localStorage?.clientRequest?.id;
+  },
+};
+
+request.initialize(module.exports, REQUEST_ID_HEADER);

api/tests/mocha/db.spec.js

@@ -1,5 +1,6 @@
 const sinon = require('sinon');
 require('chai').use(require('chai-as-promised'));
+const PouchDB = require('pouchdb-core');
 const { expect } = require('chai');
 const rewire = require('rewire');
 const request = require('@medic/couch-request');
@@ -8,6 +9,7 @@ let db;
 let unitTestEnv;
 
 const env = require('@medic/environment');
+const asyncLocalStorage = require('../../src/services/async-storage');
 
 describe('db', () => {
   beforeEach(() => {
@@ -404,4 +406,39 @@ describe('db', () => {
         .to.be.rejectedWith(Error, `Cannot add security: invalid db name dbanme or role`);
     });
   });
+
+  describe('fetch extension', () => {
+    it('should set headers where there is an active client request', async () => {
+      sinon.stub(PouchDB, 'fetch').resolves({
+        json: sinon.stub().resolves({ result: true }),
+        ok: true,
+      });
+      sinon.stub(asyncLocalStorage, 'getRequestId').returns('the_id');
+      db = rewire('../../src/db');
+
+      await db.medic.info();
+      const headers = PouchDB.fetch.args.map(arg => arg[1].headers);
+      expect(headers.length).to.equal(4);
+      headers.forEach((header) => {
+        expect(header.get('X-Medic-Service')).to.equal('api');
+        expect(header.get('X-Request-Id')).to.equal('the_id');
+      });
+    });
+
+    it('should work when call is made without an active clinet request', async () => {
+      sinon.stub(PouchDB, 'fetch').resolves({
+        json: sinon.stub().resolves({ result: true }),
+        ok: true,
+      });
+      sinon.stub(asyncLocalStorage, 'getRequestId').returns(undefined);
+      db = rewire('../../src/db');
+
+      await db.medic.info();
+      const headers = PouchDB.fetch.args.map(arg => arg[1].headers);
+      expect(headers.length).to.equal(4);
+      headers.forEach((header) => {
+        expect(header.get('X-Medic-Service')).to.equal('api');
+      });
+    });
+  });
 });

api/tests/mocha/server-utils.spec.js

@@ -282,4 +282,8 @@ describe('Server utils', () => {
       chai.expect(serverUtilsError.calledOnceWithExactly(error, req, res)).to.be.true;
     });
   });
+
+  it('should export request header', () => {
+    chai.expect(serverUtils.REQUEST_ID_HEADER).to.equal('X-Request-Id');
+  });
 });

api/tests/mocha/services/async-storage.spec.js

@@ -0,0 +1,67 @@
+const sinon = require('sinon');
+const rewire = require('rewire');
+const { expect } = require('chai');
+const asyncHooks = require('node:async_hooks');
+const request = require('@medic/couch-request');
+const serverUtils = require('../../../src/server-utils');
+
+describe('async-storage', () => {
+  let service;
+  let asyncLocalStorage;
+
+  beforeEach(() => {
+    asyncLocalStorage = sinon.spy(asyncHooks, 'AsyncLocalStorage');
+    sinon.stub(request, 'initialize');
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('should initialize async storage and initialize couch-request', async () => {
+    service = rewire('../../../src/services/async-storage');
+
+    expect(asyncLocalStorage.callCount).to.equal(1);
+    expect(request.initialize.args).to.deep.equal([[
+      service,
+      serverUtils.REQUEST_ID_HEADER
+    ]]);
+  });
+
+  it('set should set request uuid', () => {
+    service = rewire('../../../src/services/async-storage');
+    const asyncLocalStorage = service.__get__('asyncLocalStorage');
+    sinon.stub(asyncLocalStorage, 'run');
+
+    const req = { this: 'is a req' };
+    const cb = sinon.stub();
+    Object.freeze(req);
+    service.set(req, cb);
+    expect(asyncLocalStorage.run.args).to.deep.equal([[
+      { clientRequest: req },
+      cb
+    ]]);
+  });
+
+  it('getRequestId should return request id when set', done => {
+    service = rewire('../../../src/services/async-storage');
+    const req = { id: 'uuid' };
+    service.set(req, () => {
+      expect(service.getRequestId()).to.equal('uuid');
+      done();
+    });
+  });
+
+  it('getRequestId should return nothing when there is no local storage', () => {
+    service = rewire('../../../src/services/async-storage');
+    expect(service.getRequestId()).to.equal(undefined);
+  });
+
+  it('getRequestId should return nothing when there is no client request', done => {
+    service = rewire('../../../src/services/async-storage');
+    service.set(undefined, () => {
+      expect(service.getRequestId()).to.equal(undefined);
+      done();
+    });
+  });
+});

haproxy/default_frontend.cfg

@@ -63,10 +63,11 @@ frontend http-in
   http-request capture req.hdr(x-medic-service) len 200 # capture.req.hdr(1)
   http-request capture req.hdr(x-medic-user) len 200 # capture.req.hdr(2)
   http-request capture req.hdr(user-agent) len 600 # capture.req.hdr(3)
+  http-request capture req.hdr(x-request-id) len 12 # capture.req.hdr(4)
   capture response header Content-Length len 10 # capture.res.hdr(0)
   http-response set-header Connection Keep-Alive
   http-response set-header Keep-Alive timeout=18000
   log global
-  log-format "%ci,%s,%ST,%Ta,%Ti,%TR,%[capture.req.method],%[capture.req.uri],%[capture.req.hdr(1)],%[capture.req.hdr(2)],'%[capture.req.hdr(0),lua.replacePassword]',%B,%Tr,%[capture.res.hdr(0)],'%[capture.req.hdr(3)]'"
+  log-format "%ci,%s,%ST,%Ta,%Ti,%TR,%[capture.req.method],%[capture.req.uri],%[capture.req.hdr(1)],%[capture.req.hdr(2)],%[capture.req.hdr(4)],'%[capture.req.hdr(0),lua.replacePassword]',%B,%Tr,%[capture.res.hdr(0)],'%[capture.req.hdr(3)]'"
   default_backend couchdb-servers
 

shared-libs/couch-request/src/couch-request.js

@@ -2,6 +2,9 @@ const request = require('request-promise-native');
 const isPlainObject = require('lodash/isPlainObject');
 const environment = require('@medic/environment');
 const servername = environment.host;
+let asyncLocalStorage;
+let requestIdHeader;
+
 
 const isString = value => typeof value === 'string' || value instanceof String;
 const isTrue = value => isString(value) ? value.toLowerCase() === 'true' : value === true;
@@ -19,11 +22,17 @@ const methods = {
 const mergeOptions = (target, source, exclusions = []) => {
   for (const [key, value] of Object.entries(source)) {
     if (Array.isArray(exclusions) && exclusions.includes(key)) {
-      return target;
+      continue;
     }
     target[key] = value; // locally, mutation is preferable to spreading as it doesn't
     // make new objects in memory. Assuming this is a hot path.
   }
+  const requestId = asyncLocalStorage?.getRequestId();
+  if (requestId) {
+    target.headers = target.headers || {};
+    target.headers[requestIdHeader] = requestId;
+  }
+
   return target;
 };
 
@@ -104,6 +113,11 @@ const getRequestType = (method) => {
 };
 
 module.exports = {
+  initialize: (store, header) => {
+    asyncLocalStorage = store;
+    requestIdHeader = header;
+  },
+
   get: (first, second = {}) => req(methods.GET, first, second),
   post: (first, second = {}) => req(methods.POST, first, second),
   put: (first, second = {}) => req(methods.PUT, first, second),