diff --git a/CHANGELOG.md b/CHANGELOG.md index f237d818cf9..a43ee182388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,9 @@ degraded execution table operations. - **CUMULUS-3449** - Updated `@cumulus/db` package and configure knex hook postProcessResponse to convert the return string from columns ending with "cumulus_id" to number. +- **CUMULUS-3841** + - Increased `fetchRules` page size to default to 100 instead of 10. This improves overall query time when + fetching all rules such as in `sqsMessageConsumer`. ### Fixed diff --git a/packages/api/lib/rulesHelpers.js b/packages/api/lib/rulesHelpers.js index dfe8ea41a36..19a2ebb692d 100644 --- a/packages/api/lib/rulesHelpers.js +++ b/packages/api/lib/rulesHelpers.js @@ -56,12 +56,13 @@ const log = new Logger({ sender: '@cumulus/rulesHelpers' }); * * @param {Object} params - function params * @param {number} [params.pageNumber] - current page of API results + * @param {number} [params.pageSize] - database query page size used when calling listRules * @param {Array} [params.rules] - partial rules Array * @param {Object} [params.queryParams] - API query params, empty query returns all rules * @returns {Promise>} all matching rules */ -async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) { - const query = { ...queryParams, page: pageNumber }; +async function fetchRules({ pageNumber = 1, pageSize = 100, rules = [], queryParams = {} }) { + const query = { ...queryParams, page: pageNumber, limit: pageSize }; const apiResponse = await listRules({ prefix: process.env.stackName, query, @@ -70,6 +71,7 @@ async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) { if (responseBody.results.length > 0) { return fetchRules({ pageNumber: (pageNumber + 1), + pageSize, rules: rules.concat(responseBody.results), queryParams, }); diff --git a/packages/api/tests/lib/rules/test-rulesHelpers.js b/packages/api/tests/lib/rules/test-rulesHelpers.js index 6aecb38dc47..ace6fd81977 100644 --- a/packages/api/tests/lib/rules/test-rulesHelpers.js +++ b/packages/api/tests/lib/rules/test-rulesHelpers.js @@ -200,20 +200,37 @@ test.serial('fetchRules invokes API to fetch rules', async (t) => { t.true(listRulesStub.calledOnce); }); +test.serial('fetchRules allows for configurable page size', async (t) => { + listRulesStub.onFirstCall().callsFake(({ query }) => { + t.is(query.limit, 100); + return { body: JSON.stringify({ results: [] }) }; + }); + + listRulesStub.onSecondCall().callsFake(({ query }) => { + t.is(query.limit, 50); + return { body: JSON.stringify({ results: [] }) }; + }); + + await rulesHelpers.fetchRules({}); + await rulesHelpers.fetchRules({ pageSize: 50 }); + + t.true(listRulesStub.calledTwice); +}); + test.serial('fetchRules pages through results until reaching an empty list', async (t) => { const rule1 = { name: 'rule-1' }; const rule2 = { name: 'rule-2' }; const firstCallArgs = { prefix: process.env.stackName, - query: { page: 1 }, + query: { limit: 100, page: 1 }, }; const secondCallArgs = { prefix: process.env.stackName, - query: { page: 2 }, + query: { limit: 100, page: 2 }, }; const thirdCallArgs = { prefix: process.env.stackName, - query: { page: 3 }, + query: { limit: 100, page: 3 }, }; listRulesStub.onFirstCall().callsFake((params) => { t.deepEqual(params, firstCallArgs);