Merge pull request techmatters#751 from techmatters/v1.18-rc

V1.18 rc
LVS-AI · Sep 30, 2024 · cf43f70 · cf43f70
2 parents 6d21625 + 4edf1f5
commit cf43f70
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 10 deletions.
diff --git a/.github/workflows/hrm-transcript-scrubber-ecs-deploy.yml b/.github/workflows/hrm-transcript-scrubber-ecs-deploy.yml
@@ -83,7 +83,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.PRIMARY_AWS_REGION }}
+          aws-region: ${{ env.AWS_REGION }}
           # this plugin sets the AWS account ID to a secret which is not allowed in outputs
           # we have to disable that so repo output will work
           mask-aws-account-id: 'no'

diff --git a/resources-domain/lambdas/import-producer/src/config.ts b/resources-domain/lambdas/import-producer/src/config.ts
@@ -66,7 +66,7 @@ const getConfig = async () => {
     importApiKey,
     importApiAuthHeader,
     maxBatchSize: Number(process.env.RESOURCE_IMPORT_MAX_BATCH_SIZE ?? 20000),
-    maxApiSize: Number(process.env.RESOURCE_IMPORT_MAX_API_SIZE ?? 500),
+    maxApiSize: Number(process.env.RESOURCE_IMPORT_MAX_API_SIZE ?? 200),
     maxRequests: Number(process.env.RESOURCE_IMPORT_MAX_REQUESTS ?? 200),
   } as const;
 };

diff --git a/resources-domain/lambdas/import-producer/src/index.ts b/resources-domain/lambdas/import-producer/src/index.ts
@@ -30,7 +30,8 @@ import {
 import getConfig from './config';
 import { transformKhpResourceToApiResource } from './transformExternalResourceToApiResource';
 import path from 'path';
-
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Agent } from 'undici';
 declare var fetch: typeof import('undici').fetch;
 
 export type HttpError<T = any> = {
@@ -134,6 +135,10 @@ const pullUpdates =
         'x-api-key': externalApiKey,
       },
       method: 'GET',
+      dispatcher: new Agent({
+        headersTimeout: 15 * 60 * 1000, // 15 minutes
+        bodyTimeout: 15 * 60 * 1000, // 15 minutes
+      }),
     });
 
     if (response.ok) {
@@ -225,6 +230,7 @@ export const handler = async (event: ScheduledEvent): Promise<void> => {
   // Wait until the target queue is empty, otherwise the progress tracking on the DB will not account for the unprocessed messages and process the same resources again
   await waitForEmptyQueue(importResourcesSqsQueueUrl);
 
+  console.debug('Target queue empty, reading current import status.');
   const progress = await retrieveCurrentStatus(
     internalResourcesBaseUrl,
     internalResourcesApiKey,
@@ -243,6 +249,11 @@ export const handler = async (event: ScheduledEvent): Promise<void> => {
           `${parseISO(progress.lastProcessedDate).valueOf()}-0`,
       )
     : '0-0';
+  console.info(
+    `Starting import from: ${nextFrom} (${progress ? 'resuming' : 'initial'}${
+      progress?.importSequenceId ? 'import sequence supplied.' : ''
+    })`,
+  );
   let remaining = maxBatchSize;
   let totalRemaining: number | undefined;
   let requestsMade = 0;
@@ -281,6 +292,9 @@ export const handler = async (event: ScheduledEvent): Promise<void> => {
     });
     if (result.nextFrom) {
       nextFrom = result.nextFrom;
+      console.debug(
+        `Continuing import from: ${nextFrom} with another pull from the API...`,
+      );
     } else {
       console.info(
         `Import operation complete due to there being no more resources to import, ${describeRemaining(

diff --git a/resources-domain/lambdas/import-producer/src/transformExternalResourceToApiResource.ts b/resources-domain/lambdas/import-producer/src/transformExternalResourceToApiResource.ts
@@ -61,9 +61,9 @@ const pushReferenceAttributeMapping = ({
   const key = mapping.keyGenerator(context);
 
   if (value === null || value === undefined) {
-    console.debug(
-      `No value provided to referenceStringAttributes: key ${key} and value ${value} - omitting attribute`,
-    );
+    // console.debug(
+    //   `No value provided to referenceStringAttributes: key ${key} and value ${value} - omitting attribute`,
+    // );
     return;
   }
   if (typeof value !== 'string') {
@@ -102,9 +102,9 @@ const pushInlineAttributeMapping = <T extends InlineAttributeProperty>({
   }
 
   if (value === null || value === undefined) {
-    console.debug(
-      `No value provided to ${mapping.property}: key ${key} and value ${value} - omitting attribute`,
-    );
+    // console.debug(
+    //   `No value provided to ${mapping.property}: key ${key} and value ${value} - omitting attribute`,
+    // );
     return;
   }
 

diff --git a/resources-domain/lambdas/import-producer/tests/unit/handler.test.ts b/resources-domain/lambdas/import-producer/tests/unit/handler.test.ts
@@ -32,7 +32,7 @@ import {
   retrieveUnprocessedMessageCount,
 } from '../../src/clientSqs';
 import getConfig from '../../src/config';
-import { Response } from 'undici';
+import { Agent, Response } from 'undici';
 
 declare var fetch: typeof import('undici').fetch;
 
@@ -464,6 +464,7 @@ describe('resources-import-producer handler', () => {
             Authorization: MOCK_CONFIG.importApiAuthHeader,
             'x-api-key': MOCK_CONFIG.importApiKey,
           },
+          dispatcher: expect.any(Agent),
         });
       });
       expect(mockConfiguredPublisher).toHaveBeenCalledTimes(