Merge pull request #3506 from nasa/release-16.1.2

Release 16.1.2

CHANGELOG.md

@@ -6,6 +6,32 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+## [v16.1.2] 2023-11-01
+
+### Added
+
+- **CUMULUS-3218**
+  - Added optional `maxDownloadTime` field to `provider` schema
+  - Added `max_download_time` column to PostgreSQL `providers` table
+  - Updated `@cumulus/ingest/lock` to check expired locks based on `provider.maxDownloadTime`
+
+### Fixed
+
+- **@aws-sdk upgrade**
+  - Fixed TS compilation error on aws-client package caused by @aws-sdk/client-dynamodb 3.433.0 upgrade
+  - Updated mapping for collection Elasticsearch records to prevent dynamic field for keys under `meta`.
+- **CUMULUS-3286**
+  - Fixed `@cumulus/cmrjs/cmr-utils/getGranuleTemporalInfo` and `@cumulus/message/Granules/getGranuleCmrTemporalInfo`
+    to handle non-existing cmr file.
+  - Updated mapping for granule and deletedgranule Elasticsearch records to prevent dynamic field for keys under
+    `queryFields`.
+- **CUMULUS-3293**
+  - Process Dead Letter Archive is fixed to properly copy objects from `/sqs/` to `/failed-sqs/` location
+- **CUMULUS-3393**
+  - Fixed `PUT` collection endpoint to update collection configuration in S3.
+- **CUMULUS-3467**
+  - Added `childWorkflowMeta` to `QueueWorkflow` task configuration
+
 ## [v16.1.1] 2023-08-03
 
 ### Notable Changes
@@ -7259,7 +7285,8 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1.
 ## [v1.0.0] - 2018-02-23
 
 
-[unreleased]: https://github.com/nasa/cumulus/compare/v16.1.1...HEAD
+[unreleased]: https://github.com/nasa/cumulus/compare/v16.1.2...HEAD
+[v16.1.2]: https://github.com/nasa/cumulus/compare/v16.1.1...v16.1.2
 [v16.1.1]: https://github.com/nasa/cumulus/compare/v16.0.0...v16.1.1
 [v16.0.0]: https://github.com/nasa/cumulus/compare/v15.0.4...v16.0.0
 [v15.0.4]: https://github.com/nasa/cumulus/compare/v15.0.3...v15.0.4

bamboo/bootstrap-unit-tests.sh

@@ -9,19 +9,18 @@ set -ex
 export SSH_USERS=user:$(id -u):$(id -u)
 export COMPOSE_FILE=./bamboo/docker-compose.yml
 
-## Set container_id for docker-compose to use to identify the compose stack per planKey
-docker_command="docker exec -t ${container_id}_build_env_1 /bin/bash -c"
+## Set container_id for docker compose to use to identify the compose stack per planKey
+docker_command="docker exec -t ${container_id}-build_env-1 /bin/bash -c"
 
 docker ps -a
 
 ## Setup the compose stack
-docker-compose -p ${container_id} down
-docker-compose -p ${container_id} rm -f
-docker-compose -p ${container_id} up -d
+docker compose -p ${container_id} down
+docker compose -p ${container_id} rm -f
+docker compose -p ${container_id} up -d
 
 docker ps -a
-
-while ! docker container inspect ${container_id}\_build_env_1; do
+while ! docker container inspect ${container_id}-build_env-1; do
   echo 'Waiting for build env to be available';
   docker ps -a
   sleep 5;
@@ -30,8 +29,8 @@ done
 ## Setup the build env container once it's started
 $docker_command "npm install --error --no-progress -g nyc; cd $UNIT_TEST_BUILD_DIR; git fetch --all; git checkout $GIT_SHA"
 # Copy build cache of compiled TS code into cached bootstrap dir, if necessary
-docker cp $TS_BUILD_CACHE_FILE "${container_id}_build_env_1:$UNIT_TEST_BUILD_DIR"
-docker cp bamboo/extract-ts-build-cache.sh "${container_id}_build_env_1:$UNIT_TEST_BUILD_DIR/bamboo"
+docker cp $TS_BUILD_CACHE_FILE "${container_id}-build_env-1:$UNIT_TEST_BUILD_DIR"
+docker cp bamboo/extract-ts-build-cache.sh "${container_id}-build_env-1:$UNIT_TEST_BUILD_DIR/bamboo"
 
 # Extract build cache of compiled TS files
 $docker_command "cd $UNIT_TEST_BUILD_DIR; TS_BUILD_CACHE_FILE=$TS_BUILD_CACHE_FILE ./bamboo/extract-ts-build-cache.sh"

bamboo/cleanup-unit-tests.sh

@@ -10,5 +10,5 @@ container_id=${container_id/-/}
 export COMPOSE_FILE=./bamboo/docker-compose.yml
 
 docker ps -a
-docker-compose -p ${container_id} down
-docker-compose -p ${container_id} rm -f
+docker compose -p ${container_id} down
+docker compose -p ${container_id} rm -f

bamboo/docker-compose.yml

@@ -46,7 +46,7 @@ services:
     environment:
       SERVICES: "cloudformation,cloudwatch,cloudwatchlogs,dynamodb,iam,kinesis,kms,lambda,s3,secretsmanager,sns,sqs,stepfunctions,ssm,logs"
   build_env:
-    image: maven.earthdata.nasa.gov/cumulus:latest
+    image: $CUMULUS_BASE_IMAGE
     volumes:
       - /tmp/cumulus_unit_test_data:/tmp/cumulus_unit_test_data
     environment:

bamboo/set-bamboo-env-variables.sh

@@ -56,6 +56,7 @@ declare -a param_list=(
   "bamboo_USE_NPM_PACKAGES"
   "bamboo_USE_TERRAFORM_ZIPS"
   "bamboo_VERSION_FLAG"
+  "bamboo_CUMULUS_BASE_IMAGE"
 )
 
 ## Strip 'bamboo_SECRET_' from secret keys

bamboo/unit-tests.sh

@@ -7,6 +7,6 @@ set -ex
 
 docker ps -a ## Show running containers for output logs
 
-docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run db:local:reset"
-docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run test:coverage"
-docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run coverage -- --noRerun"
+docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run db:local:reset"
+docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run test:coverage"
+docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run coverage -- --noRerun"

docs/configuration/data-management-types.md

@@ -73,6 +73,7 @@ The Provider configuration is defined by a JSON object that takes different conf
 |:---:|:----|:------:|-----------|
 |id|string|Yes|Unique identifier for the provider|
 |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider.  Defaults to unlimited |
+|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. |
 |protocol|string|Yes|The protocol for this provider. Must be `s3` for this provider type. |
 |host|string|Yes|S3 Bucket to pull data from |
 
@@ -82,6 +83,7 @@ The Provider configuration is defined by a JSON object that takes different conf
 |:---:|:----|:------:|-----------|
 |id|string|Yes|Unique identifier for the provider|
 |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider.  This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider.  Defaults to unlimited |
+|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. |
 |protocol|string|Yes|The protocol for this provider.  Must be `http` for this provider type |
 |host|string|Yes|The host to pull data from (e.g. `nasa.gov`)
 |username|string|No|Configured username for basic authentication.   Cumulus encrypts this using KMS and uses it in a `Basic` auth header if needed for authentication |
@@ -96,6 +98,7 @@ The Provider configuration is defined by a JSON object that takes different conf
 |:---:|:----|:------:|-----------|
 |id|string|Yes|Unique identifier for the provider|
 |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider.  This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider.  Defaults to unlimited |
+|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. |
 |protocol|string|Yes|The protocol for this provider.  Must be `https` for this provider type |
 |host|string|Yes|The host to pull data from (e.g. `nasa.gov`) |
 |username|string|No|Configured username for basic authentication.   Cumulus encrypts this using KMS and uses it in a `Basic` auth header if needed for authentication |
@@ -110,6 +113,7 @@ The Provider configuration is defined by a JSON object that takes different conf
 |:---:|:----|:------:|-----------|
 |id|string|Yes|Unique identifier for the provider|
 |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider.  This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider.  Defaults to unlimited |
+|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. |
 |protocol|string|Yes|The protocol for this provider.  Must be `ftp` for this provider type |
 |host|string|Yes|The ftp host to pull data from (e.g. `nasa.gov`) |
 |username|string|No|Username to use to connect to the ftp server.  Cumulus encrypts this using KMS. Defaults to `anonymous` if not defined |
@@ -122,6 +126,7 @@ The Provider configuration is defined by a JSON object that takes different conf
 |:---:|:----|:------:|-----------|
 |id|string|Yes|Unique identifier for the provider|
 |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider.  This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider.  Defaults to unlimited |
+|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. |
 |protocol|string|Yes|The protocol for this provider.  Must be `sftp` for this provider type |
 |host|string|Yes|The ftp host to pull data from (e.g. `nasa.gov`) |
 |username|string|No|Username to use to connect to the sftp server.|

docs/data-cookbooks/queue-post-to-cmr.md

@@ -30,7 +30,8 @@ The last step should be the `QueuePublishWorkflow` step. It should be configured
           "workflow": "{$.meta.workflow}",
           "queueUrl": "${start_sf_queue_url}",
           "provider": "{$.meta.provider}",
-          "collection": "{$.meta.collection}"
+          "collection": "{$.meta.collection}",
+          "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" }
         }
       }
     },
@@ -97,7 +98,8 @@ Then, configure the `QueueWorkflow` task similarly to its configuration in the i
           "workflow": "PublishGranuleQueue",
           "queueUrl": "${start_sf_queue_url}",
           "provider": "{$.meta.provider}",
-          "collection": "{$.meta.collection}"
+          "collection": "{$.meta.collection}",
+          "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" }
         }
       }
     },

docs/workflows/docker.md

@@ -8,12 +8,12 @@ The software used for processing data amongst DAAC's is developed in a variety o
 
 ## Using Docker
 
-Docker images are run using the `docker` command and can be used to build a Docker image from a Dockerfile, fetch an existing image from a remote repository, or run an existing image. In Cumulus, `docker-compose` is used to help developers by making it easy to build images locally and test them.
+Docker images are run using the `docker` command and can be used to build a Docker image from a Dockerfile, fetch an existing image from a remote repository, or run an existing image. In Cumulus, `docker compose` is used to help developers by making it easy to build images locally and test them.
 
-To run a command using docker-compose use:
+To run a command using docker compose use:
 
 ```bash
-docker-compose run *command*
+docker compose run *command*
 ```
 
 where *command* is one of
@@ -34,13 +34,13 @@ pip install awscli
 aws ecr get-login --region us-east-1 | source /dev/stdin
 ```
 
-As long as you have permissions to access the NASA Cumulus AWS account, this will allow you to pull images from AWS ECR, and push rebuilt or new images there as well. Docker-compose may also be used to push images.
+As long as you have permissions to access the NASA Cumulus AWS account, this will allow you to pull images from AWS ECR, and push rebuilt or new images there as well. docker compose may also be used to push images.
 
 ```bash
-docker-compose push
+docker compose push
 ```
 
-Which will push the built image to AWS ECR. Note that the image built by docker-compose will have is the `:latest` tag, and will overwrite the `:latest` tagged docker image on the registry.  This file should be updated to push to a different tag if overwriting is not desired.
+Which will push the built image to AWS ECR. Note that the image built by docker compose will have is the `:latest` tag, and will overwrite the `:latest` tagged docker image on the registry.  This file should be updated to push to a different tag if overwriting is not desired.
 
 In normal use-cases for most production images on either repository,  CircleCI takes care of this building and deploying process
 
@@ -194,10 +194,10 @@ The docker image for a process can be used on the retrieved test data. First cre
 mkdir data/test-output
 ```
 
-Then run the docker image using docker-compose.
+Then run the docker image using docker compose.
 
 ```bash
-docker-compose run test
+docker compose run test
 ```
 
 This will process the data in the data/input directory and put the output into data/test-output. Repositories also include Python based tests which will validate this newly created output to the contents of data/output. Use Python's Nose tool to run the included tests.

example/cumulus-tf/ingest_granule_with_queue_workflow.asl.json

@@ -234,7 +234,8 @@
             "workflow": "{$.meta.workflow}",
             "queueUrl": "${start_sf_queue_url}",
             "provider": "{$.meta.provider}",
-            "collection": "{$.meta.collection}"
+            "collection": "{$.meta.collection}",
+            "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" }
           }
         }
       },

example/cumulus-tf/publish_granule_with_requeue_workflow.asl.json

@@ -13,7 +13,8 @@
             "bucket": "{$.meta.buckets.internal.name}",
             "stack": "{$.meta.stack}",
             "cmr": "{$.meta.cmr}",
-            "launchpad": "{$.meta.launchpad}"
+            "launchpad": "{$.meta.launchpad}",
+            "etags": "{$.meta.file_etags}"
           }
         }
       },
@@ -58,7 +59,8 @@
             "workflow": "PublishGranuleQueue",
             "queueUrl": "${start_sf_queue_url}",
             "provider": "{$.meta.provider}",
-            "collection": "{$.meta.collection}"
+            "collection": "{$.meta.collection}",
+            "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" }
           }
         }
       },

example/lambdas/asyncOperations/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-async-operations",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "AsyncOperations Test Lambda",
   "main": "index.js",
   "private": true,

example/lambdas/ftpPopulateTestLambda/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/ftp-populate-test-lambda",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "FTP Population Utility Lambda",
   "main": "index.js",
   "private": true,
@@ -19,12 +19,12 @@
     "access": "private"
   },
   "dependencies": {
-    "@cumulus/api": "16.1.1",
-    "@cumulus/api-client": "16.1.1",
-    "@cumulus/common": "16.1.1",
-    "@cumulus/integration-tests": "16.1.1",
-    "@cumulus/logger": "16.1.1",
-    "@cumulus/test-data": "16.1.1",
+    "@cumulus/api": "16.1.2",
+    "@cumulus/api-client": "16.1.2",
+    "@cumulus/common": "16.1.2",
+    "@cumulus/integration-tests": "16.1.2",
+    "@cumulus/logger": "16.1.2",
+    "@cumulus/test-data": "16.1.2",
     "aws-sdk": "^2.585.0",
     "fs-extra": "^9.0.0",
     "jsftp": "https://github.com/jkovarik/jsftp.git#add_288",

example/lambdas/lzardsClientTest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-lzards-api-lambda",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "LZARDS API Client Test Lambda",
   "private": true,
   "engines": {
@@ -20,7 +20,7 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/logger": "16.1.1",
-    "@cumulus/lzards-api-client": "16.1.1"
+    "@cumulus/logger": "16.1.2",
+    "@cumulus/lzards-api-client": "16.1.2"
   }
 }

example/lambdas/python-processing/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-process-activity",
   "private": true,
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

example/lambdas/python-reference-activity/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-activity",
   "private": true,
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

example/lambdas/python-reference-task/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-task",
   "private": true,
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "Python reference task",
   "main": "index.js",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task",

example/lambdas/s3AccessTest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-s3-access",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "S3 Access Test Lambda",
   "main": "index.js",
   "private": true,

example/lambdas/snsS3Test/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-sns-s3",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "SNS to S3 Test Lambda",
   "main": "index.js",
   "private": true,

example/lambdas/versionUpTest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-version-up",
-  "version": "16.1.1",
+  "version": "16.1.2",
   "description": "Version Up Test Lambda",
   "main": "index.js",
   "private": true,