diff --git a/CHANGELOG.md b/CHANGELOG.md index 7391d8a2af1..81f6323b1c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,32 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## Unreleased +## [v16.1.2] 2023-11-01 + +### Added + +- **CUMULUS-3218** + - Added optional `maxDownloadTime` field to `provider` schema + - Added `max_download_time` column to PostgreSQL `providers` table + - Updated `@cumulus/ingest/lock` to check expired locks based on `provider.maxDownloadTime` + +### Fixed + +- **@aws-sdk upgrade** + - Fixed TS compilation error on aws-client package caused by @aws-sdk/client-dynamodb 3.433.0 upgrade + - Updated mapping for collection Elasticsearch records to prevent dynamic field for keys under `meta`. +- **CUMULUS-3286** + - Fixed `@cumulus/cmrjs/cmr-utils/getGranuleTemporalInfo` and `@cumulus/message/Granules/getGranuleCmrTemporalInfo` + to handle non-existing cmr file. + - Updated mapping for granule and deletedgranule Elasticsearch records to prevent dynamic field for keys under + `queryFields`. +- **CUMULUS-3293** + - Process Dead Letter Archive is fixed to properly copy objects from `/sqs/` to `/failed-sqs/` location +- **CUMULUS-3393** + - Fixed `PUT` collection endpoint to update collection configuration in S3. +- **CUMULUS-3467** + - Added `childWorkflowMeta` to `QueueWorkflow` task configuration + ## [v16.1.1] 2023-08-03 ### Notable Changes @@ -7259,7 +7285,8 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1. ## [v1.0.0] - 2018-02-23 -[unreleased]: https://github.com/nasa/cumulus/compare/v16.1.1...HEAD +[unreleased]: https://github.com/nasa/cumulus/compare/v16.1.2...HEAD +[v16.1.2]: https://github.com/nasa/cumulus/compare/v16.1.1...v16.1.2 [v16.1.1]: https://github.com/nasa/cumulus/compare/v16.0.0...v16.1.1 [v16.0.0]: https://github.com/nasa/cumulus/compare/v15.0.4...v16.0.0 [v15.0.4]: https://github.com/nasa/cumulus/compare/v15.0.3...v15.0.4 diff --git a/bamboo/bootstrap-unit-tests.sh b/bamboo/bootstrap-unit-tests.sh index 0df20c40733..576327f25a1 100755 --- a/bamboo/bootstrap-unit-tests.sh +++ b/bamboo/bootstrap-unit-tests.sh @@ -9,19 +9,18 @@ set -ex export SSH_USERS=user:$(id -u):$(id -u) export COMPOSE_FILE=./bamboo/docker-compose.yml -## Set container_id for docker-compose to use to identify the compose stack per planKey -docker_command="docker exec -t ${container_id}_build_env_1 /bin/bash -c" +## Set container_id for docker compose to use to identify the compose stack per planKey +docker_command="docker exec -t ${container_id}-build_env-1 /bin/bash -c" docker ps -a ## Setup the compose stack -docker-compose -p ${container_id} down -docker-compose -p ${container_id} rm -f -docker-compose -p ${container_id} up -d +docker compose -p ${container_id} down +docker compose -p ${container_id} rm -f +docker compose -p ${container_id} up -d docker ps -a - -while ! docker container inspect ${container_id}\_build_env_1; do +while ! docker container inspect ${container_id}-build_env-1; do echo 'Waiting for build env to be available'; docker ps -a sleep 5; @@ -30,8 +29,8 @@ done ## Setup the build env container once it's started $docker_command "npm install --error --no-progress -g nyc; cd $UNIT_TEST_BUILD_DIR; git fetch --all; git checkout $GIT_SHA" # Copy build cache of compiled TS code into cached bootstrap dir, if necessary -docker cp $TS_BUILD_CACHE_FILE "${container_id}_build_env_1:$UNIT_TEST_BUILD_DIR" -docker cp bamboo/extract-ts-build-cache.sh "${container_id}_build_env_1:$UNIT_TEST_BUILD_DIR/bamboo" +docker cp $TS_BUILD_CACHE_FILE "${container_id}-build_env-1:$UNIT_TEST_BUILD_DIR" +docker cp bamboo/extract-ts-build-cache.sh "${container_id}-build_env-1:$UNIT_TEST_BUILD_DIR/bamboo" # Extract build cache of compiled TS files $docker_command "cd $UNIT_TEST_BUILD_DIR; TS_BUILD_CACHE_FILE=$TS_BUILD_CACHE_FILE ./bamboo/extract-ts-build-cache.sh" diff --git a/bamboo/cleanup-unit-tests.sh b/bamboo/cleanup-unit-tests.sh index e596f655cc2..1f37542af25 100755 --- a/bamboo/cleanup-unit-tests.sh +++ b/bamboo/cleanup-unit-tests.sh @@ -10,5 +10,5 @@ container_id=${container_id/-/} export COMPOSE_FILE=./bamboo/docker-compose.yml docker ps -a -docker-compose -p ${container_id} down -docker-compose -p ${container_id} rm -f +docker compose -p ${container_id} down +docker compose -p ${container_id} rm -f diff --git a/bamboo/docker-compose.yml b/bamboo/docker-compose.yml index 3a80db4fefa..75eb1e0a31c 100644 --- a/bamboo/docker-compose.yml +++ b/bamboo/docker-compose.yml @@ -46,7 +46,7 @@ services: environment: SERVICES: "cloudformation,cloudwatch,cloudwatchlogs,dynamodb,iam,kinesis,kms,lambda,s3,secretsmanager,sns,sqs,stepfunctions,ssm,logs" build_env: - image: maven.earthdata.nasa.gov/cumulus:latest + image: $CUMULUS_BASE_IMAGE volumes: - /tmp/cumulus_unit_test_data:/tmp/cumulus_unit_test_data environment: diff --git a/bamboo/set-bamboo-env-variables.sh b/bamboo/set-bamboo-env-variables.sh index 9e5417128cb..27e10a9e57d 100755 --- a/bamboo/set-bamboo-env-variables.sh +++ b/bamboo/set-bamboo-env-variables.sh @@ -56,6 +56,7 @@ declare -a param_list=( "bamboo_USE_NPM_PACKAGES" "bamboo_USE_TERRAFORM_ZIPS" "bamboo_VERSION_FLAG" + "bamboo_CUMULUS_BASE_IMAGE" ) ## Strip 'bamboo_SECRET_' from secret keys diff --git a/bamboo/unit-tests.sh b/bamboo/unit-tests.sh index beeaa1b7a81..4145c528027 100755 --- a/bamboo/unit-tests.sh +++ b/bamboo/unit-tests.sh @@ -7,6 +7,6 @@ set -ex docker ps -a ## Show running containers for output logs -docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run db:local:reset" -docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run test:coverage" -docker exec -i ${container_id}\_build_env_1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run coverage -- --noRerun" \ No newline at end of file +docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run db:local:reset" +docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run test:coverage" +docker exec -i ${container_id}-build_env-1 /bin/bash -c "cd $UNIT_TEST_BUILD_DIR && npm run coverage -- --noRerun" \ No newline at end of file diff --git a/docs/configuration/data-management-types.md b/docs/configuration/data-management-types.md index d8e39a06639..753b2a7c38f 100644 --- a/docs/configuration/data-management-types.md +++ b/docs/configuration/data-management-types.md @@ -73,6 +73,7 @@ The Provider configuration is defined by a JSON object that takes different conf |:---:|:----|:------:|-----------| |id|string|Yes|Unique identifier for the provider| |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider. Defaults to unlimited | +|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. | |protocol|string|Yes|The protocol for this provider. Must be `s3` for this provider type. | |host|string|Yes|S3 Bucket to pull data from | @@ -82,6 +83,7 @@ The Provider configuration is defined by a JSON object that takes different conf |:---:|:----|:------:|-----------| |id|string|Yes|Unique identifier for the provider| |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider. Defaults to unlimited | +|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. | |protocol|string|Yes|The protocol for this provider. Must be `http` for this provider type | |host|string|Yes|The host to pull data from (e.g. `nasa.gov`) |username|string|No|Configured username for basic authentication. Cumulus encrypts this using KMS and uses it in a `Basic` auth header if needed for authentication | @@ -96,6 +98,7 @@ The Provider configuration is defined by a JSON object that takes different conf |:---:|:----|:------:|-----------| |id|string|Yes|Unique identifier for the provider| |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider. Defaults to unlimited | +|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. | |protocol|string|Yes|The protocol for this provider. Must be `https` for this provider type | |host|string|Yes|The host to pull data from (e.g. `nasa.gov`) | |username|string|No|Configured username for basic authentication. Cumulus encrypts this using KMS and uses it in a `Basic` auth header if needed for authentication | @@ -110,6 +113,7 @@ The Provider configuration is defined by a JSON object that takes different conf |:---:|:----|:------:|-----------| |id|string|Yes|Unique identifier for the provider| |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider. Defaults to unlimited | +|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. | |protocol|string|Yes|The protocol for this provider. Must be `ftp` for this provider type | |host|string|Yes|The ftp host to pull data from (e.g. `nasa.gov`) | |username|string|No|Username to use to connect to the ftp server. Cumulus encrypts this using KMS. Defaults to `anonymous` if not defined | @@ -122,6 +126,7 @@ The Provider configuration is defined by a JSON object that takes different conf |:---:|:----|:------:|-----------| |id|string|Yes|Unique identifier for the provider| |globalConnectionLimit|integer|No|Integer specifying the connection limit for the provider. This is the maximum number of connections Cumulus compatible ingest lambdas are expected to make to a provider. Defaults to unlimited | +|maxDownloadTime|integer|No|Maximum download time in seconds for all granule files on a sync granule task. The timeout is used together with globalConnectionLimit to limit concurrent downloads. | |protocol|string|Yes|The protocol for this provider. Must be `sftp` for this provider type | |host|string|Yes|The ftp host to pull data from (e.g. `nasa.gov`) | |username|string|No|Username to use to connect to the sftp server.| diff --git a/docs/data-cookbooks/queue-post-to-cmr.md b/docs/data-cookbooks/queue-post-to-cmr.md index 522a95dec88..39c213c91cb 100644 --- a/docs/data-cookbooks/queue-post-to-cmr.md +++ b/docs/data-cookbooks/queue-post-to-cmr.md @@ -30,7 +30,8 @@ The last step should be the `QueuePublishWorkflow` step. It should be configured "workflow": "{$.meta.workflow}", "queueUrl": "${start_sf_queue_url}", "provider": "{$.meta.provider}", - "collection": "{$.meta.collection}" + "collection": "{$.meta.collection}", + "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" } } } }, @@ -97,7 +98,8 @@ Then, configure the `QueueWorkflow` task similarly to its configuration in the i "workflow": "PublishGranuleQueue", "queueUrl": "${start_sf_queue_url}", "provider": "{$.meta.provider}", - "collection": "{$.meta.collection}" + "collection": "{$.meta.collection}", + "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" } } } }, diff --git a/docs/workflows/docker.md b/docs/workflows/docker.md index 6f59fa581c5..e75849b41eb 100644 --- a/docs/workflows/docker.md +++ b/docs/workflows/docker.md @@ -8,12 +8,12 @@ The software used for processing data amongst DAAC's is developed in a variety o ## Using Docker -Docker images are run using the `docker` command and can be used to build a Docker image from a Dockerfile, fetch an existing image from a remote repository, or run an existing image. In Cumulus, `docker-compose` is used to help developers by making it easy to build images locally and test them. +Docker images are run using the `docker` command and can be used to build a Docker image from a Dockerfile, fetch an existing image from a remote repository, or run an existing image. In Cumulus, `docker compose` is used to help developers by making it easy to build images locally and test them. -To run a command using docker-compose use: +To run a command using docker compose use: ```bash -docker-compose run *command* +docker compose run *command* ``` where *command* is one of @@ -34,13 +34,13 @@ pip install awscli aws ecr get-login --region us-east-1 | source /dev/stdin ``` -As long as you have permissions to access the NASA Cumulus AWS account, this will allow you to pull images from AWS ECR, and push rebuilt or new images there as well. Docker-compose may also be used to push images. +As long as you have permissions to access the NASA Cumulus AWS account, this will allow you to pull images from AWS ECR, and push rebuilt or new images there as well. docker compose may also be used to push images. ```bash -docker-compose push +docker compose push ``` -Which will push the built image to AWS ECR. Note that the image built by docker-compose will have is the `:latest` tag, and will overwrite the `:latest` tagged docker image on the registry. This file should be updated to push to a different tag if overwriting is not desired. +Which will push the built image to AWS ECR. Note that the image built by docker compose will have is the `:latest` tag, and will overwrite the `:latest` tagged docker image on the registry. This file should be updated to push to a different tag if overwriting is not desired. In normal use-cases for most production images on either repository, CircleCI takes care of this building and deploying process @@ -194,10 +194,10 @@ The docker image for a process can be used on the retrieved test data. First cre mkdir data/test-output ``` -Then run the docker image using docker-compose. +Then run the docker image using docker compose. ```bash -docker-compose run test +docker compose run test ``` This will process the data in the data/input directory and put the output into data/test-output. Repositories also include Python based tests which will validate this newly created output to the contents of data/output. Use Python's Nose tool to run the included tests. diff --git a/example/cumulus-tf/ingest_granule_with_queue_workflow.asl.json b/example/cumulus-tf/ingest_granule_with_queue_workflow.asl.json index 3a90ca45de4..3389f7fd8b1 100644 --- a/example/cumulus-tf/ingest_granule_with_queue_workflow.asl.json +++ b/example/cumulus-tf/ingest_granule_with_queue_workflow.asl.json @@ -234,7 +234,8 @@ "workflow": "{$.meta.workflow}", "queueUrl": "${start_sf_queue_url}", "provider": "{$.meta.provider}", - "collection": "{$.meta.collection}" + "collection": "{$.meta.collection}", + "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" } } } }, diff --git a/example/cumulus-tf/publish_granule_with_requeue_workflow.asl.json b/example/cumulus-tf/publish_granule_with_requeue_workflow.asl.json index 1a496d499b6..6e5846588f4 100644 --- a/example/cumulus-tf/publish_granule_with_requeue_workflow.asl.json +++ b/example/cumulus-tf/publish_granule_with_requeue_workflow.asl.json @@ -13,7 +13,8 @@ "bucket": "{$.meta.buckets.internal.name}", "stack": "{$.meta.stack}", "cmr": "{$.meta.cmr}", - "launchpad": "{$.meta.launchpad}" + "launchpad": "{$.meta.launchpad}", + "etags": "{$.meta.file_etags}" } } }, @@ -58,7 +59,8 @@ "workflow": "PublishGranuleQueue", "queueUrl": "${start_sf_queue_url}", "provider": "{$.meta.provider}", - "collection": "{$.meta.collection}" + "collection": "{$.meta.collection}", + "childWorkflowMeta": { "file_etags": "{$.meta.file_etags}", "staticValue": "aStaticValue" } } } }, diff --git a/example/lambdas/asyncOperations/package.json b/example/lambdas/asyncOperations/package.json index ec8d4f3492f..9145d61e7df 100644 --- a/example/lambdas/asyncOperations/package.json +++ b/example/lambdas/asyncOperations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-async-operations", - "version": "16.1.1", + "version": "16.1.2", "description": "AsyncOperations Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/ftpPopulateTestLambda/package.json b/example/lambdas/ftpPopulateTestLambda/package.json index 5d280f0e774..e1ffe1a931b 100644 --- a/example/lambdas/ftpPopulateTestLambda/package.json +++ b/example/lambdas/ftpPopulateTestLambda/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ftp-populate-test-lambda", - "version": "16.1.1", + "version": "16.1.2", "description": "FTP Population Utility Lambda", "main": "index.js", "private": true, @@ -19,12 +19,12 @@ "access": "private" }, "dependencies": { - "@cumulus/api": "16.1.1", - "@cumulus/api-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/integration-tests": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/test-data": "16.1.1", + "@cumulus/api": "16.1.2", + "@cumulus/api-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/integration-tests": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/test-data": "16.1.2", "aws-sdk": "^2.585.0", "fs-extra": "^9.0.0", "jsftp": "https://github.com/jkovarik/jsftp.git#add_288", diff --git a/example/lambdas/lzardsClientTest/package.json b/example/lambdas/lzardsClientTest/package.json index d919d7b39d6..72fcbfc1f06 100644 --- a/example/lambdas/lzardsClientTest/package.json +++ b/example/lambdas/lzardsClientTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-lzards-api-lambda", - "version": "16.1.1", + "version": "16.1.2", "description": "LZARDS API Client Test Lambda", "private": true, "engines": { @@ -20,7 +20,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/logger": "16.1.1", - "@cumulus/lzards-api-client": "16.1.1" + "@cumulus/logger": "16.1.2", + "@cumulus/lzards-api-client": "16.1.2" } } diff --git a/example/lambdas/python-processing/package.json b/example/lambdas/python-processing/package.json index 437e1e9af91..781fe767b8a 100644 --- a/example/lambdas/python-processing/package.json +++ b/example/lambdas/python-processing/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-process-activity", "private": true, - "version": "16.1.1", + "version": "16.1.2", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-activity/package.json b/example/lambdas/python-reference-activity/package.json index 42f04fb40ed..40a9513e9dc 100644 --- a/example/lambdas/python-reference-activity/package.json +++ b/example/lambdas/python-reference-activity/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-activity", "private": true, - "version": "16.1.1", + "version": "16.1.2", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-task/package.json b/example/lambdas/python-reference-task/package.json index 564eaeaaafa..0666d9005c3 100644 --- a/example/lambdas/python-reference-task/package.json +++ b/example/lambdas/python-reference-task/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-task", "private": true, - "version": "16.1.1", + "version": "16.1.2", "description": "Python reference task", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task", diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json index 025630b7824..ee8a94d2660 100644 --- a/example/lambdas/s3AccessTest/package.json +++ b/example/lambdas/s3AccessTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-s3-access", - "version": "16.1.1", + "version": "16.1.2", "description": "S3 Access Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json index eada1e65ad5..4725278aa23 100644 --- a/example/lambdas/snsS3Test/package.json +++ b/example/lambdas/snsS3Test/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-sns-s3", - "version": "16.1.1", + "version": "16.1.2", "description": "SNS to S3 Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/versionUpTest/package.json b/example/lambdas/versionUpTest/package.json index 808a9b7e9fd..2298f93596c 100644 --- a/example/lambdas/versionUpTest/package.json +++ b/example/lambdas/versionUpTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-version-up", - "version": "16.1.1", + "version": "16.1.2", "description": "Version Up Test Lambda", "main": "index.js", "private": true, diff --git a/example/package.json b/example/package.json index f1006a174df..a922a278fac 100644 --- a/example/package.json +++ b/example/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-integration-tests", - "version": "16.1.1", + "version": "16.1.2", "description": "Cumulus Integration Test Deployment", "private": true, "main": "index.js", @@ -44,32 +44,32 @@ ] }, "dependencies": { - "@cumulus/api": "16.1.1", - "@cumulus/api-client": "16.1.1", - "@cumulus/async-operations": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/checksum": "16.1.1", - "@cumulus/cmr-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/discover-granules": "16.1.1", - "@cumulus/discover-pdrs": "16.1.1", - "@cumulus/files-to-granules": "16.1.1", - "@cumulus/hello-world": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/integration-tests": "16.1.1", - "@cumulus/message": "16.1.1", - "@cumulus/move-granules": "16.1.1", - "@cumulus/parse-pdr": "16.1.1", - "@cumulus/pdr-status-check": "16.1.1", - "@cumulus/post-to-cmr": "16.1.1", - "@cumulus/queue-granules": "16.1.1", - "@cumulus/queue-pdrs": "16.1.1", - "@cumulus/sf-sqs-report": "16.1.1", - "@cumulus/sync-granule": "16.1.1", - "@cumulus/test-processing": "16.1.1" + "@cumulus/api": "16.1.2", + "@cumulus/api-client": "16.1.2", + "@cumulus/async-operations": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/checksum": "16.1.2", + "@cumulus/cmr-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/discover-granules": "16.1.2", + "@cumulus/discover-pdrs": "16.1.2", + "@cumulus/files-to-granules": "16.1.2", + "@cumulus/hello-world": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/integration-tests": "16.1.2", + "@cumulus/message": "16.1.2", + "@cumulus/move-granules": "16.1.2", + "@cumulus/parse-pdr": "16.1.2", + "@cumulus/pdr-status-check": "16.1.2", + "@cumulus/post-to-cmr": "16.1.2", + "@cumulus/queue-granules": "16.1.2", + "@cumulus/queue-pdrs": "16.1.2", + "@cumulus/sf-sqs-report": "16.1.2", + "@cumulus/sync-granule": "16.1.2", + "@cumulus/test-processing": "16.1.2" }, "devDependencies": { - "@cumulus/test-data": "16.1.1" + "@cumulus/test-data": "16.1.2" } } diff --git a/example/scripts/generate_ingest/package.json b/example/scripts/generate_ingest/package.json index be69a860273..35a61f78dba 100644 --- a/example/scripts/generate_ingest/package.json +++ b/example/scripts/generate_ingest/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate_ingest", "private": true, - "version": "16.1.1", + "version": "16.1.2", "description": "Script to generate test data for scaled ingest", "keywords": [ "GIBS", @@ -22,8 +22,8 @@ "directory": "packages/types" }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1" + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2" }, "author": "Cumulus Authors", "license": "Apache-2.0" diff --git a/example/scripts/lib/package.json b/example/scripts/lib/package.json index 81e656c2d80..3c9e646b185 100644 --- a/example/scripts/lib/package.json +++ b/example/scripts/lib/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/example-lib", "private": true, - "version": "16.1.1", + "version": "16.1.2", "description": "example project libs", "homepage": "https://github.com/nasa/cumulus/tree/master/example/scripts/lib", "engines": { diff --git a/example/spec/helpers/Providers.js b/example/spec/helpers/Providers.js index 59bdc509b55..4bd25497bb5 100644 --- a/example/spec/helpers/Providers.js +++ b/example/spec/helpers/Providers.js @@ -37,6 +37,7 @@ const buildFtpProvider = async (postfix = '') => { username: 'testuser', password: 'testpass', globalConnectionLimit: 10, + maxDownloadTime: 500, }; if (process.env.PROVIDER_FTP_PORT) { @@ -59,6 +60,7 @@ const buildHttpOrHttpsProvider = async (postfix, systemBucket, protocol = 'http' host: await getProviderHost(), port: fakeProviderPortMap[protocol], globalConnectionLimit: 10, + maxDownloadTime: 360, }; if (protocol === 'https') { diff --git a/example/spec/parallel/dbRecords/DeadLetterArchiveProcessingSpec.js b/example/spec/parallel/dbRecords/DeadLetterArchiveProcessingSpec.js index 5085432ba09..bd2c43dcc36 100644 --- a/example/spec/parallel/dbRecords/DeadLetterArchiveProcessingSpec.js +++ b/example/spec/parallel/dbRecords/DeadLetterArchiveProcessingSpec.js @@ -27,7 +27,7 @@ const { findExecutionArn } = require('@cumulus/integration-tests/Executions'); const { createOneTimeRule } = require('@cumulus/integration-tests/Rules'); const { getStateMachineArnFromExecutionArn } = require('@cumulus/message/Executions'); const { randomString } = require('@cumulus/common/test-utils'); -const { putJsonS3Object, s3ObjectExists, deleteS3Object } = require('@cumulus/aws-client/S3'); +const { putJsonS3Object, s3ObjectExists, deleteS3Object, getJsonS3Object } = require('@cumulus/aws-client/S3'); const { generateNewArchiveKeyForFailedMessage } = require('@cumulus/api/lambdas/process-s3-dead-letter-archive'); const { encodedConstructCollectionId } = require('../../helpers/Collections'); @@ -291,6 +291,8 @@ describe('A dead letter record archive processing operation', () => { newArchiveKey = generateNewArchiveKeyForFailedMessage(failingMessageKey); expect(await s3ObjectExists({ Bucket: systemBucket, Key: newArchiveKey })).toBeTrue(); + // Check that the JSON object is not empty and contains at least one key + expect(Object.keys(await getJsonS3Object(systemBucket, newArchiveKey)).length).toBeGreaterThan(0); } }); }); diff --git a/example/spec/parallel/ingestGranule/ingestGranuleQueueSpec.js b/example/spec/parallel/ingestGranule/ingestGranuleQueueSpec.js index e1afcac46c7..022b1cf38b4 100644 --- a/example/spec/parallel/ingestGranule/ingestGranuleQueueSpec.js +++ b/example/spec/parallel/ingestGranule/ingestGranuleQueueSpec.js @@ -12,6 +12,7 @@ const { s3 } = require('@cumulus/aws-client/services'); const { generateChecksumFromStream } = require('@cumulus/checksum'); const { addCollections, + getExecutionInputObject, getOnlineResources, waitForCompletedExecution, } = require('@cumulus/integration-tests'); @@ -468,6 +469,12 @@ describe('The S3 Ingest Granules workflow', () => { ); expect(publishGranuleExecutionStatus).toEqual('SUCCEEDED'); }); + + it('passes through childWorkflowMeta to the PublishGranuleQueue execution', async () => { + const executionInput = await getExecutionInputObject(publishGranuleExecutionArn); + expect(executionInput.meta.staticValue).toEqual('aStaticValue'); + expect(executionInput.meta.file_tags).toEqual(lambdaOutput.meta.file_tags); + }); }); describe('the queued workflow', () => { diff --git a/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js b/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js index 68b3cce58f4..e21a7fd9dce 100644 --- a/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js +++ b/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js @@ -138,9 +138,11 @@ describe('The S3 Ingest Granules workflow', () => { // copiedToOrca contains a list of the file s3uri in primary buckets const copiedOver = await Promise.all( - filesCopiedToOrca.map((s3uri) => { + filesCopiedToOrca.map(async (s3uri) => { expect(excludedFileExtensions.filter((type) => s3uri.endsWith(type)).length).toBe(0); - return s3ObjectExists({ Bucket: config.buckets.glacier.name, Key: parseS3Uri(s3uri).Key }); + const parsedS3Uri = parseS3Uri(s3uri); + await deleteS3Object(parsedS3Uri.Bucket, parsedS3Uri.Key); + return s3ObjectExists({ Bucket: config.buckets.glacier.name, Key: parsedS3Uri.Key }); }) ); copiedOver.forEach((check) => expect(check).toEqual(true)); diff --git a/lambdas/db-migration/package.json b/lambdas/db-migration/package.json index 0c8dacb4461..f1b0c03226a 100644 --- a/lambdas/db-migration/package.json +++ b/lambdas/db-migration/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-migration-lambda", - "version": "16.1.1", + "version": "16.1.2", "description": "A Lambda function used for deploying DB migrations", "license": "Apache-2.0", "engines": { @@ -20,7 +20,7 @@ "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles" }, "dependencies": { - "@cumulus/db": "16.1.1", + "@cumulus/db": "16.1.2", "knex": "2.4.1", "pg": "~8.10" } diff --git a/lambdas/db-provision-user-database/package.json b/lambdas/db-provision-user-database/package.json index e8ce6bd2fb4..33102fc7376 100644 --- a/lambdas/db-provision-user-database/package.json +++ b/lambdas/db-provision-user-database/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-provision-user-database-lambda", - "version": "16.1.1", + "version": "16.1.2", "description": "A Lambda function used for provisioning user databases", "engines": { "node": ">=16.19.0" @@ -26,8 +26,8 @@ "timeout": "2m" }, "dependencies": { - "@cumulus/common": "16.1.1", - "@cumulus/db": "16.1.1", + "@cumulus/common": "16.1.2", + "@cumulus/db": "16.1.2", "knex": "2.4.1", "pg": "~8.10" }, diff --git a/lambdas/sqs-message-remover/package.json b/lambdas/sqs-message-remover/package.json index cf22f383389..c5e10cc0f83 100644 --- a/lambdas/sqs-message-remover/package.json +++ b/lambdas/sqs-message-remover/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sqs-message-remover-lambda", - "version": "16.1.1", + "version": "16.1.2", "description": "Remove processed messages from SQS queues", "main": "src/index.js", "private": true, @@ -37,10 +37,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/logger": "16.1.2", "lodash": "^4.17.21" } } diff --git a/lerna.json b/lerna.json index 64006d1845d..987c3a4fd6b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "3.20.2", - "version": "16.1.1", + "version": "16.1.2", "packages": [ "example", "example/lambdas/*", diff --git a/package.json b/package.json index 9bac156b380..1d2e7da817d 100644 --- a/package.json +++ b/package.json @@ -47,9 +47,9 @@ "serve-dist-oauth": "lerna run --stream serve-dist-oauth --scope @cumulus/api", "serve-dist-remote": "lerna run --stream serve-dist-remote --scope @cumulus/api", "coveralls": "nyc report --reporter=text-lcov --temp-directory=\"./.final_nyc_output\" | coveralls", - "start-unit-test-stack": "export SSH_USERS=user:$(id -u):$(id -u) && docker-compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack up -d", - "stop-unit-test-stack": "docker-compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack down", - "view-unit-test-stack-logs": "export SSH_USERS=user:$(id -u):$(id -u) && docker-compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack logs --follow" + "start-unit-test-stack": "export SSH_USERS=user:$(id -u):$(id -u) && docker compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack up -d", + "stop-unit-test-stack": "docker compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack down", + "view-unit-test-stack-logs": "export SSH_USERS=user:$(id -u):$(id -u) && docker compose -f ./bamboo/docker-compose.yml -f ./bamboo/docker-compose-local.yml -p cumulusstack logs --follow" }, "repository": { "type": "git", diff --git a/packages/api-client/package.json b/packages/api-client/package.json index cf613096b50..81db0806e10 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api-client", - "version": "16.1.1", + "version": "16.1.2", "description": "API client for working with the Cumulus archive API", "keywords": [ "GIBS", @@ -39,11 +39,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/logger": "16.1.2", "p-retry": "^2.0.0" }, "devDependencies": { - "@cumulus/types": "16.1.1" + "@cumulus/types": "16.1.2" } } diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index 78251fcf733..0f491112134 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ecs-async-operation", - "version": "16.1.1", + "version": "16.1.2", "description": "The docker image for running async operations", "keywords": [ "NASA", @@ -21,10 +21,10 @@ "coverage": "python ../../../../scripts/coverage_handler/coverage.py" }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/db": "16.1.1", - "@cumulus/es-client": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/db": "16.1.2", + "@cumulus/es-client": "16.1.2", + "@cumulus/logger": "16.1.2", "aws-sdk": "^2.585.0", "crypto-random-string": "^3.2.0", "got": "^11.8.5", diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index c45f709a40d..04ac6cbcb1b 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -182,6 +182,10 @@ async function put(req, res) { collectionPgModel = new CollectionPgModel(), knex = await getKnexClient(), esClient = await Search.es(), + collectionConfigStore = new CollectionConfigStore( + process.env.system_bucket, + process.env.stackName + ), } = req.testContext || {}; const { name, version } = req.params; @@ -218,6 +222,7 @@ async function put(req, res) { apiPgCollection = translatePostgresCollectionToApiCollection(pgCollection); await indexCollection(esClient, apiPgCollection, process.env.ES_INDEX); await publishCollectionUpdateSnsMessage(apiPgCollection); + await collectionConfigStore.put(name, version, apiPgCollection); }); } catch (error) { log.debug(`Failed to update collection with name ${name}, version ${version} and payload ${JSON.stringify(collection)} . Error: ${JSON.stringify(error)}`); diff --git a/packages/api/lambdas/process-s3-dead-letter-archive.js b/packages/api/lambdas/process-s3-dead-letter-archive.js index ab9ca89cbca..816a1d1475f 100644 --- a/packages/api/lambdas/process-s3-dead-letter-archive.js +++ b/packages/api/lambdas/process-s3-dead-letter-archive.js @@ -31,28 +31,28 @@ const generateNewArchiveKeyForFailedMessage = (failedKey) => { * Transfers unprocessed dead letters in the bucket to new location * and deletes dead letters from old archive path * - * @param {string} [deadLetterMessage] - unprocessed dead letter message + * @param {string} [deadLetterObject] - unprocessed dead letter object * @param {string} [bucket] - S3 bucket * @returns {Promise} */ -const transferUnprocessedMessage = async (deadLetterMessage, bucket) => { +const transferUnprocessedMessage = async (deadLetterObject, bucket) => { // Save allFailedKeys messages to different location - const s3KeyForFailedMessage = generateNewArchiveKeyForFailedMessage(deadLetterMessage.Key); + const s3KeyForFailedMessage = generateNewArchiveKeyForFailedMessage(deadLetterObject.Key); try { log.info(`Attempting to save messages that failed to process to ${bucket}/${s3KeyForFailedMessage}`); - await S3.s3PutObject({ + await S3.s3CopyObject({ Bucket: bucket, Key: s3KeyForFailedMessage, - Body: deadLetterMessage.Body, + CopySource: `${bucket}/${deadLetterObject.Key}`, }); log.info(`Saved message to S3 s3://${bucket}/${s3KeyForFailedMessage}`); // Delete failed key from old path - log.info(`Attempting to delete message that failed to process from old path ${bucket}/${deadLetterMessage.Key}`); - await deleteS3Object(bucket, deadLetterMessage.Key); - log.info(`Deleted archived dead letter message from S3 at ${bucket}/${deadLetterMessage.Key}`); + log.info(`Attempting to delete message that failed to process from old path ${bucket}/${deadLetterObject.Key}`); + await deleteS3Object(bucket, deadLetterObject.Key); + log.info(`Deleted archived dead letter message from S3 at ${bucket}/${deadLetterObject.Key}`); } catch (error) { - log.error(`Failed to transfer S3 Object s3://${bucket}/${deadLetterMessage.Key} due to error: ${error}`); + log.error(`Failed to transfer S3 Object s3://${bucket}/${deadLetterObject.Key} due to error: ${error}`); throw error; } }; diff --git a/packages/api/lib/schemas.js b/packages/api/lib/schemas.js index db575608178..436d8b62044 100644 --- a/packages/api/lib/schemas.js +++ b/packages/api/lib/schemas.js @@ -619,6 +619,11 @@ module.exports.provider = { title: 'Concurrent Connection Limit', type: 'integer', }, + maxDownloadTime: { + title: 'Maximum download time in seconds for all granule files on a sync granule task', + type: 'integer', + description: 'The timeout is used together with globalConnectionLimit to limit concurrent downloads', + }, protocol: { title: 'Protocol', type: 'string', diff --git a/packages/api/lib/testUtils.js b/packages/api/lib/testUtils.js index 1db0f228c8f..328ac9f4492 100644 --- a/packages/api/lib/testUtils.js +++ b/packages/api/lib/testUtils.js @@ -282,6 +282,7 @@ function fakeProviderFactory(options = {}) { return { id: randomId('id'), globalConnectionLimit: 1, + maxDownloadTime: 100, protocol: 'http', host: randomId('host'), port: 80, diff --git a/packages/api/package.json b/packages/api/package.json index c80fb566e6f..8d160bbba38 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api", - "version": "16.1.1", + "version": "16.1.2", "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "engines": { @@ -49,26 +49,26 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "16.1.1", - "@cumulus/async-operations": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmr-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/collection-config-store": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/db": "16.1.1", - "@cumulus/distribution-utils": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/es-client": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", - "@cumulus/oauth-client": "16.1.1", - "@cumulus/object-store": "16.1.1", - "@cumulus/pvl": "16.1.1", - "@cumulus/sftp-client": "16.1.1", - "@cumulus/types": "16.1.1", + "@cumulus/api-client": "16.1.2", + "@cumulus/async-operations": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmr-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/collection-config-store": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/db": "16.1.2", + "@cumulus/distribution-utils": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/es-client": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", + "@cumulus/oauth-client": "16.1.2", + "@cumulus/object-store": "16.1.2", + "@cumulus/pvl": "16.1.2", + "@cumulus/sftp-client": "16.1.2", + "@cumulus/types": "16.1.2", "@mapbox/dyno": "^1.4.2", "aggregate-error": "^3.1.0", "ajv": "^6.12.3", @@ -120,6 +120,6 @@ } }, "devDependencies": { - "@cumulus/test-data": "16.1.1" + "@cumulus/test-data": "16.1.2" } } diff --git a/packages/api/tests/endpoints/collections/update-collection.js b/packages/api/tests/endpoints/collections/update-collection.js index 284c77f22dc..7226eab5b20 100644 --- a/packages/api/tests/endpoints/collections/update-collection.js +++ b/packages/api/tests/endpoints/collections/update-collection.js @@ -28,6 +28,7 @@ const { createTestIndex, cleanupTestIndex, } = require('@cumulus/es-client/testUtils'); +const CollectionConfigStore = require('@cumulus/collection-config-store'); const { InvalidRegexError, UnmatchedRegexError, @@ -358,6 +359,42 @@ test.serial('PUT replaces an existing collection in all data stores with correct t.is(actualPgCollection.updated_at.getTime(), updatedEsRecord.updatedAt); }); +test.serial('PUT updates collection configuration store via name and version', async (t) => { + const { originalCollection } = await createCollectionTestRecords( + t.context, + { + duplicateHandling: 'replace', + process: randomString(), + createdAt: Date.now(), + updatedAt: Date.now(), + meta: { foo: randomString() }, + } + ); + + const updatedCollection = { + ...originalCollection, + updatedAt: Date.now(), + createdAt: Date.now(), + duplicateHandling: 'error', + meta: { foo: { nestedKey: randomString() } }, + }; + + const res = await request(app) + .put(`/collections/${originalCollection.name}/${originalCollection.version}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(updatedCollection) + .expect(200); + + const collectionConfigStore = new CollectionConfigStore( + process.env.system_bucket, + process.env.stackName + ); + + t.deepEqual(await collectionConfigStore.get(originalCollection.name, originalCollection.version), + res.body); +}); + test.serial('PUT returns 404 for non-existent collection', async (t) => { const originalCollection = fakeCollectionFactory(); const response = await request(app) diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index 684a67886b7..b4774ce2112 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -2059,6 +2059,9 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) => testExecutionCumulusId, } = t.context; const timestamp = Date.now(); + const oldQueryFields = { + foo: Math.random(), + }; const { newPgGranule, esRecord } = await createGranuleAndFiles({ dbClient: knex, esClient, @@ -2067,6 +2070,7 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) => status: 'running', execution: executionUrl, timestamp: Date.now(), + queryFields: oldQueryFields, }, }); const newApiGranule = await translatePostgresGranuleToApiGranule({ @@ -2075,9 +2079,9 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) => }); t.is(newPgGranule.status, 'running'); - t.is(newPgGranule.query_fields, null); + t.deepEqual(newPgGranule.query_fields, oldQueryFields); t.is(esRecord.status, 'running'); - t.is(esRecord.queryFields, undefined); + t.deepEqual(esRecord.queryFields, oldQueryFields); const newQueryFields = { foo: randomString(), @@ -3562,7 +3566,6 @@ test.serial('PUT replaces an existing granule in all data stores, removing exist executionUrl, knex, } = t.context; - console.log('foobar'); const { apiGranule, newPgGranule, diff --git a/packages/api/tests/lambdas/test-process-s3-dead-letter-archive.js b/packages/api/tests/lambdas/test-process-s3-dead-letter-archive.js index eec3d2f20d1..13a2eb091fb 100644 --- a/packages/api/tests/lambdas/test-process-s3-dead-letter-archive.js +++ b/packages/api/tests/lambdas/test-process-s3-dead-letter-archive.js @@ -187,6 +187,9 @@ test('processDeadLetterArchive saves failed dead letters to different S3 and rem // Check that failing message key exists in new location const savedDeadLetterExists = await S3.fileExists(bucket, s3KeyForFailedMessage); t.truthy(savedDeadLetterExists); + + const fileContents = await S3.getJsonS3Object(bucket, s3KeyForFailedMessage); + t.true(Object.keys(fileContents).length > 0); }); test.serial('processDeadLetterArchive does not remove message from archive S3 path if transfer to new archive path fails', async (t) => { @@ -202,8 +205,8 @@ test.serial('processDeadLetterArchive does not remove message from archive S3 pa if (getMessageExecutionName(cumulusMessage) === passingMessageExecutionName) return; throw new Error('write failure'); }; - const s3Stub = sinon.stub(S3, 's3PutObject').throws( - new Error('Failed to put object') + const s3Stub = sinon.stub(S3, 's3CopyObject').throws( + new Error('Failed to copy object') ); const output = await processDeadLetterArchive({ diff --git a/packages/api/tests/lib/writeRecords/test-write-granules.js b/packages/api/tests/lib/writeRecords/test-write-granules.js index d3c58113980..9f703dce51c 100644 --- a/packages/api/tests/lib/writeRecords/test-write-granules.js +++ b/packages/api/tests/lib/writeRecords/test-write-granules.js @@ -646,20 +646,20 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos ]; const completeGranule = fakeGranuleFactoryV2({ - beginningDateTime: new Date().toString(), + beginningDateTime: new Date().toISOString(), cmrLink: 'example.com', collectionId: constructCollectionId(collection.name, collection.version), createdAt: Date.now(), duration: 1000, - endingDateTime: new Date().toString(), + endingDateTime: new Date().toISOString(), error: { errorKey: 'errorValue' }, execution: executionUrl, files: files, - lastUpdateDateTime: new Date().toString(), + lastUpdateDateTime: new Date().toISOString(), pdrName: pdr.name, - processingEndDateTime: new Date().toString(), - processingStartDateTime: new Date().toString(), - productionDateTime: new Date().toString(), + processingEndDateTime: new Date().toISOString(), + processingStartDateTime: new Date().toISOString(), + productionDateTime: new Date().toISOString(), productVolume: '1000', provider: provider.name, published: true, @@ -842,20 +842,20 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos } = t.context; const completeGranule = fakeGranuleFactoryV2({ - beginningDateTime: new Date().toString(), + beginningDateTime: new Date().toISOString(), cmrLink: 'example.com', collectionId: constructCollectionId(collection.name, collection.version), createdAt: Date.now(), duration: 1000, - endingDateTime: new Date().toString(), + endingDateTime: new Date().toISOString(), error: { errorKey: 'errorValue' }, execution: executionUrl, files: files, - lastUpdateDateTime: new Date().toString(), + lastUpdateDateTime: new Date().toISOString(), pdrName: pdr.name, - processingEndDateTime: new Date().toString(), - processingStartDateTime: new Date().toString(), - productionDateTime: new Date().toString(), + processingEndDateTime: new Date().toISOString(), + processingStartDateTime: new Date().toISOString(), + productionDateTime: new Date().toISOString(), productVolume: '1000', provider: provider.name, published: true, @@ -989,18 +989,18 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos ]; const completeGranule = fakeGranuleFactoryV2({ - beginningDateTime: new Date().toString(), + beginningDateTime: new Date().toISOString(), cmrLink: 'example.com', collectionId: constructCollectionId(collection.name, collection.version), duration: 1000, - endingDateTime: new Date().toString(), + endingDateTime: new Date().toISOString(), error: { errorKey: 'errorValue' }, files: files, - lastUpdateDateTime: new Date().toString(), + lastUpdateDateTime: new Date().toISOString(), pdrName: pdr.name, - processingEndDateTime: new Date().toString(), - processingStartDateTime: new Date().toString(), - productionDateTime: new Date().toString(), + processingEndDateTime: new Date().toISOString(), + processingStartDateTime: new Date().toISOString(), + productionDateTime: new Date().toISOString(), productVolume: '1000', provider: provider.name, published: true, @@ -3968,21 +3968,21 @@ test.serial('writeGranuleFromApi() given a granule with all fields populated is ]; const completeGranule = fakeGranuleFactoryV2({ - beginningDateTime: new Date().toString(), + beginningDateTime: new Date().toISOString(), cmrLink: 'example.com', collectionId: constructCollectionId(collection.name, collection.version), createdAt: Date.now(), duration: 1000, - endingDateTime: new Date().toString(), + endingDateTime: new Date().toISOString(), error: { errorKey: 'errorValue' }, execution: executionUrl, files: files, granuleId: granuleId, - lastUpdateDateTime: new Date().toString(), + lastUpdateDateTime: new Date().toISOString(), pdrName: pdr.name, - processingEndDateTime: new Date().toString(), - processingStartDateTime: new Date().toString(), - productionDateTime: new Date().toString(), + processingEndDateTime: new Date().toISOString(), + processingStartDateTime: new Date().toISOString(), + productionDateTime: new Date().toISOString(), productVolume: '1000', provider: provider.name, published: true, diff --git a/packages/async-operations/package.json b/packages/async-operations/package.json index 3bf988e37b9..189ed380525 100644 --- a/packages/async-operations/package.json +++ b/packages/async-operations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/async-operations", - "version": "16.1.1", + "version": "16.1.2", "description": "Cumulus Core internal async operations module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -29,17 +29,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/db": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/es-client": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/types": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/db": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/es-client": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/types": "16.1.2", "knex": "2.4.1", "uuid": "8.3.2" }, "devDependencies": { - "@cumulus/common": "16.1.1", + "@cumulus/common": "16.1.2", "@types/aws-sdk": "2.7.0", "@types/uuid": "^8.0.0" } diff --git a/packages/aws-client/package.json b/packages/aws-client/package.json index e7921fab78d..6adcf317d39 100644 --- a/packages/aws-client/package.json +++ b/packages/aws-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/aws-client", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for working with AWS", "keywords": [ "GIBS", @@ -53,9 +53,9 @@ "@aws-sdk/s3-request-presigner": "^3.58.0", "@aws-sdk/signature-v4-crt": "^3.58.0", "@aws-sdk/types": "^3.58.0", - "@cumulus/checksum": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/checksum": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", "aws-sdk": "^2.585.0", "jsonpath-plus": "^1.1.0", "lodash": "~4.17.21", diff --git a/packages/aws-client/src/DynamoDb.ts b/packages/aws-client/src/DynamoDb.ts index 1c218f785db..c6c58055a2b 100644 --- a/packages/aws-client/src/DynamoDb.ts +++ b/packages/aws-client/src/DynamoDb.ts @@ -11,6 +11,7 @@ import { CreateTableInput, DeleteTableInput, ScanInput, + Select, } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocument, @@ -83,7 +84,7 @@ export const scan = improveStackTrace( }, fields?: string, limit?: number, - select: string, + select: Select, startKey?: ScanInput['ExclusiveStartKey'] }) => { const { diff --git a/packages/aws-client/src/DynamoDbSearchQueue.ts b/packages/aws-client/src/DynamoDbSearchQueue.ts index 923f12e46d1..caf278ae152 100644 --- a/packages/aws-client/src/DynamoDbSearchQueue.ts +++ b/packages/aws-client/src/DynamoDbSearchQueue.ts @@ -1,11 +1,15 @@ -import * as AWS from 'aws-sdk'; import { DynamoDBDocument, } from '@aws-sdk/lib-dynamodb'; +import { + AttributeValue, + ScanCommandInput, + QueryCommandInput, +} from '@aws-sdk/client-dynamodb'; import { dynamodbDocClient } from './services'; type SearchType = 'scan' | 'query'; -type SearchParams = AWS.DynamoDB.DocumentClient.ScanInput | AWS.DynamoDB.DocumentClient.QueryInput; +type SearchParams = ScanCommandInput | QueryCommandInput; /** * Class to efficiently search all of the items in a DynamoDB table, without loading them all into @@ -15,9 +19,9 @@ class DynamoDbSearchQueue { private readonly dynamodbDocClient: DynamoDBDocument; private readonly searchType: SearchType; private readonly params: SearchParams; - private items: Array; + private items: (Record | null)[]; - constructor(params: AWS.DynamoDB.DocumentClient.ScanInput, searchType: SearchType = 'scan') { + constructor(params: SearchParams, searchType: SearchType = 'scan') { this.items = []; this.params = params; this.dynamodbDocClient = dynamodbDocClient(); @@ -71,7 +75,7 @@ class DynamoDbSearchQueue { * A DynamoDbSearchQueue instance stores the list of items to be returned in * the `this.items` array. When that list is empty, the `fetchItems()` method * is called to repopulate `this.items`. Typically, the new items are fetched - * using the AWS.DynamoDB.`DocumentClient.scan()` method. + * using the `DynamoDBDocument.scan()` method. * * DynamoDB scans up to 1 MB of items at a time and then filters that 1 MB to * look for matching items. If there are more items to be search beyond that @@ -90,7 +94,7 @@ class DynamoDbSearchQueue { * items. It will continue to call `scan()` until one of those two conditions * has been satisfied. * - * Reference: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB.html#scan-property + * Reference: https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/Package/-aws-sdk-lib-dynamodb/Class/DynamoDBDocument/ * * @private */ diff --git a/packages/aws-client/tests/test-S3.js b/packages/aws-client/tests/test-S3.js index 226b28a0db4..c1b20e098e2 100644 --- a/packages/aws-client/tests/test-S3.js +++ b/packages/aws-client/tests/test-S3.js @@ -34,6 +34,7 @@ const { deleteS3Objects, promiseS3Upload, fileExists, + s3ObjectExists, } = require('../S3'); const awsServices = require('../services'); const { streamToString } = require('../test-utils'); @@ -501,3 +502,16 @@ test('fileExists() correctly returns false for non-existent file', async (t) => const { Bucket } = t.context; t.false(await fileExists(Bucket, randomString())); }); + +test('s3ObjectExists() returns true for existing file', async (t) => { + const { Bucket } = t.context; + + const { Key } = await stageTestObjectToLocalStack(Bucket, 'asdf'); + t.true(await s3ObjectExists({ Bucket, Key })); +}); + +test('s3ObjectExists() returns false for non-existent file', async (t) => { + const { Bucket } = t.context; + t.false(await s3ObjectExists({ Bucket, Key: randomString() })); + t.false(await s3ObjectExists({ Bucket: randomString(), Key: randomString() })); +}); diff --git a/packages/checksum/package.json b/packages/checksum/package.json index 7e30af60092..f912de8912a 100644 --- a/packages/checksum/package.json +++ b/packages/checksum/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/checksum", - "version": "16.1.1", + "version": "16.1.2", "description": "Cumulus checksum utilities", "engines": { "node": ">=16.19.0" diff --git a/packages/cmr-client/package.json b/packages/cmr-client/package.json index 157a0bb7378..0399cfd5b65 100644 --- a/packages/cmr-client/package.json +++ b/packages/cmr-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmr-client", - "version": "16.1.1", + "version": "16.1.2", "description": "A Node.js client to NASA's Common Metadata Repository (CMR) API.", "engines": { "node": ">=16.19.0" @@ -35,10 +35,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5", "jsonwebtoken": "^9.0.0", "lodash": "^4.17.21", diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index 9f6bb75ac24..c4b29754ded 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmrjs", - "version": "16.1.1", + "version": "16.1.2", "description": "A node SDK for CMR", "engines": { "node": ">=16.19.0" @@ -34,13 +34,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmr-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/distribution-utils": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmr-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/distribution-utils": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.1", "js2xmlparser": "^4.0.0", "lodash": "^4.17.21", diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index 3c8ad44059e..8aebecd789e 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -16,6 +16,7 @@ const { parseS3Uri, promiseS3Upload, s3GetObjectTagging, + s3ObjectExists, s3TagSetToQueryString, waitForObject, getObjectStreamContents, @@ -1100,6 +1101,11 @@ async function getGranuleTemporalInfo(granule) { const cmrFilename = getS3UrlOfFile(cmrFile[0]); + if (!(await s3ObjectExists(parseS3Uri(cmrFilename)))) { + log.warn(`getGranuleTemporalInfo cmr file does not exist ${cmrFilename}`); + return {}; + } + if (isCMRISOFilename(cmrFilename)) { const metadata = await metadataObjectFromCMRXMLFile(cmrFilename); const metadataMI = get( diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index 7a4bf55937a..9cdda1dc2eb 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -9,9 +9,11 @@ const { promisify } = require('util'); const pickAll = require('lodash/fp/pickAll'); const { buildS3Uri, + createBucket, getS3Object, parseS3Uri, promiseS3Upload, + putJsonS3Object, recursivelyDeleteS3Bucket, s3GetObjectTagging, s3TagSetToQueryString, @@ -102,6 +104,12 @@ test.before(async (t) => { t.context.distributionBucketMap = Object.fromEntries( Object.values(bucketsJson).map(({ name }) => [name, name]) ); + + t.context.cmrFileBucket = randomId('bucket'); + await createBucket(t.context.cmrFileBucket); + await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr.json', { foo: 'bar' }); + await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr.xml', { foo: 'bar' }); + await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr_iso.xml', { foo: 'bar' }); }); test.after.always(async (t) => { @@ -114,6 +122,7 @@ test.after.always(async (t) => { SecretId: cmrPasswordSecret, ForceDeleteWithoutRecovery: true, }).promise(), + await recursivelyDeleteS3Bucket(t.context.cmrFileBucket), ]); t.context.launchpadStub.restore(); @@ -741,7 +750,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.json', }], }); @@ -768,7 +777,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.json', }], }); @@ -795,7 +804,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.json', }], }); @@ -822,7 +831,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.json', }], }); @@ -849,7 +858,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.xml', }], }); @@ -876,7 +885,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr.xml', }], }); @@ -903,7 +912,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr_iso.xml', }], }); @@ -930,7 +939,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM const temporalInfo = await getGranuleTemporalInfo({ granuleId: 'testGranuleId', files: [{ - bucket: 'bucket', + bucket: t.context.cmrFileBucket, key: 'test.cmr_iso.xml', }], }); @@ -953,6 +962,18 @@ test.serial('getGranuleTemporalInfo returns empty object if cmr file s3 url is n t.deepEqual(temporalInfo, {}); }); +test.serial('getGranuleTemporalInfo returns empty object if cmr file s3 does not exist', async (t) => { + const temporalInfo = await getGranuleTemporalInfo({ + granuleId: 'testGranuleId', + files: [{ + path: t.context.cmrFileBucket, + name: 'not-exist.cmr_iso.xml', + }], + }); + + t.deepEqual(temporalInfo, {}); +}); + test.serial('generateFileUrl generates correct url for cmrGranuleUrlType distribution', (t) => { const filename = 's3://fake-bucket/folder/key.txt'; const distEndpoint = 'www.example.com/'; diff --git a/packages/collection-config-store/package.json b/packages/collection-config-store/package.json index 1ae9086e161..80696608c98 100644 --- a/packages/collection-config-store/package.json +++ b/packages/collection-config-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/collection-config-store", - "version": "16.1.1", + "version": "16.1.2", "description": "Utility for persisting collection configuration to S3 and retrieving it", "keywords": [ "CUMULUS", @@ -33,8 +33,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/message": "16.1.1" + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/message": "16.1.2" } } diff --git a/packages/common/package.json b/packages/common/package.json index f97eaca360b..e61055a62fc 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/common", - "version": "16.1.1", + "version": "16.1.2", "description": "Common utilities used across tasks", "keywords": [ "GIBS", @@ -44,9 +44,9 @@ "dependencies": { "@aws-sdk/client-s3": "^3.58.0", "@aws-sdk/signature-v4-crt": "^3.58.0", - "@cumulus/aws-client": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", "ajv": "^6.12.3", "aws-sdk": "^2.585.0", "follow-redirects": "^1.2.4", diff --git a/packages/db/package.json b/packages/db/package.json index 9aea6b6c7a5..c20636550af 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for working with the Cumulus DB", "license": "Apache-2.0", "main": "./dist/index.js", @@ -30,12 +30,12 @@ "node": ">=16.19.0" }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", - "@cumulus/types": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", + "@cumulus/types": "16.1.2", "crypto-random-string": "^3.2.0", "is-valid-hostname": "1.0.2", "knex": "2.4.1", diff --git a/packages/db/src/migrations/20230915150154_add_max_download_time_providers.ts b/packages/db/src/migrations/20230915150154_add_max_download_time_providers.ts new file mode 100644 index 00000000000..a53da84ac47 --- /dev/null +++ b/packages/db/src/migrations/20230915150154_add_max_download_time_providers.ts @@ -0,0 +1,16 @@ +import { Knex } from 'knex'; + +export const up = async (knex: Knex): Promise => + await knex.schema.table('providers', (table) => { + table + .integer('max_download_time') + .comment('Maximum download time in seconds for all granule files on a sync granule task'); + }); + +export const down = async (knex: Knex): Promise => { + if (await knex.schema.hasColumn('providers', 'max_download_time')) { + await knex.schema.table('providers', (table) => { + table.dropColumn('max_download_time'); + }); + } +}; diff --git a/packages/db/src/provider.ts b/packages/db/src/provider.ts index 1e1689de003..e2e5142d6f0 100644 --- a/packages/db/src/provider.ts +++ b/packages/db/src/provider.ts @@ -22,6 +22,7 @@ export const nullifyUndefinedProviderValues = ( 'username', 'password', 'global_connection_limit', + 'max_download_time', 'private_key', 'cm_key_id', 'certificate_uri', diff --git a/packages/db/src/translate/providers.ts b/packages/db/src/translate/providers.ts index 17d832c3e00..79dd98eeefe 100644 --- a/packages/db/src/translate/providers.ts +++ b/packages/db/src/translate/providers.ts @@ -21,6 +21,7 @@ export const translatePostgresProviderToApiProvider = ( certificateUri: record.certificate_uri, privateKey: record.private_key, globalConnectionLimit: record.global_connection_limit, + maxDownloadTime: record.max_download_time, port: record.port, host: record.host, protocol: record.protocol, @@ -63,6 +64,7 @@ export const translateApiProviderToPostgresProvider = async ( certificate_uri: record.certificateUri, private_key: record.privateKey, global_connection_limit: record.globalConnectionLimit, + max_download_time: record.maxDownloadTime, port: record.port, host: record.host, protocol: record.protocol, diff --git a/packages/db/src/types/provider.ts b/packages/db/src/types/provider.ts index 4541b6196d6..317f1942830 100644 --- a/packages/db/src/types/provider.ts +++ b/packages/db/src/types/provider.ts @@ -12,6 +12,7 @@ export interface PostgresProvider { cumulus_id?: number | null, global_connection_limit?: number | null, host: string, + max_download_time?: number | null, name: string, password?: string, port?: number | null, diff --git a/packages/db/tests/models/test-provider-model.js b/packages/db/tests/models/test-provider-model.js index 329c4ebeb5a..fc52ffa7fa0 100644 --- a/packages/db/tests/models/test-provider-model.js +++ b/packages/db/tests/models/test-provider-model.js @@ -96,3 +96,23 @@ test('ProviderPgModel.upsert() creates provider with allowed_redirects', async ( updatedRecord ); }); + +test('ProviderPgModel.upsert() creates provider with max_download_time', async (t) => { + const { + knex, + providerPgModel, + providerRecord, + } = t.context; + + const updatedRecord = { + ...providerRecord, + max_download_time: 600, + }; + + await providerPgModel.upsert(knex, updatedRecord); + const actualRecord = await providerPgModel.get(knex, updatedRecord); + t.like( + actualRecord, + updatedRecord + ); +}); diff --git a/packages/db/tests/test-providers.js b/packages/db/tests/test-providers.js index b1c91a20504..ea84d40f24a 100644 --- a/packages/db/tests/test-providers.js +++ b/packages/db/tests/test-providers.js @@ -20,6 +20,7 @@ test('nullifyUndefinedProviderValues sets undefined provider values to "null"', username: null, password: null, global_connection_limit: null, + max_download_time: null, private_key: null, cm_key_id: null, certificate_uri: null, diff --git a/packages/db/tests/translate/test-providers.js b/packages/db/tests/translate/test-providers.js index ff888d5316c..a2ab4fba3cf 100644 --- a/packages/db/tests/translate/test-providers.js +++ b/packages/db/tests/translate/test-providers.js @@ -17,6 +17,7 @@ test('translatePostgresProviderToApiProvider translates the expected API record' cm_key_id: 'fakecmId', created_at: new Date(1234), global_connection_limit: 1, + max_download_time: 100, host: 'fakeHost', name: 'testId', password: 'fakeEncryptedPasswordString', @@ -34,6 +35,7 @@ test('translatePostgresProviderToApiProvider translates the expected API record' createdAt: 1234, encrypted: true, globalConnectionLimit: 1, + maxDownloadTime: 100, host: 'fakeHost', id: 'testId', password: 'fakeEncryptedPasswordString', @@ -86,6 +88,7 @@ test('translateApiProviderToPostgresProvider translates a Cumulus Provider objec const cumulusProviderObject = { id: 'testId', globalConnectionLimit: 1, + maxDownloadTime: 100, protocol: 'fakeProtocol', host: 'fakeHost', port: 1234, @@ -105,6 +108,7 @@ test('translateApiProviderToPostgresProvider translates a Cumulus Provider objec cm_key_id: 'fakecmId', created_at: new Date(1234), global_connection_limit: 1, + max_download_time: 100, host: 'fakeHost', name: 'testId', password: 'encrypted[fakePassword]', diff --git a/packages/distribution-utils/package.json b/packages/distribution-utils/package.json index f6b0cfb5dd2..c2f7c40a08a 100644 --- a/packages/distribution-utils/package.json +++ b/packages/distribution-utils/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/distribution-utils", - "version": "16.1.1", + "version": "16.1.2", "description": "Cumulus Distribution utilities", "keywords": [ "CUMULUS" @@ -34,9 +34,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", "url-join": "^1.1.0" }, "devDependencies": { diff --git a/packages/errors/package.json b/packages/errors/package.json index f5f4ee4e846..283002c1415 100644 --- a/packages/errors/package.json +++ b/packages/errors/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/errors", - "version": "16.1.1", + "version": "16.1.2", "description": "Provides error classes for Cumulus", "keywords": [ "GIBS", diff --git a/packages/es-client/config/mappings.json b/packages/es-client/config/mappings.json index c9ebaa1f524..c41aacb1ef1 100644 --- a/packages/es-client/config/mappings.json +++ b/packages/es-client/config/mappings.json @@ -28,6 +28,10 @@ } } }, + "meta": { + "type": "object", + "dynamic": false + }, "timestamp": { "type": "date" } @@ -266,6 +270,10 @@ "published": { "type": "boolean" }, + "queryFields": { + "type": "object", + "dynamic": false + }, "duration": { "type": "float" }, @@ -349,6 +357,10 @@ "published": { "type": "boolean" }, + "queryFields": { + "type": "object", + "dynamic": false + }, "duration": { "type": "float" }, diff --git a/packages/es-client/package.json b/packages/es-client/package.json index 421317aafa7..160f5b0554b 100644 --- a/packages/es-client/package.json +++ b/packages/es-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/es-client", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for working with Elasticsearch", "keywords": [ "CUMULUS", @@ -31,10 +31,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", "@elastic/elasticsearch": "^5.6.20", "aws-elasticsearch-connector": "8.2.0", "aws-sdk": "^2.585.0", @@ -43,8 +43,8 @@ "p-limit": "^1.2.0" }, "devDependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/test-data": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/test-data": "16.1.2", "p-each-series": "^2.1.0" } } diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 63690472934..97b708c3afb 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest", - "version": "16.1.1", + "version": "16.1.2", "description": "Ingest utilities", "engines": { "node": ">=16.19.0" @@ -39,13 +39,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/db": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", - "@cumulus/sftp-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/db": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", + "@cumulus/sftp-client": "16.1.2", "aws-sdk": "^2.585.0", "cksum": "^1.3.0", "encodeurl": "^1.0.2", @@ -61,9 +61,9 @@ "tough-cookie": "~4.0.0" }, "devDependencies": { - "@cumulus/checksum": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/test-data": "16.1.1", - "@cumulus/types": "16.1.1" + "@cumulus/checksum": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/test-data": "16.1.2", + "@cumulus/types": "16.1.2" } } diff --git a/packages/ingest/src/lock.ts b/packages/ingest/src/lock.ts index 0e7bbc5060b..6de0efe514a 100644 --- a/packages/ingest/src/lock.ts +++ b/packages/ingest/src/lock.ts @@ -14,25 +14,27 @@ export interface Lock { } /** -* Checks all locks and removes those older than five minutes. Returns a count -* of locks that are not older than five minutes. -* -* @param {Object} bucket - The AWS S3 bucket with the locks to check -* @param {Array} locks - The list of locks in the bucket -* @returns {integer} - Number of locks remaining in bucket -**/ + * Checks all locks and removes those older than five minutes. Returns a count + * of locks that are not older than configured retention period or 5 minutes. + * + * @param {object} bucket - The AWS S3 bucket with the locks to check + * @param {Array} locks - The list of locks in the bucket + * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300 + * @returns {integer} - Number of locks remaining in bucket + */ export async function checkOldLocks( bucket: string, - locks: Lock[] = [] + locks: Lock[] = [], + retentionTimeInSeconds: number = 300 ): Promise { - const fiveMinutesAgo = Date.now() - (5 * 60 * 1000); + const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000); const expiredLocks = locks.filter( (lock) => { if (!lock.LastModified) { throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`); } - return lock.LastModified.getTime() < fiveMinutesAgo; + return lock.LastModified.getTime() < expirationTimestamp; } ); @@ -40,6 +42,7 @@ export async function checkOldLocks( if (!lock.Key) { throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`); } + log.debug(`Removing expired lock ${JSON.stringify(lock)}`); return deleteS3Object(bucket, lock.Key); })); @@ -47,30 +50,32 @@ export async function checkOldLocks( } /** -* Counts the number of locks in a bucket. -* -* @param {Object} bucket - The AWS S3 bucket to check -* @param {string} providerName - The provider name -* @returns {integer} - Number of current locks in the bucket -**/ + * Counts the number of locks in a bucket. + * + * @param {object} bucket - The AWS S3 bucket to check + * @param {string} providerName - The provider name + * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300 + * @returns {integer} - Number of current locks in the bucket + */ export async function countLock( bucket: string, - providerName: string + providerName: string, + retentionTimeInSeconds?: number ): Promise { const locks = await listS3ObjectsV2({ Bucket: bucket, Prefix: `${lockPrefix}/${providerName}`, }); - return checkOldLocks(bucket, locks); + return checkOldLocks(bucket, locks, retentionTimeInSeconds); } async function addLock( bucket: string, providerName: string, - filename: string + granuleId: string ): Promise { - const key = `${lockPrefix}/${providerName}/${filename}`; + const key = `${lockPrefix}/${providerName}/${granuleId}`; await s3PutObject({ Bucket: bucket, Key: key, @@ -81,54 +86,56 @@ async function addLock( export async function removeLock( bucket: string, providerName: string, - filename: string + granuleId: string ): Promise { await deleteS3Object( bucket, - `${lockPrefix}/${providerName}/${filename}` + `${lockPrefix}/${providerName}/${granuleId}` ); } /** * - * @param {string} bucket - * @param {Object} provider - * @param {string} provider.id - * @param {number} provider.globalConnectionLimit - * @param {string} filename - * @param {number} counter + * @param {string} bucket - system bucket to place the lock files + * @param {object} provider - provider object + * @param {string} provider.id - provider id + * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit + * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule + * @param {string} granuleId - id of downloading granule + * @param {number} counter - retry counter * @returns {Promise} */ export async function proceed( bucket: string, provider: { id: string, - globalConnectionLimit?: number + globalConnectionLimit?: number, + maxDownloadTime?: number, }, - filename: string, + granuleId: string, counter = 0 ): Promise { - if (provider.globalConnectionLimit === undefined) { + const { globalConnectionLimit, maxDownloadTime } = provider; + if (globalConnectionLimit === undefined) { return true; } // Fail if lock is not removed after 270 tries. if (counter > 270) { + log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`); return false; } - const globalConnectionLimit = provider.globalConnectionLimit; - - const count = await countLock(bucket, provider.id); + const count = await countLock(bucket, provider.id, maxDownloadTime); if (count >= globalConnectionLimit) { log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`); // wait for 5 second and try again await sleep(5000); - return proceed(bucket, provider, filename, counter + 1); + return proceed(bucket, provider, granuleId, counter + 1); } // add the lock - await addLock(bucket, provider.id, filename); + await addLock(bucket, provider.id, granuleId); return true; } diff --git a/packages/ingest/test/test-lock.js b/packages/ingest/test/test-lock.js index 43037670ba3..9f8216c873f 100644 --- a/packages/ingest/test/test-lock.js +++ b/packages/ingest/test/test-lock.js @@ -7,21 +7,31 @@ const { randomString } = require('@cumulus/common/test-utils'); // 5 * 60 seconds * 1000 milliseconds const fiveMinutes = 5 * 60 * 1000; +const fakeLocks = (providerName) => ([ + { + Key: `lock/${providerName}/test`, + LastModified: new Date(), + }, + { + Key: `lock/${providerName}/test1`, + LastModified: new Date(Date.now() - (fiveMinutes * 2 + 1)), + }, + { + Key: `lock/${providerName}/test2`, + LastModified: new Date(Date.now() - (fiveMinutes + 1)), + }, + { + Key: `lock/${providerName}/test3`, + LastModified: new Date(Date.now() - (fiveMinutes + 1)), + }, +]); + const { checkOldLocks, countLock, proceed } = proxyquire( '../lock', { '@cumulus/aws-client/S3': { deleteS3Object: () => Promise.resolve(), - listS3ObjectsV2: (_, providerName) => Promise.resolve([ - { - Key: `lock/${providerName}/test`, - LastModified: new Date(), - }, - { - Key: `lock/${providerName}/test2`, - LastModified: new Date(Date.now() - (fiveMinutes + 1)), - }, - ]), + listS3ObjectsV2: (_, providerName) => Promise.resolve(fakeLocks(providerName)), }, } ); @@ -37,29 +47,20 @@ test('checkOldLocks() returns correct number of locks', async (t) => { let count = await checkOldLocks(bucket, []); t.is(count, 0); - count = await checkOldLocks(bucket, [ - { - Key: `lock/${providerName}/test`, - LastModified: new Date(), - }, - { - Key: `lock/${providerName}/test2`, - LastModified: new Date(Date.now() - (fiveMinutes + 1)), - }, - { - Key: `lock/${providerName}/test3`, - LastModified: new Date(Date.now() - (fiveMinutes + 1)), - }, - ]); + count = await checkOldLocks(bucket, fakeLocks(providerName)); t.is(count, 1); + count = await checkOldLocks(bucket, fakeLocks(providerName), 10 * 60); + t.is(count, 3); }); test('countLock() returns the correct number of locks', async (t) => { const { bucket, providerName } = t.context; - const count = await countLock(bucket, providerName); - + let count = await countLock(bucket, providerName); t.is(count, 1); + + count = await countLock(bucket, fakeLocks(providerName), 10 * 60); + t.is(count, 3); }); test('proceed() returns true if globalConnectionLimit is undefined', async (t) => { diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 38859996be8..b3ffe7075d2 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/integration-tests", - "version": "16.1.1", + "version": "16.1.2", "description": "Integration tests", "bin": { "cumulus-test": "./bin/cli.js" @@ -29,16 +29,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api": "16.1.1", - "@cumulus/api-client": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmr-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", - "@cumulus/oauth-client": "16.1.1", + "@cumulus/api": "16.1.2", + "@cumulus/api-client": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmr-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", + "@cumulus/oauth-client": "16.1.2", "aws-sdk": "^2.585.0", "base-64": "^0.1.0", "commander": "^2.15.0", diff --git a/packages/launchpad-auth/package.json b/packages/launchpad-auth/package.json index ffcb233149b..4c6a1b7c289 100644 --- a/packages/launchpad-auth/package.json +++ b/packages/launchpad-auth/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/launchpad-auth", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for authentication with Launchpad", "keywords": [ "CUMULUS", @@ -38,8 +38,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5", "lodash": "^4.17.21", "uuid": "^3.2.1" diff --git a/packages/logger/package.json b/packages/logger/package.json index 867236d42ef..1b24992506f 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/logger", - "version": "16.1.1", + "version": "16.1.2", "description": "A log library for use on Cumulus", "keywords": [ "GIBS", diff --git a/packages/lzards-api-client/package.json b/packages/lzards-api-client/package.json index 4b4cd3894f7..2aff4dac1ad 100644 --- a/packages/lzards-api-client/package.json +++ b/packages/lzards-api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-api-client", - "version": "16.1.1", + "version": "16.1.2", "description": "A Node.js client to NASA's Level Zero and Repositories Data Store (LZARDS) API.", "engines": { "node": ">=16.19.0" @@ -34,11 +34,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5", "lodash": "^4.17.21" } diff --git a/packages/message/package.json b/packages/message/package.json index 2330b855859..54c87b3411a 100644 --- a/packages/message/package.json +++ b/packages/message/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/message", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for building and parsing Cumulus messages", "keywords": [ "GIBS", @@ -39,11 +39,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/types": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/types": "16.1.2", "jsonpath-plus": "^3.0.0", "lodash": "^4.17.21", "uuid": "^8.2.0" diff --git a/packages/message/src/Granules.ts b/packages/message/src/Granules.ts index 9478b0e8164..06850e177f4 100644 --- a/packages/message/src/Granules.ts +++ b/packages/message/src/Granules.ts @@ -8,10 +8,12 @@ * const Granules = require('@cumulus/message/Granules'); */ +import isEmpty from 'lodash/isEmpty'; import isInteger from 'lodash/isInteger'; import isUndefined from 'lodash/isUndefined'; import mapValues from 'lodash/mapValues'; import omitBy from 'lodash/omitBy'; +import pick from 'lodash/pick'; import { CumulusMessageError } from '@cumulus/errors'; import { Message } from '@cumulus/types'; @@ -171,7 +173,8 @@ function isGranuleTemporalInfo( } /** - * Get granule temporal information from argument or directly from CMR. + * Get granule temporal information from argument, directly from CMR + * file or from granule object. * * Converts temporal information timestamps to a standardized ISO string * format for compatibility across database systems. @@ -196,6 +199,11 @@ export const getGranuleCmrTemporalInfo = async ({ const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo) ? { ...cmrTemporalInfo } : await cmrUtils.getGranuleTemporalInfo(granule); + + if (isEmpty(temporalInfo)) { + return pick(granule, ['beginningDateTime', 'endingDateTime', 'productionDateTime', 'lastUpdateDateTime']); + } + return mapValues( temporalInfo, convertDateToISOStringPreservingNull diff --git a/packages/message/tests/test-Granules.js b/packages/message/tests/test-Granules.js index 7dded756ef2..8d007e3edfa 100644 --- a/packages/message/tests/test-Granules.js +++ b/packages/message/tests/test-Granules.js @@ -324,6 +324,17 @@ test('getGranuleCmrTemporalInfo() handles empty return from CMR', async (t) => { t.deepEqual(updatedCmrTemporalInfo, {}); }); +test('getGranuleCmrTemporalInfo() handles empty return from CMR and gets temporal info from granule object', async (t) => { + const updatedCmrTemporalInfo = await getGranuleCmrTemporalInfo({ + granule: t.context.fakeCmrMetadata, + cmrUtils: { + getGranuleTemporalInfo: () => Promise.resolve({}), + }, + }); + + t.deepEqual(updatedCmrTemporalInfo, t.context.fakeCmrMetadata); +}); + test('getGranuleProcessingTimeInfo() converts input timestamps to standardized format', (t) => { const { timestampExtraPrecision } = t.context; diff --git a/packages/oauth-client/package.json b/packages/oauth-client/package.json index debe8d50b59..2ebc6d3fcf4 100644 --- a/packages/oauth-client/package.json +++ b/packages/oauth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/oauth-client", - "version": "16.1.1", + "version": "16.1.2", "description": "A generic auth client", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/oauth-client#readme", "repository": { diff --git a/packages/object-store/package.json b/packages/object-store/package.json index 588ab263588..24f05d5d1cf 100644 --- a/packages/object-store/package.json +++ b/packages/object-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/object-store", - "version": "16.1.1", + "version": "16.1.2", "description": "Utilities for managing object stores", "keywords": [ "GIBS", @@ -40,6 +40,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1" + "@cumulus/aws-client": "16.1.2" } } diff --git a/packages/pvl/package.json b/packages/pvl/package.json index 5c9437d0378..9434a949b0f 100644 --- a/packages/pvl/package.json +++ b/packages/pvl/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pvl", - "version": "16.1.1", + "version": "16.1.2", "description": "Parse and serialize Parameter Value Language, a data markup language used by NASA", "main": "index.js", "engine": { diff --git a/packages/s3-credentials-endpoint/package.json b/packages/s3-credentials-endpoint/package.json index 48e634ef1d5..6f1e0b83d47 100644 --- a/packages/s3-credentials-endpoint/package.json +++ b/packages/s3-credentials-endpoint/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-credentials-endpoint", - "version": "16.1.1", + "version": "16.1.2", "description": "An API Gateway Lambda to return AWS credentials for fetching objects from S3", "license": "Apache-2.0", "engines": { @@ -18,12 +18,12 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/oauth-client": "16.1.1", + "@cumulus/api": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/oauth-client": "16.1.2", "aws-serverless-express": "^3.3.6", "body-parser": "^1.19.0", "cookie-parser": "^1.4.4", diff --git a/packages/schemas/package.json b/packages/schemas/package.json index 05da096cc3e..663340ebef4 100644 --- a/packages/schemas/package.json +++ b/packages/schemas/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/schemas", - "version": "16.1.1", + "version": "16.1.2", "description": "Helpers for managing Cumulus task schemas", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/schemas", "repository": { diff --git a/packages/sftp-client/package.json b/packages/sftp-client/package.json index e7db0c883bc..d12a32e8fb5 100644 --- a/packages/sftp-client/package.json +++ b/packages/sftp-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sftp-client", - "version": "16.1.1", + "version": "16.1.2", "description": "A Promise-based SFTP client", "keywords": [ "GIBS", @@ -37,16 +37,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "lodash": "^4.17.21", "mime-types": "^2.1.27", "ssh2": "^1.0.0", "ssh2-sftp-client": "^7.0.4" }, "devDependencies": { - "@cumulus/checksum": "16.1.1", - "@cumulus/test-data": "16.1.1", + "@cumulus/checksum": "16.1.2", + "@cumulus/test-data": "16.1.2", "@types/ssh2-sftp-client": "^7.0.0" } } diff --git a/packages/tea-map-cache/package.json b/packages/tea-map-cache/package.json index 853cd52c761..eed428cff85 100644 --- a/packages/tea-map-cache/package.json +++ b/packages/tea-map-cache/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tea-map-cache", - "version": "16.1.1", + "version": "16.1.2", "description": "Tea Bucket Map Cache Writer", "main": "index.js", "engines": { @@ -8,13 +8,14 @@ }, "scripts": { "clean": "git clean -d -x -e node_modules -f", - "package": "cd dist && zip -r -q ./lambda.zip * && cd .. && zip -r -q ./dist/lambda.zip node_modules", + "package": "npm run prepare && npm run webpack && (cd dist && rm -f lambda.zip && node ../../../bin/zip.js lambda.zip index.js)", "test": "../../node_modules/.bin/ava", "test:coverage": "../../node_modules/.bin/nyc npm test", "prepare": "npm run tsc", "tsc": "../../node_modules/.bin/tsc", "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles", "watch-test": "../../node_modules/.bin/tsc-watch --onsuccess 'npm test'", + "webpack": "../../node_modules/.bin/webpack", "coverage": "python ../../scripts/coverage_handler/coverage.py" }, "ava": { @@ -28,8 +29,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5", "p-retry": "^4.2.0" }, diff --git a/packages/tea-map-cache/tests/test-index.js b/packages/tea-map-cache/tests/test-index.js index 82b25aa9c25..26942511dd4 100644 --- a/packages/tea-map-cache/tests/test-index.js +++ b/packages/tea-map-cache/tests/test-index.js @@ -4,7 +4,7 @@ const test = require('ava'); const proxyquire = require('proxyquire').noPreserveCache(); test('tea-map-cache handler writes the expected bucketmap', async (t) => { - const { handler } = proxyquire('../dist/index.js', { + const { handler } = proxyquire('../dist/src', { './tea': { getTeaBucketPath: () => Promise.resolve('tea_bucket_path'), }, diff --git a/packages/tea-map-cache/tests/test-tea.js b/packages/tea-map-cache/tests/test-tea.js index e139e74afdd..37255b52927 100644 --- a/packages/tea-map-cache/tests/test-tea.js +++ b/packages/tea-map-cache/tests/test-tea.js @@ -4,7 +4,7 @@ const test = require('ava'); const proxyquire = require('proxyquire').noPreserveCache(); test('getTeaBucketPath returns a mapped bucket path on an expected response from TEA', async (t) => { - const { getTeaBucketPath } = proxyquire('../dist/tea', { + const { getTeaBucketPath } = proxyquire('../dist/src/tea', { got: { default: { get: (_param) => Promise.resolve({ body: '["fake-bucket-redirect/path"]' }), @@ -20,7 +20,7 @@ test('getTeaBucketPath returns a mapped bucket path on an expected response from }); test('getTeaBucketPath throws error if multiple paths are returned', async (t) => { - const { getTeaBucketPath } = proxyquire('../dist/tea', { + const { getTeaBucketPath } = proxyquire('../dist/src/tea', { got: { default: { get: (_param) => Promise.resolve({ body: '["fake-bucket-redirect/path", "some-other-path"]' }), @@ -34,7 +34,7 @@ test('getTeaBucketPath throws error if multiple paths are returned', async (t) = }); test('getTeaBucketPath returns empty string if TEA throws a 404', async (t) => { - const { getTeaBucketPath } = proxyquire('../dist/tea', { + const { getTeaBucketPath } = proxyquire('../dist/src/tea', { got: { default: { get: () => { @@ -54,7 +54,7 @@ test('getTeaBucketPath returns empty string if TEA throws a 404', async (t) => { }); test('getTeaBucketPath throws an error empty string if non-bucket-search 404 is thrown', async (t) => { - const { getTeaBucketPath } = proxyquire('../dist/tea', { + const { getTeaBucketPath } = proxyquire('../dist/src/tea', { got: { default: { get: () => { diff --git a/packages/tea-map-cache/tsconfig.json b/packages/tea-map-cache/tsconfig.json index 1172b8f4dc8..ec0b74d8b10 100644 --- a/packages/tea-map-cache/tsconfig.json +++ b/packages/tea-map-cache/tsconfig.json @@ -1,7 +1,15 @@ { - "extends": "../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist" - }, - "include": ["./src/*.ts"] - } + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "es2021", + "lib": [ + "DOM", + "es2021" + ], + "outDir": "dist/src", + "strictNullChecks": true + }, + "include": [ + "src" + ] +} \ No newline at end of file diff --git a/packages/tea-map-cache/webpack.config.js b/packages/tea-map-cache/webpack.config.js new file mode 100644 index 00000000000..af3eb095e4a --- /dev/null +++ b/packages/tea-map-cache/webpack.config.js @@ -0,0 +1,17 @@ +const path = require('path'); + +module.exports = { + mode: 'production', + entry: './dist/src/index.js', + output: { + libraryTarget: 'commonjs2', + path: path.resolve(__dirname, 'dist'), + filename: 'index.js' + }, + externals: ['aws-sdk'], + target: 'node', + devtool: 'eval-cheap-module-source-map', + optimization: { + nodeEnv: false + } +}; \ No newline at end of file diff --git a/packages/test-data/package.json b/packages/test-data/package.json index 0a75318c236..3c218668352 100644 --- a/packages/test-data/package.json +++ b/packages/test-data/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-data", - "version": "16.1.1", + "version": "16.1.2", "description": "Includes the test data for various packages", "keywords": [ "GIBS", diff --git a/packages/tf-inventory/package.json b/packages/tf-inventory/package.json index dcd476428c1..2943d423280 100644 --- a/packages/tf-inventory/package.json +++ b/packages/tf-inventory/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tf-inventory", - "version": "16.1.1", + "version": "16.1.2", "description": "Package to help keep track of what resources are managed by Terraform in the AWS account", "main": "index.js", "engines": { @@ -31,11 +31,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "commander": "^4.1.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "16.1.1" + "@cumulus/common": "16.1.2" } } diff --git a/packages/types/api/providers.d.ts b/packages/types/api/providers.d.ts index dac3f6dcbca..2109ce6619f 100644 --- a/packages/types/api/providers.d.ts +++ b/packages/types/api/providers.d.ts @@ -1,6 +1,7 @@ export interface ApiProvider { id: string, globalConnectionLimit?: number, + maxDownloadTime?: number, protocol: string, host: string, port?: number, diff --git a/packages/types/package.json b/packages/types/package.json index 2d0d00e8654..6895cb48261 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/types", - "version": "16.1.1", + "version": "16.1.2", "description": "TypeScript definitions for working with Cumulus data structures", "keywords": [ "GIBS", diff --git a/tasks/add-missing-file-checksums/package.json b/tasks/add-missing-file-checksums/package.json index dc17286ab9a..18176822a43 100644 --- a/tasks/add-missing-file-checksums/package.json +++ b/tasks/add-missing-file-checksums/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/add-missing-file-checksums", - "version": "16.1.1", + "version": "16.1.2", "description": "Add checksums to files in S3 which don't have one", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -43,12 +43,12 @@ } }, "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5" }, "devDependencies": { - "@cumulus/schemas": "16.1.1", - "@cumulus/types": "16.1.1", + "@cumulus/schemas": "16.1.2", + "@cumulus/types": "16.1.2", "@types/aws-lambda": "^8.10.58" } } diff --git a/tasks/discover-granules/package.json b/tasks/discover-granules/package.json index dc1c34bce0f..e8d7e634fbc 100644 --- a/tasks/discover-granules/package.json +++ b/tasks/discover-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-granules", - "version": "16.1.1", + "version": "16.1.2", "description": "Discover Granules in FTP/HTTP/HTTPS/SFTP/S3 endpoints", "main": "index.js", "directories": { @@ -36,16 +36,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "16.1.1", + "@cumulus/api-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/ingest": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5", "lodash": "^4.17.21", "p-map": "^4.0.0" }, "devDependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1" + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2" } } diff --git a/tasks/discover-pdrs/package.json b/tasks/discover-pdrs/package.json index 4e8f4c976cb..2b2c3ac2831 100644 --- a/tasks/discover-pdrs/package.json +++ b/tasks/discover-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-pdrs", - "version": "16.1.1", + "version": "16.1.2", "description": "Discover PDRs in FTP and HTTP endpoints", "main": "index.js", "directories": { @@ -35,14 +35,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", + "@cumulus/ingest": "16.1.2", "lodash": "^4.17.21", "p-filter": "^2.1.0" }, "devDependencies": { - "@cumulus/common": "16.1.1", - "@cumulus/errors": "16.1.1" + "@cumulus/common": "16.1.2", + "@cumulus/errors": "16.1.2" } } diff --git a/tasks/files-to-granules/package.json b/tasks/files-to-granules/package.json index e54b59b384e..79d9717c0eb 100644 --- a/tasks/files-to-granules/package.json +++ b/tasks/files-to-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/files-to-granules", - "version": "16.1.1", + "version": "16.1.2", "description": "Converts array-of-files input into a granules object by extracting granuleId from filename", "main": "index.js", "directories": { @@ -33,12 +33,12 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "16.1.1", - "@cumulus/schemas": "16.1.1" + "@cumulus/common": "16.1.2", + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/hello-world/package.json b/tasks/hello-world/package.json index 4a8420e412d..0eded567f90 100644 --- a/tasks/hello-world/package.json +++ b/tasks/hello-world/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hello-world", - "version": "16.1.1", + "version": "16.1.2", "description": "Example task", "main": "index.js", "directories": { @@ -33,8 +33,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5" } } diff --git a/tasks/hyrax-metadata-updates/package.json b/tasks/hyrax-metadata-updates/package.json index 135067795f1..7c0585179f4 100644 --- a/tasks/hyrax-metadata-updates/package.json +++ b/tasks/hyrax-metadata-updates/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hyrax-metadata-updates", - "version": "16.1.1", + "version": "16.1.2", "description": "Update granule metadata with hooks to OPeNDAP URL", "main": "index.js", "directories": { @@ -39,18 +39,18 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmr-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmr-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/errors": "16.1.1", + "@cumulus/errors": "16.1.2", "libxmljs": "^0.19.7", "lodash": "^4.17.21", "xml2js": "0.5.0" }, "devDependencies": { - "@cumulus/schemas": "16.1.1", + "@cumulus/schemas": "16.1.2", "jsonwebtoken": "^9.0.0", "nock": "^12.0.1", "rewire": "^6.0.0" diff --git a/tasks/lzards-backup/package.json b/tasks/lzards-backup/package.json index f70649ad594..2b31395e63d 100644 --- a/tasks/lzards-backup/package.json +++ b/tasks/lzards-backup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-backup", - "version": "16.1.1", + "version": "16.1.2", "description": "Run LZARDS backup", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -43,20 +43,20 @@ } }, "dependencies": { - "@cumulus/api-client": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/api-client": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/db": "16.1.1", - "@cumulus/distribution-utils": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", - "@cumulus/logger": "16.1.1", - "@cumulus/lzards-api-client": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/db": "16.1.2", + "@cumulus/distribution-utils": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", + "@cumulus/logger": "16.1.2", + "@cumulus/lzards-api-client": "16.1.2", + "@cumulus/message": "16.1.2", "got": "^11.8.5" }, "devDependencies": { - "@cumulus/schemas": "16.1.1", - "@cumulus/types": "16.1.1" + "@cumulus/schemas": "16.1.2", + "@cumulus/types": "16.1.2" } } diff --git a/tasks/move-granules/package.json b/tasks/move-granules/package.json index ad05dc04118..4d706bcc090 100644 --- a/tasks/move-granules/package.json +++ b/tasks/move-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/move-granules", - "version": "16.1.1", + "version": "16.1.2", "description": "Move granule files from staging to final location", "main": "index.js", "directories": { @@ -39,17 +39,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/distribution-utils": "16.1.1", - "@cumulus/errors": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/distribution-utils": "16.1.2", + "@cumulus/errors": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/schemas": "16.1.1" + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/orca-copy-to-archive-adapter/package.json b/tasks/orca-copy-to-archive-adapter/package.json index b3b96fd014c..96b665420a5 100644 --- a/tasks/orca-copy-to-archive-adapter/package.json +++ b/tasks/orca-copy-to-archive-adapter/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/orca-copy-to-archive-adapter", - "version": "16.1.1", + "version": "16.1.2", "description": "Adapter to invoke orca copy-to-archive lambda", "main": "dist/index.js", "private": true, @@ -37,13 +37,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/logger": "16.1.1", + "@cumulus/logger": "16.1.2", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/schemas": "16.1.1" + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/orca-recovery-adapter/package.json b/tasks/orca-recovery-adapter/package.json index 7d6187988f9..99c1b710d35 100644 --- a/tasks/orca-recovery-adapter/package.json +++ b/tasks/orca-recovery-adapter/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/orca-recovery-adapter", - "version": "16.1.1", + "version": "16.1.2", "description": "Adapter to invoke orca recovery workflow", "main": "dist/index.js", "private": true, @@ -37,14 +37,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/logger": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/logger": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/schemas": "16.1.1" + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/parse-pdr/package.json b/tasks/parse-pdr/package.json index 0a2378e41dc..d2efaf8ceaf 100644 --- a/tasks/parse-pdr/package.json +++ b/tasks/parse-pdr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/parse-pdr", - "version": "16.1.1", + "version": "16.1.2", "description": "Download and Parse a given PDR", "license": "Apache-2.0", "main": "index.js", @@ -31,17 +31,17 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api-client": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/collection-config-store": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/api-client": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/collection-config-store": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/errors": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/pvl": "16.1.1", + "@cumulus/errors": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/pvl": "16.1.2", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/test-data": "16.1.1" + "@cumulus/test-data": "16.1.2" } } diff --git a/tasks/pdr-status-check/package.json b/tasks/pdr-status-check/package.json index d1c51b9cc7b..0ce69516538 100644 --- a/tasks/pdr-status-check/package.json +++ b/tasks/pdr-status-check/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pdr-status-check", - "version": "16.1.1", + "version": "16.1.2", "description": "Checks execution status of granules in a PDR", "main": "index.js", "directories": { @@ -33,9 +33,9 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/errors": "16.1.1" + "@cumulus/errors": "16.1.2" } } diff --git a/tasks/post-to-cmr/package.json b/tasks/post-to-cmr/package.json index 15929f35e38..affc73c43d8 100644 --- a/tasks/post-to-cmr/package.json +++ b/tasks/post-to-cmr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/post-to-cmr", - "version": "16.1.1", + "version": "16.1.2", "description": "Post a given granule to CMR", "main": "index.js", "directories": { @@ -34,16 +34,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/errors": "16.1.1", - "@cumulus/launchpad-auth": "16.1.1", + "@cumulus/errors": "16.1.2", + "@cumulus/launchpad-auth": "16.1.2", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/cmr-client": "16.1.1", - "@cumulus/schemas": "16.1.1" + "@cumulus/cmr-client": "16.1.2", + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/queue-granules/package.json b/tasks/queue-granules/package.json index dca85c6216d..135e5500064 100644 --- a/tasks/queue-granules/package.json +++ b/tasks/queue-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-granules", - "version": "16.1.1", + "version": "16.1.2", "description": "Add discovered granules to the queue", "main": "index.js", "directories": { @@ -32,13 +32,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "16.1.1", - "@cumulus/aws-client": "16.1.1", - "@cumulus/collection-config-store": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/api-client": "16.1.2", + "@cumulus/aws-client": "16.1.2", + "@cumulus/collection-config-store": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/ingest": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.21", "p-map": "^4.0.0" } diff --git a/tasks/queue-pdrs/package.json b/tasks/queue-pdrs/package.json index 206af73f2e4..6b9126ceb16 100644 --- a/tasks/queue-pdrs/package.json +++ b/tasks/queue-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-pdrs", - "version": "16.1.1", + "version": "16.1.2", "description": "Add discovered PDRs to a queue", "main": "index.js", "directories": { @@ -32,11 +32,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/ingest": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.21" } } diff --git a/tasks/queue-workflow/package.json b/tasks/queue-workflow/package.json index c4779d25657..08be3a69660 100644 --- a/tasks/queue-workflow/package.json +++ b/tasks/queue-workflow/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-workflow", - "version": "16.1.1", + "version": "16.1.2", "description": "Add workflow to the queue", "main": "index.js", "directories": { @@ -32,11 +32,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/ingest": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.21" } } diff --git a/tasks/queue-workflow/schemas/config.json b/tasks/queue-workflow/schemas/config.json index 1d8c912e3dc..7d986ac1b7c 100644 --- a/tasks/queue-workflow/schemas/config.json +++ b/tasks/queue-workflow/schemas/config.json @@ -17,6 +17,7 @@ "stackName": { "type": "string" }, "executionNamePrefix": { "type": "string" }, "workflow": { "type": "string" }, - "workflowInput": { "type": "object" } + "workflowInput": { "type": "object" }, + "childWorkflowMeta": { "type": "object" } } } diff --git a/tasks/queue-workflow/tests/index.js b/tasks/queue-workflow/tests/index.js index 55883285a62..42e7f5e706a 100644 --- a/tasks/queue-workflow/tests/index.js +++ b/tasks/queue-workflow/tests/index.js @@ -176,6 +176,7 @@ test.serial('The correct message is enqueued', async (t) => { ); event.config.workflow = t.context.queuedWorkflow; event.config.workflowInput = { prop1: randomId('prop1'), prop2: randomId('prop2') }; + event.config.childWorkflowMeta = { metakey1: randomId('metavalue1') }; await validateConfig(t, event.config); await validateInput(t, event.input); @@ -206,6 +207,7 @@ test.serial('The correct message is enqueued', async (t) => { }, meta: { workflow_name: event.config.workflow, + ...event.config.childWorkflowMeta, }, payload: { prop1: event.config.workflowInput.prop1, diff --git a/tasks/send-pan/package.json b/tasks/send-pan/package.json index 69db777efd8..6534a6bd922 100644 --- a/tasks/send-pan/package.json +++ b/tasks/send-pan/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/send-pan", - "version": "16.1.1", + "version": "16.1.2", "description": "Sends a PAN response after parsing a PDR.", "main": "dist/index.js", "private": true, @@ -37,15 +37,15 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/api": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/ingest": "16.1.1", - "@cumulus/logger": "16.1.1", + "@cumulus/ingest": "16.1.2", + "@cumulus/logger": "16.1.2", "got": "^11.8.5" }, "devDependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "url-join": "^4.0.0" } } diff --git a/tasks/sf-sqs-report/package.json b/tasks/sf-sqs-report/package.json index ae19ffba097..65ee02f1e50 100644 --- a/tasks/sf-sqs-report/package.json +++ b/tasks/sf-sqs-report/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sf-sqs-report", - "version": "16.1.1", + "version": "16.1.2", "description": "Sends an incoming Cumulus message to SQS", "main": "index.js", "directories": { @@ -33,11 +33,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "16.1.1" + "@cumulus/common": "16.1.2" } } diff --git a/tasks/sync-granule/package.json b/tasks/sync-granule/package.json index e64cf5d6a82..19ac9ff41f0 100644 --- a/tasks/sync-granule/package.json +++ b/tasks/sync-granule/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sync-granule", - "version": "16.1.1", + "version": "16.1.2", "description": "Download a given granule", "main": "index.js", "directories": { @@ -38,19 +38,19 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/collection-config-store": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/collection-config-store": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/errors": "16.1.1", - "@cumulus/ingest": "16.1.1", - "@cumulus/message": "16.1.1", + "@cumulus/errors": "16.1.2", + "@cumulus/ingest": "16.1.2", + "@cumulus/message": "16.1.2", "lodash": "^4.17.21", "p-map": "^2.1.0", "uuid": "^3.4.0" }, "devDependencies": { - "@cumulus/schemas": "16.1.1", - "@cumulus/test-data": "16.1.1" + "@cumulus/schemas": "16.1.2", + "@cumulus/test-data": "16.1.2" } } diff --git a/tasks/test-processing/package.json b/tasks/test-processing/package.json index c372b0b7399..85ea2bd296b 100644 --- a/tasks/test-processing/package.json +++ b/tasks/test-processing/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-processing", - "version": "16.1.1", + "version": "16.1.2", "description": "Fake processing task used for integration tests", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/tasks/test-processing", @@ -20,8 +20,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "16.1.1", + "@cumulus/aws-client": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/integration-tests": "16.1.1" + "@cumulus/integration-tests": "16.1.2" } } diff --git a/tasks/update-cmr-access-constraints/package.json b/tasks/update-cmr-access-constraints/package.json index 7be616527d8..3cb3192a817 100644 --- a/tasks/update-cmr-access-constraints/package.json +++ b/tasks/update-cmr-access-constraints/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-cmr-access-constraints", - "version": "16.1.1", + "version": "16.1.2", "description": "Updates CMR metadata to set access constraints", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -35,13 +35,13 @@ "verbose": true }, "dependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/cmrjs": "16.1.1", + "@cumulus/aws-client": "16.1.2", + "@cumulus/cmrjs": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", "lodash": "^4.17.5" }, "devDependencies": { - "@cumulus/common": "16.1.1", - "@cumulus/schemas": "16.1.1" + "@cumulus/common": "16.1.2", + "@cumulus/schemas": "16.1.2" } } diff --git a/tasks/update-granules-cmr-metadata-file-links/package.json b/tasks/update-granules-cmr-metadata-file-links/package.json index d98aec943d3..a50047cb592 100644 --- a/tasks/update-granules-cmr-metadata-file-links/package.json +++ b/tasks/update-granules-cmr-metadata-file-links/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-granules-cmr-metadata-file-links", - "version": "16.1.1", + "version": "16.1.2", "description": "Update CMR metadata files with correct online access urls and etags and transfer etag info to granules' CMR files", "main": "index.js", "directories": { @@ -39,14 +39,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/cmrjs": "16.1.1", - "@cumulus/common": "16.1.1", + "@cumulus/cmrjs": "16.1.2", + "@cumulus/common": "16.1.2", "@cumulus/cumulus-message-adapter-js": "2.0.5", - "@cumulus/distribution-utils": "16.1.1", + "@cumulus/distribution-utils": "16.1.2", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/aws-client": "16.1.1", - "@cumulus/schemas": "16.1.1" + "@cumulus/aws-client": "16.1.2", + "@cumulus/schemas": "16.1.2" } } diff --git a/tf-modules/archive/sf_event_sqs_to_db_records.tf b/tf-modules/archive/sf_event_sqs_to_db_records.tf index c96d5f83c97..aba6a22b075 100644 --- a/tf-modules/archive/sf_event_sqs_to_db_records.tf +++ b/tf-modules/archive/sf_event_sqs_to_db_records.tf @@ -54,6 +54,13 @@ data "aws_iam_policy_document" "sf_event_sqs_to_db_records_lambda" { resources = ["*"] } + statement { + actions = [ + "s3:ListBucket*" + ] + resources = [for b in local.allowed_buckets: "arn:aws:s3:::${b}"] + } + statement { actions = [ "s3:GetObject*", diff --git a/tf-modules/ingest/package.json b/tf-modules/ingest/package.json index 64d636ed3de..b87fe7a860a 100644 --- a/tf-modules/ingest/package.json +++ b/tf-modules/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest-module", - "version": "16.1.1", + "version": "16.1.2", "description": "Terraform module for data ingest related functionality", "engines": { "node": ">=16.19.0" diff --git a/tf-modules/internal/cumulus-test-cleanup/package.json b/tf-modules/internal/cumulus-test-cleanup/package.json index 60bdbafc146..cea0de356a7 100644 --- a/tf-modules/internal/cumulus-test-cleanup/package.json +++ b/tf-modules/internal/cumulus-test-cleanup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-test-cleanup", - "version": "16.1.1", + "version": "16.1.2", "description": "Nightly cron job for cleaning up integration test artifacts", "main": "index.js", "engines": { diff --git a/tf-modules/s3-replicator/package.json b/tf-modules/s3-replicator/package.json index d2cd5326034..c0bed76b34e 100644 --- a/tf-modules/s3-replicator/package.json +++ b/tf-modules/s3-replicator/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-replicator", - "version": "16.1.1", + "version": "16.1.2", "description": "Replicate S3 Events to alternate bucket. Solves same-region replication.", "main": "index.js", "engines": {