release v0.4.51 from PR #690

release v0.4.51 from PR #690

.github/workflows/ci-terraform-example.yaml

@@ -1,5 +1,3 @@
-name: 'validate Terraform Examples'
-
 on:
   workflow_call:
     inputs:
@@ -8,21 +6,23 @@ on:
         type: string
         description: Java version to use # see https://github.com/actions/setup-java#supported-version-syntax
         default: 17
+      terraform_version:
+        required: false
+        type: string
+        description: Terraform version to use
+        default: '~1.6.0'
 
 jobs:
-  validate_tf_example:
-    name: 'validate tf examples'
+  validate:
     runs-on: ubuntu-latest
-    env:
-      TERRAFORM_VERSION: 1.3.9
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'setup Terraform'
-        uses: hashicorp/setup-terraform@v2
+        uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: ${{ env.TERRAFORM_VERSION }}
+          terraform_version: ${{ inputs.terraform_version }}
 
       - name: "Terraform - validate ${{ inputs.example_path }}"
         working-directory: infra/${{ inputs.example_path }}

.github/workflows/ci-terraform-examples-release.yaml

@@ -21,6 +21,8 @@ jobs:
           'examples/gcp-google-workspace',
           'examples/msft-365'
         ]
+        terraform_version: [ '~1.4.0', '~1.5.0', '~1.6.0' ]
     uses: ./.github/workflows/ci-terraform-example.yaml
     with:
+      terraform_version: ${{ matrix.terraform_version }}
       example_path: ${{ matrix.example_path }}

.github/workflows/ci-terraform-examples.yaml

@@ -1,4 +1,4 @@
-name: 'validate Terraform Examples - Dev'
+name: 'validate tf'
 
 # run everywhere; should be internally consistent
 
@@ -8,7 +8,7 @@ on:
       - '**'
 
 jobs:
-  validate_tf_examples:
+  validate_tf_example_dev:
     strategy:
       matrix:
         example_path: [
@@ -19,6 +19,8 @@ jobs:
                         'examples-dev/gcp',
                         'examples-dev/gcp-google-workspace',
         ]
+        terraform_version: [ '~1.4.0', '~1.5.0', '~1.6.0' ]
     uses: ./.github/workflows/ci-terraform-example.yaml
     with:
+      terraform_version: ${{ matrix.terraform_version }}
       example_path: ${{ matrix.example_path }}

.github/workflows/ci-terraform-modules.yaml

@@ -9,19 +9,20 @@ jobs:
   validate_tf_modules:
     name: 'validate tf modules'
     runs-on: ubuntu-latest
-    env:
-      TERRAFORM_VERSION: 1.3.9
+    strategy:
+      matrix:
+        terraform_version: [ '~1.4.0', '~1.5.0', '~1.6.0']
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'setup Terraform'
-        uses: hashicorp/setup-terraform@v2
+        uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: ${{ env.TERRAFORM_VERSION }}
+          terraform_version: ${{ matrix.terraform_version }}
 
       - name: "Terraform - validate modules/psoxy-constants"
         working-directory: infra/modules/psoxy-constants
         run: |
           terraform init -reconfigure
-          terraform validate
+          terraform validate

.github/workflows/ci-tools.yaml

@@ -10,14 +10,16 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # 16 is min version recommended to users
-        # 18 released 2022-04-19
-        # 19 released 2022-10-18
-        node-version: [ 19 ]
+        # even numbers are LTS; ods go to ~3m maintenance after next major version
+        # 16 is min version recommended to users; but unmaintained since 2023-10
+        # 18 released 2022-04-19 - maintained until June 2025
+        # 20 maintained until June 2026
+        # 21 is latest as of March 2023, with 22 coming in May
+        node-version: [ 18, 20, latest]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'

.github/workflows/publish-example.yaml

@@ -23,12 +23,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Proxy Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: 'proxy'
           ref: 'main'
       - name: Checkout Target Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: 'example'
           repository: ${{ inputs.example-repo }}
@@ -40,12 +40,12 @@ jobs:
       - name: Commit changes to example repo
         working-directory: example
         uses: planetscale/[email protected]
-          with:
-            commit_message: "update example to ${{ inputs.release }}"
-            repo: ${{ inputs.example-repo }}
-            branch: rc-${{ inputs.release }}
-          env:
-            GITHUB_TOKEN: ${{ inputs.example-repo-token }}
+        with:
+          commit_message: "update example to ${{ inputs.release }}"
+          repo: ${{ inputs.example-repo }}
+          branch: rc-${{ inputs.release }}
+        env:
+          GITHUB_TOKEN: ${{ inputs.example-repo-token }}
       - name: Open PR in example repo to merge update back to its mainline
         env:
           GH_TOKEN: ${{ inputs.example-repo-token }}

README.md

@@ -218,7 +218,7 @@ You will need all the following in your deployment environment (eg, your laptop)
 | [git](https://git-scm.com/)                     | 2.17+                  | `git --version`           |
 | [Maven](https://maven.apache.org/)              | 3.6+                   | `mvn -v`                 |
 | [Java JDK 11+](https://openjdk.org/install/) | 11, 17, 21 (see notes) | `mvn -v | grep Java` |
-| [Terraform](https://www.terraform.io/)          | 1.3.x, <= 1.5          | `terraform version`       |
+| [Terraform](https://www.terraform.io/)          | 1.3.x, <= 1.6          | `terraform version`       |
 
 NOTE: we will support Java versions for duration of official support windows, in particular the
 LTS versions. As of Nov 2023, we  still support java 11 but may end this at any time. Minor
@@ -245,11 +245,13 @@ Depending on your Cloud Host / Data Sources, you will need:
 
 For testing your psoxy instance, you will need:
 
-| Tool                                                               | Version | Test Command      |
-|--------------------------------------------------------------------|---------|-------------------|
-| [Node.js](https://nodejs.org/en/)                                  | 16+     | `node --version`  |
-| [npm](https://www.npmjs.com/package/npm) (should come with `node`) | 8+      | `npm --version`   |
+| Tool                                                               | Version                       | Test Command      |
+|--------------------------------------------------------------------|-------------------------------|-------------------|
+| [Node.js](https://nodejs.org/en/)                                  | 16+ (ideally, an LTS version) | `node --version`  |
+| [npm](https://www.npmjs.com/package/npm) (should come with `node`) | 8+                            | `npm --version`   |
 
+NOTE: NodeJS 16 is unmaintained since Oct 2023, so we recommend newer version; but in theory should
+work.
 
 We provide a script to check these prereqs, at [`tools/check-prereqs.sh`](tools/check-prereqs.sh).
 That script has no dependencies itself, so should be able to run on any plain POSIX-compliant shell

docs/aws/guides/lambdas-on-vpc.md

@@ -61,6 +61,19 @@ See the following terraform resources that you'll likely need:
 - [aws_internet_gateway](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/internet_gateway)
 
 
+## Troubleshooting
+
+Check your Cloud Watch logs for the lambda. Proxy lambda will time out in INIT phase if SSM
+Parameter Store *or* your secret store implementation (AWS Secrets Manager, Vault) is not reachable.
+
+Some potential causes of this:
+  - DNS failure - it's going to look up the SSM service by domain; if the DNS zone for the SSM
+    endpoint you've provisioned is not published on the VPC, this will fail; similarly, if the
+    endpoint wasn't configured on a subnet - then it won't have an IP to be resolved.
+  - if the IP is resolved, you should see failure to connect to it in the logs (timeouts); check
+    that your security groups for lambda/subnet/endpoint allow bidirectional traffic necessary for
+    your lambda to retrieve data from SSM via the REST API.
+
 ## Switching back from using a VPC
 
 Terraform with aws provider doesn't seem to play nice with lambdas/subnets; the subnet can't be
@@ -73,6 +86,7 @@ So:
 2. destroy the subnet `terraform destroy --target=aws_subnet.main`
 
 
+
 ## References
  - https://docs.aws.amazon.com/lambda/latest/dg/foundation-networking.html
  - https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html

docs/bulk-file-sanitization.md

@@ -99,9 +99,9 @@ This feature supports simple adaptation of existing data pipelines for use in Wo
 - Rule structure is specified in
   [`ColumnarRules`](java/gateway-core/src/main/java/com/avaulta/gateway/rules/ColumnarRules.java).
 
-### Record-Oriented Formats (RecordRules) **alpha**
+### Record-Oriented Formats (RecordRules)
 
-_As of Oct 2023, this is an alpha feature_
+_As of Oct 2023, this is a **beta** feature_
 
 `RecordRules` parses files as records, presuming the specified format. It performs transforms in
 order on each record to sanitize your data, and serializes the result back to the specified format.
@@ -124,14 +124,18 @@ root of the record object.
 
 `transforms` itself is an ordered-list of transforms. The transforms should be applied in order.
 
+CSV format is also supported, but in effect is converted to a simple JSON object before rules are
+applied; so JSON paths in transforms should all be single-level; eg, `$.email` to refer to the
+`email` column in the CSV.
+
 #### See Also
 
 - Rule structure is specified in
   [`RecordRules`](java/gateway-core/src/main/java/com/avaulta/gateway/rules/RecordRules.java).
 
-### Mixing File Formats **alpha**
+### Mixing File Formats
 
-_As of Oct 2023, this feature is in alpha and may change in backwards incompatible ways_
+_As of Oct 2023, this feature is in **beta** and may change in backwards incompatible ways_
 
 You can process multiple file formats through a single proxy instance using
 `MultiTypeBulkDataRules`.

docs/sources/microsoft-365/README.md

@@ -58,6 +58,12 @@ as an external identity provider of those credentials.
 Neither your proxy instances nor Worklytics ever hold any API key or certificate for your Microsoft
 365 tenant.
 
+See [Microsoft Workload Identity Federation](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation)
+docs for details. Specifically, the relevant scenario is workload running in either GCP or AWS (your
+proxy host platform)
+
+![Microsoft Workload Identity Federation Scenarios](msft-workload-identity-federation-scenarios.png)
+
 ### Authorization and Scopes
 
 The following Scopes are required for each connector. Note that they are all READ-only scopes.

infra/examples-dev/aws-all/google-workspace.tf

@@ -8,7 +8,7 @@ provider "google" {
 
 module "worklytics_connectors_google_workspace" {
   source = "../../modules/worklytics-connectors-google-workspace"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.4.51"
 
   providers = {
     google = google.google_workspace

infra/examples-dev/aws-all/main.tf

@@ -19,7 +19,7 @@ terraform {
 # general cases
 module "worklytics_connectors" {
   source = "../../modules/worklytics-connectors"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.4.51"
 
   enabled_connectors               = var.enabled_connectors
   jira_cloud_id                    = var.jira_cloud_id
@@ -97,7 +97,7 @@ locals {
 
 module "psoxy" {
   source = "../../modules/aws-host"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.4.51"
 
   environment_name                     = var.environment_name
   aws_account_id                       = var.aws_account_id
@@ -116,6 +116,7 @@ module "psoxy" {
   general_environment_variables        = var.general_environment_variables
   function_env_kms_key_arn             = var.project_aws_kms_key_arn
   logs_kms_key_arn                     = var.project_aws_kms_key_arn
+  log_retention_days                   = var.log_retention_days
   aws_ssm_key_id                       = var.project_aws_kms_key_arn
   use_api_gateway_v2                   = var.use_api_gateway_v2
   aws_lambda_execution_role_policy_arn = var.aws_lambda_execution_role_policy_arn
@@ -124,6 +125,7 @@ module "psoxy" {
   bulk_input_expiration_days           = var.bulk_input_expiration_days
   api_connectors                       = local.api_connectors
   bulk_connectors                      = local.bulk_connectors
+  provision_bucket_public_access_block = var.provision_bucket_public_access_block
   custom_bulk_connector_rules          = var.custom_bulk_connector_rules
   custom_bulk_connector_arguments      = var.custom_bulk_connector_arguments
   todo_step                            = local.max_auth_todo_step
@@ -149,7 +151,7 @@ module "connection_in_worklytics" {
   for_each = local.all_instances
 
   source = "../../modules/worklytics-psoxy-connection-aws"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-psoxy-connection-aws?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-psoxy-connection-aws?ref=v0.4.51"
 
   psoxy_instance_id  = each.key
   worklytics_host    = var.worklytics_host

infra/examples-dev/aws-all/msft-365.tf

@@ -2,7 +2,7 @@
 
 module "worklytics_connectors_msft_365" {
   source = "../../modules/worklytics-connectors-msft-365"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.4.51"
 
 
   enabled_connectors                  = var.enabled_connectors
@@ -47,7 +47,7 @@ module "cognito_identity_pool" {
   count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled
 
   source = "../../modules/aws-cognito-pool"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.4.51"
 
   developer_provider_name = local.developer_provider_name
   name                    = "${local.env_qualifier}-azure-ad-federation"
@@ -57,7 +57,7 @@ module "cognito_identity" {
   count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled
 
   source = "../../modules/aws-cognito-identity-cli"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.4.51"
 
   aws_region       = data.aws_region.current.id
   aws_role         = var.aws_assume_role_arn
@@ -79,7 +79,7 @@ module "msft_connection_auth_federation" {
   for_each = module.worklytics_connectors_msft_365.enabled_api_connectors
 
   source = "../../modules/azuread-federated-credentials"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.4.51"
 
   application_object_id = each.value.connector.id
   display_name          = "${local.env_qualifier}AccessFromAWS"

infra/examples-dev/aws-all/variables.tf

@@ -216,6 +216,18 @@ variable "use_api_gateway_v2" {
   default     = false
 }
 
+variable "log_retention_days" {
+  type        = number
+  description = "Number of days to retain logs in CloudWatch."
+  default     = 7
+}
+
+variable "provision_bucket_public_access_block" {
+  type        = bool
+  description = "Whether to provision public_access_block resources on all buckets; defaults to 'true', but can be 'false' if you have organizational control policies that do this at a higher level."
+  default     = true
+}
+
 variable "custom_bulk_connectors" {
   type = map(object({
     source_kind               = string

infra/examples-dev/aws-google-workspace/main.tf

@@ -59,7 +59,7 @@ data "google_project" "psoxy-google-connectors" {
 
 module "psoxy" {
   source = "../../modular-examples/aws-google-workspace"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws-google-workspace?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws-google-workspace?ref=v0.4.51"
 
   aws_account_id                 = var.aws_account_id
   aws_assume_role_arn            = var.aws_assume_role_arn # role that can test the instances (lambdas)

infra/examples-dev/aws-msft-365/main.tf

@@ -51,7 +51,7 @@ provider "azuread" {
 
 module "psoxy" {
   source = "../../modular-examples/aws-msft-365"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws-msft-365?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws-msft-365?ref=v0.4.51"
 
   aws_account_id                 = var.aws_account_id
   aws_assume_role_arn            = var.aws_assume_role_arn # role that can test the instances (lambdas)

infra/examples-dev/aws/main.tf

@@ -57,7 +57,7 @@ provider "azuread" {
 
 module "psoxy" {
   source = "../../modular-examples/aws"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws?ref=v0.4.51"
 
   aws_account_id                 = var.aws_account_id
   aws_assume_role_arn            = var.aws_assume_role_arn # role that can test the instances (lambdas)

infra/examples-dev/gcp-google-workspace/main.tf

@@ -28,7 +28,7 @@ provider "google" {
 
 module "psoxy" {
   source = "../../modular-examples/gcp-google-workspace"
-  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/gcp-google-workspace?ref=v0.4.50"
+  # source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/gcp-google-workspace?ref=v0.4.51"
 
   gcp_project_id                 = var.gcp_project_id
   environment_name               = var.environment_name