From d7d94e7468ca24f4a187f944f7bd906ec00d19af Mon Sep 17 00:00:00 2001 From: Imran Date: Thu, 30 Nov 2023 05:34:33 +0000 Subject: [PATCH 1/4] feat: add web_server_network_access_control --- examples/simple_composer_env_v2/main.tf | 10 ++++++++++ modules/create_environment_v2/README.md | 1 + modules/create_environment_v2/main.tf | 10 ++++++++++ modules/create_environment_v2/variables.tf | 9 +++++++++ 4 files changed, 30 insertions(+) diff --git a/examples/simple_composer_env_v2/main.tf b/examples/simple_composer_env_v2/main.tf index 66dfce7..0fc944b 100644 --- a/examples/simple_composer_env_v2/main.tf +++ b/examples/simple_composer_env_v2/main.tf @@ -94,4 +94,14 @@ module "simple-composer-environment" { depends_on = [ google_storage_bucket_iam_member.object_admin, ] + web_server_network_access_control = [ + { + allowed_ip_range = "192.0.2.0/24" + description = "office net 1" + }, + { + allowed_ip_range = "192.0.4.0/24" + description = "office net 2" + }, + ] } diff --git a/modules/create_environment_v2/README.md b/modules/create_environment_v2/README.md index 0d2efd8..3b1a381 100644 --- a/modules/create_environment_v2/README.md +++ b/modules/create_environment_v2/README.md @@ -92,6 +92,7 @@ module "simple-composer-environment" { | triggerer | Configuration for resources used by Airflow triggerer | 
object({
    cpu       = string
    memory_gb = number
    count     = number
  })
| `null` | no | | use\_private\_environment | Create a private environment. | `bool` | `false` | no | | web\_server | Configuration for resources used by Airflow web server. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
  })
| 
{
  "cpu": 2,
  "memory_gb": 7.5,
  "storage_gb": 5
}
| no | +| web\_server\_network\_access\_control | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied | 
list(object({
    allowed_ip_range = string
    description      = string
  }))
| `null` | no | | worker | Configuration for resources used by Airflow workers. | 
object({
    cpu        = string
    memory_gb  = number
    storage_gb = number
    min_count  = number
    max_count  = number
  })
| 
{
  "cpu": 2,
  "max_count": 6,
  "memory_gb": 7.5,
  "min_count": 2,
  "storage_gb": 5
}
| no | ## Outputs diff --git a/modules/create_environment_v2/main.tf b/modules/create_environment_v2/main.tf index 4f92311..ef12828 100644 --- a/modules/create_environment_v2/main.tf +++ b/modules/create_environment_v2/main.tf @@ -186,6 +186,16 @@ resource "google_composer_environment" "composer_env" { } } + web_server_network_access_control { + dynamic "allowed_ip_range" { + for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x } + content { + value = allowed_ip_range.value["allowed_ip_range"] + description = allowed_ip_range.value["description"] + } + } + } + } depends_on = [google_project_iam_member.composer_agent_service_account] diff --git a/modules/create_environment_v2/variables.tf b/modules/create_environment_v2/variables.tf index 50700a4..6372966 100644 --- a/modules/create_environment_v2/variables.tf +++ b/modules/create_environment_v2/variables.tf @@ -273,3 +273,12 @@ variable "cloud_data_lineage_integration" { type = bool default = false } + +variable "web_server_network_access_control" { + type = list(object({ + allowed_ip_range = string + description = string + })) + default = null + description = "The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied" +} From 8fa438c2f4a7ceef33e711728f9fd87908c77c98 Mon Sep 17 00:00:00 2001 From: Imran Date: Thu, 30 Nov 2023 11:41:18 +0000 Subject: [PATCH 2/4] fixed handle null value for web_server_network_access_control --- build/int.cloudbuild.yaml | 53 ++++++++++++++------------- modules/create_environment_v2/main.tf | 15 +++++--- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 473f086..a11d561 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -28,11 +28,35 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] + # ----- SUITE simple-composer-env-v2-local + +- id: init-simple-composer-env-v2 + waitFor: + - create allverify airflow-pool-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] +- id: apply-simple-composer-env-v2 + waitFor: + - init-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] +- id: verify-simple-composer-env-v2 + waitFor: + - apply-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] +- id: destroy-simple-composer-env-v2 + waitFor: + - verify-simple-composer-env-v2 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] + + # ----- SUITE airflow-connection-local - id: converge airflow-connection-local waitFor: - - create all + - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-connection-local'] - id: verify airflow-connection-local @@ -51,7 +75,7 @@ steps: - id: converge airflow-pool-local waitFor: - - create all + - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-pool-local'] - id: verify airflow-pool-local @@ -65,34 +89,11 @@ steps: # name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' # args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-pool-local'] - # ----- SUITE simple-composer-env-v2-local - -- id: init-simple-composer-env-v2 - waitFor: - - verify airflow-pool-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] -- id: apply-simple-composer-env-v2 - waitFor: - - init-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage apply --verbose'] -- id: verify-simple-composer-env-v2 - waitFor: - - apply-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage verify --verbose'] -- id: destroy-simple-composer-env-v2 - waitFor: - - verify-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] - # ----- SUITE composer-v2-sharedvpc-prereq-local - id: init-composer-v2-sharedvpc-prereq waitFor: - - destroy-simple-composer-env-v2 + - verify airflow-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose'] - id: apply-composer-v2-sharedvpc-prereq diff --git a/modules/create_environment_v2/main.tf b/modules/create_environment_v2/main.tf index ef12828..e2734f0 100644 --- a/modules/create_environment_v2/main.tf +++ b/modules/create_environment_v2/main.tf @@ -186,12 +186,15 @@ resource "google_composer_environment" "composer_env" { } } - web_server_network_access_control { - dynamic "allowed_ip_range" { - for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x } - content { - value = allowed_ip_range.value["allowed_ip_range"] - description = allowed_ip_range.value["description"] + dynamic "web_server_network_access_control" { + for_each = var.web_server_network_access_control == null ? [] : ["web_server_network_access_control"] + content { + dynamic "allowed_ip_range" { + for_each = { for x in var.web_server_network_access_control : x.allowed_ip_range => x } + content { + value = allowed_ip_range.value["allowed_ip_range"] + description = allowed_ip_range.value["description"] + } } } } From 3251f5174a72d71dddef0c81c31cdb70eb9675b9 Mon Sep 17 00:00:00 2001 From: Imran Date: Thu, 30 Nov 2023 11:43:45 +0000 Subject: [PATCH 3/4] fixed handle null value for web_server_network_access_control --- build/int.cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index a11d561..bcc2ab4 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -32,7 +32,7 @@ steps: - id: init-simple-composer-env-v2 waitFor: - - create allverify airflow-pool-local + - create all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage init --verbose'] - id: apply-simple-composer-env-v2 From c942869adecc3d1af34f3a0ead629dede3d1b373 Mon Sep 17 00:00:00 2001 From: Imran Date: Thu, 30 Nov 2023 13:56:29 +0000 Subject: [PATCH 4/4] change sequence in lint test --- build/int.cloudbuild.yaml | 79 +++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index bcc2ab4..d89cc4d 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -28,7 +28,7 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] - # ----- SUITE simple-composer-env-v2-local + # ----- SUITE simple-composer-env-v2 - id: init-simple-composer-env-v2 waitFor: @@ -52,48 +52,11 @@ steps: args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2Module --stage destroy --verbose'] -# ----- SUITE airflow-connection-local - -- id: converge airflow-connection-local - waitFor: - - destroy-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-connection-local'] -- id: verify airflow-connection-local - waitFor: - - converge airflow-connection-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify airflow-connection-local'] -# - id: destroy airflow-connection-local -# waitFor: -# - verify airflow-connection-local -# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' -# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-connection-local'] - - -# ----- SUITE airflow-pool-local - -- id: converge airflow-pool-local - waitFor: - - destroy-simple-composer-env-v2 - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-pool-local'] -- id: verify airflow-pool-local - waitFor: - - converge airflow-pool-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify airflow-pool-local'] -# - id: destroy airflow-pool-local -# waitFor: -# - verify airflow-pool-local -# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' -# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-pool-local'] - # ----- SUITE composer-v2-sharedvpc-prereq-local - id: init-composer-v2-sharedvpc-prereq waitFor: - - verify airflow-pool-local + - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV2SharedVpcModule --stage init --verbose'] - id: apply-composer-v2-sharedvpc-prereq @@ -116,7 +79,7 @@ steps: - id: init-simple-composer-env-v1 waitFor: - - destroy-composer-v2-sharedvpc-prereq + - destroy-simple-composer-env-v2 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage init --verbose'] - id: apply-simple-composer-env-v1 @@ -134,6 +97,42 @@ steps: - verify-simple-composer-env-v1 name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleComposerEnvV1Module --stage destroy --verbose'] +# ----- SUITE airflow-connection-local + +- id: converge airflow-connection-local + waitFor: + - destroy-simple-composer-env-v1 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-connection-local'] +- id: verify airflow-connection-local + waitFor: + - converge airflow-connection-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify airflow-connection-local'] +# - id: destroy airflow-connection-local +# waitFor: +# - verify airflow-connection-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-connection-local'] + + +# ----- SUITE airflow-pool-local + +- id: converge airflow-pool-local + waitFor: + - destroy-simple-composer-env-v1 + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge airflow-pool-local'] +- id: verify airflow-pool-local + waitFor: + - converge airflow-pool-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify airflow-pool-local'] +# - id: destroy airflow-pool-local +# waitFor: +# - verify airflow-pool-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy airflow-pool-local'] tags: - 'ci' - 'integration'