diff --git a/modules/airflow_storage/README.md b/modules/airflow_storage/README.md new file mode 100644 index 00000000..73e560c6 --- /dev/null +++ b/modules/airflow_storage/README.md @@ -0,0 +1,37 @@ +# Module Cloud Composer Storage + +This optional module is used to configure Airflow objects like dags, plugins and data. + +Unfortunately this capability is not exposed via the Composer API so we need to use `gcloud` + +```hcl +module "my-dag" { + source = "terraform-google-modules/composer/google//modules/airflow_storage" + project_id = "project-123" + environment = "Composer-Prod-Env" + location = "us-central1" + type = "dag" + source_path = "${path.root}/dags/my-dag" +} +``` + +See the examples for cloudsql support. + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| destination\_path | The optional destination path | `string` | `null` | no | +| environment | n/a | `string` | n/a | yes | +| location | n/a | `string` | n/a | yes | +| project\_id | n/a | `string` | n/a | yes | +| source\_path | The source on the local file system | `string` | n/a | yes | +| type | The type of resource to upload. Either dag, plugin or data | `string` | n/a | yes | + +## Outputs + +No output. + + diff --git a/modules/airflow_storage/main.tf b/modules/airflow_storage/main.tf new file mode 100644 index 00000000..a968032d --- /dev/null +++ b/modules/airflow_storage/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # Emulate the default behaviour when --destination isn't set otherwise we don't know how to run the destroy command + implied_destination = coalesce(var.destination_path, basename(var.source_path)) + create_parameters = { + "--project" = var.project_id + "--location" = var.location + "--environment" = var.environment + "--source" = var.source_path + "--destination" = var.destination_path + } + destroy_parameters = { + "--project" = var.project_id + "--location" = var.location + "--environment" = var.environment + } + + gcloud_cmd_body = "composer environments storage ${var.type}" + create_cmd_body = "${local.gcloud_cmd_body} import ${join(" ", [for k, v in local.create_parameters : "${k}=${v}" if v != null])}" + destroy_cmd_body = "${local.gcloud_cmd_body} delete ${join(" ", [for k, v in local.destroy_parameters : "${k}=${v}"])} ${local.implied_destination}" +} + + +module "gcloud" { + source = "terraform-google-modules/gcloud/google" + version = "~> 3.1" + platform = "linux" + create_cmd_body = local.create_cmd_body + destroy_cmd_body = local.destroy_cmd_body + + create_cmd_triggers = { + filesha1 = join(",", [ + for f in fileset(var.source_path, "**") : "${f}:${filesha1("${var.source_path}/${f}")}" + ]) + } +} diff --git a/modules/airflow_storage/outputs.tf b/modules/airflow_storage/outputs.tf new file mode 100644 index 00000000..264a34bf --- /dev/null +++ b/modules/airflow_storage/outputs.tf @@ -0,0 +1,16 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/modules/airflow_storage/variables.tf b/modules/airflow_storage/variables.tf new file mode 100644 index 00000000..7de852a7 --- /dev/null +++ b/modules/airflow_storage/variables.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string +} + +variable "location" { + type = string +} + +variable "environment" { + type = string +} + +variable "type" { + description = "The type of resource to upload. Either dag, plugin or data" + type = string +} + +variable "source_path" { + description = "The source on the local file system" + type = string +} + +variable "destination_path" { + description = "The optional destination path" + type = string + default = null +} diff --git a/modules/airflow_storage/versions.tf b/modules/airflow_storage/versions.tf new file mode 100644 index 00000000..c45e92cb --- /dev/null +++ b/modules/airflow_storage/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.14" + required_providers { + + google = { + source = "hashicorp/google" + version = "~> 3.53" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-composer:airflow_storage/v2.1.0" + } +}