From 3af6e9e77b1f88953eb15b7ff97c8e2a5514bc83 Mon Sep 17 00:00:00 2001 From: Carlos Mondo Date: Tue, 10 Jan 2023 19:28:57 -0300 Subject: [PATCH 1/3] feat: include connector params timeout in sql export workflow --- modules/backup/README.md | 2 ++ modules/backup/main.tf | 16 +++++++++------- modules/backup/templates/export.yaml.tftpl | 5 ++++- modules/backup/variables.tf | 12 ++++++++++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/modules/backup/README.md b/modules/backup/README.md index 1b90dd0e..2f7a04eb 100644 --- a/modules/backup/README.md +++ b/modules/backup/README.md @@ -56,6 +56,8 @@ fetch workflows.googleapis.com/Workflow | backup\_retention\_time | The number of days backups should be kept | `number` | `30` | no | | backup\_schedule | The cron schedule to execute the internal backup | `string` | `"45 2 * * *"` | no | | compress\_export | Whether or not to compress the export when storing in the bucket; Only valid for MySQL and PostgreSQL | `bool` | `true` | no | +| connector\_params\_timeout | The end-to-end duration the connector call is allowed to run for before throwing a timeout exception. The default value is 1800 and this should be the maximum for connector methods that are not long-running operations. Otherwise, for long-running operations, the maximum timeout for a connector call is 31536000 seconds (one year). | `number` | `1800` | no | +| enable\_connector\_params | Whether to enable connector-specific parameters for Google Workflow SQL Export. | `bool` | `false` | no | | enable\_export\_backup | Weather to create exports to GCS Buckets with this module | `bool` | `true` | no | | enable\_internal\_backup | Wether to create internal backups with this module | `bool` | `true` | no | | export\_databases | The list of databases that should be exported - if is an empty set all databases will be exported | `set(string)` | `[]` | no | diff --git a/modules/backup/main.tf b/modules/backup/main.tf index c31ed53b..8f2b8a75 100644 --- a/modules/backup/main.tf +++ b/modules/backup/main.tf @@ -103,13 +103,15 @@ resource "google_workflows_workflow" "sql_export" { project = var.project_id service_account = local.service_account source_contents = templatefile("${path.module}/templates/export.yaml.tftpl", { - project = var.project_id - instanceName = var.sql_instance - backupRetentionTime = var.backup_retention_time - databases = jsonencode(var.export_databases) - gcsBucket = var.export_uri - dbType = split("_", data.google_sql_database_instance.backup_instance.database_version)[0] - compressExport = var.compress_export + project = var.project_id + instanceName = var.sql_instance + backupRetentionTime = var.backup_retention_time + databases = jsonencode(var.export_databases) + gcsBucket = var.export_uri + dbType = split("_", data.google_sql_database_instance.backup_instance.database_version)[0] + compressExport = var.compress_export + enableConnectorParams = var.enable_connector_params + connectorParamsTimeout = var.connector_params_timeout }) } diff --git a/modules/backup/templates/export.yaml.tftpl b/modules/backup/templates/export.yaml.tftpl index ab0373ce..49e3c619 100644 --- a/modules/backup/templates/export.yaml.tftpl +++ b/modules/backup/templates/export.yaml.tftpl @@ -69,6 +69,10 @@ main: args: project: ${project} instance: ${instanceName} + %{ if enableConnectorParams } + connector_params: + timeout: ${connectorParamsTimeout} + %{ endif } body: exportContext: databases: [$${database}] @@ -87,4 +91,3 @@ main: databases: $${databases} uri: $${"${gcsBucket}/${instanceName}-" + backupTime + %{ if compressExport == true }".sql.gz"%{ else }".sql"%{ endif }} %{ endif } - diff --git a/modules/backup/variables.tf b/modules/backup/variables.tf index d78d0c05..3860e107 100644 --- a/modules/backup/variables.tf +++ b/modules/backup/variables.tf @@ -97,6 +97,18 @@ variable "compress_export" { default = true } +variable "enable_connector_params" { + description = "Whether to enable connector-specific parameters for Google Workflow SQL Export." + type = bool + default = false +} + +variable "connector_params_timeout" { + description = "The end-to-end duration the connector call is allowed to run for before throwing a timeout exception. The default value is 1800 and this should be the maximum for connector methods that are not long-running operations. Otherwise, for long-running operations, the maximum timeout for a connector call is 31536000 seconds (one year)." + type = number + default = 1800 +} + variable "unique_suffix" { description = "Unique suffix to add to scheduler jobs and workflows names." type = string From 28d95f0e5892fc49b3fd8ae28306c423f0381518 Mon Sep 17 00:00:00 2001 From: Carlos Mondo Date: Thu, 12 Jan 2023 18:04:30 -0300 Subject: [PATCH 2/3] feat: includes the possibility of execute the export workflow from a replica --- modules/backup/README.md | 2 ++ modules/backup/main.tf | 6 +++--- modules/backup/variables.tf | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/backup/README.md b/modules/backup/README.md index 2f7a04eb..d0e8a5b8 100644 --- a/modules/backup/README.md +++ b/modules/backup/README.md @@ -68,7 +68,9 @@ fetch workflows.googleapis.com/Workflow | scheduler\_timezone | The Timezone in which the Scheduler Jobs are triggered | `string` | `"Etc/GMT"` | no | | service\_account | The service account to use for running the workflow and triggering the workflow by Cloud Scheduler - If empty or null a service account will be created. If you have provided a service account you need to grant the Cloud SQL Admin and the Workflows Invoker role to that | `string` | `null` | no | | sql\_instance | The name of the SQL instance to backup | `string` | n/a | yes | +| sql\_instance\_replica | The name of the SQL instance replica to export | `string` | `null` | no | | unique\_suffix | Unique suffix to add to scheduler jobs and workflows names. | `string` | `""` | no | +| use\_sql\_instance\_replica\_in\_exporter | Whether or not to use replica instance on exporter workflow. | `bool` | `false` | no | ## Outputs diff --git a/modules/backup/main.tf b/modules/backup/main.tf index 8f2b8a75..c0a0c931 100644 --- a/modules/backup/main.tf +++ b/modules/backup/main.tf @@ -97,14 +97,14 @@ resource "google_cloud_scheduler_job" "sql_backup" { ################################ resource "google_workflows_workflow" "sql_export" { count = var.enable_export_backup ? 1 : 0 - name = "sql-export-${var.sql_instance}${var.unique_suffix}" + name = var.use_sql_instance_replica_in_exporter ? "sql-export-${var.sql_instance_replica}${var.unique_suffix}" : "sql-export-${var.sql_instance}${var.unique_suffix}" region = var.region description = "Workflow for backing up the CloudSQL Instance" project = var.project_id service_account = local.service_account source_contents = templatefile("${path.module}/templates/export.yaml.tftpl", { project = var.project_id - instanceName = var.sql_instance + instanceName = var.use_sql_instance_replica_in_exporter ? var.sql_instance_replica : var.sql_instance backupRetentionTime = var.backup_retention_time databases = jsonencode(var.export_databases) gcsBucket = var.export_uri @@ -117,7 +117,7 @@ resource "google_workflows_workflow" "sql_export" { resource "google_cloud_scheduler_job" "sql_export" { count = var.enable_export_backup ? 1 : 0 - name = "sql-export-${var.sql_instance}${var.unique_suffix}" + name = var.use_sql_instance_replica_in_exporter ? "sql-export-${var.sql_instance_replica}${var.unique_suffix}" : "sql-export-${var.sql_instance}${var.unique_suffix}" project = var.project_id region = var.region description = "Managed by Terraform - Triggers a SQL Export via Workflows" diff --git a/modules/backup/variables.tf b/modules/backup/variables.tf index 3860e107..d64eb4b9 100644 --- a/modules/backup/variables.tf +++ b/modules/backup/variables.tf @@ -114,3 +114,16 @@ variable "unique_suffix" { type = string default = "" } + +variable "use_sql_instance_replica_in_exporter" { + description = "Whether or not to use replica instance on exporter workflow." + type = bool + default = false +} + + +variable "sql_instance_replica" { + description = "The name of the SQL instance replica to export" + type = string + default = null +} From d0bac53163246490f1220a4e199f3508cf89f970 Mon Sep 17 00:00:00 2001 From: Carlos Mondo Date: Thu, 12 Jan 2023 18:30:17 -0300 Subject: [PATCH 3/3] feat: include connector params for MySQL and PostgreSQL database --- modules/backup/templates/export.yaml.tftpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/backup/templates/export.yaml.tftpl b/modules/backup/templates/export.yaml.tftpl index 49e3c619..372036e0 100644 --- a/modules/backup/templates/export.yaml.tftpl +++ b/modules/backup/templates/export.yaml.tftpl @@ -48,6 +48,10 @@ main: args: project: ${project} instance: ${instanceName} + %{ if enableConnectorParams } + connector_params: + timeout: ${connectorParamsTimeout} + %{ endif } body: exportContext: databases: [$${database}] @@ -86,6 +90,10 @@ main: args: project: ${project} instance: ${instanceName} + %{ if enableConnectorParams } + connector_params: + timeout: ${connectorParamsTimeout} + %{ endif } body: exportContext: databases: $${databases}