From 42dc67ea8d752a8c45b12c35aea48d52c9cb31f8 Mon Sep 17 00:00:00 2001 From: Georgy Malkov Date: Wed, 13 Nov 2024 11:22:21 +0300 Subject: [PATCH] Fix terraform deploy (#241) --- .github/workflows/deploy.yml | 2 - .github/workflows/e2e_tests.yml | 4 ++ deploy/scripts/tofu-opensource.sh | 14 ++-- deploy/terraform/k8s-alb-ingress.tf | 5 -- deploy/terraform/k8s-app-control-api.tf | 10 +-- deploy/terraform/k8s-app-data-api.tf | 8 --- deploy/terraform/k8s-app-ui.tf | 8 --- deploy/terraform/k8s-app-us.tf | 8 --- deploy/terraform/k8s-app-zitadel.tf | 16 ----- deploy/terraform/k8s-logs.tf | 4 -- deploy/terraform/k8s-monitoring.tf | 19 ++++++ deploy/terraform/k8s-secrets.tf | 8 --- deploy/terraform/kubeconfig.tf | 85 ++++++++++--------------- deploy/terraform/locals.tf | 5 +- deploy/terraform/provider.tf | 12 ++-- deploy/terraform/vpc.tf | 5 +- 16 files changed, 76 insertions(+), 137 deletions(-) create mode 100644 deploy/terraform/k8s-monitoring.tf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fcd7e772..7588aad3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -31,8 +31,6 @@ jobs: - uses: arduino/setup-task@v2 - uses: nightstory/setup-yc@v1 - run: | - ../scripts/tofu-opensource.sh --tofurc --cleanup --init --target data.shell_script.kubeconfig --apply --approve - ls -la ./kubeconfig.conf && sleep 10 ../scripts/tofu-opensource.sh --tofurc --cleanup --init --apply --approve working-directory: ./deploy/terraform env: diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index c23ac50d..60fd9493 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -55,6 +55,10 @@ jobs: CONTROL_API_VERSION: ${{ env.CONTROL_API_VERSION }} DATA_API_VERSION: ${{ env.DATA_API_VERSION }} UI_VERSION: ${{ env.UI_VERSION }} + E2E_RETRY_TIMES: 2 + E2E_TEST_TIMEOUT: 90000 + E2E_ACTION_TIMEOUT: 10000 + E2E_EXPECT_TIMEOUT: 10000 - name: Copy tests report from docker run if: always() diff --git a/deploy/scripts/tofu-opensource.sh b/deploy/scripts/tofu-opensource.sh index 6d58f935..03e4bed1 100755 --- a/deploy/scripts/tofu-opensource.sh +++ b/deploy/scripts/tofu-opensource.sh @@ -151,10 +151,10 @@ if [ ! "${YC_PROFILE_EXISTS}" == "true" ]; then fi fi -SERVICE_ACCOUNT_ID=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key service-account-id) +SERVICE_ACCOUNT_ID=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key service-account-id) -export AWS_ACCESS_KEY_ID=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key access-key) -export AWS_SECRET_ACCESS_KEY=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key secret-key) +export AWS_ACCESS_KEY_ID=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key access-key) +export AWS_SECRET_ACCESS_KEY=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key secret-key) export AWS_ENDPOINT_URL_S3="${STORAGE_ENDPOINT}" export YC_STORAGE_ACCESS_KEY="${AWS_ACCESS_KEY_ID}" export YC_STORAGE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY}" @@ -164,9 +164,9 @@ if [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then exit 1 fi -export BACKEND_STATE_BUCKET=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-bucket) -export BACKEND_STATE_KEY=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-key) -export BACKEND_STATE_REGION=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-region) +export BACKEND_STATE_BUCKET=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-bucket) +export BACKEND_STATE_KEY=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-key) +export BACKEND_STATE_REGION=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-region) export TF_VAR_PROFILE="${PROFILE_NAME}" @@ -183,7 +183,7 @@ export TF_VAR_BACKEND_STATE_REGION="${BACKEND_STATE_REGION}" export TF_VAR_DOMAIN="${DOMAIN}" -export TF_VAR_YC_TOKEN=$(yc --profile=${PROFILE_NAME} iam create-token --impersonate-service-account-id "${SERVICE_ACCOUNT_ID}") +export TF_VAR_YC_TOKEN=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} iam create-token --impersonate-service-account-id "${SERVICE_ACCOUNT_ID}") if [ -z "${TF_VAR_YC_TOKEN}" ]; then echo "❌ error obtain iam token for sa '${SERVICE_ACCOUNT_ID}', check profile settings, exit..." diff --git a/deploy/terraform/k8s-alb-ingress.tf b/deploy/terraform/k8s-alb-ingress.tf index 9783654d..2a6c7a2f 100644 --- a/deploy/terraform/k8s-alb-ingress.tf +++ b/deploy/terraform/k8s-alb-ingress.tf @@ -30,11 +30,6 @@ resource "helm_release" "alb_ingress" { private_key = yandex_iam_service_account_key.this["alb"].private_key }) })] - - - depends_on = [ - data.shell_script.kubeconfig, - ] } locals { diff --git a/deploy/terraform/k8s-app-control-api.tf b/deploy/terraform/k8s-app-control-api.tf index 41962b64..b2a9145a 100644 --- a/deploy/terraform/k8s-app-control-api.tf +++ b/deploy/terraform/k8s-app-control-api.tf @@ -59,7 +59,7 @@ resource "kubernetes_deployment" "control-api" { } spec { container { - image = "ghcr.io/datalens-tech/datalens-control-api:${local.data_api_version}" + image = "ghcr.io/datalens-tech/datalens-control-api:${local.control_api_version}" name = "app-control-api" port { container_port = 8080 @@ -163,10 +163,6 @@ resource "kubernetes_deployment" "control-api" { } } } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_service" "control-api_service" { @@ -185,8 +181,4 @@ resource "kubernetes_service" "control-api_service" { } type = "ClusterIP" } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-app-data-api.tf b/deploy/terraform/k8s-app-data-api.tf index 0a22521a..25d39202 100644 --- a/deploy/terraform/k8s-app-data-api.tf +++ b/deploy/terraform/k8s-app-data-api.tf @@ -195,10 +195,6 @@ resource "kubernetes_deployment" "data-api" { } } } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_service" "data-api_service" { @@ -217,8 +213,4 @@ resource "kubernetes_service" "data-api_service" { } type = "ClusterIP" } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-app-ui.tf b/deploy/terraform/k8s-app-ui.tf index 30b035c0..eac8e0f7 100644 --- a/deploy/terraform/k8s-app-ui.tf +++ b/deploy/terraform/k8s-app-ui.tf @@ -164,10 +164,6 @@ resource "kubernetes_deployment" "ui" { } } } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_service" "ui_service" { @@ -187,8 +183,4 @@ resource "kubernetes_service" "ui_service" { } type = "NodePort" } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-app-us.tf b/deploy/terraform/k8s-app-us.tf index 4f6ace8c..4320bb82 100644 --- a/deploy/terraform/k8s-app-us.tf +++ b/deploy/terraform/k8s-app-us.tf @@ -157,10 +157,6 @@ resource "kubernetes_deployment" "us" { } } } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_service" "us_service" { @@ -179,8 +175,4 @@ resource "kubernetes_service" "us_service" { } type = "ClusterIP" } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-app-zitadel.tf b/deploy/terraform/k8s-app-zitadel.tf index a38b01a6..f69c7eee 100644 --- a/deploy/terraform/k8s-app-zitadel.tf +++ b/deploy/terraform/k8s-app-zitadel.tf @@ -231,10 +231,6 @@ resource "kubernetes_deployment" "zitadel" { } } } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_job" "zitadel_init_job" { @@ -283,10 +279,6 @@ resource "kubernetes_job" "zitadel_init_job" { backoff_limit = 5 active_deadline_seconds = 600 } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_job" "zitadel_setup_job" { @@ -335,10 +327,6 @@ resource "kubernetes_job" "zitadel_setup_job" { backoff_limit = 5 active_deadline_seconds = 600 } - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_service" "zitadel_service" { @@ -360,8 +348,4 @@ resource "kubernetes_service" "zitadel_service" { } type = "NodePort" } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-logs.tf b/deploy/terraform/k8s-logs.tf index 8db834b0..32bdad50 100644 --- a/deploy/terraform/k8s-logs.tf +++ b/deploy/terraform/k8s-logs.tf @@ -38,9 +38,5 @@ resource "helm_release" "logging" { # fix helm chart error with missed escaping json }), ",", "\\,"), "\\n", "\\\\n"), "{", "\\{"), "}", "\\}") } - - depends_on = [ - data.shell_script.kubeconfig, - ] } diff --git a/deploy/terraform/k8s-monitoring.tf b/deploy/terraform/k8s-monitoring.tf new file mode 100644 index 00000000..1702701d --- /dev/null +++ b/deploy/terraform/k8s-monitoring.tf @@ -0,0 +1,19 @@ +resource "helm_release" "monitoring" { + for_each = toset(local.k8s_monitoring ? ["main"] : []) + + name = "monitoring" + + repository = "https://prometheus-community.github.io/helm-charts" + chart = "kube-prometheus-stack" + version = "62.7.0" + + timeout = 240 + namespace = "monitoring" + create_namespace = true + cleanup_on_fail = true + + set { + name = "installCRDs" + value = true + } +} diff --git a/deploy/terraform/k8s-secrets.tf b/deploy/terraform/k8s-secrets.tf index 43b9d19c..488402a7 100644 --- a/deploy/terraform/k8s-secrets.tf +++ b/deploy/terraform/k8s-secrets.tf @@ -9,10 +9,6 @@ resource "helm_release" "secrets" { version = "0.9.20" repository = "oci://${local.cr_endpoint}/yc-marketplace/yandex-cloud/external-secrets/chart" - - depends_on = [ - data.shell_script.kubeconfig, - ] } resource "kubernetes_secret" "secrets" { @@ -36,7 +32,6 @@ resource "kubernetes_secret" "secrets" { depends_on = [ helm_release.secrets, - data.shell_script.kubeconfig, ] } @@ -65,7 +60,6 @@ resource "kubernetes_manifest" "secrets" { depends_on = [ helm_release.secrets, kubernetes_secret.secrets, - data.shell_script.kubeconfig, ] } @@ -179,7 +173,6 @@ resource "kubernetes_manifest" "lockbox" { depends_on = [ helm_release.secrets, kubernetes_secret.secrets, - data.shell_script.kubeconfig, ] } @@ -224,7 +217,6 @@ resource "kubernetes_manifest" "lockbox-zitadel" { depends_on = [ helm_release.secrets, kubernetes_secret.secrets, - data.shell_script.kubeconfig, ] } diff --git a/deploy/terraform/kubeconfig.tf b/deploy/terraform/kubeconfig.tf index 22ed9c7c..f7d2be6f 100644 --- a/deploy/terraform/kubeconfig.tf +++ b/deploy/terraform/kubeconfig.tf @@ -1,7 +1,3 @@ -provider "shell" { - interpreter = ["/usr/bin/env", "bash", "-c"] -} - locals { cluster_endpoint = local.k8s_cluster_endpoint cluster_trusted_ca = base64encode(local.k8s_cluster_ca_certificate) @@ -13,59 +9,42 @@ locals { } locals { - kubeconfig = { - apiVersion = "v1", - kind = "Config" - clusters = [ - { - name = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}", - cluster = { - certificate-authority-data = local.cluster_trusted_ca - server = local.cluster_endpoint - } + kubeconfig = yamlencode({ + apiVersion = "v1" + clusters = [{ + name = "${local.service}-managed-k8s" + cluster = { + "server" = local.cluster_endpoint + "certificate-authority-data" = local.cluster_trusted_ca } - ], - contexts = [ - { - name = "yc-managed-k8s-backends", - context = { - cluster = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}", - user = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}", + }] + users = [{ + name = "yc-managed-k8s-user" + user = { + exec = { + apiVersion = "client.authentication.k8s.io/v1beta1", + command = local.cli_command, + args = local.cli_command_args, + provideClusterInfo = false + env = null } } - ], - current-context = "yc-managed-k8s-backends", - preferences = {}, - users = [ - { - name = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}", - user = { - exec = { - apiVersion = "client.authentication.k8s.io/v1beta1", - command = local.cli_command, - args = local.cli_command_args, - provideClusterInfo = false - } - } + }] + contexts = [{ + name = "yc-managed-k8s-ctx", + context = { + cluster = "${local.service}-managed-k8s", + user = "yc-managed-k8s-user", } - ] - } + }], + apiVersion = "v1", + kind = "Config", + preferences = {}, + }) } -data "shell_script" "kubeconfig" { - depends_on = [ - yandex_vpc_security_group.this - ] - - lifecycle_commands { - read = <<-CMD - set -euo pipefail - echo "$KUBECONFIG_DATA" > "$KUBECONFIG_FILE" - echo "{\"path\": \"${local.kubeconfig_path}\"}" - CMD - } - environment = { - KUBECONFIG_DATA = yamlencode(local.kubeconfig) - KUBECONFIG_FILE = local.kubeconfig_path - } +resource "local_file" "kubeconfig" { + content = local.kubeconfig + filename = local.kubeconfig_path + file_permission = "0600" } diff --git a/deploy/terraform/locals.tf b/deploy/terraform/locals.tf index 370c732d..5a092865 100644 --- a/deploy/terraform/locals.tf +++ b/deploy/terraform/locals.tf @@ -39,8 +39,11 @@ locals { # auto create github runner is_create_github_runner = true + k8s_monitoring = true + # use local k8s ipv4 by security reason + k8s_allow_from_public_net = true k8s_use_external_ipv4 = true - k8s_connect_by_internal_ipv4 = true + k8s_connect_by_internal_ipv4 = false } diff --git a/deploy/terraform/provider.tf b/deploy/terraform/provider.tf index 5c183b28..5ded0441 100644 --- a/deploy/terraform/provider.tf +++ b/deploy/terraform/provider.tf @@ -1,10 +1,6 @@ terraform { required_providers { - shell = { - source = "scottwinkler/shell" - version = "1.7.10" - } yandex = { source = "yandex-cloud/yandex" version = "0.106.0" # 23.01.2024 @@ -33,12 +29,16 @@ provider "yandex" { } provider "kubernetes" { - config_path = try(data.shell_script.kubeconfig.output["path"], "./kubeconfig.conf") + host = local.k8s_cluster_endpoint + cluster_ca_certificate = local.k8s_cluster_ca_certificate + token = var.YC_TOKEN } provider "helm" { kubernetes { - config_path = try(data.shell_script.kubeconfig.output["path"], "./kubeconfig.conf") + host = local.k8s_cluster_endpoint + cluster_ca_certificate = local.k8s_cluster_ca_certificate + token = var.YC_TOKEN } registry { diff --git a/deploy/terraform/vpc.tf b/deploy/terraform/vpc.tf index bd2c2a90..75407925 100644 --- a/deploy/terraform/vpc.tf +++ b/deploy/terraform/vpc.tf @@ -83,14 +83,15 @@ locals { { proto = "ANY", cidr_v4 = local.v4_k8s_cidr_blocks, from_port = 0, to_port = 65535, desc = "k8s" }, { proto = "ICMP", cidr_v4 = local.v4_icmp_cidr_blocks, from_port = 0, to_port = 65535, desc = "icmp" }, { proto = "TCP", target = "loadbalancer_healthchecks", from_port = 0, to_port = 65535, desc = "alb" }, - ], local.k8s_use_external_ipv4 && !local.k8s_connect_by_internal_ipv4 ? [{ proto = "TCP", cidr_v4 = ["${local.v4_public_ip}/32"], port = 443, desc = "deploy" }] : []) + ], local.k8s_use_external_ipv4 && !local.k8s_connect_by_internal_ipv4 ? [{ proto = "TCP", cidr_v4 = ["${local.v4_public_ip}/32"], port = 443, desc = "deploy" }] : [], + local.k8s_use_external_ipv4 && local.k8s_allow_from_public_net ? [{ proto = "TCP", cidr_v4 = ["0.0.0.0/0"], port = 443, desc = "public" }] : []) egress = concat( [ { proto = "ANY", target = "self_security_group", from_port = 0, to_port = 65535, desc = "self" }, { proto = "ANY", cidr_v4 = local.v4_subnets_cidr_blocks, from_port = 0, to_port = 65535, desc = "subnets" }, { proto = "ANY", cidr_v4 = local.v4_k8s_cidr_blocks, from_port = 0, to_port = 65535, desc = "k8s" }, { proto = "TCP", cidr_v4 = local.v4_gh_api_cidr_blocks, port = 443, desc = "github api" }, - { proto = "TCP", cidr_v4 = ["${data.dns_a_record_set.domain.addrs[0]}/32"], port = 443, desc = "domain" } + { proto = "TCP", cidr_v4 = ["${data.dns_a_record_set.domain.addrs[0]}/32"], port = 443, desc = "domain" }, ], [for e in local.endpoints : { proto = "TCP", cidr_v4 = ["${data.dns_a_record_set.this[e].addrs[0]}/32"], port = 443, desc = e }] ) }