Fix terraform deploy (#241)

datalens-tech · Nov 13, 2024 · 42dc67e · 42dc67e
1 parent 23544f3
commit 42dc67e
Show file tree

Hide file tree

Showing 16 changed files with 76 additions and 137 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -31,8 +31,6 @@ jobs:
       - uses: arduino/setup-task@v2
       - uses: nightstory/setup-yc@v1
       - run: |
-          ../scripts/tofu-opensource.sh --tofurc --cleanup --init --target data.shell_script.kubeconfig --apply --approve
-          ls -la ./kubeconfig.conf && sleep 10
           ../scripts/tofu-opensource.sh --tofurc --cleanup --init --apply --approve
         working-directory: ./deploy/terraform
         env:

diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml
@@ -55,6 +55,10 @@ jobs:
           CONTROL_API_VERSION: ${{ env.CONTROL_API_VERSION }}
           DATA_API_VERSION: ${{ env.DATA_API_VERSION }}
           UI_VERSION: ${{ env.UI_VERSION }}
+          E2E_RETRY_TIMES: 2
+          E2E_TEST_TIMEOUT: 90000
+          E2E_ACTION_TIMEOUT: 10000
+          E2E_EXPECT_TIMEOUT: 10000
 
       - name: Copy tests report from docker run
         if: always()

diff --git a/deploy/scripts/tofu-opensource.sh b/deploy/scripts/tofu-opensource.sh
@@ -151,10 +151,10 @@ if [ ! "${YC_PROFILE_EXISTS}" == "true" ]; then
   fi
 fi
 
-SERVICE_ACCOUNT_ID=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key service-account-id)
+SERVICE_ACCOUNT_ID=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key service-account-id)
 
-export AWS_ACCESS_KEY_ID=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key access-key)
-export AWS_SECRET_ACCESS_KEY=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key secret-key)
+export AWS_ACCESS_KEY_ID=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key access-key)
+export AWS_SECRET_ACCESS_KEY=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key secret-key)
 export AWS_ENDPOINT_URL_S3="${STORAGE_ENDPOINT}"
 export YC_STORAGE_ACCESS_KEY="${AWS_ACCESS_KEY_ID}"
 export YC_STORAGE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY}"
@@ -164,9 +164,9 @@ if [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then
   exit 1
 fi
 
-export BACKEND_STATE_BUCKET=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-bucket)
-export BACKEND_STATE_KEY=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-key)
-export BACKEND_STATE_REGION=$(yc --profile=${PROFILE_NAME} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-region)
+export BACKEND_STATE_BUCKET=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-bucket)
+export BACKEND_STATE_KEY=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-key)
+export BACKEND_STATE_REGION=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} lockbox payload get --name "${LOCKBOX_NAME}" --key backend-state-region)
 
 export TF_VAR_PROFILE="${PROFILE_NAME}"
 
@@ -183,7 +183,7 @@ export TF_VAR_BACKEND_STATE_REGION="${BACKEND_STATE_REGION}"
 
 export TF_VAR_DOMAIN="${DOMAIN}"
 
-export TF_VAR_YC_TOKEN=$(yc --profile=${PROFILE_NAME} iam create-token --impersonate-service-account-id "${SERVICE_ACCOUNT_ID}")
+export TF_VAR_YC_TOKEN=$(yc --profile=${PROFILE_NAME} --folder-id=${FOLDER_ID} iam create-token --impersonate-service-account-id "${SERVICE_ACCOUNT_ID}")
 
 if [ -z "${TF_VAR_YC_TOKEN}" ]; then
   echo "❌ error obtain iam token for sa '${SERVICE_ACCOUNT_ID}', check profile settings, exit..."

diff --git a/deploy/terraform/k8s-alb-ingress.tf b/deploy/terraform/k8s-alb-ingress.tf
@@ -30,11 +30,6 @@ resource "helm_release" "alb_ingress" {
       private_key        = yandex_iam_service_account_key.this["alb"].private_key
     })
   })]
-
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 locals {

diff --git a/deploy/terraform/k8s-app-control-api.tf b/deploy/terraform/k8s-app-control-api.tf
@@ -59,7 +59,7 @@ resource "kubernetes_deployment" "control-api" {
       }
       spec {
         container {
-          image = "ghcr.io/datalens-tech/datalens-control-api:${local.data_api_version}"
+          image = "ghcr.io/datalens-tech/datalens-control-api:${local.control_api_version}"
           name  = "app-control-api"
           port {
             container_port = 8080
@@ -163,10 +163,6 @@ resource "kubernetes_deployment" "control-api" {
       }
     }
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_service" "control-api_service" {
@@ -185,8 +181,4 @@ resource "kubernetes_service" "control-api_service" {
     }
     type = "ClusterIP"
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
diff --git a/deploy/terraform/k8s-app-data-api.tf b/deploy/terraform/k8s-app-data-api.tf
@@ -195,10 +195,6 @@ resource "kubernetes_deployment" "data-api" {
       }
     }
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_service" "data-api_service" {
@@ -217,8 +213,4 @@ resource "kubernetes_service" "data-api_service" {
     }
     type = "ClusterIP"
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
diff --git a/deploy/terraform/k8s-app-ui.tf b/deploy/terraform/k8s-app-ui.tf
@@ -164,10 +164,6 @@ resource "kubernetes_deployment" "ui" {
       }
     }
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_service" "ui_service" {
@@ -187,8 +183,4 @@ resource "kubernetes_service" "ui_service" {
     }
     type = "NodePort"
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
diff --git a/deploy/terraform/k8s-app-us.tf b/deploy/terraform/k8s-app-us.tf
@@ -157,10 +157,6 @@ resource "kubernetes_deployment" "us" {
       }
     }
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_service" "us_service" {
@@ -179,8 +175,4 @@ resource "kubernetes_service" "us_service" {
     }
     type = "ClusterIP"
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
diff --git a/deploy/terraform/k8s-app-zitadel.tf b/deploy/terraform/k8s-app-zitadel.tf
@@ -231,10 +231,6 @@ resource "kubernetes_deployment" "zitadel" {
       }
     }
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_job" "zitadel_init_job" {
@@ -283,10 +279,6 @@ resource "kubernetes_job" "zitadel_init_job" {
     backoff_limit           = 5
     active_deadline_seconds = 600
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_job" "zitadel_setup_job" {
@@ -335,10 +327,6 @@ resource "kubernetes_job" "zitadel_setup_job" {
     backoff_limit           = 5
     active_deadline_seconds = 600
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_service" "zitadel_service" {
@@ -360,8 +348,4 @@ resource "kubernetes_service" "zitadel_service" {
     }
     type = "NodePort"
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
diff --git a/deploy/terraform/k8s-logs.tf b/deploy/terraform/k8s-logs.tf
@@ -38,9 +38,5 @@ resource "helm_release" "logging" {
       # fix helm chart error with missed escaping json
     }), ",", "\\,"), "\\n", "\\\\n"), "{", "\\{"), "}", "\\}")
   }
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
diff --git a/deploy/terraform/k8s-monitoring.tf b/deploy/terraform/k8s-monitoring.tf
@@ -0,0 +1,19 @@
+resource "helm_release" "monitoring" {
+  for_each = toset(local.k8s_monitoring ? ["main"] : [])
+
+  name = "monitoring"
+
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "kube-prometheus-stack"
+  version    = "62.7.0"
+
+  timeout          = 240
+  namespace        = "monitoring"
+  create_namespace = true
+  cleanup_on_fail  = true
+
+  set {
+    name  = "installCRDs"
+    value = true
+  }
+}
diff --git a/deploy/terraform/k8s-secrets.tf b/deploy/terraform/k8s-secrets.tf
@@ -9,10 +9,6 @@ resource "helm_release" "secrets" {
 
   version    = "0.9.20"
   repository = "oci://${local.cr_endpoint}/yc-marketplace/yandex-cloud/external-secrets/chart"
-
-  depends_on = [
-    data.shell_script.kubeconfig,
-  ]
 }
 
 resource "kubernetes_secret" "secrets" {
@@ -36,7 +32,6 @@ resource "kubernetes_secret" "secrets" {
 
   depends_on = [
     helm_release.secrets,
-    data.shell_script.kubeconfig,
   ]
 }
 
@@ -65,7 +60,6 @@ resource "kubernetes_manifest" "secrets" {
   depends_on = [
     helm_release.secrets,
     kubernetes_secret.secrets,
-    data.shell_script.kubeconfig,
   ]
 }
 
@@ -179,7 +173,6 @@ resource "kubernetes_manifest" "lockbox" {
   depends_on = [
     helm_release.secrets,
     kubernetes_secret.secrets,
-    data.shell_script.kubeconfig,
   ]
 }
 
@@ -224,7 +217,6 @@ resource "kubernetes_manifest" "lockbox-zitadel" {
   depends_on = [
     helm_release.secrets,
     kubernetes_secret.secrets,
-    data.shell_script.kubeconfig,
   ]
 }
 

diff --git a/deploy/terraform/kubeconfig.tf b/deploy/terraform/kubeconfig.tf
@@ -1,7 +1,3 @@
-provider "shell" {
-  interpreter = ["/usr/bin/env", "bash", "-c"]
-}
-
 locals {
   cluster_endpoint   = local.k8s_cluster_endpoint
   cluster_trusted_ca = base64encode(local.k8s_cluster_ca_certificate)
@@ -13,59 +9,42 @@ locals {
 }
 
 locals {
-  kubeconfig = {
-    apiVersion = "v1",
-    kind       = "Config"
-    clusters = [
-      {
-        name = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}",
-        cluster = {
-          certificate-authority-data = local.cluster_trusted_ca
-          server                     = local.cluster_endpoint
-        }
+  kubeconfig = yamlencode({
+    apiVersion = "v1"
+    clusters = [{
+      name = "${local.service}-managed-k8s"
+      cluster = {
+        "server"                     = local.cluster_endpoint
+        "certificate-authority-data" = local.cluster_trusted_ca
       }
-    ],
-    contexts = [
-      {
-        name = "yc-managed-k8s-backends",
-        context = {
-          cluster = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}",
-          user    = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}",
+    }]
+    users = [{
+      name = "yc-managed-k8s-user"
+      user = {
+        exec = {
+          apiVersion         = "client.authentication.k8s.io/v1beta1",
+          command            = local.cli_command,
+          args               = local.cli_command_args,
+          provideClusterInfo = false
+          env                = null
         }
       }
-    ],
-    current-context = "yc-managed-k8s-backends",
-    preferences     = {},
-    users = [
-      {
-        name = "yc-managed-k8s-${yandex_kubernetes_cluster.this.id}",
-        user = {
-          exec = {
-            apiVersion         = "client.authentication.k8s.io/v1beta1",
-            command            = local.cli_command,
-            args               = local.cli_command_args,
-            provideClusterInfo = false
-          }
-        }
+    }]
+    contexts = [{
+      name = "yc-managed-k8s-ctx",
+      context = {
+        cluster = "${local.service}-managed-k8s",
+        user    = "yc-managed-k8s-user",
       }
-    ]
-  }
+    }],
+    apiVersion  = "v1",
+    kind        = "Config",
+    preferences = {},
+  })
 }
 
-data "shell_script" "kubeconfig" {
-  depends_on = [
-    yandex_vpc_security_group.this
-  ]
-
-  lifecycle_commands {
-    read = <<-CMD
-    set -euo pipefail
-    echo "$KUBECONFIG_DATA" > "$KUBECONFIG_FILE"
-    echo "{\"path\": \"${local.kubeconfig_path}\"}"
-    CMD
-  }
-  environment = {
-    KUBECONFIG_DATA = yamlencode(local.kubeconfig)
-    KUBECONFIG_FILE = local.kubeconfig_path
-  }
+resource "local_file" "kubeconfig" {
+  content         = local.kubeconfig
+  filename        = local.kubeconfig_path
+  file_permission = "0600"
 }
diff --git a/deploy/terraform/locals.tf b/deploy/terraform/locals.tf
@@ -39,8 +39,11 @@ locals {
   # auto create github runner
   is_create_github_runner = true
 
+  k8s_monitoring = true
+
   # use local k8s ipv4 by security reason
+  k8s_allow_from_public_net    = true
   k8s_use_external_ipv4        = true
-  k8s_connect_by_internal_ipv4 = true
+  k8s_connect_by_internal_ipv4 = false
 }