[ci][terraform][cloud] tags/labels with metadata (#792)

.ci/Jenkinsfile

@@ -5,8 +5,10 @@
 pipeline {
   agent { label 'ubuntu-20 && immutable' }
   environment {
+    BRANCH_NAME_LOWER_CASE = "${env.BRANCH_NAME.toLowerCase()}"
+    CREATED_DATE = "${new Date().getTime()}"
+    ENVIRONMENT = "ci"
     REPO = "elastic-package"
-
     BASE_DIR="src/github.com/elastic/elastic-package"
     JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
     GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7"

docs/howto/system_testing.md

@@ -173,11 +173,13 @@ Injecting credentials can be achieved with functions from the [`apm-pipeline-lib
 
 Leveraging Terraform to create cloud resources is useful but risks creating leftover resources that are difficult to remove.
 
-There are 4 environment variables that should be leveraged to overcome this issue; these variables are already injected to be used by Terraform (through `TF_VAR_`):
+There are some specific environment variables that should be leveraged to overcome this issue; these variables are already injected to be used by Terraform (through `TF_VAR_`):
 - `TF_VAR_TEST_RUN_ID`: a unique identifier for the test run, allows to distinguish each run
-- `REPO_NAME`: the repository name the CI run is linked to
-- `CHANGE_ID`: the PR number the CI run is linked to
-- `BUILD_NUMBER`: incremental number providing the current CI run number
+- `BRANCH_NAME_LOWER_CASE`: the branch name or PR number the CI run is linked to
+- `BUILD_ID`: incremental number providing the current CI run number
+- `CREATED_DATE`: the creation date in epoch time, milliseconds, when the resource was created
+- `ENVIRONMENT`: what environment created the resource (`ci`)
+- `REPO`: the GitHub repository name (`elastic-package`)
 
 ### Kubernetes service deployer
 

internal/install/_static/terraform_deployer.yml

@@ -6,8 +6,10 @@ services:
     stop_grace_period: 5m
     environment:
       - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached}
-      - TF_VAR_REPO_NAME=${REPO_NAME:-unknown}
-      - TF_VAR_PULL_REQUEST=pr-${CHANGE_ID:-unknown}
-      - TF_VAR_CI_BUILD_NUMBER=${BUILD_NUMBER:-unknown}
+      - TF_VAR_CREATED_DATE=${CREATED_DATE:-unknown}
+      - TF_VAR_BRANCH=${BRANCH_NAME_LOWER_CASE:-unknown}
+      - TF_VAR_BUILD_ID=${BUILD_ID:-unknown}
+      - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown}
+      - TF_VAR_REPO=${REPO:-unknown}
     volumes:
       - ${TF_DIR}:/stage

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf

@@ -2,7 +2,17 @@ variable "TEST_RUN_ID" {
   default = "detached"
 }
 
-provider "aws" {}
+provider "aws" {
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
+  }
+}
 
 resource "aws_instance" "i" {
   ami           = data.aws_ami.latest-amzn.id
@@ -20,4 +30,4 @@ data "aws_ami" "latest-amzn" {
     name   = "name"
     values = ["amzn2-ami-minimal-hvm-*-ebs"]
   }
-}
+}

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf

@@ -0,0 +1,22 @@
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/compute.tf

@@ -6,17 +6,18 @@ data "google_compute_image" "default" {
 
 resource "google_compute_instance" "default" {
   name = "elastic-package-system-test-${var.TEST_RUN_ID}"
-  // NOTE: e2 instance type is required to collect instance/memory/balloon/* 
+  // NOTE: e2 instance type is required to collect instance/memory/balloon/*
   // metrics, available only on those instances.
   // https://cloud.google.com/monitoring/api/metrics_gcp
   machine_type = "e2-micro"
   zone         = var.zone
 
   labels = {
-    run_id = var.TEST_RUN_ID
-    repo_name = var.REPO_NAME
-    pull_request = var.PULL_REQUEST
-    ci_build_number = var.CI_BUILD_NUMBER
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = var.CREATED_DATE
   }
 
   boot_disk {

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/vars.tf

@@ -2,16 +2,27 @@ variable "TEST_RUN_ID" {
   default = "detached"
 }
 
-variable "REPO_NAME" {
-  default = "unknown-repo"
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default = "unknown-branch"
 }
 
-variable "PULL_REQUEST" {
-  default = "unknown-pr"
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default = "unknown-build"
 }
 
-variable "CI_BUILD_NUMBER" {
-  default = "unknown-build"
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo"
 }
 
 variable "gcp_project_id" {