Skip to content

Commit ec75f6c

Browse files
IshwarKanseIshwarKanse
authored
Add test cases for using GCP WIF with Tempo (#1208)
1 parent f90cee4 commit ec75f6c

20 files changed

+1110
-0
lines changed

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/chainsaw-test.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha1
2+
kind: Test
3+
metadata:
4+
name: gcp-wif-monolithic
5+
labels:
6+
type: gcp-wif
7+
spec:
8+
description: Test TempoStack support for GCP WIF using STS.
9+
namespace: chainsaw-gcpwif-mono
10+
steps:
11+
- name: Create AWS S3 bucket, IAM policy and role required for STS
12+
try:
13+
- script:
14+
timeout: 2m
15+
content: ./gcp-wif-create.sh
16+
- assert:
17+
file: gcp-wif-create-assert.yaml
18+
- name: Install Tempo Monolithic
19+
try:
20+
- apply:
21+
file: install-monolithic.yaml
22+
- assert:
23+
file: install-monolithic-assert.yaml
24+
- name: Wait for the TempoStack to be ready
25+
try:
26+
- script:
27+
timeout: 5m
28+
content: oc get --namespace chainsaw-gcpwif-mono tempomonolithic gcpwifmn -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep True
29+
- name: Generate traces
30+
try:
31+
- apply:
32+
file: generate-traces.yaml
33+
- assert:
34+
file: generate-traces-assert.yaml
35+
- name: Verify traces
36+
try:
37+
- apply:
38+
file: verify-traces.yaml
39+
- assert:
40+
file: verify-traces-assert.yaml
41+
cleanup:
42+
- script:
43+
timeout: 2m
44+
content: ./gcp-wif-delete.sh

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/gcp-wif-create-assert.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: gcs-secret
5+
namespace: chainsaw-gcpwif-mono
6+
type: Opaque

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/gcp-wif-create.sh‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
PROJECT_ID=$(gcloud config get-value project)
5+
SERVICE_ACCOUNT_NAME="ikanse-gcp-wif-mono-sa"
6+
GCS_KEY_FILE="/tmp/gcp-wif-mono.json"
7+
BUCKET_NAME="ikanse-gcp-wif-mono"
8+
9+
# Create GCP service account
10+
SERVICE_ACCOUNT_EMAIL=$(gcloud iam service-accounts create "$SERVICE_ACCOUNT_NAME" \
11+
--display-name="TempoMonolithic Account" \
12+
--project "$PROJECT_ID" \
13+
--format='value(email)' \
14+
--quiet)
15+
16+
# Wait for the service account to be ready
17+
echo "Waiting for service account $SERVICE_ACCOUNT_EMAIL to be ready..."
18+
MAX_RETRIES=10
19+
RETRY_COUNT=0
20+
while ! gcloud iam service-accounts describe "$SERVICE_ACCOUNT_EMAIL" --project "$PROJECT_ID" &> /dev/null; do
21+
if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
22+
echo "Error: Service account $SERVICE_ACCOUNT_EMAIL not found after $MAX_RETRIES retries. Exiting."
23+
exit 1
24+
fi
25+
echo "Service account not yet available. Retrying in 5 seconds..."
26+
sleep 5
27+
RETRY_COUNT=$((RETRY_COUNT + 1))
28+
done
29+
echo "Service account $SERVICE_ACCOUNT_EMAIL is ready."
30+
31+
# Set the GCP and TempoStack vars
32+
TEMPO_NAME="gcpwifmn"
33+
TEMPO_NAMESPACE="chainsaw-gcpwif-mono"
34+
PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format='value(projectNumber)')
35+
OIDC_ISSUER=$(oc get authentication.config cluster -o jsonpath='{.spec.serviceAccountIssuer}')
36+
POOL_ID=$(echo "$OIDC_ISSUER" | awk -F'/' '{print $NF}' | sed 's/-oidc$//')
37+
38+
# Bind the required GCP roles to the created SA at the project level
39+
gcloud projects add-iam-policy-binding "$PROJECT_ID" \
40+
--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \
41+
--role="roles/storage.objectAdmin" \
42+
--format=none \
43+
--quiet
44+
45+
# Workload Identity Bindings: Allow Kubernetes Service Accounts to impersonate the Google Service Account
46+
gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" \
47+
--role="roles/iam.workloadIdentityUser" \
48+
--member="principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:${TEMPO_NAMESPACE}:tempo-${TEMPO_NAME}" \
49+
--project="$PROJECT_ID" \
50+
--quiet
51+
52+
gcloud iam service-accounts add-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" \
53+
--role="roles/iam.workloadIdentityUser" \
54+
--member="principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:${TEMPO_NAMESPACE}:tempo-${TEMPO_NAME}-query-frontend" \
55+
--project="$PROJECT_ID" \
56+
--quiet
57+
58+
# Get provider ID from GCP
59+
PROVIDER_ID=$(gcloud iam workload-identity-pools providers list \
60+
--project="$PROJECT_ID" \
61+
--location="global" \
62+
--workload-identity-pool="$POOL_ID" \
63+
--filter="displayName:$POOL_ID" \
64+
--format="value(name)" | awk -F'/' '{print $NF}')
65+
66+
# Create a credentials configuration file for the managed identity to be used by TempoStack
67+
gcloud iam workload-identity-pools create-cred-config \
68+
"projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/providers/$PROVIDER_ID" \
69+
--service-account="$SERVICE_ACCOUNT_EMAIL" \
70+
--credential-source-file=/var/run/secrets/storage/serviceaccount/token \
71+
--credential-source-type=text \
72+
--output-file="$GCS_KEY_FILE"
73+
74+
echo "Checking if bucket $BUCKET_NAME exists..."
75+
if gsutil ls "gs://$BUCKET_NAME" > /dev/null 2>&1; then
76+
echo "Bucket $BUCKET_NAME found. Attempting to remove..."
77+
gcloud alpha storage rm --recursive "gs://$BUCKET_NAME"
78+
if [ $? -ne 0 ]; then
79+
echo "Failed to remove bucket $BUCKET_NAME."
80+
exit 1
81+
fi
82+
echo "Bucket $BUCKET_NAME removed successfully."
83+
else
84+
echo "Bucket $BUCKET_NAME does not exist (as expected). Proceeding to create."
85+
fi
86+
87+
echo "Waiting for the bucket to be confirmed deleted (if it was removed)."
88+
BUCKET_DELETION_RETRIES=6
89+
DELETE_RETRY_COUNT=0
90+
while gsutil ls "gs://$BUCKET_NAME" > /dev/null 2>&1 && [ $DELETE_RETRY_COUNT -lt $BUCKET_DELETION_RETRIES ]; do
91+
echo "Bucket $BUCKET_NAME still detected. Waiting 5 seconds for deletion..."
92+
sleep 5
93+
DELETE_RETRY_COUNT=$((DELETE_RETRY_COUNT + 1))
94+
done
95+
96+
if [ $DELETE_RETRY_COUNT -ge $BUCKET_DELETION_RETRIES ]; then
97+
echo "Warning: Bucket $BUCKET_NAME still exists after waiting period. This might cause issues for creation."
98+
fi
99+
100+
echo "Attempting to create a new bucket: gs://$BUCKET_NAME in us-central1..."
101+
gsutil mb -l us-central1 -p "$PROJECT_ID" "gs://$BUCKET_NAME"
102+
if [ $? -ne 0 ]; then
103+
echo "Failed to create bucket $BUCKET_NAME."
104+
exit 1
105+
fi
106+
echo "Bucket $BUCKET_NAME created successfully."
107+
108+
echo "Grant access to the bucket by service account."
109+
gcloud storage buckets add-iam-policy-binding "gs://$BUCKET_NAME" \
110+
--role="roles/storage.admin" \
111+
--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \
112+
--condition=None
113+
114+
PROVIDER_NAME="projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/providers/$PROVIDER_ID"
115+
AUDIENCE=$(gcloud iam workload-identity-pools providers describe "$PROVIDER_NAME" --format='value(oidc.allowedAudiences[0])')
116+
117+
# Create Kubernetes secret to be used with TempoStack
118+
kubectl -n "$TEMPO_NAMESPACE" create secret generic gcs-secret \
119+
--from-literal=bucketname="$BUCKET_NAME" \
120+
--from-literal=audience="$AUDIENCE" \
121+
--from-file=key.json="$GCS_KEY_FILE"
122+
if [ $? -ne 0 ]; then
123+
echo "Failed to create secret"
124+
exit 1
125+
fi

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/gcp-wif-delete.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
set -uo pipefail
3+
4+
PROJECT_ID=$(gcloud config get-value project)
5+
SERVICE_ACCOUNT_NAME="ikanse-gcp-wif-mono-sa"
6+
TEMPO_NAME="gcpwifmn"
7+
TEMPO_NAMESPACE="chainsaw-gcpwif-mono"
8+
PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format='value(projectNumber)')
9+
OIDC_ISSUER=$(oc get authentication.config cluster -o jsonpath='{.spec.serviceAccountIssuer}')
10+
POOL_ID=$(echo "$OIDC_ISSUER" | awk -F'/' '{print $NF}' | sed 's/-oidc$//')
11+
BUCKET_NAME="ikanse-gcp-wif-mono"
12+
GCS_KEY_FILE="/tmp/gcp-wif-mono.json"
13+
14+
# Fetch the service account email using the name
15+
SERVICE_ACCOUNT_EMAIL=$(gcloud iam service-accounts list \
16+
--project="$PROJECT_ID" \
17+
--filter="displayName:TempoMonolithic Account" \
18+
--format='value(email)')
19+
20+
if [ -z "$SERVICE_ACCOUNT_EMAIL" ]; then
21+
echo "Error: Service account with display name 'TempoStack Account' not found. Cannot proceed with cleanup."
22+
exit 1
23+
fi
24+
25+
echo "Starting cleanup (excluding Workload Identity Pool Provider)..."
26+
echo "Target Service Account Email: **$SERVICE_ACCOUNT_EMAIL**"
27+
28+
echo "Removing bucket-level IAM policy binding for service account '**$SERVICE_ACCOUNT_EMAIL**' on bucket '**$BUCKET_NAME**'..."
29+
gcloud storage buckets remove-iam-policy-binding "gs://$BUCKET_NAME" \
30+
--role="roles/storage.admin" \
31+
--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \
32+
--quiet
33+
echo "Bucket-level IAM policy binding removed."
34+
35+
# Remove IAM policy bindings for the created service account (project level)
36+
echo "Removing project-level IAM policy bindings for service account '**$SERVICE_ACCOUNT_EMAIL**'..."
37+
gcloud projects remove-iam-policy-binding "$PROJECT_ID" \
38+
--member="serviceAccount:$SERVICE_ACCOUNT_EMAIL" \
39+
--role="roles/storage.objectAdmin" --quiet
40+
echo "Project-level IAM policy bindings for service account '**$SERVICE_ACCOUNT_EMAIL**' removed."
41+
42+
# Remove IAM policy bindings for TempoStack service accounts (Workload Identity Principals)
43+
# These were added directly to the Google Service Account's IAM policy
44+
echo "Removing Workload Identity bindings from Google Service Account '**$SERVICE_ACCOUNT_EMAIL**'..."
45+
46+
# tempo-${TEMPO_NAME} bindings
47+
gcloud iam service-accounts remove-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" \
48+
--role="roles/iam.workloadIdentityUser" \
49+
--member="principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:${TEMPO_NAMESPACE}:tempo-${TEMPO_NAME}" \
50+
--project="$PROJECT_ID" \
51+
--quiet
52+
53+
# tempo-${TEMPO_NAME}-query-frontend bindings
54+
gcloud iam service-accounts remove-iam-policy-binding "$SERVICE_ACCOUNT_EMAIL" \
55+
--role="roles/iam.workloadIdentityUser" \
56+
--member="principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:${TEMPO_NAMESPACE}:tempo-${TEMPO_NAME}-query-frontend" \
57+
--project="$PROJECT_ID" \
58+
--quiet
59+
echo "Workload Identity bindings removed from Google Service Account '**$SERVICE_ACCOUNT_EMAIL**'."
60+
61+
# Delete the GCP service account
62+
echo "Deleting service account '**$SERVICE_ACCOUNT_EMAIL**'..."
63+
gcloud iam service-accounts delete "$SERVICE_ACCOUNT_EMAIL" --quiet
64+
if [ $? -eq 0 ]; then
65+
echo "Service account '**$SERVICE_ACCOUNT_EMAIL**' deleted successfully."
66+
else
67+
echo "Failed to delete service account '**$SERVICE_ACCOUNT_EMAIL**'."
68+
fi
69+
70+
# Remove GCS bucket
71+
echo "Deleting GCS bucket '**$BUCKET_NAME**'..."
72+
gcloud alpha storage rm --recursive gs://"$BUCKET_NAME" --quiet
73+
if [ $? -eq 0 ]; then
74+
echo "GCS bucket '**$BUCKET_NAME**' deleted successfully."
75+
else
76+
echo "Failed to delete GCS bucket '**$BUCKET_NAME**'."
77+
fi
78+
79+
echo "Deleting **gcs-secret** Kubernetes secret..."
80+
if kubectl get secret gcs-secret -n "$TEMPO_NAMESPACE" &> /dev/null; then
81+
kubectl delete secret gcs-secret -n "$TEMPO_NAMESPACE" --ignore-not-found=true
82+
echo "Kubernetes secret '**gcs-secret**' deleted."
83+
else
84+
echo "Kubernetes secret '**gcs-secret**' not found in namespace '**$TEMPO_NAMESPACE**'. Skipping deletion."
85+
fi
86+
87+
echo "Delete the GCS keyfile"
88+
rm "$GCS_KEY_FILE"
89+
90+
echo "Cleanup completed."

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/generate-traces-assert.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: generate-traces
5+
namespace: chainsaw-gcpwif-mono
6+
status:
7+
succeeded: 1

‎tests/e2e-openshift-object-stores/gcp-wif-monolithic/generate-traces.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: generate-traces
5+
namespace: chainsaw-gcpwif-mono
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: telemetrygen
11+
image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0
12+
args:
13+
- traces
14+
- --otlp-endpoint=tempo-gcpwifmn.chainsaw-gcpwif-mono.svc:4317
15+
- --otlp-insecure
16+
- --traces=10
17+
restartPolicy: Never
18+
backoffLimit: 4

0 commit comments

Comments
 (0)