Skip to content

Commit b0778ca

Browse files
vprashar2929vprashar2929
authored
Add support to validate Logs on CI (#462)
1 parent 25cffa4 commit b0778ca

23 files changed

+328
-92
lines changed

‎.circleci/config.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ jobs:
5858
command: |
5959
echo $KUBECONFIG
6060
cd tests/ci
61-
./ci_test.sh metrics
61+
./ci_test.sh deploy
62+
./ci_test.sh tests
6263
- store_artifacts:
6364
path: /tmp/artifacts
6465
workflows:

‎resources/services/jaeger-template.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ objects:
320320
- ReadWriteOnce
321321
resources:
322322
requests:
323-
storage: 50Gi
323+
storage: ${JAEGER_PVC_STORAGE}
324324
storageClassName: ${STORAGE_CLASS}
325325
parameters:
326326
- name: NAMESPACE
@@ -357,3 +357,5 @@ parameters:
357357
value: prometheus-telemeter
358358
- name: STORAGE_CLASS
359359
value: gp2
360+
- name: JAEGER_PVC_STORAGE
361+
value: 50Gi

‎resources/services/observatorium-logs-template.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -775,6 +775,8 @@ objects:
775775
"max_join_retries": 10
776776
"min_join_backoff": "1s"
777777
"querier":
778+
"engine":
779+
"max_look_back_period": "5m"
778780
"max_concurrent": ${LOKI_QUERIER_MAX_CONCURRENCY}
779781
"query_ingesters_within": "2h"
780782
"query_range":

‎services/dex-template.jsonnet

+10
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,15 @@ local dex = (import 'github.com/observatorium/observatorium/configuration/compon
7878
},
7979
},
8080
},
81+
pvc+: {
82+
spec+: {
83+
resources: {
84+
requests: {
85+
storage: '${DEX_STORAGE}',
86+
},
87+
},
88+
},
89+
},
8190
};
8291

8392
{
@@ -103,5 +112,6 @@ local dex = (import 'github.com/observatorium/observatorium/configuration/compon
103112
{ name: 'DEX_MEMORY_REQUEST', value: '200Mi' },
104113
{ name: 'DEX_CPU_LIMITS', value: '100m' },
105114
{ name: 'DEX_MEMORY_LIMITS', value: '200Mi' },
115+
{ name: 'DEX_STORAGE', value: '1Gi' },
106116
],
107117
}

‎services/jaeger-template.jsonnet

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ local jaeger = (import 'components/jaeger-collector.libsonnet')({
66
image:: '${IMAGE}:${IMAGE_TAG}',
77
version:: '${IMAGE_TAG}',
88
replicas: 1,
9-
pvc+:: { class: '${STORAGE_CLASS}' },
9+
pvc+:: { class: '${STORAGE_CLASS}', size: '${JAEGER_PVC_STORAGE}' },
1010
serviceMonitor: true,
1111
}) + {
1212
local j = self,
@@ -147,5 +147,6 @@ local jaeger = (import 'components/jaeger-collector.libsonnet')({
147147
{ name: 'OAUTH_PROXY_MEMORY_LIMITS', value: '200Mi' },
148148
{ name: 'SERVICE_ACCOUNT_NAME', value: 'prometheus-telemeter' },
149149
{ name: 'STORAGE_CLASS', value: 'gp2' },
150+
{ name: 'JAEGER_PVC_STORAGE', value: '50Gi' },
150151
],
151152
}

‎services/minio-template.jsonnet

+11-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ local minio = (import 'github.com/observatorium/observatorium/configuration/comp
55
version:: '${IMAGE_TAG}',
66
accessKey:: '${MINIO_ACCESS_KEY}',
77
secretKey:: '${MINIO_SECRET_KEY}',
8-
buckets:: ['thanos'],
8+
buckets:: ['thanos', 'loki', 'rules'],
99
replicas: 1,
1010
}) + {
1111
deployment+: {
@@ -31,6 +31,15 @@ local minio = (import 'github.com/observatorium/observatorium/configuration/comp
3131
},
3232
},
3333
},
34+
pvc+: {
35+
spec+: {
36+
resources: {
37+
requests: {
38+
storage: '${MINIO_STORAGE}',
39+
},
40+
},
41+
},
42+
},
3443
};
3544

3645
{
@@ -58,5 +67,6 @@ local minio = (import 'github.com/observatorium/observatorium/configuration/comp
5867
{ name: 'MINIO_MEMORY_LIMITS', value: '200Mi' },
5968
{ name: 'MINIO_ACCESS_KEY', value: 'minio' },
6069
{ name: 'MINIO_SECRET_KEY', value: 'minio123' },
70+
{ name: 'MINIO_STORAGE', value: '10Gi' },
6171
],
6272
}

‎services/observatorium-logs.libsonnet

+5
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,11 @@ local lokiCaches = (import 'components/loki-caches.libsonnet');
270270
tracing: {
271271
enabled: true,
272272
},
273+
querier+: {
274+
engine+: {
275+
max_look_back_period: '5m',
276+
},
277+
},
273278
},
274279
}),
275280
}

‎tests/ci/ci_test.sh

+54-64
Original file line numberDiff line numberDiff line change
@@ -14,92 +14,90 @@ prereq() {
1414
oc apply -f pre-requisites.yaml
1515
oc create -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/bundle.yaml
1616
oc process -f ../../resources/crds/observatorium-logs-crds-template.yaml | oc apply -f -
17+
}
1718

19+
ns() {
20+
oc create ns minio || true
21+
oc create ns dex || true
22+
oc create ns observatorium-metrics || true
23+
oc create ns observatorium || true
24+
oc create ns telemeter || true
25+
oc create ns observatorium-logs || true
26+
oc create ns observatorium-mst || true
1827
}
1928

2029
minio() {
21-
oc create ns minio || true
2230
oc wait --for=jsonpath='{.status.phase}=Active' namespace/minio --timeout=5s
23-
oc process -f ../minio-template.yaml \
24-
-p MINIO_CPU_REQUEST=30m \
25-
-p MINIO_CPU_LIMITS=50m \
26-
-p MINIO_MEMORY_REQUEST=50Mi \
27-
-p MINIO_MEMORY_LIMITS=100Mi \
28-
--local -o yaml | \
29-
sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | \
31+
oc process --param-file=minio.test.ci.env -f ../minio-template.yaml | \
3032
oc apply -n minio -f -
31-
check_status deployment/minio minio
3233
}
3334

3435
dex() {
35-
oc create ns dex || true
3636
oc wait --for=jsonpath='{.status.phase}=Active' namespace/dex --timeout=5s
37-
oc process -f ../dex-template.yaml \
38-
-p DEX_CPU_REQUEST=30m \
39-
-p DEX_CPU_LIMITS=50m \
40-
-p DEX_MEMORY_REQUEST=50Mi \
41-
-p DEX_MEMORY_LIMITS=100Mi \
42-
--local -o yaml | \
43-
sed -e 's/storage: [0-9].Gi/storage: 0.25Gi/g' | \
37+
oc process --param-file=dex.test.ci.env -f ../dex-template.yaml | \
4438
oc apply -n dex -f -
45-
check_status deployment/dex dex
4639
}
4740

4841
observatorium_metrics() {
49-
oc create ns observatorium-metrics || true
5042
oc wait --for=jsonpath='{.status.phase}=Active' namespace/observatorium-metrics --timeout=5s
51-
oc process -f ../observatorium-metrics-thanos-objectstorage-secret-template.yaml | oc apply --namespace observatorium-metrics -f -
43+
oc process -f ../observatorium-metrics-thanos-objectstorage-secret-template.yaml | \
44+
oc apply --namespace observatorium-metrics -f -
5245
oc apply -f ../observatorium-alertmanager-config-secret.yaml --namespace observatorium-metrics
5346
oc apply -f ../observatorium-cluster-role.yaml
5447
oc apply -f ../observatorium-cluster-role-binding.yaml
5548
oc apply --namespace observatorium-metrics -f ../observatorium-service-account.yaml
5649
oc process --param-file=observatorium-metrics.ci.env \
5750
-f ../../resources/services/observatorium-metrics-template.yaml | \
5851
oc apply --namespace observatorium-metrics -f -
59-
resources=$(
60-
oc get statefulsets -o name -n observatorium-metrics
61-
oc get deployments -o name -n observatorium-metrics
62-
)
63-
for res in $resources; do
64-
check_status $res observatorium-metrics
65-
done
52+
oc process --param-file=observatorium-metric-federation-rule.test.ci.env \
53+
-f ../../resources/services/metric-federation-rule-template.yaml| \
54+
oc apply --namespace observatorium-metrics -f -
6655
}
6756

6857
observatorium() {
69-
oc create ns observatorium || true
7058
oc wait --for=jsonpath='{.status.phase}=Active' namespace/observatorium --timeout=5s
7159
oc apply -f ../observatorium-rules-objstore-secret.yaml --namespace observatorium
7260
oc apply -f ../observatorium-rhobs-tenant-secret.yaml --namespace observatorium
61+
oc apply --namespace observatorium -f ../observatorium-service-account.yaml
62+
oc apply -f ../observatorium-parca-secret.yaml --namespace observatorium
63+
oc process -f ../../resources/services/parca-observatorium-remote-ns-rbac-template.yaml | \
64+
oc apply -f -
7365
oc process --param-file=observatorium.test.ci.env \
7466
-f ../../resources/services/observatorium-template.yaml | \
7567
oc apply --namespace observatorium -f -
76-
resources=$(
77-
oc get statefulsets -o name -n observatorium
78-
oc get deployments -o name -n observatorium
79-
)
80-
for res in $resources; do
81-
check_status $res observatorium
82-
done
83-
68+
oc process --param-file=observatorium-parca.test.ci.env \
69+
-f ../../resources/services/parca-template.yaml| \
70+
oc apply --namespace observatorium -f -
71+
oc process --param-file=observatorium-jaeger.test.ci.env \
72+
-f ../../resources/services/jaeger-template.yaml| \
73+
oc apply --namespace observatorium -f -
8474
}
8575

8676
telemeter() {
87-
oc create ns telemeter || true
8877
oc wait --for=jsonpath='{.status.phase}=Active' namespace/telemeter --timeout=5s
8978
oc apply --namespace telemeter -f ../telemeter-token-refersher-oidc-secret.yaml
9079
oc process --param-file=telemeter.ci.env \
9180
-f ../../resources/services/telemeter-template.yaml | \
9281
oc apply --namespace telemeter -f -
93-
resources=$(
94-
oc get statefulsets -o name -n telemeter
95-
oc get deployments -o name -n telemeter
96-
)
97-
for res in $resources; do
98-
check_status $res telemeter
99-
done
82+
}
83+
84+
observatorium_logs() {
85+
oc apply --namespace observatorium-logs -f ../observatorium-logs-secret.yaml
86+
oc process --param-file=observatorium-logs.test.ci.env -f \
87+
../../resources/services/observatorium-logs-template.yaml | \
88+
oc apply --namespace observatorium-logs -f -
10089
}
10190

10291
run_test() {
92+
for namespace in minio dex observatorium observatorium-metrics observatorium-logs telemeter ; do
93+
resources=$(
94+
oc get statefulsets -o name -n $namespace
95+
oc get deployments -o name -n $namespace
96+
)
97+
for res in $resources; do
98+
check_status $res $namespace
99+
done
100+
done
103101
oc apply -n observatorium -f test-tenant.yaml
104102
oc apply -n observatorium -f rbac.yaml
105103
oc rollout restart deployment/observatorium-observatorium-api -n observatorium
@@ -110,14 +108,12 @@ run_test() {
110108
must_gather "$ARTIFACT_DIR"
111109
exit 1
112110
}
113-
for namespace in minio dex observatorium observatorium-metrics telemeter; do
114-
115-
out=$(oc get pods -n $namespace -o jsonpath='{.items[?(@.status.containerStatuses[].restartCount>=3)].metadata.name}')
116-
if [ -n "$out" ]; then
117-
must_gather "$ARTIFACT_DIR"
118-
exit 1
119-
fi
120-
done
111+
oc apply -n observatorium-logs -f observatorium-up-logs.yaml
112+
oc wait --for=condition=complete --timeout=5m \
113+
-n observatorium-logs job/observatorium-up-logs || {
114+
must_gather "$ARTIFACT_DIR"
115+
exit 1
116+
}
121117
}
122118

123119
must_gather() {
@@ -148,24 +144,19 @@ must_gather() {
148144
oc get endpoints --all-namespaces > "$artifact_dir/endpoints"
149145
}
150146

151-
ci.metrics() {
147+
ci.deploy() {
152148
prereq
149+
ns
153150
minio
154151
dex
155-
observatorium_metrics
156152
observatorium
157-
run_test
153+
observatorium_metrics
158154
telemeter
155+
observatorium_logs
159156
}
160157

161-
ci.logs() {
162-
#TODO
163-
:
164-
}
165-
166-
ci.traces(){
167-
#TODO
168-
:
158+
ci.tests() {
159+
run_test
169160
}
170161

171162
ci.help() {
@@ -198,4 +189,3 @@ main() {
198189
return $?
199190
}
200191
main "$@"
201-

‎tests/ci/dex.test.ci.env

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
DEX_CPU_REQUEST=30m
2+
DEX_CPU_LIMITS=50m
3+
DEX_MEMORY_REQUEST=50Mi
4+
DEX_MEMORY_LIMITS=100Mi
5+
DEX_STORAGE=0.25Gi

‎tests/ci/minio.test.ci.env

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
MINIO_CPU_REQUEST=30m
2+
MINIO_CPU_LIMITS=50m
3+
MINIO_MEMORY_REQUEST=50Mi
4+
MINIO_MEMORY_LIMITS=100Mi
5+
MINIO_STORAGE=0.25Gi

‎tests/ci/observatorium-jaeger.test.ci.env

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
JAEGER_CPU_REQUEST=25m
2+
JAEGER_MEMORY_REQUEST=50Mi
3+
JAEGER_CPU_LIMITS=50m
4+
JAEGER_MEMORY_LIMITS=50Mi
5+
OAUTH_PROXY_CPU_REQUEST=25m
6+
OAUTH_PROXY_MEMORY_REQUEST=50Mi
7+
OAUTH_PROXY_CPU_LIMITS=50m
8+
OAUTH_PROXY_MEMORY_LIMITS=50Mi
9+
SERVICE_ACCOUNT_NAME=observatorium
10+
STORAGE_CLASS=kubevirt-hostpath-provisioner
11+
JAEGER_PVC_STORAGE=0.25Gi

0 commit comments

Comments
 (0)