feat: add Prometheus

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,4 +3,14 @@
 ## Why
 
 ## Notes
-<!-- Add any notes here -->
+<!-- Add any notes here -->
+
+## Labels
+
+Assign the following labels to the PR:
+
+## PR Comments
+
+Add the following comments to the PR:
+
+`/e2e` - to trigger E2E build

charts/gitops-runtime/Chart.yaml

@@ -22,6 +22,8 @@ annotations:
       description: "fix default version config"
     - kind: fixed
       description: "support better concurrency in argocd session manager"
+    - kind: added
+      description: "Add optional Prometheus subchart"
 dependencies:
 - name: argo-cd
   repository: https://codefresh-io.github.io/argo-helm
@@ -50,3 +52,7 @@ dependencies:
   version: 1.0.4
   alias: gitops-operator
   condition: gitops-operator.enabled
+- name: prometheus
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 0.12.1
+  condition: prometheus.enabled

charts/gitops-runtime/README.md

@@ -141,21 +141,41 @@ sealed-secrets:
 | app-proxy.serviceAccount.create | bool | `true` |  |
 | app-proxy.serviceAccount.name | string | `"cap-app-proxy"` |  |
 | app-proxy.tolerations | list | `[]` |  |
+| argo-cd.applicationSet.metrics.enabled | bool | `true` |  |
+| argo-cd.applicationSet.podAnnotations."prometheus.io/port" | string | `"8080"` |  |
 | argo-cd.applicationVersioning.enabled | bool | `true` | Enable application versioning |
 | argo-cd.applicationVersioning.useApplicationConfiguration | bool | `true` | Extract application version based on ApplicationConfiguration CRD |
 | argo-cd.configs.cm."accounts.admin" | string | `"apiKey,login"` |  |
 | argo-cd.configs.cm."application.resourceTrackingMethod" | string | `"annotation+label"` |  |
 | argo-cd.configs.cm."timeout.reconciliation" | string | `"20s"` |  |
 | argo-cd.configs.params."application.namespaces" | string | `"cf-*"` |  |
 | argo-cd.configs.params."server.insecure" | bool | `true` |  |
+| argo-cd.controller.metrics.enabled | bool | `true` |  |
+| argo-cd.controller.podAnnotations."prometheus.io/port" | string | `"8082"` |  |
 | argo-cd.crds.install | bool | `true` |  |
+| argo-cd.dex.metrics.enabled | bool | `true` |  |
+| argo-cd.dex.podAnnotations."prometheus.io/port" | string | `"5558"` |  |
 | argo-cd.eventReporter.enabled | bool | `true` | Installs new event reporter component to cluster |
+| argo-cd.eventReporter.metrics.enabled | bool | `true` |  |
+| argo-cd.eventReporter.podAnnotations."prometheus.io/port" | string | `"8087"` |  |
 | argo-cd.eventReporter.replicas | int | `3` | Amount of shards to handle applications events |
 | argo-cd.eventReporter.version | string | `"v2"` | Switches between old and new reporter version. Possible values: v1, v2. For v2 `argo-cd.eventReporter.enabled=true` is required |
 | argo-cd.fullnameOverride | string | `"argo-cd"` |  |
+| argo-cd.global.podAnnotations."prometheus.io/path" | string | `"/metrics"` |  |
+| argo-cd.global.podAnnotations."prometheus.io/scrape" | string | `"true"` |  |
 | argo-cd.notifications | object | `{}` |  |
+| argo-cd.redis.exporter.enabled | bool | `true` |  |
+| argo-cd.redis.metrics.enabled | bool | `true` |  |
+| argo-cd.redis.podAnnotations."prometheus.io/port" | string | `"9121"` |  |
+| argo-cd.repoServer.metrics.enabled | bool | `true` |  |
+| argo-cd.repoServer.podAnnotations."prometheus.io/port" | string | `"8084"` |  |
+| argo-cd.server.metrics.enabled | bool | `true` |  |
+| argo-cd.server.podAnnotations."prometheus.io/port" | string | `"8083"` |  |
 | argo-events.crds.install | bool | `false` |  |
 | argo-events.fullnameOverride | string | `"argo-events"` |  |
+| argo-rollouts.controller.podAnnotations."prometheus.io/path" | string | `"/metrics"` |  |
+| argo-rollouts.controller.podAnnotations."prometheus.io/port" | string | `"8090"` |  |
+| argo-rollouts.controller.podAnnotations."prometheus.io/scrape" | string | `"true"` |  |
 | argo-rollouts.controller.replicas | int | `1` |  |
 | argo-rollouts.enabled | bool | `true` |  |
 | argo-rollouts.fullnameOverride | string | `"argo-rollouts"` |  |
@@ -187,13 +207,19 @@ sealed-secrets:
 | event-reporters.events.serviceAccount.create | bool | `true` |  |
 | event-reporters.rollout.eventSource.affinity | object | `{}` |  |
 | event-reporters.rollout.eventSource.nodeSelector | object | `{}` |  |
+| event-reporters.rollout.eventSource.podAnnotations."prometheus.io/path" | string | `"/metrics"` |  |
+| event-reporters.rollout.eventSource.podAnnotations."prometheus.io/port" | string | `"7777"` |  |
+| event-reporters.rollout.eventSource.podAnnotations."prometheus.io/scrape" | string | `"true"` |  |
 | event-reporters.rollout.eventSource.replicas | int | `1` |  |
 | event-reporters.rollout.eventSource.resources | object | `{}` |  |
 | event-reporters.rollout.eventSource.tolerations | list | `[]` |  |
 | event-reporters.rollout.sensor.affinity | object | `{}` |  |
 | event-reporters.rollout.sensor.env | object | `{}` | Environment variables for sensor pods - add DEBUG_LOG: "true" to add debug level logs |
 | event-reporters.rollout.sensor.logging | object | `{"enabled":false,"intervalSeconds":0}` | Set to true to enable logging. Set intervalSeconds to add logging interval to moderate log flow. |
 | event-reporters.rollout.sensor.nodeSelector | object | `{}` |  |
+| event-reporters.rollout.sensor.podAnnotations."prometheus.io/path" | string | `"/metrics"` |  |
+| event-reporters.rollout.sensor.podAnnotations."prometheus.io/port" | string | `"7777"` |  |
+| event-reporters.rollout.sensor.podAnnotations."prometheus.io/scrape" | string | `"true"` |  |
 | event-reporters.rollout.sensor.replicas | int | `1` |  |
 | event-reporters.rollout.sensor.resources | object | `{}` |  |
 | event-reporters.rollout.sensor.retryStrategy | object | `{"duration":0,"factor":1,"jitter":1,"steps":3}` | Retry strategy for events sent to Codefresh |
@@ -313,6 +339,7 @@ sealed-secrets:
 | internal-router.serviceAccount.create | bool | `true` |  |
 | internal-router.serviceAccount.name | string | `""` |  |
 | internal-router.tolerations | list | `[]` |  |
+| prometheus | object | `{"alertmanager":{"enabled":false},"enabled":false,"fullnameOverride":"prometheus","server":{"extraScrapeConfigs":[{"job_name":"gitops-runtime-pods","kubernetes_sd_configs":[{"namespaces":{"names":["{{ .Release.Namespace }}"]},"role":"pod"}],"metrics_path":"/metrics","relabel_configs":[{"action":"keep","regex":true,"source_labels":["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]},{"action":"replace","regex":"(.+)","source_labels":["__meta_kubernetes_pod_annotation_prometheus_io_path"],"target_label":"__metrics_path__"},{"action":"replace","regex":"([^:]+)(?::\\d+)?;(\\d+)","replacement":"$1:$2","source_labels":["__address__","__meta_kubernetes_pod_annotation_prometheus_io_port"],"target_label":"__address__"},{"action":"labelmap","regex":"__meta_kubernetes_pod_label_(.+)"},{"action":"replace","source_labels":["__meta_kubernetes_namespace"],"target_label":"kubernetes_namespace"},{"action":"replace","source_labels":["__meta_kubernetes_pod_name"],"target_label":"kubernetes_pod_name"}]}],"persistence":{"enabled":true},"service":{"type":"ClusterIP"}}}` | Prometheus |
 | sealed-secrets | object | `{"fullnameOverride":"sealed-secrets-controller","image":{"registry":"quay.io","repository":"codefresh/sealed-secrets-controller","tag":"v0.24.5"},"keyrenewperiod":"720h","resources":{"limits":{"cpu":"500m","memory":"1Gi"},"requests":{"cpu":"200m","memory":"512Mi"}}}` | --------------------------------------------------------------------------------------------------------------------- |
 | tunnel-client | object | `{"enabled":true,"libraryMode":true,"tunnelServer":{"host":"register-tunnels.cf-cd.com","subdomainHost":"tunnels.cf-cd.com"}}` | Tunnel based runtime. Not supported for on-prem platform. In on-prem use ingress based runtimes. |
 | tunnel-client.enabled | bool | `true` | Will only be used if global.runtime.ingress.enabled = false |

charts/gitops-runtime/ci/default-values.yaml

@@ -24,3 +24,6 @@ global:
 
 argo-rollouts:
   enabled: true
+
+prometheus:
+  enabled: true

charts/gitops-runtime/templates/_components/event-reporters/rollout-reporter/_event-source.yaml

@@ -40,6 +40,11 @@ spec:
       resource: rollouts
       version: v1alpha1
   template:
+    metadata:
+      annotations:
+      {{- with .Values.rollout.eventSource.podAnnotations }}
+        {{- . | toYaml | nindent 8 }}
+      {{- end }}
     container:
       name: ""
       resources:

charts/gitops-runtime/templates/_components/event-reporters/rollout-reporter/_sensor.yaml

@@ -19,6 +19,11 @@ spec:
     eventName: analysisruns
   eventBusName: {{ default "codefresh-eventbus" .Values.global.runtime.eventBus.name }}
   template:
+    metadata:
+      annotations:
+      {{- with .Values.rollout.sensor.podAnnotations }}
+        {{- . | toYaml | nindent 8 }}
+      {{- end }}
     serviceAccountName: {{ include "event-reporters.rollout-reporter.serviceAccountName" .}}
     container:
       name: ""

charts/gitops-runtime/templates/_components/internal-router/_configmap.yaml

@@ -40,6 +40,12 @@ data:
       }
       {{- end }}
 
+      {{- if .Values.prometheus.enabled }}
+      location /prometheus/ {
+        proxy_pass http://{{ include "prometheus.server.fullname" .Subcharts.prometheus }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.prometheus.server.service.ports.http }}/;
+      }
+      {{- end }}
+
       location ~ /webhooks/([^/]+)/([^/]+) {
         proxy_pass http://$2-eventsource-svc.$1.svc.cluster.local;
       }
@@ -52,4 +58,4 @@ data:
         return 200 'ok';
       }
     }
-{{- end }}
+{{- end }}

charts/gitops-runtime/templates/hooks/pre-install/validate-values.yaml

@@ -25,8 +25,8 @@ spec:
         - name: VERSION
           value: {{ .Chart.Version }}
         command: ["sh", "-c"]
-        args: 
+        args:
         - |
-          echo {{ .Values | toYaml | quote }} > values.yaml
+          echo {{ .Values | toYaml }} > values.yaml
           cf helm validate --values values.yaml --namespace ${NAMESPACE} --version ${VERSION} --hook --log-level debug
 {{- end }}

charts/gitops-runtime/templates/ingress.yaml

@@ -63,6 +63,15 @@ spec:
               port:
                 number: 80
           path: /app-proxy
-          pathType: Prefix  
+          pathType: Prefix
+      {{- if (index (get $.Values "prometheus") "enabled") }}
+        - backend:
+            service:
+              name: internal-router
+              port:
+                number: 80
+          path: /prometheus
+          pathType: Prefix
+        {{- end }}
     {{- end }}
 {{- end }}

charts/gitops-runtime/templates/internal-router/config.yaml

@@ -17,4 +17,8 @@ Set app-proxy routing
 {{- if not (index $internalRouterContext.Values "routing" "app-proxy") }}
   {{ $_ := set $internalRouterContext.Values.routing "app-proxy" (dict "enabled" true "internalUrl" ( include "codefresh-gitops-runtime.app-proxy.url" . )) }}
 {{- end }}
+
+{{- $_ := set $internalRouterContext.Values "prometheus" (get .Values "prometheus") }}
+{{- $_ := set $internalRouterContext.Subcharts "prometheus" (get .Subcharts "prometheus") }}
+
 {{- include "internal-router.resources.configmap" $internalRouterContext }}

charts/gitops-runtime/values.yaml

@@ -137,6 +137,10 @@ sealed-secrets:
 # ArgoCD
 #-----------------------------------------------------------------------------------------------------------------------
 argo-cd:
+  global:
+    podAnnotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/path: "/metrics"
   fullnameOverride: argo-cd
   crds:
     install: true
@@ -149,6 +153,10 @@ argo-cd:
       server.insecure: true
       application.namespaces: 'cf-*'
   eventReporter:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "8087"
     # -- Installs new event reporter component to cluster
     enabled: true
     # -- Amount of shards to handle applications events
@@ -164,6 +172,44 @@ argo-cd:
     useApplicationConfiguration: true
   notifications: {}
 
+  controller:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "8082"
+
+  applicationSet:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "8080"
+
+  dex:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "5558"
+
+  server:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "8083"
+
+  repoServer:
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "8084"
+
+  redis:
+    exporter:
+      enabled: true
+    metrics:
+      enabled: true
+    podAnnotations:
+      prometheus.io/port: "9121"
+
 #-----------------------------------------------------------------------------------------------------------------------
 # Argo Events
 #-----------------------------------------------------------------------------------------------------------------------
@@ -196,6 +242,10 @@ argo-rollouts:
   fullnameOverride: argo-rollouts
   controller:
     replicas: 1
+    podAnnotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/path: "/metrics"
+      prometheus.io/port: "8090"
   installCRDs: true
 
 #-----------------------------------------------------------------------------------------------------------------------
@@ -246,6 +296,10 @@ event-reporters:
       tolerations: []
       nodeSelector: {}
       affinity: {}
+      podAnnotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "7777"
     sensor:
       # -- Set to true to enable logging. Set intervalSeconds to add logging interval to moderate log flow.
       logging:
@@ -268,6 +322,10 @@ event-reporters:
       tolerations: []
       nodeSelector: {}
       affinity: {}
+      podAnnotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "7777"
   workflow:
     serviceAccount:
       create: true
@@ -603,3 +661,47 @@ gitops-operator:
       requests:
         cpu: 100m
         memory: 64Mi
+
+# -- Prometheus
+prometheus:
+  enabled: false
+  fullnameOverride: prometheus
+  server:
+    service:
+      type: ClusterIP
+
+    persistence:
+      enabled: true
+
+    extraScrapeConfigs:
+      - job_name: gitops-runtime-pods
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              names:
+                - "{{ .Release.Namespace }}"
+        metrics_path: /metrics
+        relabel_configs:
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+          action: keep
+          regex: true
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+          action: replace
+          target_label: __metrics_path__
+          regex: (.+)
+        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+          action: replace
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          target_label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - source_labels: [__meta_kubernetes_namespace]
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels: [__meta_kubernetes_pod_name]
+          action: replace
+          target_label: kubernetes_pod_name
+
+  alertmanager:
+    enabled: false