Adding store-gw and memcached configuration to match existing datahub store path

Signed-off-by: Moad Zardab <[email protected]>

Author: moadz

resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-store-telemeter-template.yaml

@@ -83,7 +83,7 @@ objects:
     name: observatorium-thanos-store-bucket-cache-memcached-telemeter
     namespace: rhobs
   spec:
-    replicas: 1
+    replicas: 5
     selector:
       matchLabels:
         app.kubernetes.io/component: store-bucket-cache
@@ -135,10 +135,11 @@ objects:
             protocol: TCP
           resources:
             limits:
-              memory: 3Gi
+              cpu: "7"
+              memory: 12Gi
             requests:
-              cpu: 500m
-              memory: 2Gi
+              cpu: "6"
+              memory: 10Gi
           terminationMessagePolicy: FallbackToLogsOnError
         - args:
           - --memcached.address=localhost:0
@@ -240,7 +241,7 @@ objects:
     name: observatorium-thanos-store-index-cache-memcached-telemeter
     namespace: rhobs
   spec:
-    replicas: 1
+    replicas: 5
     selector:
       matchLabels:
         app.kubernetes.io/component: store-index-cache
@@ -292,10 +293,11 @@ objects:
             protocol: TCP
           resources:
             limits:
-              memory: 3Gi
+              cpu: "7"
+              memory: 12Gi
             requests:
-              cpu: 500m
-              memory: 2Gi
+              cpu: "6"
+              memory: 10Gi
           terminationMessagePolicy: FallbackToLogsOnError
         - args:
           - --memcached.address=localhost:0
@@ -517,7 +519,7 @@ objects:
               dns_provider_update_interval: 10s
           - --log.format=logfmt
           - --log.level=${STORE_LOG_LEVEL}
-          - --max-time=-22h0m0s
+          - --max-time=-336h0m0s
           - --objstore.config=$(OBJSTORE_CONFIG)
           - --selector.relabel-config-file=/etc/thanos/hashmod/hashmod-config.yaml
           - |
@@ -607,6 +609,7 @@ objects:
             periodSeconds: 5
           resources:
             limits:
+              cpu: "7"
               memory: ${STORE_MEMORY_LIMIT}
             requests:
               cpu: ${STORE_CPU_REQUEST}
@@ -706,7 +709,7 @@ objects:
         - ReadWriteOnce
         resources:
           requests:
-            storage: 5Gi
+            storage: 50Gi
         storageClassName: gp2
 - apiVersion: monitoring.coreos.com/v1
   kind: ServiceMonitor
@@ -809,12 +812,12 @@ objects:
         observatorium/tenant: telemeter
 parameters:
 - name: STORE_CPU_REQUEST
-  value: "2"
+  value: "5"
 - name: STORE_LOG_LEVEL
   value: warn
 - name: STORE_MEMORY_LIMIT
-  value: 20Gi
+  value: 60Gi
 - name: STORE_MEMORY_REQUEST
-  value: 5Gi
+  value: 40Gi
 - name: STORE_REPLICAS
-  value: "1"
+  value: "5"

services_go/instances/rhobs/rhobs.go

@@ -6,6 +6,9 @@ import (
 	"sort"
 	"time"
 
+	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/memcached"
+	thanostime "github.com/observatorium/observatorium/configuration_go/schemas/thanos/time"
+
 	"github.com/google/go-jsonnet"
 	observatoriumapi "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/observatorium/api"
 	"github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/thanos/receive"
@@ -331,7 +334,22 @@ func prodConfig() observatorium.Observatorium {
 						ingestor.VolumeSize = "5Gi"
 					},
 					StorePreManifestsHook: func(store *store.StoreStatefulSet) {
-						store.VolumeSize = "5Gi"
+						store.VolumeSize = "50Gi"
+						store.Replicas = 5
+						store.ContainerResources = kghelpers.NewResourcesRequirements("5", "7", "40Gi", "60Gi")
+					},
+					StoreOpts: func(opts *store.StoreOptions) {
+						// Telemeter Lookback is -14 days (336 hours) during migration to account for an overlap with our hot telemeter instance
+						telemeterMaxLookBack := time.Duration(-336) * time.Hour
+						opts.MaxTime = &thanostime.TimeOrDurationValue{Dur: &telemeterMaxLookBack}
+					},
+					IndexCachePreManifestsHook: func(indexCache *memcached.MemcachedDeployment) {
+						indexCache.Replicas = 5
+						indexCache.ContainerResources = kghelpers.NewResourcesRequirements("6", "7", "10Gi", "12Gi")
+					},
+					BucketCachePreManifestsHook: func(bucketCache *memcached.MemcachedDeployment) {
+						bucketCache.Replicas = 5
+						bucketCache.ContainerResources = kghelpers.NewResourcesRequirements("6", "7", "10Gi", "12Gi")
 					},
 				},
 			},

services_go/observatorium/metrics.go

@@ -88,6 +88,7 @@ type ObservatoriumMetricsInstance struct {
 	ObjStoreSecret                  string
 	Tenants                         []Tenants
 	StorePreManifestsHook           func(*store.StoreStatefulSet)
+	StoreOpts                       func(opts *store.StoreOptions)
 	IndexCachePreManifestsHook      func(*memcached.MemcachedDeployment)
 	BucketCachePreManifestsHook     func(*memcached.MemcachedDeployment)
 	CompactorPreManifestsHook       func(*compactor.CompactorStatefulSet)
@@ -939,6 +940,7 @@ func (o *ObservatoriumMetrics) makeStore(instanceCfg *ObservatoriumMetricsInstan
 		},
 	}
 	opts.AddExtraOpts("--store.enable-index-header-lazy-reader")
+	executeIfNotNil(instanceCfg.StoreOpts, opts)
 
 	indexCacheName := fmt.Sprintf("observatorium-thanos-store-index-cache-memcached-%s", instanceCfg.InstanceName)
 	bucketCacheName := fmt.Sprintf("observatorium-thanos-store-bucket-cache-memcached-%s", instanceCfg.InstanceName)