From 96abe83413664231b4d014a417e83b261159bd73 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Thu, 19 Oct 2023 08:37:29 +0530 Subject: [PATCH 01/47] secrets to be used instead of default values --- .../syft/templates/mongo-statefulset.yaml | 12 +++++++-- packages/grid/helm/syft/values.yaml | 12 ++++++--- packages/grid/k8s/manifests/secrets.yaml | 27 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 packages/grid/k8s/manifests/secrets.yaml diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 9da6335d889..948dec20214 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -32,9 +32,17 @@ spec: command: null env: - name: MONGO_INITDB_ROOT_USERNAME - value: {{ .Values.secrets.db.mongo.mongoInitdbRootUsername }} + # value: {{ .Values.secrets.db.mongo.mongoInitdbRootUsername }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongo-secret }} + key: mongo-username-secret - name: MONGO_INITDB_ROOT_PASSWORD - value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} + # value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongo-secret }} + key: mongo-password-secret envFrom: null image: mongo:7.0.0 lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index bf42d768b8f..dd6acda30cc 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -9,10 +9,14 @@ secrets: defaultRootPassword: "changethis" db: mongo: - mongoPassword: "example" - mongoInitdbRootPassword: "example" - mongoUsername: "root" - mongoInitdbRootUsername: "root" + # mongoPassword: "example" + # mongoInitdbRootPassword: "example" + # mongoUsername: "root" + # mongoInitdbRootUsername: "root" + mongo-secret: "mongo-secret" + +configuration: + devmode: false db: mongo: diff --git a/packages/grid/k8s/manifests/secrets.yaml b/packages/grid/k8s/manifests/secrets.yaml new file mode 100644 index 00000000000..8fe4ec362f8 --- /dev/null +++ b/packages/grid/k8s/manifests/secrets.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "mongo-secret" +type: Opaque +data: + # generate 32 chars long random string, base64 encode it and then double-quote the result string. + {{- if not .Values.configration.devmode }} + mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} + mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} + {{- else }} + mongo-username-secret: root + mongo-password-secret: example + + + + +# apiVersion: v1 +# kind: Secret +# metadata: +# name: "backend-secret" +# type: Opaque +# data: +# # generate 32 chars long random string, base64 encode it and then double-quote the result string. +# mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} +# mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} + \ No newline at end of file From fe976a9589a48bd6d5ceff402ba75788cd4a8171 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Thu, 19 Oct 2023 08:55:55 +0530 Subject: [PATCH 02/47] Support for all mongo secrets --- .../helm/syft/templates/backend-statefulset.yaml | 10 ++++++++-- .../helm/syft/templates/mongo-statefulset.yaml | 16 ++++++++-------- packages/grid/helm/syft/values.yaml | 2 +- packages/grid/k8s/manifests/secrets.yaml | 7 ++++++- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 4635147d65f..22226d78846 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -36,9 +36,15 @@ spec: - name: MONGO_HOST value: {{ .Values.db.mongo.settings.mongoHost }} - name: MONGO_USERNAME - value: {{ .Values.secrets.db.mongo.mongoUsername }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongo-username-secret - name: MONGO_PASSWORD - value: {{ .Values.secrets.db.mongo.mongoPassword }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongo-password-secret - name: SERVICE_NAME value: backend - name: RELEASE diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 948dec20214..10d4f57924c 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -33,16 +33,16 @@ spec: env: - name: MONGO_INITDB_ROOT_USERNAME # value: {{ .Values.secrets.db.mongo.mongoInitdbRootUsername }} - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongo-secret }} - key: mongo-username-secret + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongoInitdbRootUsername - name: MONGO_INITDB_ROOT_PASSWORD # value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongo-secret }} - key: mongo-password-secret + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongoInitdbRootPassword envFrom: null image: mongo:7.0.0 lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index dd6acda30cc..82066353c48 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -13,7 +13,7 @@ secrets: # mongoInitdbRootPassword: "example" # mongoUsername: "root" # mongoInitdbRootUsername: "root" - mongo-secret: "mongo-secret" + mongoSecret: "mongoSecret" configuration: devmode: false diff --git a/packages/grid/k8s/manifests/secrets.yaml b/packages/grid/k8s/manifests/secrets.yaml index 8fe4ec362f8..d1129c6b212 100644 --- a/packages/grid/k8s/manifests/secrets.yaml +++ b/packages/grid/k8s/manifests/secrets.yaml @@ -1,16 +1,21 @@ apiVersion: v1 kind: Secret metadata: - name: "mongo-secret" + name: "mongoSecret" type: Opaque data: # generate 32 chars long random string, base64 encode it and then double-quote the result string. {{- if not .Values.configration.devmode }} mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} + mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} + mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} mongo-username-secret: root mongo-password-secret: example + mongoInitdbRootPassword: "example" + mongoInitdbRootUsername: "root" + {{ end }} From 2c07be1fca6296c5545e2d15f4b99800fc1239c4 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:21:02 +0530 Subject: [PATCH 03/47] removed extra comments in mongo statefulset --- .../helm/syft/templates/mongo-statefulset.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 10d4f57924c..5a670fd5592 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -32,17 +32,15 @@ spec: command: null env: - name: MONGO_INITDB_ROOT_USERNAME - # value: {{ .Values.secrets.db.mongo.mongoInitdbRootUsername }} - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} - key: mongoInitdbRootUsername + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongoInitdbRootUsername - name: MONGO_INITDB_ROOT_PASSWORD - # value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} - key: mongoInitdbRootPassword + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.db.mongo.mongoSecret }} + key: mongoInitdbRootPassword envFrom: null image: mongo:7.0.0 lifecycle: null From 2db333f66939aab0edb90ea30cf8f1d457258371 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:49:43 +0530 Subject: [PATCH 04/47] added secrets to the templates file for debugging fixed error with mongosecret naming --- .../syft/templates/backend-statefulset.yaml | 4 ++-- .../syft/templates/mongo-statefulset.yaml | 4 ++-- .../grid/helm/syft/templates/secrets.yaml | 17 +++++++++++++ packages/grid/helm/syft/values.yaml | 2 +- packages/grid/k8s/manifests/secrets.yaml | 24 ++++--------------- 5 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 packages/grid/helm/syft/templates/secrets.yaml diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 22226d78846..eb855e676c2 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -38,12 +38,12 @@ spec: - name: MONGO_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} + name: {{ .Values.secrets.db.mongo.mongosecret }} key: mongo-username-secret - name: MONGO_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} + name: {{ .Values.secrets.db.mongo.mongosecret }} key: mongo-password-secret - name: SERVICE_NAME value: backend diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 5a670fd5592..171034fee01 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -34,12 +34,12 @@ spec: - name: MONGO_INITDB_ROOT_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} + name: {{ .Values.secrets.db.mongo.mongosecret }} key: mongoInitdbRootUsername - name: MONGO_INITDB_ROOT_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongoSecret }} + name: {{ .Values.secrets.db.mongo.mongosecret }} key: mongoInitdbRootPassword envFrom: null image: mongo:7.0.0 diff --git a/packages/grid/helm/syft/templates/secrets.yaml b/packages/grid/helm/syft/templates/secrets.yaml new file mode 100644 index 00000000000..f0035d3192c --- /dev/null +++ b/packages/grid/helm/syft/templates/secrets.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "mongosecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} + mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} + mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} + mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} + {{- else }} + mongo-username-secret: "root" + mongo-password-secret: "example" + mongoInitdbRootPassword: "example" + mongoInitdbRootUsername: "root" + {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 82066353c48..b9f40a1984c 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -13,7 +13,7 @@ secrets: # mongoInitdbRootPassword: "example" # mongoUsername: "root" # mongoInitdbRootUsername: "root" - mongoSecret: "mongoSecret" + mongosecret: "mongosecret" configuration: devmode: false diff --git a/packages/grid/k8s/manifests/secrets.yaml b/packages/grid/k8s/manifests/secrets.yaml index d1129c6b212..57a5ed24065 100644 --- a/packages/grid/k8s/manifests/secrets.yaml +++ b/packages/grid/k8s/manifests/secrets.yaml @@ -1,32 +1,18 @@ apiVersion: v1 kind: Secret metadata: - name: "mongoSecret" + name: "mongosecret" type: Opaque data: - # generate 32 chars long random string, base64 encode it and then double-quote the result string. - {{- if not .Values.configration.devmode }} + {{- if not .Values.configuration.devmode }} mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} - mongo-username-secret: root - mongo-password-secret: example + mongo-username-secret: "root" + mongo-password-secret: "example" mongoInitdbRootPassword: "example" mongoInitdbRootUsername: "root" - {{ end }} - - + {{- end }} - -# apiVersion: v1 -# kind: Secret -# metadata: -# name: "backend-secret" -# type: Opaque -# data: -# # generate 32 chars long random string, base64 encode it and then double-quote the result string. -# mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} -# mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} - \ No newline at end of file From 63364c6e67433be3d54e36171019e4438e37d7cc Mon Sep 17 00:00:00 2001 From: Param-29 Date: Thu, 2 Nov 2023 08:22:31 +0530 Subject: [PATCH 05/47] moved secrets.txt from secrets.yaml --- packages/grid/helm/helm.py | 12 +++++++++++- .../grid/k8s/manifests/{secrets.yaml => secrets.txt} | 3 +-- 2 files changed, 12 insertions(+), 3 deletions(-) rename packages/grid/k8s/manifests/{secrets.yaml => secrets.txt} (97%) diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index 9dd0ad1a618..6e62a6fbe30 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -104,6 +104,16 @@ def ingress_with_tls() -> str: with open(ingress_tls) as fp: return fp.read() +def add_secrets(helm_chart_template_dir: str) -> None: + """ + Add secrets.txt to secrets.yaml during runtime build. + """ + secrets_path = os.path.join(helm_chart_template_dir, "secrets.yaml") + secrets_source = os.path.join('k8s/manifests', "secrets.txt") + with open(secrets_path, "w") as fp: + with open(secrets_source, "r") as fp2: + fp.write(fp2.read()) + def add_notes(helm_chart_template_dir: str) -> None: """Add notes or information post helm install or upgrade.""" @@ -263,7 +273,7 @@ def main() -> None: # Add notes add_notes(helm_chart_template_dir) - + add_secrets(helm_chart_template_dir) if file_count > 0: print(f"✅ Done: Generated {file_count} template files") else: diff --git a/packages/grid/k8s/manifests/secrets.yaml b/packages/grid/k8s/manifests/secrets.txt similarity index 97% rename from packages/grid/k8s/manifests/secrets.yaml rename to packages/grid/k8s/manifests/secrets.txt index 57a5ed24065..f0035d3192c 100644 --- a/packages/grid/k8s/manifests/secrets.yaml +++ b/packages/grid/k8s/manifests/secrets.txt @@ -14,5 +14,4 @@ data: mongo-password-secret: "example" mongoInitdbRootPassword: "example" mongoInitdbRootUsername: "root" - {{- end }} - + {{- end }} \ No newline at end of file From 89c19ecefd9930d2318749847792e9b8ef933253 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Thu, 2 Nov 2023 08:29:09 +0530 Subject: [PATCH 06/47] removing secrets.yaml --- packages/grid/helm/syft/templates/secrets.yaml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 packages/grid/helm/syft/templates/secrets.yaml diff --git a/packages/grid/helm/syft/templates/secrets.yaml b/packages/grid/helm/syft/templates/secrets.yaml deleted file mode 100644 index f0035d3192c..00000000000 --- a/packages/grid/helm/syft/templates/secrets.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: "mongosecret" -type: Opaque -data: - {{- if not .Values.configuration.devmode }} - mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} - mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} - mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} - mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} - {{- else }} - mongo-username-secret: "root" - mongo-password-secret: "example" - mongoInitdbRootPassword: "example" - mongoInitdbRootUsername: "root" - {{- end }} \ No newline at end of file From bac6a2d91e309f9996d24063d7d8eae0bd431f08 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Tue, 7 Nov 2023 08:29:59 +0530 Subject: [PATCH 07/47] lookup added for 4 secrets in mongodb --- packages/grid/k8s/manifests/secrets.txt | 30 +++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/grid/k8s/manifests/secrets.txt b/packages/grid/k8s/manifests/secrets.txt index f0035d3192c..d6a31db5981 100644 --- a/packages/grid/k8s/manifests/secrets.txt +++ b/packages/grid/k8s/manifests/secrets.txt @@ -5,10 +5,32 @@ metadata: type: Opaque data: {{- if not .Values.configuration.devmode }} - mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} - mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} - mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} - mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "mongosecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + # set $jwtSecret to existing secret data or generate a random one when not exists + + # Sample below + # {{- $jwtSecret := (get $secretData "mongosecret") | default (randAlphaNum 32 | b64enc) }} + # mongosecret: {{ $jwtSecret | quote }} + + + {{- $mongoUnameSecret := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} + mongo-username-secret: {{ $mongoUnameSecret | quote }} + + {{- $mongoPassSecret := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} + mongo-password-secret: {{ $mongoPassSecret | quote }} + + {{- $mongoDbRootUser := (get $secretData "mongoInitdbRootUsername") | default (randAlphaNum 32 | b64enc) }} + mongoInitdbRootUsername: {{ $mongoDbRootUser | quote }} + + {{- $mongoDbRootPass := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} + mongoInitdbRootPassword: {{ $mongoDbRootPass | quote }} + + # mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} + # mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} + # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} + # mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} mongo-username-secret: "root" mongo-password-secret: "example" From abd6ddeb7d34020a5c6336ce8a81fd86399bcc04 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Tue, 7 Nov 2023 08:32:34 +0530 Subject: [PATCH 08/47] precommit fix --- packages/grid/helm/helm.py | 7 ++++--- packages/grid/k8s/manifests/secrets.txt | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index 6e62a6fbe30..03e1d221b33 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -104,14 +104,15 @@ def ingress_with_tls() -> str: with open(ingress_tls) as fp: return fp.read() + def add_secrets(helm_chart_template_dir: str) -> None: """ - Add secrets.txt to secrets.yaml during runtime build. + Add secrets.txt to secrets.yaml during runtime build. """ secrets_path = os.path.join(helm_chart_template_dir, "secrets.yaml") - secrets_source = os.path.join('k8s/manifests', "secrets.txt") + secrets_source = os.path.join("k8s/manifests", "secrets.txt") with open(secrets_path, "w") as fp: - with open(secrets_source, "r") as fp2: + with open(secrets_source) as fp2: fp.write(fp2.read()) diff --git a/packages/grid/k8s/manifests/secrets.txt b/packages/grid/k8s/manifests/secrets.txt index d6a31db5981..cd62fb3db0d 100644 --- a/packages/grid/k8s/manifests/secrets.txt +++ b/packages/grid/k8s/manifests/secrets.txt @@ -9,7 +9,7 @@ data: {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "mongosecret") | default dict }} {{- $secretData := (get $secretObj "data") | default dict }} # set $jwtSecret to existing secret data or generate a random one when not exists - + # Sample below # {{- $jwtSecret := (get $secretData "mongosecret") | default (randAlphaNum 32 | b64enc) }} # mongosecret: {{ $jwtSecret | quote }} @@ -26,7 +26,7 @@ data: {{- $mongoDbRootPass := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} mongoInitdbRootPassword: {{ $mongoDbRootPass | quote }} - + # mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} # mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} From 9a642fb3a6b0aae6d9351e707cc7cac40424de1a Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:06:15 +0530 Subject: [PATCH 09/47] base64 fix for default root email and password --- packages/grid/k8s/manifests/secrets.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/grid/k8s/manifests/secrets.txt b/packages/grid/k8s/manifests/secrets.txt index cd62fb3db0d..5a4abde6d9a 100644 --- a/packages/grid/k8s/manifests/secrets.txt +++ b/packages/grid/k8s/manifests/secrets.txt @@ -32,8 +32,8 @@ data: # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} # mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} - mongo-username-secret: "root" - mongo-password-secret: "example" - mongoInitdbRootPassword: "example" - mongoInitdbRootUsername: "root" + mongo-username-secret: cm9vdA== # Base64-encoded "root" + mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" + mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" + mongoInitdbRootUsername: cm9vdA== # Base64-encoded "root" {{- end }} \ No newline at end of file From 74a9890b8317cd48959562bc4f39e7e38c601c5c Mon Sep 17 00:00:00 2001 From: Param-29 Date: Wed, 15 Nov 2023 18:26:01 +0530 Subject: [PATCH 10/47] created 2 seperate secrets: one for mongo, other for syft --- packages/grid/helm/helm.py | 14 +++++++++---- .../syft/templates/backend-statefulset.yaml | 10 +++++++-- packages/grid/helm/syft/values.yaml | 7 ++++--- .../{secrets.txt => mongo_secrets.txt} | 1 + packages/grid/k8s/manifests/syft_secrets.txt | 21 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) rename packages/grid/k8s/manifests/{secrets.txt => mongo_secrets.txt} (99%) create mode 100644 packages/grid/k8s/manifests/syft_secrets.txt diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index 03e1d221b33..5800dec52d6 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -109,10 +109,16 @@ def add_secrets(helm_chart_template_dir: str) -> None: """ Add secrets.txt to secrets.yaml during runtime build. """ - secrets_path = os.path.join(helm_chart_template_dir, "secrets.yaml") - secrets_source = os.path.join("k8s/manifests", "secrets.txt") - with open(secrets_path, "w") as fp: - with open(secrets_source) as fp2: + mongo_secrets_path = os.path.join(helm_chart_template_dir, "mongo_secrets.yaml") + mongo_secrets_source = os.path.join("k8s/manifests", "mongo_secrets.txt") + with open(mongo_secrets_path, "w") as fp: + with open(mongo_secrets_source) as fp2: + fp.write(fp2.read()) + + syft_secrets_path = os.path.join(helm_chart_template_dir, "syft_secrets.yaml") + syft_secrets_source = os.path.join("k8s/manifests", "syft_secrets.txt") + with open(syft_secrets_path, "w") as fp: + with open(syft_secrets_source) as fp2: fp.write(fp2.read()) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 78fb86cd720..7b1939ac842 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -58,7 +58,10 @@ spec: - name: NODE_NAME value: {{ .Values.node.settings.nodeName }} - name: STACK_API_KEY - value: {{ .Values.secrets.syft.stackApiKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.syft.syftsecret }} + key: stackApiKey - name: PORT value: '80' - name: IGNORE_TLS_ERRORS @@ -84,7 +87,10 @@ spec: - name: DEFAULT_ROOT_EMAIL value: {{ .Values.secrets.syft.defaultRootEmail }} - name: DEFAULT_ROOT_PASSWORD - value: {{ .Values.secrets.syft.defaultRootPassword }} + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.syft.syftsecret }} + key: defaultRootPassword envFrom: null image: docker.io/openmined/grid-backend:0.8.2-beta.40 lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index b9f40a1984c..14864c19776 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -4,9 +4,10 @@ secrets: syft: - stackApiKey: "changeme" - defaultRootEmail: "info@openmined.org" - defaultRootPassword: "changethis" + syftsecret: "syftsecret" + # stackApiKey: "changeme" + # defaultRootEmail: "info@openmined.org" + # defaultRootPassword: "changethis" db: mongo: # mongoPassword: "example" diff --git a/packages/grid/k8s/manifests/secrets.txt b/packages/grid/k8s/manifests/mongo_secrets.txt similarity index 99% rename from packages/grid/k8s/manifests/secrets.txt rename to packages/grid/k8s/manifests/mongo_secrets.txt index 5a4abde6d9a..3ecb24aeb45 100644 --- a/packages/grid/k8s/manifests/secrets.txt +++ b/packages/grid/k8s/manifests/mongo_secrets.txt @@ -32,6 +32,7 @@ data: # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} # mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} + mongo-username-secret: cm9vdA== # Base64-encoded "root" mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" diff --git a/packages/grid/k8s/manifests/syft_secrets.txt b/packages/grid/k8s/manifests/syft_secrets.txt new file mode 100644 index 00000000000..b71c0bd9e01 --- /dev/null +++ b/packages/grid/k8s/manifests/syft_secrets.txt @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "syftsecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "syftsecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + + {{- $stackApiKey := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} + stackApiKey: {{ $stackApiKey | quote }} + + {{- $defaultRootPassword := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} + defaultRootPassword: {{ $defaultRootPassword | quote }} + + {{- else }} + stackApiKey: Y2hhbmdlbWU= # Base64-encoded "changeme" + defaultRootPassword: Y2hhbmdldGhpcw== # Base64-encoded "changethis" + {{- end }} \ No newline at end of file From cce8d1f885b1b81992fc06e04132b7eb6dd70a5b Mon Sep 17 00:00:00 2001 From: Param-29 Date: Wed, 15 Nov 2023 18:49:46 +0530 Subject: [PATCH 11/47] precommit fix --- packages/grid/k8s/manifests/mongo_secrets.txt | 2 +- packages/grid/k8s/manifests/syft_secrets.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grid/k8s/manifests/mongo_secrets.txt b/packages/grid/k8s/manifests/mongo_secrets.txt index 3ecb24aeb45..9c74da02ba7 100644 --- a/packages/grid/k8s/manifests/mongo_secrets.txt +++ b/packages/grid/k8s/manifests/mongo_secrets.txt @@ -32,7 +32,7 @@ data: # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} # mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} - + mongo-username-secret: cm9vdA== # Base64-encoded "root" mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" diff --git a/packages/grid/k8s/manifests/syft_secrets.txt b/packages/grid/k8s/manifests/syft_secrets.txt index b71c0bd9e01..cfe2212da5a 100644 --- a/packages/grid/k8s/manifests/syft_secrets.txt +++ b/packages/grid/k8s/manifests/syft_secrets.txt @@ -14,7 +14,7 @@ data: {{- $defaultRootPassword := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} defaultRootPassword: {{ $defaultRootPassword | quote }} - + {{- else }} stackApiKey: Y2hhbmdlbWU= # Base64-encoded "changeme" defaultRootPassword: Y2hhbmdldGhpcw== # Base64-encoded "changethis" From d3ae8be823b9d24cd38b375da768c846f3c60d83 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Thu, 23 Nov 2023 08:33:59 +0530 Subject: [PATCH 12/47] seaweedfs randomizing --- packages/grid/default.env | 4 ++-- .../syft/templates/seaweedfs-statefulset.yaml | 14 +++++++---- packages/grid/helm/syft/values.yaml | 10 ++++++++ .../grid/k8s/manifests/seaweedfs_secrets.txt | 23 +++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 packages/grid/k8s/manifests/seaweedfs_secrets.txt diff --git a/packages/grid/default.env b/packages/grid/default.env index 95c6329e0f1..47479064f0a 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -61,9 +61,9 @@ VITE_PUBLIC_API_BASE_URL="/api/v2" S3_ENDPOINT="seaweedfs" S3_PORT=8333 S3_ROOT_USER="admin" -S3_ROOT_PWD="admin" +S3_ROOT_PWD="admin" # needs randomizing S3_REGION="us-east-1" -S3_PRESIGNED_TIMEOUT_SECS=1800 +S3_PRESIGNED_TIMEOUT_SECS=1800 #not-using S3_VOLUME_SIZE_MB=1024 # Jax diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 7927ac34acc..9e35b8e724a 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -33,15 +33,19 @@ spec: command: - sh - /etc/seaweedfs/start.sh - env: + env: # move this in values.yaml - name: S3_VOLUME_SIZE_MB - value: '1024' + value: {{ .Values.seaweedfs.start_variables.S3_VOLUME_SIZE_MB }} - name: S3_ROOT_USER - value: admin + secretKeyRef: + name: {{ .Values.secrets.seaweedfs.start_vars.seaweedfsecret }} + key: S3_ROOT_USER_SECRET - name: S3_ROOT_PWD - value: admin + secretKeyRef: + name: {{ .Values.secrets.seaweedfs.start_vars.seaweedfsecret }} + key: S3_ROOT_PWD_SECRET - name: S3_PORT - value: '8333' + value: {{ .Values.seaweedfs.start_variables.S3_PORT }} envFrom: null image: chrislusf/seaweedfs:3.57 lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 14864c19776..bf79f5320a8 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -15,6 +15,9 @@ secrets: # mongoUsername: "root" # mongoInitdbRootUsername: "root" mongosecret: "mongosecret" + seaweedfs: + start_vars: + seaweedfsecret: "seaweedfsecret" configuration: devmode: false @@ -25,6 +28,13 @@ db: mongoPort: "'27017'" mongoHost: "mongo" +seaweedfs: + start_variables: + S3_VOLUME_SIZE_MB: '1024' + S3_ROOT_USER: admin + S3_ROOT_PWD: admin + S3_PORT: '8333' + node: settings: tls: false diff --git a/packages/grid/k8s/manifests/seaweedfs_secrets.txt b/packages/grid/k8s/manifests/seaweedfs_secrets.txt new file mode 100644 index 00000000000..d48ebc6ae77 --- /dev/null +++ b/packages/grid/k8s/manifests/seaweedfs_secrets.txt @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "seaweedfsecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "seaweedfsecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + + + {{- $S3_ROOT_USER_SECRET := (get $secretData "S3_ROOT_USER_SECRET") | default (randAlphaNum 32 | b64enc) }} + S3_ROOT_USER_SECRET: {{ $S3_ROOT_USER_SECRET | quote }} + + {{- $S3_ROOT_PWD_SECRET := (get $secretData "S3_ROOT_PWD_SECRET") | default (randAlphaNum 32 | b64enc) }} + S3_ROOT_PWD_SECRET: {{ $mongoPassSecret | quote }} + + {{- else }} + + S3_ROOT_USER_SECRET: YWRtaW4= # Base64-encoded "admin" + S3_ROOT_PWD_SECRET: YWRtaW4= # Base64-encoded "admin" + {{- end }} \ No newline at end of file From 2e214d658dc12efbbaae1c863f0066aac76e6e84 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Tue, 28 Nov 2023 16:28:15 +0530 Subject: [PATCH 13/47] seeweedfs changes v2, start_vars removed --- packages/grid/helm/helm.py | 6 ++++++ .../helm/syft/templates/seaweedfs-statefulset.yaml | 8 ++++---- packages/grid/helm/syft/values.yaml | 10 +++------- packages/grid/k8s/manifests/seaweedfs_secrets.txt | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index 5800dec52d6..ce8b48660a0 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -120,6 +120,12 @@ def add_secrets(helm_chart_template_dir: str) -> None: with open(syft_secrets_path, "w") as fp: with open(syft_secrets_source) as fp2: fp.write(fp2.read()) + + seeweedfs_secrets_path = os.path.join(helm_chart_template_dir, "seaweedfs_secrets.yaml") + seeweedfs_secrets_source = os.path.join("k8s/manifests", "seaweedfs_secrets.txt") + with open(seeweedfs_secrets_path, "w") as fp: + with open(seeweedfs_secrets_source) as fp2: + fp.write(fp2.read()) def add_notes(helm_chart_template_dir: str) -> None: diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 9e35b8e724a..4c8bdf1ac36 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -35,17 +35,17 @@ spec: - /etc/seaweedfs/start.sh env: # move this in values.yaml - name: S3_VOLUME_SIZE_MB - value: {{ .Values.seaweedfs.start_variables.S3_VOLUME_SIZE_MB }} + value: {{ .Values.seaweedfs.S3_VOLUME_SIZE_MB }} - name: S3_ROOT_USER secretKeyRef: - name: {{ .Values.secrets.seaweedfs.start_vars.seaweedfsecret }} + name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} key: S3_ROOT_USER_SECRET - name: S3_ROOT_PWD secretKeyRef: - name: {{ .Values.secrets.seaweedfs.start_vars.seaweedfsecret }} + name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} key: S3_ROOT_PWD_SECRET - name: S3_PORT - value: {{ .Values.seaweedfs.start_variables.S3_PORT }} + value: {{ .Values.seaweedfs.S3_PORT }} envFrom: null image: chrislusf/seaweedfs:3.57 lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index bf79f5320a8..73608720d81 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -16,8 +16,7 @@ secrets: # mongoInitdbRootUsername: "root" mongosecret: "mongosecret" seaweedfs: - start_vars: - seaweedfsecret: "seaweedfsecret" + seaweedfsecret: "seaweedfsecret" configuration: devmode: false @@ -29,11 +28,8 @@ db: mongoHost: "mongo" seaweedfs: - start_variables: - S3_VOLUME_SIZE_MB: '1024' - S3_ROOT_USER: admin - S3_ROOT_PWD: admin - S3_PORT: '8333' + S3_VOLUME_SIZE_MB: '1024' + S3_PORT: '8333' node: settings: diff --git a/packages/grid/k8s/manifests/seaweedfs_secrets.txt b/packages/grid/k8s/manifests/seaweedfs_secrets.txt index d48ebc6ae77..d027e5fc864 100644 --- a/packages/grid/k8s/manifests/seaweedfs_secrets.txt +++ b/packages/grid/k8s/manifests/seaweedfs_secrets.txt @@ -14,7 +14,7 @@ data: S3_ROOT_USER_SECRET: {{ $S3_ROOT_USER_SECRET | quote }} {{- $S3_ROOT_PWD_SECRET := (get $secretData "S3_ROOT_PWD_SECRET") | default (randAlphaNum 32 | b64enc) }} - S3_ROOT_PWD_SECRET: {{ $mongoPassSecret | quote }} + S3_ROOT_PWD_SECRET: {{ $S3_ROOT_PWD_SECRET | quote }} {{- else }} From 69a6907c5df5bfe50d01354b41c54f7364956ff8 Mon Sep 17 00:00:00 2001 From: Param-29 Date: Tue, 5 Dec 2023 15:31:07 +0530 Subject: [PATCH 14/47] Fix for Syft_secrets + cleanup --- packages/grid/k8s/manifests/mongo_secrets.txt | 11 +---------- packages/grid/k8s/manifests/syft_secrets.txt | 4 ++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/grid/k8s/manifests/mongo_secrets.txt b/packages/grid/k8s/manifests/mongo_secrets.txt index 9c74da02ba7..c8abd44be22 100644 --- a/packages/grid/k8s/manifests/mongo_secrets.txt +++ b/packages/grid/k8s/manifests/mongo_secrets.txt @@ -8,12 +8,7 @@ data: {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "mongosecret") | default dict }} {{- $secretData := (get $secretObj "data") | default dict }} - # set $jwtSecret to existing secret data or generate a random one when not exists - - # Sample below - # {{- $jwtSecret := (get $secretData "mongosecret") | default (randAlphaNum 32 | b64enc) }} - # mongosecret: {{ $jwtSecret | quote }} - + {{- $mongoUnameSecret := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} mongo-username-secret: {{ $mongoUnameSecret | quote }} @@ -27,10 +22,6 @@ data: {{- $mongoDbRootPass := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} mongoInitdbRootPassword: {{ $mongoDbRootPass | quote }} - # mongo-username-secret: {{ randAlphaNum 32 | b64enc | quote }} - # mongo-password-secret: {{ randAlphaNum 32 | b64enc | quote }} - # mongoInitdbRootPassword: {{ randAlphaNum 32 | b64enc | quote }} - # mongoInitdbRootUsername: {{ randAlphaNum 32 | b64enc | quote }} {{- else }} mongo-username-secret: cm9vdA== # Base64-encoded "root" diff --git a/packages/grid/k8s/manifests/syft_secrets.txt b/packages/grid/k8s/manifests/syft_secrets.txt index cfe2212da5a..67cf3d28e9d 100644 --- a/packages/grid/k8s/manifests/syft_secrets.txt +++ b/packages/grid/k8s/manifests/syft_secrets.txt @@ -9,10 +9,10 @@ data: {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "syftsecret") | default dict }} {{- $secretData := (get $secretObj "data") | default dict }} - {{- $stackApiKey := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} + {{- $stackApiKey := (get $secretData "stackApiKey") | default (randAlphaNum 32 | b64enc) }} stackApiKey: {{ $stackApiKey | quote }} - {{- $defaultRootPassword := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} + {{- $defaultRootPassword := (get $secretData "defaultRootPassword") | default (randAlphaNum 32 | b64enc) }} defaultRootPassword: {{ $defaultRootPassword | quote }} {{- else }} From 875bb180df0b7c33b7fe8d73b2ecf0a59684d192 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:35:15 +0530 Subject: [PATCH 15/47] updated helm template files --- packages/grid/helm/manifests.yaml | 10 +++++----- .../grid/helm/syft/templates/backend-statefulset.yaml | 4 ++-- .../grid/helm/syft/templates/frontend-deployment.yaml | 4 ++-- .../helm/syft/templates/seaweedfs-statefulset.yaml | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index a762b398d48..4a991bb63d7 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -65,7 +65,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.3-beta.3 + value: 0.8.3-beta.4 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -103,7 +103,7 @@ spec: - name: DEFAULT_ROOT_PASSWORD value: changethis envFrom: null - image: docker.io/openmined/grid-backend:0.8.3-beta.3 + image: docker.io/openmined/grid-backend:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 @@ -220,7 +220,7 @@ spec: command: null env: - name: VERSION - value: 0.8.3-beta.3 + value: 0.8.3-beta.4 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -228,7 +228,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.3-beta.3 + image: docker.io/openmined/grid-frontend:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 @@ -586,7 +586,7 @@ spec: - name: SEAWEED_MOUNT_PORT value: '4001' envFrom: null - image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.3 + image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 5aacb056db2..e314e1d8874 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -44,7 +44,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.3-beta.3 + value: 0.8.3-beta.4 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -82,7 +82,7 @@ spec: - name: DEFAULT_ROOT_PASSWORD value: {{ .Values.secrets.syft.defaultRootPassword }} envFrom: null - image: docker.io/openmined/grid-backend:0.8.3-beta.3 + image: docker.io/openmined/grid-backend:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index f07f737f1dc..4abbddcb720 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -32,7 +32,7 @@ spec: command: null env: - name: VERSION - value: 0.8.3-beta.3 + value: 0.8.3-beta.4 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -40,7 +40,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.3-beta.3 + image: docker.io/openmined/grid-frontend:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 6afc4af8800..51d5b53a25c 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -43,7 +43,7 @@ spec: - name: SEAWEED_MOUNT_PORT value: '4001' envFrom: null - image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.3 + image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.4 lifecycle: null livenessProbe: null name: container-0 From a5124e425b9c7222446314ec1b464882c49ce493 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:45:42 +0530 Subject: [PATCH 16/47] updated helm charts for configuring registry and syft version --- packages/grid/devspace.yaml | 228 +----------------- .../syft/templates/backend-statefulset.yaml | 4 +- .../syft/templates/frontend-deployment.yaml | 4 +- .../syft/templates/seaweedfs-statefulset.yaml | 2 +- packages/grid/helm/syft/values.yaml | 4 + 5 files changed, 11 insertions(+), 231 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 95c6927993c..fdb062f15a9 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -57,234 +57,10 @@ images: # This is a list of `deployments` that DevSpace can create for this project deployments: - grid-stack-ingress: - kubectl: - manifests: - - k8s/manifests/ingress.yaml - - traefik-main-config: - kubectl: - manifests: - - "k8s/manifests/traefik-domain.yaml" - - seaweedfs-config: - kubectl: - manifests: - - "k8s/manifests/seaweedfs.yaml" - - proxy: - helm: - upgradeArgs: - - --dependency-update - chart: - name: component-chart - repo: https://charts.devspace.sh - values: - containers: - - image: "${DOCKER_IMAGE_TRAEFIK}:${TRAEFIK_VERSION}" - volumeMounts: - - containerPath: /etc/traefik - volume: - name: traefik-conf - env: - - name: SERVICE_NAME - value: "proxy" - volumes: - - name: traefik-conf - configMap: - name: traefik-main-config - service: - name: "proxy" - ports: - - name: proxy - port: 80 - # - name: api - # port: 4000 - - backend: - helm: - upgradeArgs: - - --dependency-update - chart: - name: component-chart - repo: https://charts.devspace.sh - values: - # PodSecurityContext (uncomment for rootless "syftuser") - # securityContext: - # runAsNonRoot: true - # runAsUser: 1000 - # runAsGroup: 1000 - # fsGroup: 1000 - # fsGroupChangePolicy: "Always" - containers: - - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" - volumeMounts: - - containerPath: /root/data/creds/ - volume: - name: credentials-data - subPath: /credentials-data - readOnly: false - env: - - name: MONGO_PORT - value: "${MONGO_PORT}" - - name: MONGO_HOST - value: "${MONGO_HOST}" - - name: MONGO_USERNAME - value: "${MONGO_USERNAME}" - - name: MONGO_PASSWORD - value: "${MONGO_PASSWORD}" - - name: SERVICE_NAME - value: "backend" - - name: RELEASE - value: "${RELEASE}" - - name: VERSION - value: "${VERSION}" - - name: VERSION_HASH - value: "${VERSION_HASH}" - - name: NODE_TYPE - value: "${NODE_TYPE}" - - name: NODE_NAME - value: "${NODE_NAME}" - - name: NODE_SIDE_TYPE - value: "${NODE_SIDE_TYPE}" - - name: STACK_API_KEY - value: "changeme" - - name: PORT - value: "${HTTP_PORT}" - - name: IGNORE_TLS_ERRORS - value: "${IGNORE_TLS_ERRORS}" - - name: HTTP_PORT - value: "${HTTP_PORT}" - - name: HTTPS_PORT - value: "${HTTPS_PORT}" - - name: CONTAINER_HOST - value: "k8s" - - name: TRACE - value: "${TRACE}" - - name: JAEGER_HOST - value: "${JAEGER_HOST}" - - name: JAEGER_PORT - value: "${JAEGER_PORT}" - - name: DEV_MODE - value: "${DEV_MODE}" - - name: DOMAIN_CONNECTION_PORT - value: "${DOMAIN_CONNECTION_PORT}" - - name: ENABLE_OBLV - value: "${ENABLE_OBLV}" - - name: DEFAULT_ROOT_EMAIL - value: "${DEFAULT_ROOT_EMAIL}" - - name: DEFAULT_ROOT_PASSWORD - value: "${DEFAULT_ROOT_PASSWORD}" - volumes: - - name: credentials-data - size: "100Mi" - service: - name: "backend" - ports: - - port: "${HTTP_PORT}" - - mongo: - helm: - upgradeArgs: - - --dependency-update - chart: - name: component-chart - repo: https://charts.devspace.sh - values: - containers: - - image: "${MONGO_IMAGE}:${MONGO_VERSION}" - env: - - name: MONGO_INITDB_ROOT_USERNAME - value: "${MONGO_USERNAME}" - - name: MONGO_INITDB_ROOT_PASSWORD - value: "${MONGO_PASSWORD}" - volumeMounts: - - containerPath: /data/db - volume: - name: mongo-data - subPath: / - readOnly: false - volumes: - - name: mongo-data - size: "5Gi" - service: - name: mongo - ports: - - port: "27017" - - seaweedfs: - helm: - upgradeArgs: - - --dependency-update - chart: - name: component-chart - repo: https://charts.devspace.sh - values: - containers: - - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}:${VERSION}" - env: - - name: S3_VOLUME_SIZE_MB - value: "${S3_VOLUME_SIZE_MB}" - - name: S3_ROOT_USER - value: "${S3_ROOT_USER}" - - name: S3_ROOT_PWD - value: "${S3_ROOT_PWD}" - - name: S3_PORT - value: "${S3_PORT}" - - name: SEAWEED_MOUNT_PORT - value: "${SEAWEED_MOUNT_PORT}" - volumeMounts: - - containerPath: /etc/seaweedfs/filer.toml - volume: - name: seaweedfs-config - subPath: /filer.toml - readOnly: false - - containerPath: /etc/seaweedfs/start.sh - volume: - name: seaweedfs-config - subPath: /start.sh - readOnly: false - - containerPath: /data/blob - volume: - name: seaweedfs-data - subPath: / - readOnly: false - volumes: - - name: seaweedfs-data - size: "5Gi" - - name: seaweedfs-config - configMap: - name: seaweedfs-config - service: - name: seaweedfs - ports: - - port: "8888" # filer - - port: "8333" # S3 - - port: "4001" # mount azure - - frontend: + syft: helm: - upgradeArgs: - - --dependency-update chart: - name: component-chart - repo: https://charts.devspace.sh - values: - containers: - - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}:${VERSION}" - env: - - name: VERSION - value: "${VERSION}" - - name: VERSION_HASH - value: "${VERSION_HASH}" - - name: NODE_TYPE - value: "${NODE_TYPE}" - - name: NEXT_PUBLIC_API_URL - value: "${NEXT_PUBLIC_API_URL}" - service: - name: "frontend" - ports: - - port: "80" + name: ./helm/syft dev: mongo: diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index e314e1d8874..93048d41029 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -44,7 +44,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.3-beta.4 + value: {{ .Values.container.syft_version }} - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -82,7 +82,7 @@ spec: - name: DEFAULT_ROOT_PASSWORD value: {{ .Values.secrets.syft.defaultRootPassword }} envFrom: null - image: docker.io/openmined/grid-backend:0.8.3-beta.4 + image: {{ .Values.container.registry }}/openmined/grid-backend:{{ .Values.container.syft_version }} lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index 4abbddcb720..84aa6026f14 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -32,7 +32,7 @@ spec: command: null env: - name: VERSION - value: 0.8.3-beta.4 + value: {{ .Values.container.syft_version }} - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -40,7 +40,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.3-beta.4 + image: {{ .Values.container.registry }}/openmined/grid-frontend:{{ .Values.container.syft_version }} lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 51d5b53a25c..f26a70040ed 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -43,7 +43,7 @@ spec: - name: SEAWEED_MOUNT_PORT value: '4001' envFrom: null - image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.4 + image: {{ .Values.container.registry }}/openmined/grid-seaweedfs:{{ .Values.container.syft_version }} lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 90d892163a4..d097afbcc00 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -20,6 +20,10 @@ db: mongoPort: "'27017'" mongoHost: "mongo" +container: + registry: "docker.io" + syft_version: 0.8.3-beta.4 + node: settings: tls: false From bee8b9efa31e69989da8434c23f383e3a1549058 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:48:35 +0530 Subject: [PATCH 17/47] added registry and version to devspace --- packages/grid/devspace.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index fdb062f15a9..e12d1d02980 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -61,6 +61,9 @@ deployments: helm: chart: name: ./helm/syft + values: + container.registry: "${CONTAINER_REGISTRY}" + container.syft_version: "${VERSION}" dev: mongo: From df8c3ce7baab5afa8915bbf003475014db834476 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:31:16 +0530 Subject: [PATCH 18/47] fixed tag setting in devspace file --- packages/grid/devspace.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index e12d1d02980..fda65b93c32 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -62,8 +62,9 @@ deployments: chart: name: ./helm/syft values: - container.registry: "${CONTAINER_REGISTRY}" - container.syft_version: "${VERSION}" + container: + registry: ${CONTAINER_REGISTRY} + syft_version: "dev-latest" dev: mongo: From 44e3ae9d7cb85a553979e9d21ffe1055fdbe086b Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:35:35 +0530 Subject: [PATCH 19/47] remove unsed variable names --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index 1467f7f52b4..f69c302b450 100644 --- a/tox.ini +++ b/tox.ini @@ -689,8 +689,6 @@ commands = devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ -p gateway \ --var NODE_NAME=$NODE_NAME \ - --var NETWORK_CHECK_INTERVAL=5 \ - --var ASSOCIATION_TIMEOUT=100 \ --var TEST_MODE=1 \ --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ --var NODE_TYPE=gateway \ @@ -707,8 +705,6 @@ commands = (r=5;while ! \ devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ --var NODE_NAME=$NODE_NAME \ - --var DOMAIN_CHECK_INTERVAL=5 \ - --var ASSOCIATION_TIMEOUT=100 \ --var TEST_MODE=1 \ --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ deploy -b; \ From efb93ced158750e0c59512558e09074050af77c1 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:45:12 +0530 Subject: [PATCH 20/47] added node name attribute in devspace --- packages/grid/devspace.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index fda65b93c32..c6f7bd5bc46 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -22,6 +22,7 @@ vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io" VERSION: "0.8.3-beta.4" + NODE_NAME: "mynode" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible @@ -65,6 +66,10 @@ deployments: container: registry: ${CONTAINER_REGISTRY} syft_version: "dev-latest" + node: + settings: + nodeName: ${NODE_NAME} + nodeType: "domain" dev: mongo: @@ -91,10 +96,9 @@ dev: profiles: - name: gateway patches: - - op: remove - path: deployments.seaweedfs - - op: remove - path: deployments.seaweedfs-config + - op: replace + path: deployments.syft.helm.values.node.setting.nodeType + value: "gateway" commands: start: From da05ca788b3d59f6d989637cee4ef17839dcb317 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:55:17 +0530 Subject: [PATCH 21/47] fixed gateway profile in devspace --- packages/grid/devspace.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index c6f7bd5bc46..ae9b8c2ebd1 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -97,7 +97,7 @@ profiles: - name: gateway patches: - op: replace - path: deployments.syft.helm.values.node.setting.nodeType + path: deployments.syft.helm.values.node.settings.nodeType value: "gateway" commands: From b5a4beaa8edc763a7ee8a179fac26b18adfc461d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 19:47:54 +0530 Subject: [PATCH 22/47] fixed devspace dev mode --- packages/grid/devspace.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index ae9b8c2ebd1..51ae3f6828a 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -15,7 +15,7 @@ pipelines: run: |- run_dependencies --all ensure_pull_secrets --all - build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.3-beta.4 -t dev-latest + build_images --all create_deployments --all vars: @@ -35,6 +35,7 @@ images: tags: - dev-latest - "${VERSION}" + - "${devspace.git.commit}" frontend: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}" buildKit: @@ -45,6 +46,7 @@ images: tags: - dev-latest - "${VERSION}" + - "${devspace.git.commit}" seaweedfs: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}" buildKit: {} @@ -55,6 +57,7 @@ images: tags: - dev-latest - "${VERSION}" + - "${devspace.git.commit}" # This is a list of `deployments` that DevSpace can create for this project deployments: From 2bb924b4c925ba0db04a6911f05aaf87e556e9a4 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:14:06 +0530 Subject: [PATCH 23/47] addded helm syft version to bump2version and cd pipeline --- .bumpversion.cfg | 2 ++ .github/workflows/cd-syft.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 73831d64a7f..dbe950e5fac 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -53,3 +53,5 @@ first_value = 1 [bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube-config.yaml] [bumpversion:file:packages/syftcli/manifest.yml] + +[bumpversion:file:packages/grid/helm/syft/values.yaml] \ No newline at end of file diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 3ae5e414d92..60036eaefbf 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -358,7 +358,7 @@ jobs: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[syft]bump version" - add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile']" + add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml']" - name: Scheduled Build and Publish if: github.event_name == 'schedule' From bd5ec33d2cb22a81dc664e7ac647c2e479fc9da0 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:26:59 +0530 Subject: [PATCH 24/47] shifted custom notes generation function --- packages/grid/helm/generate_helm_notes.py | 72 +++++++++++++++++++++++ tox.ini | 35 +---------- 2 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 packages/grid/helm/generate_helm_notes.py diff --git a/packages/grid/helm/generate_helm_notes.py b/packages/grid/helm/generate_helm_notes.py new file mode 100644 index 00000000000..6869a79a97d --- /dev/null +++ b/packages/grid/helm/generate_helm_notes.py @@ -0,0 +1,72 @@ +# stdlib +import json +import os +from pathlib import Path +import sys + + +def add_notes(helm_chart_template_dir: str) -> None: + """Add notes or information post helm install or upgrade.""" + + notes = """ + Thank you for installing {{ .Chart.Name }}. + Your release is named {{ .Release.Name }}. + To learn more about the release, try: + + $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} + """ + + notes_path = os.path.join(helm_chart_template_dir, "NOTES.txt") + + protocol_changelog = get_protocol_changes() + + notes += "\n" + protocol_changelog + + with open(notes_path, "w") as fp: + fp.write(notes) + + +def get_protocol_changes() -> str: + """Generate change log of the dev protocol state.""" + script_path = os.path.dirname(os.path.realpath(__file__)) + protocol_path = Path( + os.path.normpath( + os.path.join( + script_path, + "../../", + "syft/src/syft/protocol", + "protocol_version.json", + ) + ) + ) + + protocol_changes = "" + if protocol_path.exists(): + dev_protocol_changes = json.loads(protocol_path.read_text()).get("dev", {}) + protocol_changes = json.dumps( + dev_protocol_changes.get("object_versions", {}), indent=4 + ) + + protocol_changelog = f""" + Following class versions are either added/removed. + + {protocol_changes} + + This means the existing data will be automatically be migrated to + their latest class versions during the upgrade. + """ + + return protocol_changelog + + +if __name__ == "__main__": + # write code to path from user and pass to generate notes + if len(sys.argv) != 2: + print("Please provide helm chart template directory path") + sys.exit(1) + helm_chart_template_dir = sys.argv[1] + add_notes(helm_chart_template_dir) + print("=" * 50) + print("Notes Generated Successfully") + print("=" * 50) diff --git a/tox.ini b/tox.ini index f69c302b450..acd949785b4 100644 --- a/tox.ini +++ b/tox.ini @@ -810,47 +810,18 @@ commands = [testenv:syft.build.helm] description = Build Helm Chart for Kubernetes -deps = - pyyaml changedir = {toxinidir} -passenv=HOME, USER, CONTAINER_REGISTRY , VERSION +passenv=HOME, USER allowlist_externals = - devspace - kubectl - grep - sleep bash - kubectx - k3d - echo - rm helm commands = - k3d version - devspace version - - bash -c "k3d cluster delete build ; \ - docker volume rm k3d-build-images --force || true" - - bash -c 'k3d cluster create build && \ - k3d cluster start build && \ - echo "Waiting for cluster to be ready..." && \ - sleep 20' - - bash -c 'cd packages/grid && \ - [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \ - [[ -n "$VERSION" ]] && VERSION_FLAG="--var VERSION=$VERSION" || VERSION_FLAG="" && \ - devspace deploy --render --skip-build ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" --no-warn --no-colors > out.txt ; \ - EXITCODE=$?; OUTPUT=$(cat out.txt); printf "Devspace exit code: $EXITCODE\nDevspace output:\n$OUTPUT\n"; exit $EXITCODE' - - bash -c 'cd packages/grid && \ - python3 helm/helm.py out.txt && \ - rm out.txt' + bash -c 'cd packages/grid/helm && \ + python3 generate_helm_notes.py ./syft/templates' bash -c 'cd packages/grid/helm && \ helm lint syft' - bash -c "k3d cluster delete build; docker volume rm k3d-build-images --force; echo Done" [testenv:syft.package.helm] description = Package Helm Chart for Kubernetes From 2791cfa85a159ccb36620ae7d8005c5afd9a52f7 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:28:43 +0530 Subject: [PATCH 25/47] removed unused packages --- tox.ini | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tox.ini b/tox.ini index acd949785b4..82e8e678a8b 100644 --- a/tox.ini +++ b/tox.ini @@ -829,15 +829,7 @@ deps = changedir = {toxinidir} passenv=HOME, USER allowlist_externals = - devspace - kubectl - grep - sleep bash - kubectx - k3d - echo - rm helm commands = bash -c 'cd packages/grid/helm && \ From 3205da5c9d91ca4a56074d0c0f77716371ef09b0 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:29:18 +0530 Subject: [PATCH 26/47] updated syft version --- packages/grid/helm/syft/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index d097afbcc00..2337bea7a6a 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -22,7 +22,7 @@ db: container: registry: "docker.io" - syft_version: 0.8.3-beta.4 + syft_version: 0.8.3-beta.5 node: settings: From 38a4abb7fca483928108a0434dc0fcd9a0c93c9f Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:35:26 +0530 Subject: [PATCH 27/47] removed tag during build images --- packages/grid/devspace.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 07c4e27c93f..d59617b2b67 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -15,7 +15,7 @@ pipelines: run: |- run_dependencies --all ensure_pull_secrets --all - build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.3-beta.5 -t dev-latest + build_images --all create_deployments --all vars: From 6a9051e84d0aa7a2465debb0962dfc779f191a36 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 7 Dec 2023 07:56:15 +0530 Subject: [PATCH 28/47] trying removing component-chart --- packages/grid/helm/syft/Chart.yaml | 7 +------ .../grid/helm/syft/templates/backend-headless-service.yaml | 2 -- packages/grid/helm/syft/templates/backend-service.yaml | 2 -- packages/grid/helm/syft/templates/backend-statefulset.yaml | 6 ------ packages/grid/helm/syft/templates/frontend-deployment.yaml | 4 ---- packages/grid/helm/syft/templates/frontend-service.yaml | 2 -- .../grid/helm/syft/templates/mongo-headless-service.yaml | 2 -- packages/grid/helm/syft/templates/mongo-service.yaml | 2 -- packages/grid/helm/syft/templates/mongo-statefulset.yaml | 6 ------ packages/grid/helm/syft/templates/proxy-deployment.yaml | 4 ---- packages/grid/helm/syft/templates/proxy-service.yaml | 2 -- .../helm/syft/templates/seaweedfs-headless-service.yaml | 2 -- packages/grid/helm/syft/templates/seaweedfs-service.yaml | 2 -- .../grid/helm/syft/templates/seaweedfs-statefulset.yaml | 6 ------ 14 files changed, 1 insertion(+), 48 deletions(-) diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index cb67443fd0f..95f7e1f207b 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -4,9 +4,4 @@ description: Perform numpy-like analysis on data that remains in someone elses s type: application version: "0.8.3-beta.5" appVersion: "0.8.3-beta.5" -icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png - -dependencies: - - name: component-chart - version: "0.9.1" - repository: "https://charts.devspace.sh" +icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/backend-headless-service.yaml b/packages/grid/helm/syft/templates/backend-headless-service.yaml index 5cd1e6668ac..f59d89a905b 100644 --- a/packages/grid/helm/syft/templates/backend-headless-service.yaml +++ b/packages/grid/helm/syft/templates/backend-headless-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: backend diff --git a/packages/grid/helm/syft/templates/backend-service.yaml b/packages/grid/helm/syft/templates/backend-service.yaml index f1e676707bb..5326659d116 100644 --- a/packages/grid/helm/syft/templates/backend-service.yaml +++ b/packages/grid/helm/syft/templates/backend-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: backend diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 93048d41029..7ae473b4a85 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -1,8 +1,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: backend app.kubernetes.io/managed-by: Helm @@ -19,8 +17,6 @@ spec: serviceName: backend-headless template: metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: backend app.kubernetes.io/managed-by: Helm @@ -111,8 +107,6 @@ spec: volumes: null volumeClaimTemplates: - metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: backend app.kubernetes.io/managed-by: Helm diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index 84aa6026f14..a1bdaddb2e6 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -1,8 +1,6 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: frontend app.kubernetes.io/managed-by: Helm @@ -19,8 +17,6 @@ spec: type: Recreate template: metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: frontend app.kubernetes.io/managed-by: Helm diff --git a/packages/grid/helm/syft/templates/frontend-service.yaml b/packages/grid/helm/syft/templates/frontend-service.yaml index 5b9916d409c..6a85a40a31a 100644 --- a/packages/grid/helm/syft/templates/frontend-service.yaml +++ b/packages/grid/helm/syft/templates/frontend-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: frontend diff --git a/packages/grid/helm/syft/templates/mongo-headless-service.yaml b/packages/grid/helm/syft/templates/mongo-headless-service.yaml index 685f3c81f96..e69be319f7f 100644 --- a/packages/grid/helm/syft/templates/mongo-headless-service.yaml +++ b/packages/grid/helm/syft/templates/mongo-headless-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: mongo diff --git a/packages/grid/helm/syft/templates/mongo-service.yaml b/packages/grid/helm/syft/templates/mongo-service.yaml index 23699f74f21..c0af82c7641 100644 --- a/packages/grid/helm/syft/templates/mongo-service.yaml +++ b/packages/grid/helm/syft/templates/mongo-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: mongo diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 0efc62f142d..7bae5c7b04a 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -1,8 +1,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: mongo app.kubernetes.io/managed-by: Helm @@ -19,8 +17,6 @@ spec: serviceName: mongo-headless template: metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: mongo app.kubernetes.io/managed-by: Helm @@ -65,8 +61,6 @@ spec: volumes: null volumeClaimTemplates: - metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: mongo app.kubernetes.io/managed-by: Helm diff --git a/packages/grid/helm/syft/templates/proxy-deployment.yaml b/packages/grid/helm/syft/templates/proxy-deployment.yaml index 4197618d8ef..8a561b2d953 100644 --- a/packages/grid/helm/syft/templates/proxy-deployment.yaml +++ b/packages/grid/helm/syft/templates/proxy-deployment.yaml @@ -1,8 +1,6 @@ apiVersion: apps/v1 kind: Deployment metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: proxy app.kubernetes.io/managed-by: Helm @@ -19,8 +17,6 @@ spec: type: Recreate template: metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: proxy app.kubernetes.io/managed-by: Helm diff --git a/packages/grid/helm/syft/templates/proxy-service.yaml b/packages/grid/helm/syft/templates/proxy-service.yaml index 987607391df..18da1e72287 100644 --- a/packages/grid/helm/syft/templates/proxy-service.yaml +++ b/packages/grid/helm/syft/templates/proxy-service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: proxy diff --git a/packages/grid/helm/syft/templates/seaweedfs-headless-service.yaml b/packages/grid/helm/syft/templates/seaweedfs-headless-service.yaml index c2e9678e1f5..d6baa9e0a3d 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-headless-service.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-headless-service.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: seaweedfs diff --git a/packages/grid/helm/syft/templates/seaweedfs-service.yaml b/packages/grid/helm/syft/templates/seaweedfs-service.yaml index 78bccff1967..c6c26b0a589 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-service.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-service.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: seaweedfs diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index f26a70040ed..62b24d87460 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -2,8 +2,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: seaweedfs app.kubernetes.io/managed-by: Helm @@ -20,8 +18,6 @@ spec: serviceName: seaweedfs-headless template: metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: seaweedfs app.kubernetes.io/managed-by: Helm @@ -83,8 +79,6 @@ spec: name: seaweedfs-config volumeClaimTemplates: - metadata: - annotations: - helm.sh/chart: component-chart-0.9.1 labels: app.kubernetes.io/component: seaweedfs app.kubernetes.io/managed-by: Helm From 719af508d6c054180b7f6747668ac2c01897b542 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 7 Dec 2023 07:58:47 +0530 Subject: [PATCH 29/47] replace buildjet cache with actions/cache --- .github/workflows/pr-tests-stack.yml | 8 ++++---- .github/workflows/pr-tests-syft.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index 8ac817640bf..7294ea88377 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -82,7 +82,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.stack == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -272,7 +272,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.stack == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -354,7 +354,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.stack == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -560,7 +560,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.stack == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index bc908249d26..84759670500 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -75,7 +75,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.syft == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -160,7 +160,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -240,7 +240,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} @@ -338,7 +338,7 @@ jobs: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - name: pip cache - uses: buildjet/cache@v3 + uses: actions/cache@v3 if: steps.changes.outputs.syft == 'true' with: path: ${{ steps.pip-cache.outputs.dir }} From 01fc78b24429b2c13b84eb4dab9b295c55ca78b6 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 7 Dec 2023 08:30:42 +0530 Subject: [PATCH 30/47] removed component chart and updated helm notes.txt --- packages/grid/helm/syft/Chart.lock | 6 - .../syft/charts/component-chart-0.9.1.tgz | Bin 5089 -> 0 bytes packages/grid/helm/syft/templates/NOTES.txt | 827 ++---------------- 3 files changed, 65 insertions(+), 768 deletions(-) delete mode 100644 packages/grid/helm/syft/Chart.lock delete mode 100644 packages/grid/helm/syft/charts/component-chart-0.9.1.tgz diff --git a/packages/grid/helm/syft/Chart.lock b/packages/grid/helm/syft/Chart.lock deleted file mode 100644 index e171adaed3c..00000000000 --- a/packages/grid/helm/syft/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: component-chart - repository: https://charts.devspace.sh - version: 0.9.1 -digest: sha256:fbc2b8447b97f168c9199a0bbdc3a95501a19a3213a5ddbd7a78bc4ef2ab999f -generated: "2023-09-28T21:45:16.869504405+05:30" diff --git a/packages/grid/helm/syft/charts/component-chart-0.9.1.tgz b/packages/grid/helm/syft/charts/component-chart-0.9.1.tgz deleted file mode 100644 index fa9d380b7e1bbbd19e6c99fa8f490d02353fbcc4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5089 zcmV<76CUgziwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKD1ZyUFkct7i}Amm=4`Js9xPST%&eDJkzZ;dvN*G{`wEEWrT zhLQ-6$SKK@qq_dz7bNGw=gi2K?LIUg5{o=9l81+fhckx>UFDP@B1Xv)GBG)k|Bg3s zm3_6dXF8ou-yI&R|EJTb`~URyZ1&ac@bLZN?C{;2>ETz?+2QN=Z@vQ4-H8x?3N9e~ zYWiqiZsYzW56tl|h;dBG5m?U#DdGvkxlq54!2%J)UtpVtm z|FgG;@7($S_VDd~{y)caaBy${PSN@*hY12d$oZ|703a&{2jKG(bAUO3V2KyYYy%QX z=6F#sc%7B-N^!>Ed&&SNbIMj~)&okwlHLMAYZL^8q&CMH`gU*t1iV22f{fA|00j_B z1X7w5D?|j;252cn&W|RO6s@_GZ=7VbNGB;x_$0~jMB5*UFlh|Xq?S+;nb0+2Ym9D3 zJVyzhS2k&I0FFV1xsdiMp+rDT5aU2Bp#XeIi!24KAWEv7rpl}->1#%V2I~m?_x}tq z!Q#X&8gR)wK}Q1s#3n~a;3O-!KQO1wJ-TeRnNC?%_2^+XB<^YogL!3*dDduvV(jzd38Ashr6U4*{61YI@2nbe46jo%YWGhu=jtK&} zG~QewBEXtv#R_q-L<}i~K?JO5LDYz@048Sl!fU!lgA&E2$pK!$1p;}IWq_lEA;AZR zZl3&f25xbd$)yF66u>OAV8QJYCrbbs0bS2JQC$?d#d}tz-Y+AHJUL?f-K;cXuPeAXy;rbv;ebNQ%CEYZUxU3XW26^Z|T5 z{zW?o#+<&tA4n>A4$!YI6)=_6rjrTtoKMy>h4S?cCW`Fvv{trZMXyt;7I^?LC}?Z* zQJSQRhEb%b)YxOArlh24;kkuX!i^b%=_itHOi+|@RQ(PJV#{DPqB5m5a?P7i0luLO ze6xi7N3^N2huYu_U->n98T<(IP$gRBVsnaFsBTHQxPW3gd<9+wM3I9W=n3iMR~7n) zfvS=9aDM>9D**+X7uglk!)*dd)81C(6j(ovAI_g&!z^nSXW8j$?YC78r`ze5L-a$n zGCAc2MPD$BwdF!s>}}^_dl78Pwit3$j=|`6n*X~O? zy56`1uCzDi(lCZWwL|jxw)|UlzcmuNsSU1yWDq`TYBY%e&suFwH%zxm?z6{yIZ^apA9h*7Vb8BQ2mBtm^-w)juHMGlSH3fd#3_;8W+BCdYx)kG0l`gn!Aww!Sz{~L| zxmTs&caWkvEHVLRji#5DN2#d0TmZlYg90r!N0z~^Dbp;&Wbt#JLR+pV&=?WrI9I^h zd%rY6RJnz8RGO$2{Bl*W1#+5a(XU{P#tvV<{fA)=SMbZv1g;^@q$C}*3`mpWv?1uO zOl6CZ_F;Xc+#bXGVGWsY^Lvca7GqSXvMWmuwi>k@GJ*BJGv3uTkkQN2Nb~C-IZZz* zqa+pff@V0`*gOIN<|uw*DPkzSL@?b9?dj!qP`Tkeuh(Crz~*T}dL)}+xA2F)J-)vm zEfGvJ#QCtb6FACHA}G_{y@DcH{$#8gTdo_AUpaQOc06wNc=-C^bZ)h4D?8U%zU7ly z{k(Ng@1n1F)!UEl@5l6bz!~*c?p>_jJ6^$evx*0rU#;fyQ6Z2EBkB=qiQcPJ`yc&F z$zry^2%VU`M}{M6VD=d#LQ`~ATc5i-An1R=Ro1XH0l@D7lN1pFzOyMP3mC&LW$`iL zd@fz@O%SuH3}OnJAy%nY0NJPy;-`e4s2yMsyRHr5${pZwh9O5`-bQ^8pRN&;t3nXD zstjVv{WKSEIg=QbkC8vrR>J0T^9QEJ?LIMF3wJE`WI)XK5YxC7hOB5RF(LYZMaU-IYVfr zj=Wu6acdl}J&MTQ&B?Nh=7MIH2RCva&X&xawt01J zOt)^O_@SJp$Nc;MI6XJ{I#n*)C}ua;C1mJ=(FAD)Q2QK%bEw+k-cF%gg5o-*D~OG? zt}b#o`H;QWErHky6XhBHo52KKAckq`I})vMwm7Yzj)>ObM+c9$v9_UniB^a~BZeJx zxdL!OXUjOM4{b~yc*dlkk>3319%~Mg^kH^jf70kh1MHudG%K3}6{TqRd_!Yc6Bt?K z^!c2*=?osggv70Pd#?2c=2p3iV_40sMj3`&q?inC zXjSI&Q-zOb^@$`X#7>q6S;~q>u@z$DFciWYLm~Q%TS8YWNYW#t$eDGjzL|6Cr*j@# zm0FbLq|*}V%HW)cNCnw~M@52cQHcZD!VyJchUJj3+sNQ+v|%VeBF^O$fnwvL7#-Mr#*4F|cvbM~BLllv9WAJCMJ+$u`UByvlZ5-Y25cNuy4W!@F8U(dT z)_sI|rBUTiN2>xf8`dRCV12M7Vr_8YwV}ZuO5~A2CA2U^2BtiO{$JUqf^a07Rt{B2 z8K|S?y7#Nfrq2(1YNl7%7UMCDVl2wz4t9p;DA^<#iV&*n+eoYtL7ZPOdL1WYz{E+_ zF|IaRHc*_H+GiDc8z~bcPDsDpaf+np&m)v9m@ZQKHJhk4e)NkwjjF4tQ)4d8_g88X z%1%bixap9^UVe$9w!A;Yi)-IJ1HCb*#T5*%jzADxM(VZF2d;SxaaynhaaZ@tVqY(Au~-Hvj5EB#{pnGpRROSAsIRsqr9dh!NKynzM(v ztqd*25mg#9^lO25ctwq?PYkOne<~FUOrG}Uh4s7-PK&I=>2MpJ*uG+?s`{LL)+O|_ z^xSjJzvt}O2es)5d%LYTvchfGD;<{PStjoDD*KCSz&6E!La)gAxW@wAE{GUUoPlSE+Io{oqa%mA5{OW6Cpe}4MI#Un^Xu^`UBt9_6T$g+y|9==W#}-k2W?F zEGCH+pLF*W9f8DV4^a(?yCY`E-U!>!t#9V|*3@Cy47(FYDvLvm8=Qynj_Q{aN+^p% zB2;xdlAzXN8@ehENl?{oD?yVJN~_XI2Jy;XrV?3nzMU5JdmZSJQ=l$dH==*8QXw@- z1s-UqJ=6ANCdwTd%gR&@>RKiqy+rJKd3gNN;0||e`aIO4_v+`6RqvysPC?+OX~ARM z75K>eIJ~D#{~LZoUiQb|E(>RQv0F?q$vxn@&-oi^JH63lLT=t8_gp_WHndGPV;Z%>;NRMlGqwF3jYfmP0XR_!aX^&^rTTbIrARi? z-k4eP!Z~rAWXw7-S82zrq+EAhXPB9T>7FQEbPNUQFD~W$g*^TmIB=IBg z-Bbysb@mor59AJ`Ot5@*xOsXnBF8URGlQT@j91Q=#he4cIHi(LQyn5Am zLR!LY$;f@ZO~gYoqXvQYerO1ajE{nRC5cxdr!C?RK8B}BBFM8pMRSETNmU+`|D%gZ zOc^X%;Xh3mo9OH|m@LLO->KEPcDm0r;Z~>HA6eh)wqRP^?ilSJj(rGp3r@bSME4Gz zt>~32xMS@9^m~eGJpL?ui;&tG*Fabk5vFN%g9Yb>4g#Ao%?hZTJ}L|of_cb(rjjkV zuc_k0at&~|I78W*QP~{j!|NuDuxi8bRUE5AD$PTGM3ddtY-$86GU9iBAZ+i)g=3up zlNB1fqArEVr?XMn)uTj?-xh*L(7k3@>e1DT9d22TZXw87Jn0)Ee zEltvIa-a29C#G*`V8D}u-bfi z4xUa%GJU1~FbU}{F$me4wAL2by;~!PTcE09Eok>!UFt!|w$7;$Er-(U1?R^|y-cB% z%xsV>b!VG2L`}YVv8<0LM&P=dGsbyEb9-3x7-^78i6bv#st~DF0stvi*VVdEX@he- z=MG}-yzZaZA0>EDu>(Knuw7G_biI-h66EtH_egzKN&Z6uDd;VP(v;3td4}ps=&s6J zYLWKcD4smsL69S9-()!B?Ypyrc~O zMu~u#5r2V2x*!~K#{aXu9vdOJ>wMxxCL$*NT63um+L)TQwa{q!n+YxV3j}<1D?U$Wtu*N3( zHzsu0P)z^{h7%szuSZD0HBxWNUHn{aW2*bT^t)85jn#`)_RFG4UX;Iu>cD1~p-CKT z(u#jw1&zX9ZFB`}jR>^UxI`Hgc>N@e{aN4)|E4dMcG_xGZPQ{?+TQ^t;?eE@Xnc^h zMbq{7TF*Ds9|K=)?qJlPuJunh_H|c;hJTkY<;4B}Ds1x9?|)3+O=s@?&+pz(r+feZ zIi9<&bj5+YpY3R&{c4AmL)g3g`5`a0zR=0^pZ5H@>XN^*msqv&xxB@gYSwR?YFy*x z(~y!n&{o%?F9IFjrDQ1V)dE9 z`5Y%^Qm0^5JM#dQUQ#R(%*66!iISUhni^*?fr_vMQ3}2Ih6O{{GIU+r;Km7iUegea)mbuFRtS{0nl=V34(E~1J*3=2 zFj+Sl%PcUxr}8C;fcpW|7y(}zTYN?ZO=#vIm8IreAD>Sy+TZzjv2RO6edy`wygu}- z+fcphW4GinTgLtK7kc9R|MJM;>G%I{4&VCUf4q6O-~T_y)8h~-a0YevED0@4i?0r> z67`|&FSTU#i=%(YQ^8%NtMCr_`lqGvd(6#0vHfvgwX$1eV=jOZrWuL$lmEfZ{m>`xL!Tp+mtL!%;&J`I@d69HAe5oq@#ygXrqh}0{~f-0 z^M3FDJ;yTO8wj8g<(T9*Cu{Es*PrH8O%1JF7D-@bO||J&*Q`~PQoq`jXE_ Date: Thu, 7 Dec 2023 08:37:23 +0530 Subject: [PATCH 31/47] added helm templates to cd-pipeline --- .github/workflows/cd-syft.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 60036eaefbf..c2bdc315f95 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -358,7 +358,7 @@ jobs: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[syft]bump version" - add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml']" + add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']" - name: Scheduled Build and Publish if: github.event_name == 'schedule' From e7bb0931b967de73fcc06eaa0082b284729278af Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 08:10:44 +0530 Subject: [PATCH 32/47] - Moved the secrets randomization to syft templates folder Co-authored-by: Param-29 --- .../helm/syft/templates/mongo_secrets.yaml | 31 +++++++++++++++++++ .../syft/templates/seaweedfs_secrets.yaml | 23 ++++++++++++++ .../helm/syft/templates/syft_secrets.yaml | 21 +++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/grid/helm/syft/templates/mongo_secrets.yaml create mode 100644 packages/grid/helm/syft/templates/seaweedfs_secrets.yaml create mode 100644 packages/grid/helm/syft/templates/syft_secrets.yaml diff --git a/packages/grid/helm/syft/templates/mongo_secrets.yaml b/packages/grid/helm/syft/templates/mongo_secrets.yaml new file mode 100644 index 00000000000..09a0df21900 --- /dev/null +++ b/packages/grid/helm/syft/templates/mongo_secrets.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "mongosecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "mongosecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + + # Helm does not allow hypen character, as a variable name , so we conform to camelCase + {{- $mongoUnameSecret_env := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} + mongo-username-secret: {{ $mongoUnameSecret_env | quote }} + + {{- $mongoPassSecret_env := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} + mongo-password-secret: {{ $mongoPassSecret_env | quote }} + + {{- $mongoDbRootUser_env := (get $secretData "mongoInitdbRootUsername") | default (randAlphaNum 32 | b64enc) }} + mongoInitdbRootUsername: {{ $mongoDbRootUser_env | quote }} + + {{- $mongoDbRootPass_env := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} + mongoInitdbRootPassword: {{ $mongoDbRootPass_env | quote }} + + {{- else }} + + mongo-username-secret: cm9vdA== # Base64-encoded "root" + mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" + mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" + mongoInitdbRootUsername: cm9vdA== # Base64-encoded "root" + {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml new file mode 100644 index 00000000000..db51593b73d --- /dev/null +++ b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "seaweedfsecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "seaweedfsecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + + + {{- $S3_ROOT_USER_SECRET_env := (get $secretData "S3_ROOT_USER_SECRET") | default (randAlphaNum 32 | b64enc) }} + S3_ROOT_USER_SECRET: {{ $S3_ROOT_USER_SECRET_env | quote }} + + {{- $S3_ROOT_PWD_SECRET_env := (get $secretData "S3_ROOT_PWD_SECRET") | default (randAlphaNum 32 | b64enc) }} + S3_ROOT_PWD_SECRET: {{ $S3_ROOT_PWD_SECRET_env | quote }} + + {{- else }} + + S3_ROOT_USER_SECRET: YWRtaW4= # Base64-encoded "admin" + S3_ROOT_PWD_SECRET: YWRtaW4= # Base64-encoded "admin" + {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/syft_secrets.yaml b/packages/grid/helm/syft/templates/syft_secrets.yaml new file mode 100644 index 00000000000..28c0eeb5e68 --- /dev/null +++ b/packages/grid/helm/syft/templates/syft_secrets.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "syftsecret" +type: Opaque +data: + {{- if not .Values.configuration.devmode }} + + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "syftsecret") | default dict }} + {{- $secretData := (get $secretObj "data") | default dict }} + + {{- $stackApiKey_env := (get $secretData "stackApiKey") | default (randAlphaNum 32 | b64enc) }} + stackApiKey: {{ $stackApiKey_env | quote }} + + {{- $defaultRootPassword_env := (get $secretData "defaultRootPassword") | default (randAlphaNum 32 | b64enc) }} + defaultRootPassword: {{ $defaultRootPassword_env | quote }} + + {{- else }} + stackApiKey: Y2hhbmdlbWU= # Base64-encoded "changeme" + defaultRootPassword: Y2hhbmdldGhpcw== # Base64-encoded "changethis" + {{- end }} \ No newline at end of file From 66157a4b22fcd842e3cec0a574e39b46f23d951e Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 09:56:19 +0530 Subject: [PATCH 33/47] fixed helm string requirement --- .../grid/helm/syft/templates/seaweedfs-statefulset.yaml | 6 +++--- packages/grid/helm/syft/values.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 59738d78adf..d17995b5dc0 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -29,7 +29,7 @@ spec: command: null env: - name: S3_VOLUME_SIZE_MB - value: {{ .Values.seaweedfs.S3_VOLUME_SIZE_MB }} + value: "{{ .Values.seaweedfs.S3_VOLUME_SIZE_MB }}" - name: S3_ROOT_USER secretKeyRef: name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} @@ -39,9 +39,9 @@ spec: name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} key: S3_ROOT_PWD_SECRET - name: S3_PORT - value: {{ .Values.seaweedfs.S3_PORT }} + value: "{{ .Values.seaweedfs.S3_PORT }}" - name: SEAWEED_MOUNT_PORT - value: {{ .Values.seaweedfs.SEAWEED_MOUNT_PORT }} + value: "{{ .Values.seaweedfs.SEAWEED_MOUNT_PORT }}" envFrom: null image: {{ .Values.container.registry }}/openmined/grid-seaweedfs:{{ .Values.container.syft_version }} lifecycle: null diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index afc27aff9eb..99def2c1bd5 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -28,9 +28,9 @@ db: mongoHost: "mongo" seaweedfs: - S3_VOLUME_SIZE_MB: '1024' - S3_PORT: '8333' - SEAWEED_MOUNT_PORT: '4001' + S3_VOLUME_SIZE_MB: 1024 + S3_PORT: 8333 + SEAWEED_MOUNT_PORT: 4001 container: registry: "docker.io" From f92117d5134fee669a71a026217fc19ae336e308 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:24:27 +0530 Subject: [PATCH 34/47] fix syntax error in helm templates for referencing secrets --- .../helm/syft/templates/backend-statefulset.yaml | 2 +- .../helm/syft/templates/seaweedfs-statefulset.yaml | 14 ++++++++------ packages/grid/helm/syft/values.yaml | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 663aea830aa..969f1267b1f 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -85,7 +85,7 @@ spec: - name: DEFAULT_ROOT_EMAIL value: {{ .Values.secrets.syft.defaultRootEmail }} - name: DEFAULT_ROOT_PASSWORD - valueFrom: + valueFrom: secretKeyRef: name: {{ .Values.secrets.syft.syftsecret }} key: defaultRootPassword diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index d17995b5dc0..2da23dba8f8 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -31,13 +31,15 @@ spec: - name: S3_VOLUME_SIZE_MB value: "{{ .Values.seaweedfs.S3_VOLUME_SIZE_MB }}" - name: S3_ROOT_USER - secretKeyRef: - name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} - key: S3_ROOT_USER_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} + key: S3_ROOT_USER_SECRET - name: S3_ROOT_PWD - secretKeyRef: - name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} - key: S3_ROOT_PWD_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} + key: S3_ROOT_PWD_SECRET - name: S3_PORT value: "{{ .Values.seaweedfs.S3_PORT }}" - name: SEAWEED_MOUNT_PORT diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 99def2c1bd5..c23ed924899 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -6,7 +6,7 @@ secrets: syft: syftsecret: "syftsecret" # stackApiKey: "changeme" - # defaultRootEmail: "info@openmined.org" + defaultRootEmail: "info@openmined.org" # defaultRootPassword: "changethis" db: mongo: From dcba683fbd85877a705d0aa3d646d6be6a11457e Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:18:43 +0530 Subject: [PATCH 35/47] simplified base64 conversion automatically during secrets creation --- packages/grid/helm/syft/templates/mongo_secrets.yaml | 8 ++++---- packages/grid/helm/syft/templates/seaweedfs_secrets.yaml | 4 ++-- packages/grid/helm/syft/templates/syft_secrets.yaml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/grid/helm/syft/templates/mongo_secrets.yaml b/packages/grid/helm/syft/templates/mongo_secrets.yaml index 09a0df21900..bb89bb0bc8e 100644 --- a/packages/grid/helm/syft/templates/mongo_secrets.yaml +++ b/packages/grid/helm/syft/templates/mongo_secrets.yaml @@ -24,8 +24,8 @@ data: {{- else }} - mongo-username-secret: cm9vdA== # Base64-encoded "root" - mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" - mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" - mongoInitdbRootUsername: cm9vdA== # Base64-encoded "root" + mongo-username-secret: {{ "root" | b64enc }} # Base64-encoded "root" + mongo-password-secret: {{ "example" | b64enc }} # Base64-encoded "example" + mongoInitdbRootPassword: {{ "example" | b64enc }} # Base64-encoded "example" + mongoInitdbRootUsername: {{ "root" | b64enc }} # Base64-encoded "root" {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml index db51593b73d..d1bd5fd5b58 100644 --- a/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml @@ -18,6 +18,6 @@ data: {{- else }} - S3_ROOT_USER_SECRET: YWRtaW4= # Base64-encoded "admin" - S3_ROOT_PWD_SECRET: YWRtaW4= # Base64-encoded "admin" + S3_ROOT_USER_SECRET: {{ "admin" | b64enc }} # Base64-encoded "admin" + S3_ROOT_PWD_SECRET: {{ "admin" | b64enc }} # Base64-encoded "admin" {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/syft_secrets.yaml b/packages/grid/helm/syft/templates/syft_secrets.yaml index 28c0eeb5e68..b09116df291 100644 --- a/packages/grid/helm/syft/templates/syft_secrets.yaml +++ b/packages/grid/helm/syft/templates/syft_secrets.yaml @@ -16,6 +16,6 @@ data: defaultRootPassword: {{ $defaultRootPassword_env | quote }} {{- else }} - stackApiKey: Y2hhbmdlbWU= # Base64-encoded "changeme" - defaultRootPassword: Y2hhbmdldGhpcw== # Base64-encoded "changethis" + stackApiKey: {{ "changeme" | b64enc }} # Base64-encoded "changeme" + defaultRootPassword: {{ "changethis" | b64enc}} # Base64-encoded "changethis" {{- end }} \ No newline at end of file From 846cd09fecdcabc1016810ee2dfe9bf834fb9252 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:28:30 +0530 Subject: [PATCH 36/47] - Moved Non-Public Information to node settings - Deduplicated values in mongo-secret --- .../grid/helm/syft/templates/backend-statefulset.yaml | 2 +- packages/grid/helm/syft/templates/mongo-statefulset.yaml | 4 ++-- packages/grid/helm/syft/templates/mongo_secrets.yaml | 8 +------- packages/grid/helm/syft/values.yaml | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 969f1267b1f..97a72903cbf 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -83,7 +83,7 @@ spec: - name: ENABLE_OBLV value: 'false' - name: DEFAULT_ROOT_EMAIL - value: {{ .Values.secrets.syft.defaultRootEmail }} + value: {{ .Values.node.settings.defaultRootEmail }} - name: DEFAULT_ROOT_PASSWORD valueFrom: secretKeyRef: diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index e2519ed1c2c..846d262c9f2 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -31,12 +31,12 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.secrets.db.mongo.mongosecret }} - key: mongoInitdbRootUsername + key: mongo-username-secret - name: MONGO_INITDB_ROOT_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.secrets.db.mongo.mongosecret }} - key: mongoInitdbRootPassword + key: mongo-password-secret envFrom: null image: mongo:7.0.3 lifecycle: null diff --git a/packages/grid/helm/syft/templates/mongo_secrets.yaml b/packages/grid/helm/syft/templates/mongo_secrets.yaml index bb89bb0bc8e..deca94aaeb5 100644 --- a/packages/grid/helm/syft/templates/mongo_secrets.yaml +++ b/packages/grid/helm/syft/templates/mongo_secrets.yaml @@ -16,16 +16,10 @@ data: {{- $mongoPassSecret_env := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} mongo-password-secret: {{ $mongoPassSecret_env | quote }} - {{- $mongoDbRootUser_env := (get $secretData "mongoInitdbRootUsername") | default (randAlphaNum 32 | b64enc) }} - mongoInitdbRootUsername: {{ $mongoDbRootUser_env | quote }} - - {{- $mongoDbRootPass_env := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} - mongoInitdbRootPassword: {{ $mongoDbRootPass_env | quote }} {{- else }} mongo-username-secret: {{ "root" | b64enc }} # Base64-encoded "root" mongo-password-secret: {{ "example" | b64enc }} # Base64-encoded "example" - mongoInitdbRootPassword: {{ "example" | b64enc }} # Base64-encoded "example" - mongoInitdbRootUsername: {{ "root" | b64enc }} # Base64-encoded "root" + {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index c23ed924899..fa1340a3c01 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -6,7 +6,6 @@ secrets: syft: syftsecret: "syftsecret" # stackApiKey: "changeme" - defaultRootEmail: "info@openmined.org" # defaultRootPassword: "changethis" db: mongo: @@ -44,6 +43,7 @@ node: nodeType: "domain" versionHash: "abc" nodeSideType: "high" + defaultRootEmail: "info@openmined.org" ingress: ingressClass: "" From aa2195848a49cfd89de306590f63e741f0e3401c Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:18:16 +0530 Subject: [PATCH 37/47] added devmode to devspace configuration --- packages/grid/devspace.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 31eef63572d..24cb5c0674d 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -73,6 +73,8 @@ deployments: settings: nodeName: ${NODE_NAME} nodeType: "domain" + configuration: + devmode: True dev: mongo: From b99e541d67cb355777e51797e346197657e2f484 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:06:39 +0530 Subject: [PATCH 38/47] removed unsed secrets text file --- packages/grid/k8s/manifests/mongo_secrets.txt | 31 ------------------- .../grid/k8s/manifests/seaweedfs_secrets.txt | 23 -------------- packages/grid/k8s/manifests/syft_secrets.txt | 21 ------------- 3 files changed, 75 deletions(-) delete mode 100644 packages/grid/k8s/manifests/mongo_secrets.txt delete mode 100644 packages/grid/k8s/manifests/seaweedfs_secrets.txt delete mode 100644 packages/grid/k8s/manifests/syft_secrets.txt diff --git a/packages/grid/k8s/manifests/mongo_secrets.txt b/packages/grid/k8s/manifests/mongo_secrets.txt deleted file mode 100644 index c8abd44be22..00000000000 --- a/packages/grid/k8s/manifests/mongo_secrets.txt +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: "mongosecret" -type: Opaque -data: - {{- if not .Values.configuration.devmode }} - - {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "mongosecret") | default dict }} - {{- $secretData := (get $secretObj "data") | default dict }} - - - {{- $mongoUnameSecret := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} - mongo-username-secret: {{ $mongoUnameSecret | quote }} - - {{- $mongoPassSecret := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} - mongo-password-secret: {{ $mongoPassSecret | quote }} - - {{- $mongoDbRootUser := (get $secretData "mongoInitdbRootUsername") | default (randAlphaNum 32 | b64enc) }} - mongoInitdbRootUsername: {{ $mongoDbRootUser | quote }} - - {{- $mongoDbRootPass := (get $secretData "mongoInitdbRootPassword") | default (randAlphaNum 32 | b64enc) }} - mongoInitdbRootPassword: {{ $mongoDbRootPass | quote }} - - {{- else }} - - mongo-username-secret: cm9vdA== # Base64-encoded "root" - mongo-password-secret: ZXhhbXBsZQ== # Base64-encoded "example" - mongoInitdbRootPassword: ZXhhbXBsZQ== # Base64-encoded "example" - mongoInitdbRootUsername: cm9vdA== # Base64-encoded "root" - {{- end }} \ No newline at end of file diff --git a/packages/grid/k8s/manifests/seaweedfs_secrets.txt b/packages/grid/k8s/manifests/seaweedfs_secrets.txt deleted file mode 100644 index d027e5fc864..00000000000 --- a/packages/grid/k8s/manifests/seaweedfs_secrets.txt +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: "seaweedfsecret" -type: Opaque -data: - {{- if not .Values.configuration.devmode }} - - {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "seaweedfsecret") | default dict }} - {{- $secretData := (get $secretObj "data") | default dict }} - - - {{- $S3_ROOT_USER_SECRET := (get $secretData "S3_ROOT_USER_SECRET") | default (randAlphaNum 32 | b64enc) }} - S3_ROOT_USER_SECRET: {{ $S3_ROOT_USER_SECRET | quote }} - - {{- $S3_ROOT_PWD_SECRET := (get $secretData "S3_ROOT_PWD_SECRET") | default (randAlphaNum 32 | b64enc) }} - S3_ROOT_PWD_SECRET: {{ $S3_ROOT_PWD_SECRET | quote }} - - {{- else }} - - S3_ROOT_USER_SECRET: YWRtaW4= # Base64-encoded "admin" - S3_ROOT_PWD_SECRET: YWRtaW4= # Base64-encoded "admin" - {{- end }} \ No newline at end of file diff --git a/packages/grid/k8s/manifests/syft_secrets.txt b/packages/grid/k8s/manifests/syft_secrets.txt deleted file mode 100644 index 67cf3d28e9d..00000000000 --- a/packages/grid/k8s/manifests/syft_secrets.txt +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: "syftsecret" -type: Opaque -data: - {{- if not .Values.configuration.devmode }} - - {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace "syftsecret") | default dict }} - {{- $secretData := (get $secretObj "data") | default dict }} - - {{- $stackApiKey := (get $secretData "stackApiKey") | default (randAlphaNum 32 | b64enc) }} - stackApiKey: {{ $stackApiKey | quote }} - - {{- $defaultRootPassword := (get $secretData "defaultRootPassword") | default (randAlphaNum 32 | b64enc) }} - defaultRootPassword: {{ $defaultRootPassword | quote }} - - {{- else }} - stackApiKey: Y2hhbmdlbWU= # Base64-encoded "changeme" - defaultRootPassword: Y2hhbmdldGhpcw== # Base64-encoded "changethis" - {{- end }} \ No newline at end of file From 5cec5d532220535ad2dc7373deee170974afc2ae Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 09:24:22 +0530 Subject: [PATCH 39/47] - Added seaweedfs specific arguments to backend statefulset - Removed seaweedfs username during randomization --- .../grid/helm/syft/templates/backend-statefulset.yaml | 11 +++++++++++ .../helm/syft/templates/seaweedfs-statefulset.yaml | 5 +---- .../grid/helm/syft/templates/seaweedfs_secrets.yaml | 5 +---- packages/grid/helm/syft/values.yaml | 1 + 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 97a72903cbf..146a3238e7f 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -89,6 +89,17 @@ spec: secretKeyRef: name: {{ .Values.secrets.syft.syftsecret }} key: defaultRootPassword + - name: S3_ROOT_USER + value: "{{ .Values.seaweedfs.S3_ROOT_USER }}" + - name: S3_ROOT_PWD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} + key: S3_ROOT_PWD_SECRET + - name: S3_PORT + value: "{{ .Values.seaweedfs.S3_PORT }}" + - name: SEAWEED_MOUNT_PORT + value: "{{ .Values.seaweedfs.SEAWEED_MOUNT_PORT }}" envFrom: null image: {{ .Values.container.registry }}/openmined/grid-backend:{{ .Values.container.syft_version }} lifecycle: null diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index 2da23dba8f8..04753974e17 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -31,10 +31,7 @@ spec: - name: S3_VOLUME_SIZE_MB value: "{{ .Values.seaweedfs.S3_VOLUME_SIZE_MB }}" - name: S3_ROOT_USER - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.seaweedfs.seaweedfsecret }} - key: S3_ROOT_USER_SECRET + value: "{{ .Values.seaweedfs.S3_ROOT_USER }}" - name: S3_ROOT_PWD valueFrom: secretKeyRef: diff --git a/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml index d1bd5fd5b58..8ad84b642bd 100644 --- a/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs_secrets.yaml @@ -10,14 +10,11 @@ data: {{- $secretData := (get $secretObj "data") | default dict }} - {{- $S3_ROOT_USER_SECRET_env := (get $secretData "S3_ROOT_USER_SECRET") | default (randAlphaNum 32 | b64enc) }} - S3_ROOT_USER_SECRET: {{ $S3_ROOT_USER_SECRET_env | quote }} - {{- $S3_ROOT_PWD_SECRET_env := (get $secretData "S3_ROOT_PWD_SECRET") | default (randAlphaNum 32 | b64enc) }} S3_ROOT_PWD_SECRET: {{ $S3_ROOT_PWD_SECRET_env | quote }} {{- else }} - S3_ROOT_USER_SECRET: {{ "admin" | b64enc }} # Base64-encoded "admin" S3_ROOT_PWD_SECRET: {{ "admin" | b64enc }} # Base64-encoded "admin" + {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index fa1340a3c01..e2854bbe346 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -30,6 +30,7 @@ seaweedfs: S3_VOLUME_SIZE_MB: 1024 S3_PORT: 8333 SEAWEED_MOUNT_PORT: 4001 + S3_ROOT_USER: "admin" container: registry: "docker.io" From 43d12fb0ed3b260e8631e5a4177fb02e108df809 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 09:31:55 +0530 Subject: [PATCH 40/47] - Removed mongo user name from randomization - Added a new field for mongo username in values.yaml --- packages/grid/helm/syft/templates/backend-statefulset.yaml | 5 +---- packages/grid/helm/syft/templates/mongo-statefulset.yaml | 5 +---- packages/grid/helm/syft/templates/mongo_secrets.yaml | 5 ----- packages/grid/helm/syft/values.yaml | 1 + 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 146a3238e7f..5c793361884 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -32,10 +32,7 @@ spec: - name: MONGO_HOST value: {{ .Values.db.mongo.settings.mongoHost }} - name: MONGO_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongosecret }} - key: mongo-username-secret + value: {{ .Values.db.mongo.settings.mongoUsername }} - name: MONGO_PASSWORD valueFrom: secretKeyRef: diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 846d262c9f2..17eb69577b1 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -28,10 +28,7 @@ spec: command: null env: - name: MONGO_INITDB_ROOT_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.db.mongo.mongosecret }} - key: mongo-username-secret + value: {{ .Values.db.mongo.settings.mongoUsername }} - name: MONGO_INITDB_ROOT_PASSWORD valueFrom: secretKeyRef: diff --git a/packages/grid/helm/syft/templates/mongo_secrets.yaml b/packages/grid/helm/syft/templates/mongo_secrets.yaml index deca94aaeb5..ae294033ab6 100644 --- a/packages/grid/helm/syft/templates/mongo_secrets.yaml +++ b/packages/grid/helm/syft/templates/mongo_secrets.yaml @@ -10,16 +10,11 @@ data: {{- $secretData := (get $secretObj "data") | default dict }} # Helm does not allow hypen character, as a variable name , so we conform to camelCase - {{- $mongoUnameSecret_env := (get $secretData "mongo-username-secret") | default (randAlphaNum 32 | b64enc) }} - mongo-username-secret: {{ $mongoUnameSecret_env | quote }} - {{- $mongoPassSecret_env := (get $secretData "mongo-password-secret") | default (randAlphaNum 32 | b64enc) }} mongo-password-secret: {{ $mongoPassSecret_env | quote }} - {{- else }} - mongo-username-secret: {{ "root" | b64enc }} # Base64-encoded "root" mongo-password-secret: {{ "example" | b64enc }} # Base64-encoded "example" {{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index e2854bbe346..e58186c5eb5 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -25,6 +25,7 @@ db: settings: mongoPort: "'27017'" mongoHost: "mongo" + mongoUsername: "root" seaweedfs: S3_VOLUME_SIZE_MB: 1024 From d1c92a64c055c4d0eba7e1084d49cd6a764d363f Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:12:34 +0530 Subject: [PATCH 41/47] added retrieval of password by environment variables for login --- packages/syft/src/syft/client/client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index d2db4cca983..4c2580a3fd6 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -5,6 +5,7 @@ from enum import Enum from getpass import getpass import json +import os from typing import Any from typing import Callable from typing import Dict @@ -673,6 +674,14 @@ def login( register: bool = False, **kwargs: Any, ) -> Self: + # If SYFT_LOGIN_{NODE_NAME}_PASSWORD is set, use that as the password + # for the login. This is useful for CI/CD environments to test password + # randomization that is implemented by helm charts + if self.name is not None: + pass_env_var = f"SYFT_LOGIN_{self.name}_PASSWORD" + if pass_env_var in os.environ: + password = os.environ[pass_env_var] + if email is None: email = input("Email: ") if password is None: From 4755776d29cac99311fcd8d2ad4251fdd8f56703 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:15:43 +0530 Subject: [PATCH 42/47] removed unused fields in kubernetes tox test --- tox.ini | 45 ++++----------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/tox.ini b/tox.ini index 82e8e678a8b..665df65d1b4 100644 --- a/tox.ini +++ b/tox.ini @@ -667,13 +667,12 @@ commands = # Deleting current cluster bash -c "k3d cluster delete test-gateway-1 || true" bash -c "k3d cluster delete test-domain-1 || true" - # bash -c "k3d cluster delete test-domain-2 || true" # Deleting registery & volumes bash -c "k3d registry delete k3d-registry.localhost || true" bash -c "docker volume rm k3d-test-gateway-1-images --force || true" bash -c "docker volume rm k3d-test-domain-1-images --force || true" - # bash -c "docker volume rm k3d-test-domain-2-images --force || true" + # Creating registory bash -c 'k3d registry create registry.localhost --port 12345 -v `pwd`/k3d-registry:/var/lib/registry || true' @@ -710,21 +709,6 @@ commands = deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' - # bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \ - # k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ - # k3d cluster start $NODE_NAME' - - # bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \ - # cd packages/grid && \ - # (r=5;while ! \ - # devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ - # --var DOMAIN_NAME=$NODE_NAME \ - # --var DOMAIN_CHECK_INTERVAL=5 \ - # --var ASSOCIATION_TIMEOUT=100 \ - # --var TEST_MODE=1 \ - # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ - # deploy -b -p domain; \ - # do ((--r))||exit;echo "retrying" && sleep 20;done)' # free up build cache after build of images bash -c 'if [[ "$GITHUB_CI" != "false" ]]; then \ @@ -742,33 +726,14 @@ commands = bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-gateway-1 --namespace test-gateway-1 bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-gateway-1 --namespace test-gateway-1 bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-gateway-1 --namespace test-gateway-1 - # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-gateway-1 --namespace test-gateway-1 - # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-gateway-1 --namespace test-gateway-1 - # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-gateway-1 --namespace test-gateway-1 # wait for test domain 1 bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-domain-1 --namespace test-domain-1 bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-1 --namespace test-domain-1 bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-1 --namespace test-domain-1 - # bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-1 --namespace test-domain-1 - - # wait for test domain 2 - # bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service db --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-2 --namespace test-domain-2 - # bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-2 --namespace test-domain-2 - - # pytest tests/integration -m frontend -p no:randomly --co - # bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no" + bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-1 --namespace test-domain-1 + + # Checking logs generated & startup of test-domain 1 bash -c '(kubectl logs service/backend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q "Application startup complete" || true' @@ -799,12 +764,10 @@ commands = # deleting clusters created bash -c "k3d cluster delete test-gateway-1 || true" bash -c "k3d cluster delete test-domain-1 || true" - # bash -c "k3d cluster delete test-domain-2 || true" bash -c "k3d registry delete k3d-registry.localhost || true" bash -c "docker rm $(docker ps -aq) --force || true" bash -c "docker volume rm k3d-test-gateway-1-images --force || true" bash -c "docker volume rm k3d-test-domain-1-images --force || true" - # bash -c "docker volume rm k3d-test-domain-2-images --force || true" From eee9a01673c56ed831490861c16ecb284bdc4631 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:31:15 +0530 Subject: [PATCH 43/47] - Combined Variables names in tox - As environment variables cannot have hypens - And cluster names cannot have underscores - These are required for passing the password as environment variables to test randomization --- tox.ini | 54 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tox.ini b/tox.ini index 665df65d1b4..fa914a3b83d 100644 --- a/tox.ini +++ b/tox.ini @@ -663,26 +663,28 @@ commands = python -c 'import syft as sy; sy.stage_protocol_changes()' k3d version + # Since cluster name cannot have underscore and environment variable cannot have hyphen + # we are passing a grouped name for node names # bash -c "docker rm $(docker ps -aq) --force || true" # Deleting current cluster - bash -c "k3d cluster delete test-gateway-1 || true" - bash -c "k3d cluster delete test-domain-1 || true" + bash -c "k3d cluster delete testgateway1 || true" + bash -c "k3d cluster delete testdomain1 || true" # Deleting registery & volumes bash -c "k3d registry delete k3d-registry.localhost || true" - bash -c "docker volume rm k3d-test-gateway-1-images --force || true" - bash -c "docker volume rm k3d-test-domain-1-images --force || true" + bash -c "docker volume rm k3d-testgateway1-images --force || true" + bash -c "docker volume rm k3d-testdomain1-images --force || true" # Creating registory bash -c 'k3d registry create registry.localhost --port 12345 -v `pwd`/k3d-registry:/var/lib/registry || true' - # Creating test-gateway-1 cluster on port 9081 - bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \ + # Creating testgateway1 cluster on port 9081 + bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \ k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start $NODE_NAME' - bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \ + bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \ cd packages/grid && \ (r=5;while ! \ devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ @@ -694,12 +696,12 @@ commands = deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' - # Creating test-domain-1 cluster on port 9082 - bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \ + # Creating testdomain1 cluster on port 9082 + bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \ k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ k3d cluster start $NODE_NAME' - bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \ + bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \ cd packages/grid && \ (r=5;while ! \ devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \ @@ -719,26 +721,26 @@ commands = sleep 30 # wait for front end - bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1 - bash -c '(kubectl logs service/frontend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true' + bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace testdomain1 + bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true' # wait for test gateway 1 - bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-gateway-1 --namespace test-gateway-1 - bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-gateway-1 --namespace test-gateway-1 - bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-gateway-1 --namespace test-gateway-1 + bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testgateway1 --namespace testgateway1 + bash packages/grid/scripts/wait_for.sh service backend --context k3d-testgateway1 --namespace testgateway1 + bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testgateway1 --namespace testgateway1 # wait for test domain 1 - bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-domain-1 --namespace test-domain-1 - bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-1 --namespace test-domain-1 - bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1 - bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-1 --namespace test-domain-1 + bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testdomain1 --namespace testdomain1 + bash packages/grid/scripts/wait_for.sh service backend --context k3d-testdomain1 --namespace testdomain1 + bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testdomain1 --namespace testdomain1 + bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-testdomain1 --namespace testdomain1 # Checking logs generated & startup of test-domain 1 - bash -c '(kubectl logs service/backend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q "Application startup complete" || true' - # Checking logs generated & startup of test-gateway-1 - bash -c '(kubectl logs service/backend --context k3d-test-gateway-1 --namespace test-gateway-1 -f &) | grep -q "Application startup complete" || true' + bash -c '(kubectl logs service/backend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q "Application startup complete" || true' + # Checking logs generated & startup of testgateway1 + bash -c '(kubectl logs service/backend --context k3d-testgateway1 --namespace testgateway1 -f &) | grep -q "Application startup complete" || true' # frontend @@ -762,12 +764,12 @@ commands = pytest tests/integration/network -k "not test_domain_gateway_user_code" -p no:randomly -vvvv # deleting clusters created - bash -c "k3d cluster delete test-gateway-1 || true" - bash -c "k3d cluster delete test-domain-1 || true" + bash -c "k3d cluster delete testgateway1 || true" + bash -c "k3d cluster delete testdomain1 || true" bash -c "k3d registry delete k3d-registry.localhost || true" bash -c "docker rm $(docker ps -aq) --force || true" - bash -c "docker volume rm k3d-test-gateway-1-images --force || true" - bash -c "docker volume rm k3d-test-domain-1-images --force || true" + bash -c "docker volume rm k3d-testgateway1-images --force || true" + bash -c "docker volume rm k3d-testdomain1-images --force || true" From fedc977cac97b7375038a91a99d490544a6cca70 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:42:37 +0530 Subject: [PATCH 44/47] - Fixed Node Name intergration test - Add environment variable mapping for randomized password in tox --- tests/integration/network/client_test.py | 11 ----------- tox.ini | 9 +++++++++ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/integration/network/client_test.py b/tests/integration/network/client_test.py index 92c03510f6e..018d66eab90 100644 --- a/tests/integration/network/client_test.py +++ b/tests/integration/network/client_test.py @@ -19,14 +19,3 @@ def test_client_type(node_metadata): client = sy.login(port=port, email="info@openmined.org", password="changethis") assert isinstance(client, client_type) - - -@pytest.mark.parametrize( - "node_metadata", [(NETWORK_PORT, "test-gateway-1"), (DOMAIN_PORT, "test-domain-1")] -) -@pytest.mark.network -def test_client_name(node_metadata): - port, node_name = node_metadata - client = sy.login(port=port, email="info@openmined.org", password="changethis") - - assert client.name == node_name diff --git a/tox.ini b/tox.ini index fa914a3b83d..aacfc6faf25 100644 --- a/tox.ini +++ b/tox.ini @@ -753,6 +753,15 @@ commands = exit $return; \ fi' + # Since we randomize the password, we retrieve them and store as environment variables + # which would then be used by the notebook + #Randomized password for testgateway1 + bash -c 'export SYFT_LOGIN_testgateway1_PASSWORD=$(kubectl --context=k3d-testgateway1 get secret syftsecret -n testgateway1 \ + -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)' + + #Randomized password for testdomain1 + bash -c 'export SYFT_LOGIN_testdomain1_PASSWORD=$(kubectl get --context=k3d-testdomain1 secret syftsecret -n testdomain1 \ + -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)' # ignore 06 because of opendp on arm64 # Run 0.8 notebooks From 0a061e73e6e599784b84af46b3de08f72c26cee6 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 11:41:50 +0530 Subject: [PATCH 45/47] fixed randomized testing on CI --- packages/grid/devspace.yaml | 4 ++-- packages/syft/src/syft/client/client.py | 2 +- scripts/get_k8s_secret_ci.sh | 6 ++++++ tox.ini | 10 ++-------- 4 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 scripts/get_k8s_secret_ci.sh diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 24cb5c0674d..8a74e003714 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -73,8 +73,8 @@ deployments: settings: nodeName: ${NODE_NAME} nodeType: "domain" - configuration: - devmode: True + # configuration: + # devmode: True dev: mongo: diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 4c2580a3fd6..239c33f57a3 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -677,7 +677,7 @@ def login( # If SYFT_LOGIN_{NODE_NAME}_PASSWORD is set, use that as the password # for the login. This is useful for CI/CD environments to test password # randomization that is implemented by helm charts - if self.name is not None: + if self.name is not None and email == "info@openmined.org": pass_env_var = f"SYFT_LOGIN_{self.name}_PASSWORD" if pass_env_var in os.environ: password = os.environ[pass_env_var] diff --git a/scripts/get_k8s_secret_ci.sh b/scripts/get_k8s_secret_ci.sh new file mode 100644 index 00000000000..8974780d87f --- /dev/null +++ b/scripts/get_k8s_secret_ci.sh @@ -0,0 +1,6 @@ +#!/bin/bash +export SYFT_LOGIN_testgateway1_PASSWORD=$(kubectl --context=k3d-testgateway1 get secret syftsecret -n testgateway1 \ + -o jsonpath='{.data.defaultRootPassword}' | base64 --decode) + +export SYFT_LOGIN_testdomain1_PASSWORD=$(kubectl get --context=k3d-testdomain1 secret syftsecret -n testdomain1 \ + -o jsonpath='{.data.defaultRootPassword}' | base64 --decode) \ No newline at end of file diff --git a/tox.ini b/tox.ini index aacfc6faf25..35594225324 100644 --- a/tox.ini +++ b/tox.ini @@ -755,22 +755,16 @@ commands = # Since we randomize the password, we retrieve them and store as environment variables # which would then be used by the notebook - #Randomized password for testgateway1 - bash -c 'export SYFT_LOGIN_testgateway1_PASSWORD=$(kubectl --context=k3d-testgateway1 get secret syftsecret -n testgateway1 \ - -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)' - #Randomized password for testdomain1 - bash -c 'export SYFT_LOGIN_testdomain1_PASSWORD=$(kubectl get --context=k3d-testdomain1 secret syftsecret -n testdomain1 \ - -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)' # ignore 06 because of opendp on arm64 # Run 0.8 notebooks - pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv + bash -c " source ./scripts/get_k8s_secret_ci.sh; pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv" #Integration + Gateway Connection Tests # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure # high/low side warning flag. - pytest tests/integration/network -k "not test_domain_gateway_user_code" -p no:randomly -vvvv + bash -c " source ./scripts/get_k8s_secret_ci.sh;pytest tests/integration/network -k 'not test_domain_gateway_user_code' -p no:randomly -vvvv" # deleting clusters created bash -c "k3d cluster delete testgateway1 || true" From 4374b97723203d5be5a939796739f648052c7b0b Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:23:50 +0530 Subject: [PATCH 46/47] removed unused k8s files --- packages/grid/helm/helm.py | 301 -- packages/grid/k8s/manifests/ingress-tls.yaml | 26 - packages/grid/k8s/manifests/ingress.yaml | 22 - packages/grid/k8s/manifests/loadbalancer.yaml | 17 - packages/grid/k8s/manifests/redis.yaml | 2057 ---------- packages/grid/k8s/manifests/seaweedfs.yaml | 17 - .../grid/k8s/manifests/traefik-domain.yaml | 69 - .../grid/k8s/manifests/traefik-network.yaml | 57 - packages/grid/k8s/rendered/domain.yaml | 3472 ----------------- packages/grid/k8s/rendered/network.yaml | 3339 ---------------- 10 files changed, 9377 deletions(-) delete mode 100644 packages/grid/helm/helm.py delete mode 100644 packages/grid/k8s/manifests/ingress-tls.yaml delete mode 100644 packages/grid/k8s/manifests/ingress.yaml delete mode 100644 packages/grid/k8s/manifests/loadbalancer.yaml delete mode 100644 packages/grid/k8s/manifests/redis.yaml delete mode 100644 packages/grid/k8s/manifests/seaweedfs.yaml delete mode 100644 packages/grid/k8s/manifests/traefik-domain.yaml delete mode 100644 packages/grid/k8s/manifests/traefik-network.yaml delete mode 100644 packages/grid/k8s/rendered/domain.yaml delete mode 100644 packages/grid/k8s/rendered/network.yaml diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py deleted file mode 100644 index 743190d1636..00000000000 --- a/packages/grid/helm/helm.py +++ /dev/null @@ -1,301 +0,0 @@ -# stdlib -import argparse -import json -import os -from pathlib import Path -import shutil -import sys -from typing import Any - -# third party -import yaml - - -# Preserve those beautiful multi-line strings with | -# https://stackoverflow.com/a/33300001 -def str_presenter(dumper: Any, data: Any) -> Any: - if len(data.splitlines()) > 1: # check for multiline string - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") - return dumper.represent_scalar("tag:yaml.org,2002:str", data) - - -yaml.add_representer(str, str_presenter) -yaml.representer.SafeRepresenter.add_representer(str, str_presenter) - -template_variables = { - "STACK_API_KEY": "secrets.syft", - "DEFAULT_ROOT_EMAIL": "secrets.syft", - "DEFAULT_ROOT_PASSWORD": "secrets.syft", - "MONGO_PASSWORD": "secrets.db.mongo", - "MONGO_USERNAME": "secrets.db.mongo", - "MONGO_INITDB_ROOT_PASSWORD": "secrets.db.mongo", - "MONGO_INITDB_ROOT_USERNAME": "secrets.db.mongo", - "MONGO_PORT": "db.mongo.settings", - "MONGO_HOST": "db.mongo.settings", - "HOSTNAME": "node.settings", - "NODE_TYPE": "node.settings", - "VERSION_HASH": "node.settings", - "NODE_NAME": "node.settings", - "NODE_SIDE_TYPE": "node.settings", -} - - -def to_lower_camel_case(s: str) -> str: - words = s.replace("-", "_").split("_") - return words[0].lower() + "".join(word.capitalize() for word in words[1:]) - - -def remove_yaml(d: Any) -> None: - if "namespace" in d: - del d["namespace"] - if ( - "kind" in d - and d["kind"] == "Deployment" - and "spec" in d - and "volumeClaimTemplates" in d["spec"] - and d["spec"]["volumeClaimTemplates"] is None - ): - del d["spec"]["volumeClaimTemplates"] - - -def replace_variables(d: Any) -> None: - if "name" in d and "value" in d and d["name"] in template_variables: - variable_name = d["name"] - path = template_variables[variable_name] - camel_case_name = to_lower_camel_case(variable_name) - d["value"] = f"{{{{ .Values.{path}.{camel_case_name} }}}}" - - if "kubernetes.io/ingress.class" in d: - d["kubernetes.io/ingress.class"] = "{{ .Values.ingress.ingressClass }}" - - # ONLY FOR TLS - if d.get("kind") == "Ingress" and "tls" in d.get("spec", {}): - d["spec"]["tls"][0]["hosts"][0] = "{{ .Values.node.settings.hostname }}" - d["spec"]["rules"][0]["host"] = "{{ .Values.node.settings.hostname }}" - - -# parse whole tree -def fix_devspace_yaml(d: Any) -> None: - if isinstance(d, dict): - remove_yaml(d) - replace_variables(d) - - for _, v in d.items(): - fix_devspace_yaml(v) - - elif isinstance(d, list): - for item in d: - fix_devspace_yaml(item) - - -def get_yaml_name(doc: dict) -> Any: - try: - return doc.get("metadata", {}).get("name", "") - except Exception: # nosec - return "" - - -def ingress_with_tls() -> str: - script_path = os.path.dirname(os.path.realpath(__file__)) - manifest_path = os.path.normpath( - os.path.join(script_path, "..", "k8s", "manifests") - ) - ingress_tls = os.path.join(manifest_path, "ingress-tls.yaml") - - with open(ingress_tls) as fp: - return fp.read() - - -def add_secrets(helm_chart_template_dir: str) -> None: - """ - Add secrets.txt to secrets.yaml during runtime build. - """ - mongo_secrets_path = os.path.join(helm_chart_template_dir, "mongo_secrets.yaml") - mongo_secrets_source = os.path.join("k8s/manifests", "mongo_secrets.txt") - with open(mongo_secrets_path, "w") as fp: - with open(mongo_secrets_source) as fp2: - fp.write(fp2.read()) - - syft_secrets_path = os.path.join(helm_chart_template_dir, "syft_secrets.yaml") - syft_secrets_source = os.path.join("k8s/manifests", "syft_secrets.txt") - with open(syft_secrets_path, "w") as fp: - with open(syft_secrets_source) as fp2: - fp.write(fp2.read()) - - seeweedfs_secrets_path = os.path.join( - helm_chart_template_dir, "seaweedfs_secrets.yaml" - ) - seeweedfs_secrets_source = os.path.join("k8s/manifests", "seaweedfs_secrets.txt") - with open(seeweedfs_secrets_path, "w") as fp: - with open(seeweedfs_secrets_source) as fp2: - fp.write(fp2.read()) - - -def add_notes(helm_chart_template_dir: str) -> None: - """Add notes or information post helm install or upgrade.""" - - notes = """ - Thank you for installing {{ .Chart.Name }}. - Your release is named {{ .Release.Name }}. - To learn more about the release, try: - - $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} - $ helm get all {{ .Release.Name }} - """ - - notes_path = os.path.join(helm_chart_template_dir, "NOTES.txt") - - protocol_changelog = get_protocol_changes() - - notes += "\n" + protocol_changelog - - with open(notes_path, "w") as fp: - fp.write(notes) - - -def get_protocol_changes() -> str: - """Generate change log of the dev protocol state.""" - script_path = os.path.dirname(os.path.realpath(__file__)) - protocol_path = Path( - os.path.normpath( - os.path.join( - script_path, - "../../", - "syft/src/syft/protocol", - "protocol_version.json", - ) - ) - ) - - protocol_changes = "" - if protocol_path.exists(): - dev_protocol_changes = json.loads(protocol_path.read_text()).get("dev", {}) - protocol_changes = json.dumps( - dev_protocol_changes.get("object_versions", {}), indent=4 - ) - - protocol_changelog = f""" - Following class versions are either added/removed. - - {protocol_changes} - - This means the existing data will be automatically be migrated to - their latest class versions during the upgrade. - """ - - return protocol_changelog - - -def apply_patches(yaml: str, resource_name: str, resource_kind: str) -> str: - # print(resource_kind, resource_name) - # apply resource specific patches - if resource_name.startswith("seaweedfs"): - yaml = ( - '{{- if ne .Values.node.settings.nodeType "gateway"}}\n' - + yaml.rstrip() - + "\n{{ end }}\n" - ) - elif resource_kind == "ingress" and resource_name.endswith("tls"): - yaml = "{{- if .Values.node.settings.tls }}\n" + yaml.rstrip() + "\n{{ end }}\n" - elif resource_kind == "ingress" and not resource_name.endswith("tls"): - yaml = ( - "{{- if not .Values.node.settings.tls }}\n" - + yaml.rstrip() - + "\n{{ end }}\n" - ) - - # global patches - yaml = ( - yaml.replace("'{{", "{{") - .replace("}}'", "}}") - .replace("''{{", "{{") - .replace("}}''", "}}") - ) - - return yaml - - -def main() -> None: - # Argument parsing - parser = argparse.ArgumentParser(description="Process devspace yaml file.") - parser.add_argument( - "file", nargs="?", type=argparse.FileType("r"), default=sys.stdin - ) - args = parser.parse_args() - text = args.file.read() - - file_count = 0 - helm_dir = "helm" - manifest_file = f"{helm_dir}/manifests.yaml" - helm_chart_template_dir = f"{helm_dir}/syft/templates" - - # Read input from file or stdin - lines = text.splitlines() - - # Find first line that starts with 'apiVersion' and slice list from that point - try: - first_index = next( - i for i, line in enumerate(lines) if line.strip().startswith("apiVersion") - ) - input_data = "\n".join(lines[first_index:]) - except StopIteration: - print("❌ Error: No line starting with 'apiVersion' found in the input.") - exit(1) - - # Load the multi-doc yaml file - try: - # append custom docs - input_data = "\n---\n".join( - [ - input_data, - ingress_with_tls(), - ] - ) - - yaml_docs = list(yaml.safe_load_all(input_data)) - except Exception as e: - print(f"❌ Error while parsing yaml file: {e}") - exit(1) - - # clear templates dir - shutil.rmtree(helm_chart_template_dir, ignore_errors=True) - - # Create directories if they don't exist - os.makedirs(helm_chart_template_dir, exist_ok=True) - - # Cleanup YAML docs - yaml_docs = [doc for doc in yaml_docs if doc] - - # Sort YAML docs based on metadata name - yaml_docs.sort(key=get_yaml_name) - - # Save sorted YAML docs to file - with open(manifest_file, "w") as f: - yaml.dump_all(yaml_docs, f) - - for doc in yaml_docs: - fix_devspace_yaml(doc) - name = doc.get("metadata", {}).get("name") - kind = doc.get("kind", "").lower() - if name: - # Create new file with name or append if it already exists - new_file = os.path.join(helm_chart_template_dir, f"{name}-{kind}.yaml") - yaml_dump = yaml.dump(doc) - yaml_dump = apply_patches(yaml_dump, name, kind) - - with open(new_file, "w") as f: - f.write(yaml_dump) # add document separator - file_count += 1 - - # Add notes - add_notes(helm_chart_template_dir) - add_secrets(helm_chart_template_dir) - if file_count > 0: - print(f"✅ Done: Generated {file_count} template files") - else: - print("❌ Failed: No files were generated. Check input for errors.") - exit(1) - - -if __name__ == "__main__": - main() diff --git a/packages/grid/k8s/manifests/ingress-tls.yaml b/packages/grid/k8s/manifests/ingress-tls.yaml deleted file mode 100644 index 6476e4c846a..00000000000 --- a/packages/grid/k8s/manifests/ingress-tls.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: grid-stack-ingress-tls - annotations: - kubernetes.io/ingress.class: "" -spec: - defaultBackend: - service: - name: proxy - port: - number: 80 - rules: - - host: "" - http: - paths: - - backend: - service: - name: proxy - port: - number: 80 - path: / - pathType: Prefix - tls: - - hosts: - - "" diff --git a/packages/grid/k8s/manifests/ingress.yaml b/packages/grid/k8s/manifests/ingress.yaml deleted file mode 100644 index 072e54dedb0..00000000000 --- a/packages/grid/k8s/manifests/ingress.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: grid-stack-ingress - annotations: - kubernetes.io/ingress.class: "" -spec: - defaultBackend: - service: - name: proxy - port: - number: 80 - rules: - - http: - paths: - - backend: - service: - name: proxy - port: - number: 80 - path: / - pathType: Prefix diff --git a/packages/grid/k8s/manifests/loadbalancer.yaml b/packages/grid/k8s/manifests/loadbalancer.yaml deleted file mode 100644 index e16ac13e09c..00000000000 --- a/packages/grid/k8s/manifests/loadbalancer.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: public-loadbalancer -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/component: proxy - type: LoadBalancer diff --git a/packages/grid/k8s/manifests/redis.yaml b/packages/grid/k8s/manifests/redis.yaml deleted file mode 100644 index 9a22c83105b..00000000000 --- a/packages/grid/k8s/manifests/redis.yaml +++ /dev/null @@ -1,2057 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - redis.conf: | - # Redis configuration file example. - # - # Note that in order to read the configuration file, Redis must be - # started with the file path as first argument: - # - # ./redis-server /path/to/redis.conf - - # Note on units: when memory size is needed, it is possible to specify - # it in the usual form of 1k 5GB 4M and so forth: - # - # 1k => 1000 bytes - # 1kb => 1024 bytes - # 1m => 1000000 bytes - # 1mb => 1024*1024 bytes - # 1g => 1000000000 bytes - # 1gb => 1024*1024*1024 bytes - # - # units are case insensitive so 1GB 1Gb 1gB are all the same. - - ################################## INCLUDES ################################### - - # Include one or more other config files here. This is useful if you - # have a standard template that goes to all Redis servers but also need - # to customize a few per-server settings. Include files can include - # other files, so use this wisely. - # - # Note that option "include" won't be rewritten by command "CONFIG REWRITE" - # from admin or Redis Sentinel. Since Redis always uses the last processed - # line as value of a configuration directive, you'd better put includes - # at the beginning of this file to avoid overwriting config change at runtime. - # - # If instead you are interested in using includes to override configuration - # options, it is better to use include as the last line. - # - # include /path/to/local.conf - # include /path/to/other.conf - - ################################## MODULES ##################################### - - # Load modules at startup. If the server is not able to load modules - # it will abort. It is possible to use multiple loadmodule directives. - # - # loadmodule /path/to/my_module.so - # loadmodule /path/to/other_module.so - - ################################## NETWORK ##################################### - - # By default, if no "bind" configuration directive is specified, Redis listens - # for connections from all available network interfaces on the host machine. - # It is possible to listen to just one or multiple selected interfaces using - # the "bind" configuration directive, followed by one or more IP addresses. - # Each address can be prefixed by "-", which means that redis will not fail to - # start if the address is not available. Being not available only refers to - # addresses that does not correspond to any network interfece. Addresses that - # are already in use will always fail, and unsupported protocols will always BE - # silently skipped. - # - # Examples: - # - # bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses - # bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 - # bind * -::* # like the default, all available interfaces - # - # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the - # internet, binding to all the interfaces is dangerous and will expose the - # instance to everybody on the internet. So by default we uncomment the - # following bind directive, that will force Redis to listen only on the - # IPv4 and IPv6 (if available) loopback interface addresses (this means Redis - # will only be able to accept client connections from the same host that it is - # running on). - # - # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES - # JUST COMMENT OUT THE FOLLOWING LINE. - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - bind 127.0.0.1 -::1 - - # Protected mode is a layer of security protection, in order to avoid that - # Redis instances left open on the internet are accessed and exploited. - # - # When protected mode is on and if: - # - # 1) The server is not binding explicitly to a set of addresses using the - # "bind" directive. - # 2) No password is configured. - # - # The server only accepts connections from clients connecting from the - # IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain - # sockets. - # - # By default protected mode is enabled. You should disable it only if - # you are sure you want clients from other hosts to connect to Redis - # even if no authentication is configured, nor a specific set of interfaces - # are explicitly listed using the "bind" directive. - protected-mode yes - - # Accept connections on the specified port, default is 6379 (IANA #815344). - # If port 0 is specified Redis will not listen on a TCP socket. - port 6379 - - # TCP listen() backlog. - # - # In high requests-per-second environments you need a high backlog in order - # to avoid slow clients connection issues. Note that the Linux kernel - # will silently truncate it to the value of /proc/sys/net/core/somaxconn so - # make sure to raise both the value of somaxconn and tcp_max_syn_backlog - # in order to get the desired effect. - tcp-backlog 511 - - # Unix socket. - # - # Specify the path for the Unix socket that will be used to listen for - # incoming connections. There is no default, so Redis will not listen - # on a unix socket when not specified. - # - # unixsocket /run/redis.sock - # unixsocketperm 700 - - # Close the connection after a client is idle for N seconds (0 to disable) - timeout 0 - - # TCP keepalive. - # - # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence - # of communication. This is useful for two reasons: - # - # 1) Detect dead peers. - # 2) Force network equipment in the middle to consider the connection to be - # alive. - # - # On Linux, the specified value (in seconds) is the period used to send ACKs. - # Note that to close the connection the double of the time is needed. - # On other kernels the period depends on the kernel configuration. - # - # A reasonable value for this option is 300 seconds, which is the new - # Redis default starting with Redis 3.2.1. - tcp-keepalive 300 - - ################################# TLS/SSL ##################################### - - # By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration - # directive can be used to define TLS-listening ports. To enable TLS on the - # default port, use: - # - # port 0 - # tls-port 6379 - - # Configure a X.509 certificate and private key to use for authenticating the - # server to connected clients, masters or cluster peers. These files should be - # PEM formatted. - # - # tls-cert-file redis.crt - # tls-key-file redis.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-key-file-pass secret - - # Normally Redis uses the same certificate for both server functions (accepting - # connections) and client functions (replicating from a master, establishing - # cluster bus connections, etc.). - # - # Sometimes certificates are issued with attributes that designate them as - # client-only or server-only certificates. In that case it may be desired to use - # different certificates for incoming (server) and outgoing (client) - # connections. To do that, use the following directives: - # - # tls-client-cert-file client.crt - # tls-client-key-file client.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-client-key-file-pass secret - - # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: - # - # tls-dh-params-file redis.dh - - # Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL - # clients and peers. Redis requires an explicit configuration of at least one - # of these, and will not implicitly use the system wide configuration. - # - # tls-ca-cert-file ca.crt - # tls-ca-cert-dir /etc/ssl/certs - - # By default, clients (including replica servers) on a TLS port are required - # to authenticate using valid client side certificates. - # - # If "no" is specified, client certificates are not required and not accepted. - # If "optional" is specified, client certificates are accepted and must be - # valid if provided, but are not required. - # - # tls-auth-clients no - # tls-auth-clients optional - - # By default, a Redis replica does not attempt to establish a TLS connection - # with its master. - # - # Use the following directive to enable TLS on replication links. - # - # tls-replication yes - - # By default, the Redis Cluster bus uses a plain TCP connection. To enable - # TLS for the bus protocol, use the following directive: - # - # tls-cluster yes - - # By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended - # that older formally deprecated versions are kept disabled to reduce the attack surface. - # You can explicitly specify TLS versions to support. - # Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", - # "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. - # To enable only TLSv1.2 and TLSv1.3, use: - # - # tls-protocols "TLSv1.2 TLSv1.3" - - # Configure allowed ciphers. See the ciphers(1ssl) manpage for more information - # about the syntax of this string. - # - # Note: this configuration applies only to <= TLSv1.2. - # - # tls-ciphers DEFAULT:!MEDIUM - - # Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more - # information about the syntax of this string, and specifically for TLSv1.3 - # ciphersuites. - # - # tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 - - # When choosing a cipher, use the server's preference instead of the client - # preference. By default, the server follows the client's preference. - # - # tls-prefer-server-ciphers yes - - # By default, TLS session caching is enabled to allow faster and less expensive - # reconnections by clients that support it. Use the following directive to disable - # caching. - # - # tls-session-caching no - - # Change the default number of TLS sessions cached. A zero value sets the cache - # to unlimited size. The default size is 20480. - # - # tls-session-cache-size 5000 - - # Change the default timeout of cached TLS sessions. The default timeout is 300 - # seconds. - # - # tls-session-cache-timeout 60 - - ################################# GENERAL ##################################### - - # By default Redis does not run as a daemon. Use 'yes' if you need it. - # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. - # When Redis is supervised by upstart or systemd, this parameter has no impact. - daemonize no - - # If you run Redis from upstart or systemd, Redis can interact with your - # supervision tree. Options: - # supervised no - no supervision interaction - # supervised upstart - signal upstart by putting Redis into SIGSTOP mode - # requires "expect stop" in your upstart job config - # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET - # on startup, and updating Redis status on a regular - # basis. - # supervised auto - detect upstart or systemd method based on - # UPSTART_JOB or NOTIFY_SOCKET environment variables - # Note: these supervision methods only signal "process is ready." - # They do not enable continuous pings back to your supervisor. - # - # The default is "no". To run under upstart/systemd, you can simply uncomment - # the line below: - # - # supervised auto - - # If a pid file is specified, Redis writes it where specified at startup - # and removes it at exit. - # - # When the server runs non daemonized, no pid file is created if none is - # specified in the configuration. When the server is daemonized, the pid file - # is used even if not specified, defaulting to "/var/run/redis.pid". - # - # Creating a pid file is best effort: if Redis is not able to create it - # nothing bad happens, the server will start and run normally. - # - # Note that on modern Linux systems "/run/redis.pid" is more conforming - # and should be used instead. - pidfile /var/run/redis_6379.pid - - # Specify the server verbosity level. - # This can be one of: - # debug (a lot of information, useful for development/testing) - # verbose (many rarely useful info, but not a mess like the debug level) - # notice (moderately verbose, what you want in production probably) - # warning (only very important / critical messages are logged) - loglevel notice - - # Specify the log file name. Also the empty string can be used to force - # Redis to log on the standard output. Note that if you use standard - # output for logging but daemonize, logs will be sent to /dev/null - logfile "" - - # To enable logging to the system logger, just set 'syslog-enabled' to yes, - # and optionally update the other syslog parameters to suit your needs. - # syslog-enabled no - - # Specify the syslog identity. - # syslog-ident redis - - # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. - # syslog-facility local0 - - # To disable the built in crash log, which will possibly produce cleaner core - # dumps when they are needed, uncomment the following: - # - # crash-log-enabled no - - # To disable the fast memory check that's run as part of the crash log, which - # will possibly let redis terminate sooner, uncomment the following: - # - # crash-memcheck-enabled no - - # Set the number of databases. The default database is DB 0, you can select - # a different one on a per-connection basis using SELECT where - # dbid is a number between 0 and 'databases'-1 - databases 16 - - # By default Redis shows an ASCII art logo only when started to log to the - # standard output and if the standard output is a TTY and syslog logging is - # disabled. Basically this means that normally a logo is displayed only in - # interactive sessions. - # - # However it is possible to force the pre-4.0 behavior and always show a - # ASCII art logo in startup logs by setting the following option to yes. - always-show-logo no - - # By default, Redis modifies the process title (as seen in 'top' and 'ps') to - # provide some runtime information. It is possible to disable this and leave - # the process name as executed by setting the following to no. - set-proc-title yes - - # When changing the process title, Redis uses the following template to construct - # the modified title. - # - # Template variables are specified in curly brackets. The following variables are - # supported: - # - # {title} Name of process as executed if parent, or type of child process. - # {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or - # Unix socket if only that's available. - # {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". - # {port} TCP port listening on, or 0. - # {tls-port} TLS port listening on, or 0. - # {unixsocket} Unix domain socket listening on, or "". - # {config-file} Name of configuration file used. - # - proc-title-template "{title} {listen-addr} {server-mode}" - - ################################ SNAPSHOTTING ################################ - - # Save the DB to disk. - # - # save - # - # Redis will save the DB if both the given number of seconds and the given - # number of write operations against the DB occurred. - # - # Snapshotting can be completely disabled with a single empty string argument - # as in following example: - # - # save "" - # - # Unless specified otherwise, by default Redis will save the DB: - # * After 3600 seconds (an hour) if at least 1 key changed - # * After 300 seconds (5 minutes) if at least 100 keys changed - # * After 60 seconds if at least 10000 keys changed - # - # You can set these explicitly by uncommenting the three following lines. - # - # save 3600 1 - # save 300 100 - # save 60 10000 - - # By default Redis will stop accepting writes if RDB snapshots are enabled - # (at least one save point) and the latest background save failed. - # This will make the user aware (in a hard way) that data is not persisting - # on disk properly, otherwise chances are that no one will notice and some - # disaster will happen. - # - # If the background saving process will start working again Redis will - # automatically allow writes again. - # - # However if you have setup your proper monitoring of the Redis server - # and persistence, you may want to disable this feature so that Redis will - # continue to work as usual even if there are problems with disk, - # permissions, and so forth. - stop-writes-on-bgsave-error yes - - # Compress string objects using LZF when dump .rdb databases? - # By default compression is enabled as it's almost always a win. - # If you want to save some CPU in the saving child set it to 'no' but - # the dataset will likely be bigger if you have compressible values or keys. - rdbcompression yes - - # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. - # This makes the format more resistant to corruption but there is a performance - # hit to pay (around 10%) when saving and loading RDB files, so you can disable it - # for maximum performances. - # - # RDB files created with checksum disabled have a checksum of zero that will - # tell the loading code to skip the check. - rdbchecksum yes - - # Enables or disables full sanitation checks for ziplist and listpack etc when - # loading an RDB or RESTORE payload. This reduces the chances of a assertion or - # crash later on while processing commands. - # Options: - # no - Never perform full sanitation - # yes - Always perform full sanitation - # clients - Perform full sanitation only for user connections. - # Excludes: RDB files, RESTORE commands received from the master - # connection, and client connections which have the - # skip-sanitize-payload ACL flag. - # The default should be 'clients' but since it currently affects cluster - # resharding via MIGRATE, it is temporarily set to 'no' by default. - # - # sanitize-dump-payload no - - # The filename where to dump the DB - dbfilename dump.rdb - - # Remove RDB files used by replication in instances without persistence - # enabled. By default this option is disabled, however there are environments - # where for regulations or other security concerns, RDB files persisted on - # disk by masters in order to feed replicas, or stored on disk by replicas - # in order to load them for the initial synchronization, should be deleted - # ASAP. Note that this option ONLY WORKS in instances that have both AOF - # and RDB persistence disabled, otherwise is completely ignored. - # - # An alternative (and sometimes better) way to obtain the same effect is - # to use diskless replication on both master and replicas instances. However - # in the case of replicas, diskless is not always an option. - rdb-del-sync-files no - - # The working directory. - # - # The DB will be written inside this directory, with the filename specified - # above using the 'dbfilename' configuration directive. - # - # The Append Only File will also be created inside this directory. - # - # Note that you must specify a directory here, not a file name. - dir ./ - - ################################# REPLICATION ################################# - - # Master-Replica replication. Use replicaof to make a Redis instance a copy of - # another Redis server. A few things to understand ASAP about Redis replication. - # - # +------------------+ +---------------+ - # | Master | ---> | Replica | - # | (receive writes) | | (exact copy) | - # +------------------+ +---------------+ - # - # 1) Redis replication is asynchronous, but you can configure a master to - # stop accepting writes if it appears to be not connected with at least - # a given number of replicas. - # 2) Redis replicas are able to perform a partial resynchronization with the - # master if the replication link is lost for a relatively small amount of - # time. You may want to configure the replication backlog size (see the next - # sections of this file) with a sensible value depending on your needs. - # 3) Replication is automatic and does not need user intervention. After a - # network partition replicas automatically try to reconnect to masters - # and resynchronize with them. - # - # replicaof - - # If the master is password protected (using the "requirepass" configuration - # directive below) it is possible to tell the replica to authenticate before - # starting the replication synchronization process, otherwise the master will - # refuse the replica request. - # - # masterauth - # - # However this is not enough if you are using Redis ACLs (for Redis version - # 6 or greater), and the default user is not capable of running the PSYNC - # command and/or other commands needed for replication. In this case it's - # better to configure a special user to use with replication, and specify the - # masteruser configuration as such: - # - # masteruser - # - # When masteruser is specified, the replica will authenticate against its - # master using the new AUTH form: AUTH . - - # When a replica loses its connection with the master, or when the replication - # is still in progress, the replica can act in two different ways: - # - # 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will - # still reply to client requests, possibly with out of date data, or the - # data set may just be empty if this is the first synchronization. - # - # 2) If replica-serve-stale-data is set to 'no' the replica will reply with - # an error "SYNC with master in progress" to all commands except: - # INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, - # UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, - # HOST and LATENCY. - # - replica-serve-stale-data yes - - # You can configure a replica instance to accept writes or not. Writing against - # a replica instance may be useful to store some ephemeral data (because data - # written on a replica will be easily deleted after resync with the master) but - # may also cause problems if clients are writing to it because of a - # misconfiguration. - # - # Since Redis 2.6 by default replicas are read-only. - # - # Note: read only replicas are not designed to be exposed to untrusted clients - # on the internet. It's just a protection layer against misuse of the instance. - # Still a read only replica exports by default all the administrative commands - # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve - # security of read only replicas using 'rename-command' to shadow all the - # administrative / dangerous commands. - replica-read-only yes - - # Replication SYNC strategy: disk or socket. - # - # New replicas and reconnecting replicas that are not able to continue the - # replication process just receiving differences, need to do what is called a - # "full synchronization". An RDB file is transmitted from the master to the - # replicas. - # - # The transmission can happen in two different ways: - # - # 1) Disk-backed: The Redis master creates a new process that writes the RDB - # file on disk. Later the file is transferred by the parent - # process to the replicas incrementally. - # 2) Diskless: The Redis master creates a new process that directly writes the - # RDB file to replica sockets, without touching the disk at all. - # - # With disk-backed replication, while the RDB file is generated, more replicas - # can be queued and served with the RDB file as soon as the current child - # producing the RDB file finishes its work. With diskless replication instead - # once the transfer starts, new replicas arriving will be queued and a new - # transfer will start when the current one terminates. - # - # When diskless replication is used, the master waits a configurable amount of - # time (in seconds) before starting the transfer in the hope that multiple - # replicas will arrive and the transfer can be parallelized. - # - # With slow disks and fast (large bandwidth) networks, diskless replication - # works better. - repl-diskless-sync no - - # When diskless replication is enabled, it is possible to configure the delay - # the server waits in order to spawn the child that transfers the RDB via socket - # to the replicas. - # - # This is important since once the transfer starts, it is not possible to serve - # new replicas arriving, that will be queued for the next RDB transfer, so the - # server waits a delay in order to let more replicas arrive. - # - # The delay is specified in seconds, and by default is 5 seconds. To disable - # it entirely just set it to 0 seconds and the transfer will start ASAP. - repl-diskless-sync-delay 5 - - # ----------------------------------------------------------------------------- - # WARNING: RDB diskless load is experimental. Since in this setup the replica - # does not immediately store an RDB on disk, it may cause data loss during - # failovers. RDB diskless load + Redis modules not handling I/O reads may also - # cause Redis to abort in case of I/O errors during the initial synchronization - # stage with the master. Use only if you know what you are doing. - # ----------------------------------------------------------------------------- - # - # Replica can load the RDB it reads from the replication link directly from the - # socket, or store the RDB to a file and read that file after it was completely - # received from the master. - # - # In many cases the disk is slower than the network, and storing and loading - # the RDB file may increase replication time (and even increase the master's - # Copy on Write memory and salve buffers). - # However, parsing the RDB file directly from the socket may mean that we have - # to flush the contents of the current database before the full rdb was - # received. For this reason we have the following options: - # - # "disabled" - Don't use diskless load (store the rdb file to the disk first) - # "on-empty-db" - Use diskless load only when it is completely safe. - # "swapdb" - Keep a copy of the current db contents in RAM while parsing - # the data directly from the socket. note that this requires - # sufficient memory, if you don't have it, you risk an OOM kill. - repl-diskless-load disabled - - # Replicas send PINGs to server in a predefined interval. It's possible to - # change this interval with the repl_ping_replica_period option. The default - # value is 10 seconds. - # - # repl-ping-replica-period 10 - - # The following option sets the replication timeout for: - # - # 1) Bulk transfer I/O during SYNC, from the point of view of replica. - # 2) Master timeout from the point of view of replicas (data, pings). - # 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). - # - # It is important to make sure that this value is greater than the value - # specified for repl-ping-replica-period otherwise a timeout will be detected - # every time there is low traffic between the master and the replica. The default - # value is 60 seconds. - # - # repl-timeout 60 - - # Disable TCP_NODELAY on the replica socket after SYNC? - # - # If you select "yes" Redis will use a smaller number of TCP packets and - # less bandwidth to send data to replicas. But this can add a delay for - # the data to appear on the replica side, up to 40 milliseconds with - # Linux kernels using a default configuration. - # - # If you select "no" the delay for data to appear on the replica side will - # be reduced but more bandwidth will be used for replication. - # - # By default we optimize for low latency, but in very high traffic conditions - # or when the master and replicas are many hops away, turning this to "yes" may - # be a good idea. - repl-disable-tcp-nodelay no - - # Set the replication backlog size. The backlog is a buffer that accumulates - # replica data when replicas are disconnected for some time, so that when a - # replica wants to reconnect again, often a full resync is not needed, but a - # partial resync is enough, just passing the portion of data the replica - # missed while disconnected. - # - # The bigger the replication backlog, the longer the replica can endure the - # disconnect and later be able to perform a partial resynchronization. - # - # The backlog is only allocated if there is at least one replica connected. - # - # repl-backlog-size 1mb - - # After a master has no connected replicas for some time, the backlog will be - # freed. The following option configures the amount of seconds that need to - # elapse, starting from the time the last replica disconnected, for the backlog - # buffer to be freed. - # - # Note that replicas never free the backlog for timeout, since they may be - # promoted to masters later, and should be able to correctly "partially - # resynchronize" with other replicas: hence they should always accumulate backlog. - # - # A value of 0 means to never release the backlog. - # - # repl-backlog-ttl 3600 - - # The replica priority is an integer number published by Redis in the INFO - # output. It is used by Redis Sentinel in order to select a replica to promote - # into a master if the master is no longer working correctly. - # - # A replica with a low priority number is considered better for promotion, so - # for instance if there are three replicas with priority 10, 100, 25 Sentinel - # will pick the one with priority 10, that is the lowest. - # - # However a special priority of 0 marks the replica as not able to perform the - # role of master, so a replica with priority of 0 will never be selected by - # Redis Sentinel for promotion. - # - # By default the priority is 100. - replica-priority 100 - - # ----------------------------------------------------------------------------- - # By default, Redis Sentinel includes all replicas in its reports. A replica - # can be excluded from Redis Sentinel's announcements. An unannounced replica - # will be ignored by the 'sentinel replicas ' command and won't be - # exposed to Redis Sentinel's clients. - # - # This option does not change the behavior of replica-priority. Even with - # replica-announced set to 'no', the replica can be promoted to master. To - # prevent this behavior, set replica-priority to 0. - # - # replica-announced yes - - # It is possible for a master to stop accepting writes if there are less than - # N replicas connected, having a lag less or equal than M seconds. - # - # The N replicas need to be in "online" state. - # - # The lag in seconds, that must be <= the specified value, is calculated from - # the last ping received from the replica, that is usually sent every second. - # - # This option does not GUARANTEE that N replicas will accept the write, but - # will limit the window of exposure for lost writes in case not enough replicas - # are available, to the specified number of seconds. - # - # For example to require at least 3 replicas with a lag <= 10 seconds use: - # - # min-replicas-to-write 3 - # min-replicas-max-lag 10 - # - # Setting one or the other to 0 disables the feature. - # - # By default min-replicas-to-write is set to 0 (feature disabled) and - # min-replicas-max-lag is set to 10. - - # A Redis master is able to list the address and port of the attached - # replicas in different ways. For example the "INFO replication" section - # offers this information, which is used, among other tools, by - # Redis Sentinel in order to discover replica instances. - # Another place where this info is available is in the output of the - # "ROLE" command of a master. - # - # The listed IP address and port normally reported by a replica is - # obtained in the following way: - # - # IP: The address is auto detected by checking the peer address - # of the socket used by the replica to connect with the master. - # - # Port: The port is communicated by the replica during the replication - # handshake, and is normally the port that the replica is using to - # listen for connections. - # - # However when port forwarding or Network Address Translation (NAT) is - # used, the replica may actually be reachable via different IP and port - # pairs. The following two options can be used by a replica in order to - # report to its master a specific set of IP and port, so that both INFO - # and ROLE will report those values. - # - # There is no need to use both the options if you need to override just - # the port or the IP address. - # - # replica-announce-ip 5.5.5.5 - # replica-announce-port 1234 - - ############################### KEYS TRACKING ################################# - - # Redis implements server assisted support for client side caching of values. - # This is implemented using an invalidation table that remembers, using - # a radix key indexed by key name, what clients have which keys. In turn - # this is used in order to send invalidation messages to clients. Please - # check this page to understand more about the feature: - # - # https://redis.io/topics/client-side-caching - # - # When tracking is enabled for a client, all the read only queries are assumed - # to be cached: this will force Redis to store information in the invalidation - # table. When keys are modified, such information is flushed away, and - # invalidation messages are sent to the clients. However if the workload is - # heavily dominated by reads, Redis could use more and more memory in order - # to track the keys fetched by many clients. - # - # For this reason it is possible to configure a maximum fill value for the - # invalidation table. By default it is set to 1M of keys, and once this limit - # is reached, Redis will start to evict keys in the invalidation table - # even if they were not modified, just to reclaim memory: this will in turn - # force the clients to invalidate the cached values. Basically the table - # maximum size is a trade off between the memory you want to spend server - # side to track information about who cached what, and the ability of clients - # to retain cached objects in memory. - # - # If you set the value to 0, it means there are no limits, and Redis will - # retain as many keys as needed in the invalidation table. - # In the "stats" INFO section, you can find information about the number of - # keys in the invalidation table at every given moment. - # - # Note: when key tracking is used in broadcasting mode, no memory is used - # in the server side so this setting is useless. - # - # tracking-table-max-keys 1000000 - - ################################## SECURITY ################################### - - # Warning: since Redis is pretty fast, an outside user can try up to - # 1 million passwords per second against a modern box. This means that you - # should use very strong passwords, otherwise they will be very easy to break. - # Note that because the password is really a shared secret between the client - # and the server, and should not be memorized by any human, the password - # can be easily a long string from /dev/urandom or whatever, so by using a - # long and unguessable password no brute force attack will be possible. - - # Redis ACL users are defined in the following format: - # - # user ... acl rules ... - # - # For example: - # - # user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 - # - # The special username "default" is used for new connections. If this user - # has the "nopass" rule, then new connections will be immediately authenticated - # as the "default" user without the need of any password provided via the - # AUTH command. Otherwise if the "default" user is not flagged with "nopass" - # the connections will start in not authenticated state, and will require - # AUTH (or the HELLO command AUTH option) in order to be authenticated and - # start to work. - # - # The ACL rules that describe what a user can do are the following: - # - # on Enable the user: it is possible to authenticate as this user. - # off Disable the user: it's no longer possible to authenticate - # with this user, however the already authenticated connections - # will still work. - # skip-sanitize-payload RESTORE dump-payload sanitation is skipped. - # sanitize-payload RESTORE dump-payload is sanitized (default). - # + Allow the execution of that command - # - Disallow the execution of that command - # +@ Allow the execution of all the commands in such category - # with valid categories are like @admin, @set, @sortedset, ... - # and so forth, see the full list in the server.c file where - # the Redis command table is described and defined. - # The special category @all means all the commands, but currently - # present in the server, and that will be loaded in the future - # via modules. - # +|subcommand Allow a specific subcommand of an otherwise - # disabled command. Note that this form is not - # allowed as negative like -DEBUG|SEGFAULT, but - # only additive starting with "+". - # allcommands Alias for +@all. Note that it implies the ability to execute - # all the future commands loaded via the modules system. - # nocommands Alias for -@all. - # ~ Add a pattern of keys that can be mentioned as part of - # commands. For instance ~* allows all the keys. The pattern - # is a glob-style pattern like the one of KEYS. - # It is possible to specify multiple patterns. - # allkeys Alias for ~* - # resetkeys Flush the list of allowed keys patterns. - # & Add a glob-style pattern of Pub/Sub channels that can be - # accessed by the user. It is possible to specify multiple channel - # patterns. - # allchannels Alias for &* - # resetchannels Flush the list of allowed channel patterns. - # > Add this password to the list of valid password for the user. - # For example >mypass will add "mypass" to the list. - # This directive clears the "nopass" flag (see later). - # < Remove this password from the list of valid passwords. - # nopass All the set passwords of the user are removed, and the user - # is flagged as requiring no password: it means that every - # password will work against this user. If this directive is - # used for the default user, every new connection will be - # immediately authenticated with the default user without - # any explicit AUTH command required. Note that the "resetpass" - # directive will clear this condition. - # resetpass Flush the list of allowed passwords. Moreover removes the - # "nopass" status. After "resetpass" the user has no associated - # passwords and there is no way to authenticate without adding - # some password (or setting it as "nopass" later). - # reset Performs the following actions: resetpass, resetkeys, off, - # -@all. The user returns to the same state it has immediately - # after its creation. - # - # ACL rules can be specified in any order: for instance you can start with - # passwords, then flags, or key patterns. However note that the additive - # and subtractive rules will CHANGE MEANING depending on the ordering. - # For instance see the following example: - # - # user alice on +@all -DEBUG ~* >somepassword - # - # This will allow "alice" to use all the commands with the exception of the - # DEBUG command, since +@all added all the commands to the set of the commands - # alice can use, and later DEBUG was removed. However if we invert the order - # of two ACL rules the result will be different: - # - # user alice on -DEBUG +@all ~* >somepassword - # - # Now DEBUG was removed when alice had yet no commands in the set of allowed - # commands, later all the commands are added, so the user will be able to - # execute everything. - # - # Basically ACL rules are processed left-to-right. - # - # For more information about ACL configuration please refer to - # the Redis web site at https://redis.io/topics/acl - - # ACL LOG - # - # The ACL Log tracks failed commands and authentication events associated - # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked - # by ACLs. The ACL Log is stored in memory. You can reclaim memory with - # ACL LOG RESET. Define the maximum entry length of the ACL Log below. - acllog-max-len 128 - - # Using an external ACL file - # - # Instead of configuring users here in this file, it is possible to use - # a stand-alone file just listing users. The two methods cannot be mixed: - # if you configure users here and at the same time you activate the external - # ACL file, the server will refuse to start. - # - # The format of the external ACL user file is exactly the same as the - # format that is used inside redis.conf to describe users. - # - # aclfile /etc/redis/users.acl - - # IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility - # layer on top of the new ACL system. The option effect will be just setting - # the password for the default user. Clients will still authenticate using - # AUTH as usually, or more explicitly with AUTH default - # if they follow the new protocol: both will work. - # - # The requirepass is not compatable with aclfile option and the ACL LOAD - # command, these will cause requirepass to be ignored. - # - # requirepass foobared - - # New users are initialized with restrictive permissions by default, via the - # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it - # is possible to manage access to Pub/Sub channels with ACL rules as well. The - # default Pub/Sub channels permission if new users is controlled by the - # acl-pubsub-default configuration directive, which accepts one of these values: - # - # allchannels: grants access to all Pub/Sub channels - # resetchannels: revokes access to all Pub/Sub channels - # - # To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default - # defaults to the 'allchannels' permission. - # - # Future compatibility note: it is very likely that in a future version of Redis - # the directive's default of 'allchannels' will be changed to 'resetchannels' in - # order to provide better out-of-the-box Pub/Sub security. Therefore, it is - # recommended that you explicitly define Pub/Sub permissions for all users - # rather then rely on implicit default values. Once you've set explicit - # Pub/Sub for all existing users, you should uncomment the following line. - # - # acl-pubsub-default resetchannels - - # Command renaming (DEPRECATED). - # - # ------------------------------------------------------------------------ - # WARNING: avoid using this option if possible. Instead use ACLs to remove - # commands from the default user, and put them only in some admin user you - # create for administrative purposes. - # ------------------------------------------------------------------------ - # - # It is possible to change the name of dangerous commands in a shared - # environment. For instance the CONFIG command may be renamed into something - # hard to guess so that it will still be available for internal-use tools - # but not available for general clients. - # - # Example: - # - # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 - # - # It is also possible to completely kill a command by renaming it into - # an empty string: - # - # rename-command CONFIG "" - # - # Please note that changing the name of commands that are logged into the - # AOF file or transmitted to replicas may cause problems. - - ################################### CLIENTS #################################### - - # Set the max number of connected clients at the same time. By default - # this limit is set to 10000 clients, however if the Redis server is not - # able to configure the process file limit to allow for the specified limit - # the max number of allowed clients is set to the current file limit - # minus 32 (as Redis reserves a few file descriptors for internal uses). - # - # Once the limit is reached Redis will close all the new connections sending - # an error 'max number of clients reached'. - # - # IMPORTANT: When Redis Cluster is used, the max number of connections is also - # shared with the cluster bus: every node in the cluster will use two - # connections, one incoming and another outgoing. It is important to size the - # limit accordingly in case of very large clusters. - # - # maxclients 10000 - - ############################## MEMORY MANAGEMENT ################################ - - # Set a memory usage limit to the specified amount of bytes. - # When the memory limit is reached Redis will try to remove keys - # according to the eviction policy selected (see maxmemory-policy). - # - # If Redis can't remove keys according to the policy, or if the policy is - # set to 'noeviction', Redis will start to reply with errors to commands - # that would use more memory, like SET, LPUSH, and so on, and will continue - # to reply to read-only commands like GET. - # - # This option is usually useful when using Redis as an LRU or LFU cache, or to - # set a hard memory limit for an instance (using the 'noeviction' policy). - # - # WARNING: If you have replicas attached to an instance with maxmemory on, - # the size of the output buffers needed to feed the replicas are subtracted - # from the used memory count, so that network problems / resyncs will - # not trigger a loop where keys are evicted, and in turn the output - # buffer of replicas is full with DELs of keys evicted triggering the deletion - # of more keys, and so forth until the database is completely emptied. - # - # In short... if you have replicas attached it is suggested that you set a lower - # limit for maxmemory so that there is some free RAM on the system for replica - # output buffers (but this is not needed if the policy is 'noeviction'). - # - # maxmemory - - # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory - # is reached. You can select one from the following behaviors: - # - # volatile-lru -> Evict using approximated LRU, only keys with an expire set. - # allkeys-lru -> Evict any key using approximated LRU. - # volatile-lfu -> Evict using approximated LFU, only keys with an expire set. - # allkeys-lfu -> Evict any key using approximated LFU. - # volatile-random -> Remove a random key having an expire set. - # allkeys-random -> Remove a random key, any key. - # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) - # noeviction -> Don't evict anything, just return an error on write operations. - # - # LRU means Least Recently Used - # LFU means Least Frequently Used - # - # Both LRU, LFU and volatile-ttl are implemented using approximated - # randomized algorithms. - # - # Note: with any of the above policies, when there are no suitable keys for - # eviction, Redis will return an error on write operations that require - # more memory. These are usually commands that create new keys, add data or - # modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, - # SORT (due to the STORE argument), and EXEC (if the transaction includes any - # command that requires memory). - # - # The default is: - # - # maxmemory-policy noeviction - - # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated - # algorithms (in order to save memory), so you can tune it for speed or - # accuracy. By default Redis will check five keys and pick the one that was - # used least recently, you can change the sample size using the following - # configuration directive. - # - # The default of 5 produces good enough results. 10 Approximates very closely - # true LRU but costs more CPU. 3 is faster but not very accurate. - # - # maxmemory-samples 5 - - # Eviction processing is designed to function well with the default setting. - # If there is an unusually large amount of write traffic, this value may need to - # be increased. Decreasing this value may reduce latency at the risk of - # eviction processing effectiveness - # 0 = minimum latency, 10 = default, 100 = process without regard to latency - # - # maxmemory-eviction-tenacity 10 - - # Starting from Redis 5, by default a replica will ignore its maxmemory setting - # (unless it is promoted to master after a failover or manually). It means - # that the eviction of keys will be just handled by the master, sending the - # DEL commands to the replica as keys evict in the master side. - # - # This behavior ensures that masters and replicas stay consistent, and is usually - # what you want, however if your replica is writable, or you want the replica - # to have a different memory setting, and you are sure all the writes performed - # to the replica are idempotent, then you may change this default (but be sure - # to understand what you are doing). - # - # Note that since the replica by default does not evict, it may end using more - # memory than the one set via maxmemory (there are certain buffers that may - # be larger on the replica, or data structures may sometimes take more memory - # and so forth). So make sure you monitor your replicas and make sure they - # have enough memory to never hit a real out-of-memory condition before the - # master hits the configured maxmemory setting. - # - # replica-ignore-maxmemory yes - - # Redis reclaims expired keys in two ways: upon access when those keys are - # found to be expired, and also in background, in what is called the - # "active expire key". The key space is slowly and interactively scanned - # looking for expired keys to reclaim, so that it is possible to free memory - # of keys that are expired and will never be accessed again in a short time. - # - # The default effort of the expire cycle will try to avoid having more than - # ten percent of expired keys still in memory, and will try to avoid consuming - # more than 25% of total memory and to add latency to the system. However - # it is possible to increase the expire "effort" that is normally set to - # "1", to a greater value, up to the value "10". At its maximum value the - # system will use more CPU, longer cycles (and technically may introduce - # more latency), and will tolerate less already expired keys still present - # in the system. It's a tradeoff between memory, CPU and latency. - # - # active-expire-effort 1 - - ############################# LAZY FREEING #################################### - - # Redis has two primitives to delete keys. One is called DEL and is a blocking - # deletion of the object. It means that the server stops processing new commands - # in order to reclaim all the memory associated with an object in a synchronous - # way. If the key deleted is associated with a small object, the time needed - # in order to execute the DEL command is very small and comparable to most other - # O(1) or O(log_N) commands in Redis. However if the key is associated with an - # aggregated value containing millions of elements, the server can block for - # a long time (even seconds) in order to complete the operation. - # - # For the above reasons Redis also offers non blocking deletion primitives - # such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and - # FLUSHDB commands, in order to reclaim memory in background. Those commands - # are executed in constant time. Another thread will incrementally free the - # object in the background as fast as possible. - # - # DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. - # It's up to the design of the application to understand when it is a good - # idea to use one or the other. However the Redis server sometimes has to - # delete keys or flush the whole database as a side effect of other operations. - # Specifically Redis deletes objects independently of a user call in the - # following scenarios: - # - # 1) On eviction, because of the maxmemory and maxmemory policy configurations, - # in order to make room for new data, without going over the specified - # memory limit. - # 2) Because of expire: when a key with an associated time to live (see the - # EXPIRE command) must be deleted from memory. - # 3) Because of a side effect of a command that stores data on a key that may - # already exist. For example the RENAME command may delete the old key - # content when it is replaced with another one. Similarly SUNIONSTORE - # or SORT with STORE option may delete existing keys. The SET command - # itself removes any old content of the specified key in order to replace - # it with the specified string. - # 4) During replication, when a replica performs a full resynchronization with - # its master, the content of the whole database is removed in order to - # load the RDB file just transferred. - # - # In all the above cases the default is to delete objects in a blocking way, - # like if DEL was called. However you can configure each case specifically - # in order to instead release memory in a non-blocking way like if UNLINK - # was called, using the following configuration directives. - - lazyfree-lazy-eviction no - lazyfree-lazy-expire no - lazyfree-lazy-server-del no - replica-lazy-flush no - - # It is also possible, for the case when to replace the user code DEL calls - # with UNLINK calls is not easy, to modify the default behavior of the DEL - # command to act exactly like UNLINK, using the following configuration - # directive: - - lazyfree-lazy-user-del no - - # FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous - # deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the - # commands. When neither flag is passed, this directive will be used to determine - # if the data should be deleted asynchronously. - - lazyfree-lazy-user-flush no - - ################################ THREADED I/O ################################# - - # Redis is mostly single threaded, however there are certain threaded - # operations such as UNLINK, slow I/O accesses and other things that are - # performed on side threads. - # - # Now it is also possible to handle Redis clients socket reads and writes - # in different I/O threads. Since especially writing is so slow, normally - # Redis users use pipelining in order to speed up the Redis performances per - # core, and spawn multiple instances in order to scale more. Using I/O - # threads it is possible to easily speedup two times Redis without resorting - # to pipelining nor sharding of the instance. - # - # By default threading is disabled, we suggest enabling it only in machines - # that have at least 4 or more cores, leaving at least one spare core. - # Using more than 8 threads is unlikely to help much. We also recommend using - # threaded I/O only if you actually have performance problems, with Redis - # instances being able to use a quite big percentage of CPU time, otherwise - # there is no point in using this feature. - # - # So for instance if you have a four cores boxes, try to use 2 or 3 I/O - # threads, if you have a 8 cores, try to use 6 threads. In order to - # enable I/O threads use the following configuration directive: - # - # io-threads 4 - # - # Setting io-threads to 1 will just use the main thread as usual. - # When I/O threads are enabled, we only use threads for writes, that is - # to thread the write(2) syscall and transfer the client buffers to the - # socket. However it is also possible to enable threading of reads and - # protocol parsing using the following configuration directive, by setting - # it to yes: - # - # io-threads-do-reads no - # - # Usually threading reads doesn't help much. - # - # NOTE 1: This configuration directive cannot be changed at runtime via - # CONFIG SET. Aso this feature currently does not work when SSL is - # enabled. - # - # NOTE 2: If you want to test the Redis speedup using redis-benchmark, make - # sure you also run the benchmark itself in threaded mode, using the - # --threads option to match the number of Redis threads, otherwise you'll not - # be able to notice the improvements. - - ############################ KERNEL OOM CONTROL ############################## - - # On Linux, it is possible to hint the kernel OOM killer on what processes - # should be killed first when out of memory. - # - # Enabling this feature makes Redis actively control the oom_score_adj value - # for all its processes, depending on their role. The default scores will - # attempt to have background child processes killed before all others, and - # replicas killed before masters. - # - # Redis supports three options: - # - # no: Don't make changes to oom-score-adj (default). - # yes: Alias to "relative" see below. - # absolute: Values in oom-score-adj-values are written as is to the kernel. - # relative: Values are used relative to the initial value of oom_score_adj when - # the server starts and are then clamped to a range of -1000 to 1000. - # Because typically the initial value is 0, they will often match the - # absolute values. - oom-score-adj no - - # When oom-score-adj is used, this directive controls the specific values used - # for master, replica and background child processes. Values range -2000 to - # 2000 (higher means more likely to be killed). - # - # Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) - # can freely increase their value, but not decrease it below its initial - # settings. This means that setting oom-score-adj to "relative" and setting the - # oom-score-adj-values to positive values will always succeed. - oom-score-adj-values 0 200 800 - - - #################### KERNEL transparent hugepage CONTROL ###################### - - # Usually the kernel Transparent Huge Pages control is set to "madvise" or - # or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which - # case this config has no effect. On systems in which it is set to "always", - # redis will attempt to disable it specifically for the redis process in order - # to avoid latency problems specifically with fork(2) and CoW. - # If for some reason you prefer to keep it enabled, you can set this config to - # "no" and the kernel global to "always". - - disable-thp yes - - ############################## APPEND ONLY MODE ############################### - - # By default Redis asynchronously dumps the dataset on disk. This mode is - # good enough in many applications, but an issue with the Redis process or - # a power outage may result into a few minutes of writes lost (depending on - # the configured save points). - # - # The Append Only File is an alternative persistence mode that provides - # much better durability. For instance using the default data fsync policy - # (see later in the config file) Redis can lose just one second of writes in a - # dramatic event like a server power outage, or a single write if something - # wrong with the Redis process itself happens, but the operating system is - # still running correctly. - # - # AOF and RDB persistence can be enabled at the same time without problems. - # If the AOF is enabled on startup Redis will load the AOF, that is the file - # with the better durability guarantees. - # - # Please check https://redis.io/topics/persistence for more information. - - appendonly no - - # The name of the append only file (default: "appendonly.aof") - - appendfilename "appendonly.aof" - - # The fsync() call tells the Operating System to actually write data on disk - # instead of waiting for more data in the output buffer. Some OS will really flush - # data on disk, some other OS will just try to do it ASAP. - # - # Redis supports three different modes: - # - # no: don't fsync, just let the OS flush the data when it wants. Faster. - # always: fsync after every write to the append only log. Slow, Safest. - # everysec: fsync only one time every second. Compromise. - # - # The default is "everysec", as that's usually the right compromise between - # speed and data safety. It's up to you to understand if you can relax this to - # "no" that will let the operating system flush the output buffer when - # it wants, for better performances (but if you can live with the idea of - # some data loss consider the default persistence mode that's snapshotting), - # or on the contrary, use "always" that's very slow but a bit safer than - # everysec. - # - # More details please check the following article: - # http://antirez.com/post/redis-persistence-demystified.html - # - # If unsure, use "everysec". - - # appendfsync always - appendfsync everysec - # appendfsync no - - # When the AOF fsync policy is set to always or everysec, and a background - # saving process (a background save or AOF log background rewriting) is - # performing a lot of I/O against the disk, in some Linux configurations - # Redis may block too long on the fsync() call. Note that there is no fix for - # this currently, as even performing fsync in a different thread will block - # our synchronous write(2) call. - # - # In order to mitigate this problem it's possible to use the following option - # that will prevent fsync() from being called in the main process while a - # BGSAVE or BGREWRITEAOF is in progress. - # - # This means that while another child is saving, the durability of Redis is - # the same as "appendfsync none". In practical terms, this means that it is - # possible to lose up to 30 seconds of log in the worst scenario (with the - # default Linux settings). - # - # If you have latency problems turn this to "yes". Otherwise leave it as - # "no" that is the safest pick from the point of view of durability. - - no-appendfsync-on-rewrite no - - # Automatic rewrite of the append only file. - # Redis is able to automatically rewrite the log file implicitly calling - # BGREWRITEAOF when the AOF log size grows by the specified percentage. - # - # This is how it works: Redis remembers the size of the AOF file after the - # latest rewrite (if no rewrite has happened since the restart, the size of - # the AOF at startup is used). - # - # This base size is compared to the current size. If the current size is - # bigger than the specified percentage, the rewrite is triggered. Also - # you need to specify a minimal size for the AOF file to be rewritten, this - # is useful to avoid rewriting the AOF file even if the percentage increase - # is reached but it is still pretty small. - # - # Specify a percentage of zero in order to disable the automatic AOF - # rewrite feature. - - auto-aof-rewrite-percentage 100 - auto-aof-rewrite-min-size 64mb - - # An AOF file may be found to be truncated at the end during the Redis - # startup process, when the AOF data gets loaded back into memory. - # This may happen when the system where Redis is running - # crashes, especially when an ext4 filesystem is mounted without the - # data=ordered option (however this can't happen when Redis itself - # crashes or aborts but the operating system still works correctly). - # - # Redis can either exit with an error when this happens, or load as much - # data as possible (the default now) and start if the AOF file is found - # to be truncated at the end. The following option controls this behavior. - # - # If aof-load-truncated is set to yes, a truncated AOF file is loaded and - # the Redis server starts emitting a log to inform the user of the event. - # Otherwise if the option is set to no, the server aborts with an error - # and refuses to start. When the option is set to no, the user requires - # to fix the AOF file using the "redis-check-aof" utility before to restart - # the server. - # - # Note that if the AOF file will be found to be corrupted in the middle - # the server will still exit with an error. This option only applies when - # Redis will try to read more data from the AOF file but not enough bytes - # will be found. - aof-load-truncated yes - - # When rewriting the AOF file, Redis is able to use an RDB preamble in the - # AOF file for faster rewrites and recoveries. When this option is turned - # on the rewritten AOF file is composed of two different stanzas: - # - # [RDB file][AOF tail] - # - # When loading, Redis recognizes that the AOF file starts with the "REDIS" - # string and loads the prefixed RDB file, then continues loading the AOF - # tail. - aof-use-rdb-preamble yes - - ################################ LUA SCRIPTING ############################### - - # Max execution time of a Lua script in milliseconds. - # - # If the maximum execution time is reached Redis will log that a script is - # still in execution after the maximum allowed time and will start to - # reply to queries with an error. - # - # When a long running script exceeds the maximum execution time only the - # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be - # used to stop a script that did not yet call any write commands. The second - # is the only way to shut down the server in the case a write command was - # already issued by the script but the user doesn't want to wait for the natural - # termination of the script. - # - # Set it to 0 or a negative value for unlimited execution without warnings. - lua-time-limit 5000 - - ################################ REDIS CLUSTER ############################### - - # Normal Redis instances can't be part of a Redis Cluster; only nodes that are - # started as cluster nodes can. In order to start a Redis instance as a - # cluster node enable the cluster support uncommenting the following: - # - # cluster-enabled yes - - # Every cluster node has a cluster configuration file. This file is not - # intended to be edited by hand. It is created and updated by Redis nodes. - # Every Redis Cluster node requires a different cluster configuration file. - # Make sure that instances running in the same system do not have - # overlapping cluster configuration file names. - # - # cluster-config-file nodes-6379.conf - - # Cluster node timeout is the amount of milliseconds a node must be unreachable - # for it to be considered in failure state. - # Most other internal time limits are a multiple of the node timeout. - # - # cluster-node-timeout 15000 - - # A replica of a failing master will avoid to start a failover if its data - # looks too old. - # - # There is no simple way for a replica to actually have an exact measure of - # its "data age", so the following two checks are performed: - # - # 1) If there are multiple replicas able to failover, they exchange messages - # in order to try to give an advantage to the replica with the best - # replication offset (more data from the master processed). - # Replicas will try to get their rank by offset, and apply to the start - # of the failover a delay proportional to their rank. - # - # 2) Every single replica computes the time of the last interaction with - # its master. This can be the last ping or command received (if the master - # is still in the "connected" state), or the time that elapsed since the - # disconnection with the master (if the replication link is currently down). - # If the last interaction is too old, the replica will not try to failover - # at all. - # - # The point "2" can be tuned by user. Specifically a replica will not perform - # the failover if, since the last interaction with the master, the time - # elapsed is greater than: - # - # (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period - # - # So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor - # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the - # replica will not try to failover if it was not able to talk with the master - # for longer than 310 seconds. - # - # A large cluster-replica-validity-factor may allow replicas with too old data to failover - # a master, while a too small value may prevent the cluster from being able to - # elect a replica at all. - # - # For maximum availability, it is possible to set the cluster-replica-validity-factor - # to a value of 0, which means, that replicas will always try to failover the - # master regardless of the last time they interacted with the master. - # (However they'll always try to apply a delay proportional to their - # offset rank). - # - # Zero is the only value able to guarantee that when all the partitions heal - # the cluster will always be able to continue. - # - # cluster-replica-validity-factor 10 - - # Cluster replicas are able to migrate to orphaned masters, that are masters - # that are left without working replicas. This improves the cluster ability - # to resist to failures as otherwise an orphaned master can't be failed over - # in case of failure if it has no working replicas. - # - # Replicas migrate to orphaned masters only if there are still at least a - # given number of other working replicas for their old master. This number - # is the "migration barrier". A migration barrier of 1 means that a replica - # will migrate only if there is at least 1 other working replica for its master - # and so forth. It usually reflects the number of replicas you want for every - # master in your cluster. - # - # Default is 1 (replicas migrate only if their masters remain with at least - # one replica). To disable migration just set it to a very large value or - # set cluster-allow-replica-migration to 'no'. - # A value of 0 can be set but is useful only for debugging and dangerous - # in production. - # - # cluster-migration-barrier 1 - - # Turning off this option allows to use less automatic cluster configuration. - # It both disables migration to orphaned masters and migration from masters - # that became empty. - # - # Default is 'yes' (allow automatic migrations). - # - # cluster-allow-replica-migration yes - - # By default Redis Cluster nodes stop accepting queries if they detect there - # is at least a hash slot uncovered (no available node is serving it). - # This way if the cluster is partially down (for example a range of hash slots - # are no longer covered) all the cluster becomes, eventually, unavailable. - # It automatically returns available as soon as all the slots are covered again. - # - # However sometimes you want the subset of the cluster which is working, - # to continue to accept queries for the part of the key space that is still - # covered. In order to do so, just set the cluster-require-full-coverage - # option to no. - # - # cluster-require-full-coverage yes - - # This option, when set to yes, prevents replicas from trying to failover its - # master during master failures. However the replica can still perform a - # manual failover, if forced to do so. - # - # This is useful in different scenarios, especially in the case of multiple - # data center operations, where we want one side to never be promoted if not - # in the case of a total DC failure. - # - # cluster-replica-no-failover no - - # This option, when set to yes, allows nodes to serve read traffic while the - # the cluster is in a down state, as long as it believes it owns the slots. - # - # This is useful for two cases. The first case is for when an application - # doesn't require consistency of data during node failures or network partitions. - # One example of this is a cache, where as long as the node has the data it - # should be able to serve it. - # - # The second use case is for configurations that don't meet the recommended - # three shards but want to enable cluster mode and scale later. A - # master outage in a 1 or 2 shard configuration causes a read/write outage to the - # entire cluster without this option set, with it set there is only a write outage. - # Without a quorum of masters, slot ownership will not change automatically. - # - # cluster-allow-reads-when-down no - - # In order to setup your cluster make sure to read the documentation - # available at https://redis.io web site. - - ########################## CLUSTER DOCKER/NAT support ######################## - - # In certain deployments, Redis Cluster nodes address discovery fails, because - # addresses are NAT-ted or because ports are forwarded (the typical case is - # Docker and other containers). - # - # In order to make Redis Cluster working in such environments, a static - # configuration where each node knows its public address is needed. The - # following four options are used for this scope, and are: - # - # * cluster-announce-ip - # * cluster-announce-port - # * cluster-announce-tls-port - # * cluster-announce-bus-port - # - # Each instructs the node about its address, client ports (for connections - # without and with TLS) and cluster message bus port. The information is then - # published in the header of the bus packets so that other nodes will be able to - # correctly map the address of the node publishing the information. - # - # If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set - # to zero, then cluster-announce-port refers to the TLS port. Note also that - # cluster-announce-tls-port has no effect if cluster-tls is set to no. - # - # If the above options are not used, the normal Redis Cluster auto-detection - # will be used instead. - # - # Note that when remapped, the bus port may not be at the fixed offset of - # clients port + 10000, so you can specify any port and bus-port depending - # on how they get remapped. If the bus-port is not set, a fixed offset of - # 10000 will be used as usual. - # - # Example: - # - # cluster-announce-ip 10.1.1.5 - # cluster-announce-tls-port 6379 - # cluster-announce-port 0 - # cluster-announce-bus-port 6380 - - ################################## SLOW LOG ################################### - - # The Redis Slow Log is a system to log queries that exceeded a specified - # execution time. The execution time does not include the I/O operations - # like talking with the client, sending the reply and so forth, - # but just the time needed to actually execute the command (this is the only - # stage of command execution where the thread is blocked and can not serve - # other requests in the meantime). - # - # You can configure the slow log with two parameters: one tells Redis - # what is the execution time, in microseconds, to exceed in order for the - # command to get logged, and the other parameter is the length of the - # slow log. When a new command is logged the oldest one is removed from the - # queue of logged commands. - - # The following time is expressed in microseconds, so 1000000 is equivalent - # to one second. Note that a negative number disables the slow log, while - # a value of zero forces the logging of every command. - slowlog-log-slower-than 10000 - - # There is no limit to this length. Just be aware that it will consume memory. - # You can reclaim memory used by the slow log with SLOWLOG RESET. - slowlog-max-len 128 - - ################################ LATENCY MONITOR ############################## - - # The Redis latency monitoring subsystem samples different operations - # at runtime in order to collect data related to possible sources of - # latency of a Redis instance. - # - # Via the LATENCY command this information is available to the user that can - # print graphs and obtain reports. - # - # The system only logs operations that were performed in a time equal or - # greater than the amount of milliseconds specified via the - # latency-monitor-threshold configuration directive. When its value is set - # to zero, the latency monitor is turned off. - # - # By default latency monitoring is disabled since it is mostly not needed - # if you don't have latency issues, and collecting data has a performance - # impact, that while very small, can be measured under big load. Latency - # monitoring can easily be enabled at runtime using the command - # "CONFIG SET latency-monitor-threshold " if needed. - latency-monitor-threshold 0 - - ############################# EVENT NOTIFICATION ############################## - - # Redis can notify Pub/Sub clients about events happening in the key space. - # This feature is documented at https://redis.io/topics/notifications - # - # For instance if keyspace events notification is enabled, and a client - # performs a DEL operation on key "foo" stored in the Database 0, two - # messages will be published via Pub/Sub: - # - # PUBLISH __keyspace@0__:foo del - # PUBLISH __keyevent@0__:del foo - # - # It is possible to select the events that Redis will notify among a set - # of classes. Every class is identified by a single character: - # - # K Keyspace events, published with __keyspace@__ prefix. - # E Keyevent events, published with __keyevent@__ prefix. - # g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... - # $ String commands - # l List commands - # s Set commands - # h Hash commands - # z Sorted set commands - # x Expired events (events generated every time a key expires) - # e Evicted events (events generated when a key is evicted for maxmemory) - # t Stream commands - # d Module key type events - # m Key-miss events (Note: It is not included in the 'A' class) - # A Alias for g$lshzxetd, so that the "AKE" string means all the events - # (Except key-miss events which are excluded from 'A' due to their - # unique nature). - # - # The "notify-keyspace-events" takes as argument a string that is composed - # of zero or multiple characters. The empty string means that notifications - # are disabled. - # - # Example: to enable list and generic events, from the point of view of the - # event name, use: - # - # notify-keyspace-events Elg - # - # Example 2: to get the stream of the expired keys subscribing to channel - # name __keyevent@0__:expired use: - # - # notify-keyspace-events Ex - # - # By default all notifications are disabled because most users don't need - # this feature and the feature has some overhead. Note that if you don't - # specify at least one of K or E, no events will be delivered. - notify-keyspace-events "" - - ############################### GOPHER SERVER ################################# - - # Redis contains an implementation of the Gopher protocol, as specified in - # the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). - # - # The Gopher protocol was very popular in the late '90s. It is an alternative - # to the web, and the implementation both server and client side is so simple - # that the Redis server has just 100 lines of code in order to implement this - # support. - # - # What do you do with Gopher nowadays? Well Gopher never *really* died, and - # lately there is a movement in order for the Gopher more hierarchical content - # composed of just plain text documents to be resurrected. Some want a simpler - # internet, others believe that the mainstream internet became too much - # controlled, and it's cool to create an alternative space for people that - # want a bit of fresh air. - # - # Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol - # as a gift. - # - # --- HOW IT WORKS? --- - # - # The Redis Gopher support uses the inline protocol of Redis, and specifically - # two kind of inline requests that were anyway illegal: an empty request - # or any request that starts with "/" (there are no Redis commands starting - # with such a slash). Normal RESP2/RESP3 requests are completely out of the - # path of the Gopher protocol implementation and are served as usual as well. - # - # If you open a connection to Redis when Gopher is enabled and send it - # a string like "/foo", if there is a key named "/foo" it is served via the - # Gopher protocol. - # - # In order to create a real Gopher "hole" (the name of a Gopher site in Gopher - # talking), you likely need a script like the following: - # - # https://github.com/antirez/gopher2redis - # - # --- SECURITY WARNING --- - # - # If you plan to put Redis on the internet in a publicly accessible address - # to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. - # Once a password is set: - # - # 1. The Gopher server (when enabled, not by default) will still serve - # content via Gopher. - # 2. However other commands cannot be called before the client will - # authenticate. - # - # So use the 'requirepass' option to protect your instance. - # - # Note that Gopher is not currently supported when 'io-threads-do-reads' - # is enabled. - # - # To enable Gopher support, uncomment the following line and set the option - # from no (the default) to yes. - # - # gopher-enabled no - - ############################### ADVANCED CONFIG ############################### - - # Hashes are encoded using a memory efficient data structure when they have a - # small number of entries, and the biggest entry does not exceed a given - # threshold. These thresholds can be configured using the following directives. - hash-max-ziplist-entries 512 - hash-max-ziplist-value 64 - - # Lists are also encoded in a special way to save a lot of space. - # The number of entries allowed per internal list node can be specified - # as a fixed maximum size or a maximum number of elements. - # For a fixed maximum size, use -5 through -1, meaning: - # -5: max size: 64 Kb <-- not recommended for normal workloads - # -4: max size: 32 Kb <-- not recommended - # -3: max size: 16 Kb <-- probably not recommended - # -2: max size: 8 Kb <-- good - # -1: max size: 4 Kb <-- good - # Positive numbers mean store up to _exactly_ that number of elements - # per list node. - # The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), - # but if your use case is unique, adjust the settings as necessary. - list-max-ziplist-size -2 - - # Lists may also be compressed. - # Compress depth is the number of quicklist ziplist nodes from *each* side of - # the list to *exclude* from compression. The head and tail of the list - # are always uncompressed for fast push/pop operations. Settings are: - # 0: disable all list compression - # 1: depth 1 means "don't start compressing until after 1 node into the list, - # going from either the head or tail" - # So: [head]->node->node->...->node->[tail] - # [head], [tail] will always be uncompressed; inner nodes will compress. - # 2: [head]->[next]->node->node->...->node->[prev]->[tail] - # 2 here means: don't compress head or head->next or tail->prev or tail, - # but compress all nodes between them. - # 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] - # etc. - list-compress-depth 0 - - # Sets have a special encoding in just one case: when a set is composed - # of just strings that happen to be integers in radix 10 in the range - # of 64 bit signed integers. - # The following configuration setting sets the limit in the size of the - # set in order to use this special memory saving encoding. - set-max-intset-entries 512 - - # Similarly to hashes and lists, sorted sets are also specially encoded in - # order to save a lot of space. This encoding is only used when the length and - # elements of a sorted set are below the following limits: - zset-max-ziplist-entries 128 - zset-max-ziplist-value 64 - - # HyperLogLog sparse representation bytes limit. The limit includes the - # 16 bytes header. When an HyperLogLog using the sparse representation crosses - # this limit, it is converted into the dense representation. - # - # A value greater than 16000 is totally useless, since at that point the - # dense representation is more memory efficient. - # - # The suggested value is ~ 3000 in order to have the benefits of - # the space efficient encoding without slowing down too much PFADD, - # which is O(N) with the sparse encoding. The value can be raised to - # ~ 10000 when CPU is not a concern, but space is, and the data set is - # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. - hll-sparse-max-bytes 3000 - - # Streams macro node max size / items. The stream data structure is a radix - # tree of big nodes that encode multiple items inside. Using this configuration - # it is possible to configure how big a single node can be in bytes, and the - # maximum number of items it may contain before switching to a new node when - # appending new stream entries. If any of the following settings are set to - # zero, the limit is ignored, so for instance it is possible to set just a - # max entries limit by setting max-bytes to 0 and max-entries to the desired - # value. - stream-node-max-bytes 4096 - stream-node-max-entries 100 - - # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in - # order to help rehashing the main Redis hash table (the one mapping top-level - # keys to values). The hash table implementation Redis uses (see dict.c) - # performs a lazy rehashing: the more operation you run into a hash table - # that is rehashing, the more rehashing "steps" are performed, so if the - # server is idle the rehashing is never complete and some more memory is used - # by the hash table. - # - # The default is to use this millisecond 10 times every second in order to - # actively rehash the main dictionaries, freeing memory when possible. - # - # If unsure: - # use "activerehashing no" if you have hard latency requirements and it is - # not a good thing in your environment that Redis can reply from time to time - # to queries with 2 milliseconds delay. - # - # use "activerehashing yes" if you don't have such hard requirements but - # want to free memory asap when possible. - activerehashing yes - - # The client output buffer limits can be used to force disconnection of clients - # that are not reading data from the server fast enough for some reason (a - # common reason is that a Pub/Sub client can't consume messages as fast as the - # publisher can produce them). - # - # The limit can be set differently for the three different classes of clients: - # - # normal -> normal clients including MONITOR clients - # replica -> replica clients - # pubsub -> clients subscribed to at least one pubsub channel or pattern - # - # The syntax of every client-output-buffer-limit directive is the following: - # - # client-output-buffer-limit - # - # A client is immediately disconnected once the hard limit is reached, or if - # the soft limit is reached and remains reached for the specified number of - # seconds (continuously). - # So for instance if the hard limit is 32 megabytes and the soft limit is - # 16 megabytes / 10 seconds, the client will get disconnected immediately - # if the size of the output buffers reach 32 megabytes, but will also get - # disconnected if the client reaches 16 megabytes and continuously overcomes - # the limit for 10 seconds. - # - # By default normal clients are not limited because they don't receive data - # without asking (in a push way), but just after a request, so only - # asynchronous clients may create a scenario where data is requested faster - # than it can read. - # - # Instead there is a default limit for pubsub and replica clients, since - # subscribers and replicas receive data in a push fashion. - # - # Both the hard or the soft limit can be disabled by setting them to zero. - client-output-buffer-limit normal 0 0 0 - client-output-buffer-limit replica 256mb 64mb 60 - client-output-buffer-limit pubsub 32mb 8mb 60 - - # Client query buffers accumulate new commands. They are limited to a fixed - # amount by default in order to avoid that a protocol desynchronization (for - # instance due to a bug in the client) will lead to unbound memory usage in - # the query buffer. However you can configure it here if you have very special - # needs, such us huge multi/exec requests or alike. - # - # client-query-buffer-limit 1gb - - # In the Redis protocol, bulk requests, that are, elements representing single - # strings, are normally limited to 512 mb. However you can change this limit - # here, but must be 1mb or greater - # - # proto-max-bulk-len 512mb - - # Redis calls an internal function to perform many background tasks, like - # closing connections of clients in timeout, purging expired keys that are - # never requested, and so forth. - # - # Not all tasks are performed with the same frequency, but Redis checks for - # tasks to perform according to the specified "hz" value. - # - # By default "hz" is set to 10. Raising the value will use more CPU when - # Redis is idle, but at the same time will make Redis more responsive when - # there are many keys expiring at the same time, and timeouts may be - # handled with more precision. - # - # The range is between 1 and 500, however a value over 100 is usually not - # a good idea. Most users should use the default of 10 and raise this up to - # 100 only in environments where very low latency is required. - hz 10 - - # Normally it is useful to have an HZ value which is proportional to the - # number of clients connected. This is useful in order, for instance, to - # avoid too many clients are processed for each background task invocation - # in order to avoid latency spikes. - # - # Since the default HZ value by default is conservatively set to 10, Redis - # offers, and enables by default, the ability to use an adaptive HZ value - # which will temporarily raise when there are many connected clients. - # - # When dynamic HZ is enabled, the actual configured HZ will be used - # as a baseline, but multiples of the configured HZ value will be actually - # used as needed once more clients are connected. In this way an idle - # instance will use very little CPU time while a busy instance will be - # more responsive. - dynamic-hz yes - - # When a child rewrites the AOF file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - aof-rewrite-incremental-fsync yes - - # When redis saves RDB file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - rdb-save-incremental-fsync yes - - # Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good - # idea to start with the default settings and only change them after investigating - # how to improve the performances and how the keys LFU change over time, which - # is possible to inspect via the OBJECT FREQ command. - # - # There are two tunable parameters in the Redis LFU implementation: the - # counter logarithm factor and the counter decay time. It is important to - # understand what the two parameters mean before changing them. - # - # The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis - # uses a probabilistic increment with logarithmic behavior. Given the value - # of the old counter, when a key is accessed, the counter is incremented in - # this way: - # - # 1. A random number R between 0 and 1 is extracted. - # 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). - # 3. The counter is incremented only if R < P. - # - # The default lfu-log-factor is 10. This is a table of how the frequency - # counter changes with a different number of accesses with different - # logarithmic factors: - # - # +--------+------------+------------+------------+------------+------------+ - # | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | - # +--------+------------+------------+------------+------------+------------+ - # | 0 | 104 | 255 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 1 | 18 | 49 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 10 | 10 | 18 | 142 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 100 | 8 | 11 | 49 | 143 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # - # NOTE: The above table was obtained by running the following commands: - # - # redis-benchmark -n 1000000 incr foo - # redis-cli object freq foo - # - # NOTE 2: The counter initial value is 5 in order to give new objects a chance - # to accumulate hits. - # - # The counter decay time is the time, in minutes, that must elapse in order - # for the key counter to be divided by two (or decremented if it has a value - # less <= 10). - # - # The default value for the lfu-decay-time is 1. A special value of 0 means to - # decay the counter every time it happens to be scanned. - # - # lfu-log-factor 10 - # lfu-decay-time 1 - - ########################### ACTIVE DEFRAGMENTATION ####################### - # - # What is active defragmentation? - # ------------------------------- - # - # Active (online) defragmentation allows a Redis server to compact the - # spaces left between small allocations and deallocations of data in memory, - # thus allowing to reclaim back memory. - # - # Fragmentation is a natural process that happens with every allocator (but - # less so with Jemalloc, fortunately) and certain workloads. Normally a server - # restart is needed in order to lower the fragmentation, or at least to flush - # away all the data and create it again. However thanks to this feature - # implemented by Oran Agra for Redis 4.0 this process can happen at runtime - # in a "hot" way, while the server is running. - # - # Basically when the fragmentation is over a certain level (see the - # configuration options below) Redis will start to create new copies of the - # values in contiguous memory regions by exploiting certain specific Jemalloc - # features (in order to understand if an allocation is causing fragmentation - # and to allocate it in a better place), and at the same time, will release the - # old copies of the data. This process, repeated incrementally for all the keys - # will cause the fragmentation to drop back to normal values. - # - # Important things to understand: - # - # 1. This feature is disabled by default, and only works if you compiled Redis - # to use the copy of Jemalloc we ship with the source code of Redis. - # This is the default with Linux builds. - # - # 2. You never need to enable this feature if you don't have fragmentation - # issues. - # - # 3. Once you experience fragmentation, you can enable this feature when - # needed with the command "CONFIG SET activedefrag yes". - # - # The configuration parameters are able to fine tune the behavior of the - # defragmentation process. If you are not sure about what they mean it is - # a good idea to leave the defaults untouched. - - # Enabled active defragmentation - # activedefrag no - - # Minimum amount of fragmentation waste to start active defrag - # active-defrag-ignore-bytes 100mb - - # Minimum percentage of fragmentation to start active defrag - # active-defrag-threshold-lower 10 - - # Maximum percentage of fragmentation at which we use maximum effort - # active-defrag-threshold-upper 100 - - # Minimal effort for defrag in CPU percentage, to be used when the lower - # threshold is reached - # active-defrag-cycle-min 1 - - # Maximal effort for defrag in CPU percentage, to be used when the upper - # threshold is reached - # active-defrag-cycle-max 25 - - # Maximum number of set/hash/zset/list fields that will be processed from - # the main dictionary scan - # active-defrag-max-scan-fields 1000 - - # Jemalloc background thread for purging will be enabled by default - jemalloc-bg-thread yes - - # It is possible to pin different threads and processes of Redis to specific - # CPUs in your system, in order to maximize the performances of the server. - # This is useful both in order to pin different Redis threads in different - # CPUs, but also in order to make sure that multiple Redis instances running - # in the same host will be pinned to different CPUs. - # - # Normally you can do this using the "taskset" command, however it is also - # possible to this via Redis configuration directly, both in Linux and FreeBSD. - # - # You can pin the server/IO threads, bio threads, aof rewrite child process, and - # the bgsave child process. The syntax to specify the cpu list is the same as - # the taskset command: - # - # Set redis server/io threads to cpu affinity 0,2,4,6: - # server_cpulist 0-7:2 - # - # Set bio threads to cpu affinity 1,3: - # bio_cpulist 1,3 - # - # Set aof rewrite child process to cpu affinity 8,9,10,11: - # aof_rewrite_cpulist 8-11 - # - # Set bgsave child process to cpu affinity 1,10,11 - # bgsave_cpulist 1,10-11 - - # In some cases redis will emit warnings and even refuse to start if it detects - # that the system is in bad state, it is possible to suppress these warnings - # by setting the following config which takes a space delimited list of warnings - # to suppress - # - # ignore-warnings ARM64-COW-BUG -metadata: - name: redis-config diff --git a/packages/grid/k8s/manifests/seaweedfs.yaml b/packages/grid/k8s/manifests/seaweedfs.yaml deleted file mode 100644 index 994da439b83..00000000000 --- a/packages/grid/k8s/manifests/seaweedfs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - filer.toml: | - [leveldb2] - enabled = true - dir = "./filerldb2" - - start.sh: | - #! /usr/bin/env bash - - sleep 30 && - echo "s3.configure -access_key ${S3_ROOT_USER} -secret_key ${S3_ROOT_PWD} -user iam -actions Read,Write,List,Tagging,Admin -apply" \ - | weed shell > /dev/null 2>&1 \ - & weed server -s3 -s3.port=${S3_PORT} -master.volumeSizeLimitMB=${S3_VOLUME_SIZE_MB} -metadata: - name: seaweedfs-config diff --git a/packages/grid/k8s/manifests/traefik-domain.yaml b/packages/grid/k8s/manifests/traefik-domain.yaml deleted file mode 100644 index a71c42df545..00000000000 --- a/packages/grid/k8s/manifests/traefik-domain.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - dynamic.yml: | - http: - services: - frontend: - loadBalancer: - servers: - - url: "http://frontend" - backend: - loadBalancer: - servers: - - url: "http://backend" - seaweedfs: - loadBalancer: - servers: - - url: "http://seaweedfs:8333" - routers: - frontend: - rule: "PathPrefix(`/`)" - entryPoints: - - "web" - service: "frontend" - backend: - rule: "PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend" - blob-storage: - rule: "PathPrefix(`/blob`)" - entryPoints: - - "web" - service: "seaweedfs" - middlewares: - - "blob-storage-url" - - "blob-storage-host" - middlewares: - blob-storage-host: - headers: - customrequestheaders: - Host: seaweedfs:8333 - blob-storage-url: - stripprefix: - prefixes: /blob - forceslash: true - - traefik.yml: | - global: - checkNewVersion: false - sendAnonymousUsage: false - - log: - level: INFO - - entryPoints: - web: - address: :80 - ping: - address: :8082 - - ping: - entryPoint: "ping" - - providers: - file: - filename: /etc/traefik/dynamic.yml -metadata: - name: traefik-main-config diff --git a/packages/grid/k8s/manifests/traefik-network.yaml b/packages/grid/k8s/manifests/traefik-network.yaml deleted file mode 100644 index 71a6538650f..00000000000 --- a/packages/grid/k8s/manifests/traefik-network.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - dynamic.yml: | - http: - services: - frontend: - loadBalancer: - servers: - - url: "http://frontend" - backend: - loadBalancer: - servers: - - url: "http://backend" - backend-stream: - loadBalancer: - servers: - - url: "http://backend-stream" - routers: - frontend: - rule: "PathPrefix(`/`)" - entryPoints: - - "web" - service: "frontend" - backend: - rule: "PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend" - backend-stream: - rule: "PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend-stream" - - traefik.yml: | - global: - checkNewVersion: false - sendAnonymousUsage: false - - log: - level: INFO - - entryPoints: - web: - address: :80 - ping: - address: :8082 - - ping: - entryPoint: "ping" - - providers: - file: - filename: /etc/traefik/dynamic.yml -metadata: - name: traefik-main-config diff --git a/packages/grid/k8s/rendered/domain.yaml b/packages/grid/k8s/rendered/domain.yaml deleted file mode 100644 index 6c20147bdbd..00000000000 --- a/packages/grid/k8s/rendered/domain.yaml +++ /dev/null @@ -1,3472 +0,0 @@ -apiVersion: v1 -data: - dynamic.yml: | - http: - services: - frontend: - loadBalancer: - servers: - - url: "http://frontend" - backend: - loadBalancer: - servers: - - url: "http://backend" - backend-stream: - loadBalancer: - servers: - - url: "http://backend-stream" - seaweedfs: - loadBalancer: - servers: - - url: "http://seaweedfs:8333" - routers: - frontend: - rule: "PathPrefix(`/`)" - entryPoints: - - "web" - service: "frontend" - backend: - rule: "PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend" - backend-stream: - rule: "PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend-stream" - blob-storage: - rule: "PathPrefix(`/blob`)" - entryPoints: - - "web" - service: "seaweedfs" - middlewares: - - "blob-storage-url" - - "blob-storage-host" - middlewares: - blob-storage-host: - headers: - customrequestheaders: - Host: seaweedfs:8333 - blob-storage-url: - stripprefix: - prefixes: /blob - forceslash: true - traefik.yml: | - global: - checkNewVersion: false - sendAnonymousUsage: false - - log: - level: INFO - - entryPoints: - web: - address: :80 - ping: - address: :8082 - - ping: - entryPoint: "ping" - - providers: - file: - filename: /etc/traefik/dynamic.yml -kind: ConfigMap -metadata: - name: traefik-main-config - namespace: default - ---- -apiVersion: v1 -data: - redis.conf: | - # Redis configuration file example. - # - # Note that in order to read the configuration file, Redis must be - # started with the file path as first argument: - # - # ./redis-server /path/to/redis.conf - - # Note on units: when memory size is needed, it is possible to specify - # it in the usual form of 1k 5GB 4M and so forth: - # - # 1k => 1000 bytes - # 1kb => 1024 bytes - # 1m => 1000000 bytes - # 1mb => 1024*1024 bytes - # 1g => 1000000000 bytes - # 1gb => 1024*1024*1024 bytes - # - # units are case insensitive so 1GB 1Gb 1gB are all the same. - - ################################## INCLUDES ################################### - - # Include one or more other config files here. This is useful if you - # have a standard template that goes to all Redis servers but also need - # to customize a few per-server settings. Include files can include - # other files, so use this wisely. - # - # Note that option "include" won't be rewritten by command "CONFIG REWRITE" - # from admin or Redis Sentinel. Since Redis always uses the last processed - # line as value of a configuration directive, you'd better put includes - # at the beginning of this file to avoid overwriting config change at runtime. - # - # If instead you are interested in using includes to override configuration - # options, it is better to use include as the last line. - # - # include /path/to/local.conf - # include /path/to/other.conf - - ################################## MODULES ##################################### - - # Load modules at startup. If the server is not able to load modules - # it will abort. It is possible to use multiple loadmodule directives. - # - # loadmodule /path/to/my_module.so - # loadmodule /path/to/other_module.so - - ################################## NETWORK ##################################### - - # By default, if no "bind" configuration directive is specified, Redis listens - # for connections from all available network interfaces on the host machine. - # It is possible to listen to just one or multiple selected interfaces using - # the "bind" configuration directive, followed by one or more IP addresses. - # Each address can be prefixed by "-", which means that redis will not fail to - # start if the address is not available. Being not available only refers to - # addresses that does not correspond to any network interfece. Addresses that - # are already in use will always fail, and unsupported protocols will always BE - # silently skipped. - # - # Examples: - # - # bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses - # bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 - # bind * -::* # like the default, all available interfaces - # - # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the - # internet, binding to all the interfaces is dangerous and will expose the - # instance to everybody on the internet. So by default we uncomment the - # following bind directive, that will force Redis to listen only on the - # IPv4 and IPv6 (if available) loopback interface addresses (this means Redis - # will only be able to accept client connections from the same host that it is - # running on). - # - # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES - # JUST COMMENT OUT THE FOLLOWING LINE. - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - bind 127.0.0.1 -::1 - - # Protected mode is a layer of security protection, in order to avoid that - # Redis instances left open on the internet are accessed and exploited. - # - # When protected mode is on and if: - # - # 1) The server is not binding explicitly to a set of addresses using the - # "bind" directive. - # 2) No password is configured. - # - # The server only accepts connections from clients connecting from the - # IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain - # sockets. - # - # By default protected mode is enabled. You should disable it only if - # you are sure you want clients from other hosts to connect to Redis - # even if no authentication is configured, nor a specific set of interfaces - # are explicitly listed using the "bind" directive. - protected-mode yes - - # Accept connections on the specified port, default is 6379 (IANA #815344). - # If port 0 is specified Redis will not listen on a TCP socket. - port 6379 - - # TCP listen() backlog. - # - # In high requests-per-second environments you need a high backlog in order - # to avoid slow clients connection issues. Note that the Linux kernel - # will silently truncate it to the value of /proc/sys/net/core/somaxconn so - # make sure to raise both the value of somaxconn and tcp_max_syn_backlog - # in order to get the desired effect. - tcp-backlog 511 - - # Unix socket. - # - # Specify the path for the Unix socket that will be used to listen for - # incoming connections. There is no default, so Redis will not listen - # on a unix socket when not specified. - # - # unixsocket /run/redis.sock - # unixsocketperm 700 - - # Close the connection after a client is idle for N seconds (0 to disable) - timeout 0 - - # TCP keepalive. - # - # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence - # of communication. This is useful for two reasons: - # - # 1) Detect dead peers. - # 2) Force network equipment in the middle to consider the connection to be - # alive. - # - # On Linux, the specified value (in seconds) is the period used to send ACKs. - # Note that to close the connection the double of the time is needed. - # On other kernels the period depends on the kernel configuration. - # - # A reasonable value for this option is 300 seconds, which is the new - # Redis default starting with Redis 3.2.1. - tcp-keepalive 300 - - ################################# TLS/SSL ##################################### - - # By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration - # directive can be used to define TLS-listening ports. To enable TLS on the - # default port, use: - # - # port 0 - # tls-port 6379 - - # Configure a X.509 certificate and private key to use for authenticating the - # server to connected clients, masters or cluster peers. These files should be - # PEM formatted. - # - # tls-cert-file redis.crt - # tls-key-file redis.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-key-file-pass secret - - # Normally Redis uses the same certificate for both server functions (accepting - # connections) and client functions (replicating from a master, establishing - # cluster bus connections, etc.). - # - # Sometimes certificates are issued with attributes that designate them as - # client-only or server-only certificates. In that case it may be desired to use - # different certificates for incoming (server) and outgoing (client) - # connections. To do that, use the following directives: - # - # tls-client-cert-file client.crt - # tls-client-key-file client.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-client-key-file-pass secret - - # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: - # - # tls-dh-params-file redis.dh - - # Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL - # clients and peers. Redis requires an explicit configuration of at least one - # of these, and will not implicitly use the system wide configuration. - # - # tls-ca-cert-file ca.crt - # tls-ca-cert-dir /etc/ssl/certs - - # By default, clients (including replica servers) on a TLS port are required - # to authenticate using valid client side certificates. - # - # If "no" is specified, client certificates are not required and not accepted. - # If "optional" is specified, client certificates are accepted and must be - # valid if provided, but are not required. - # - # tls-auth-clients no - # tls-auth-clients optional - - # By default, a Redis replica does not attempt to establish a TLS connection - # with its master. - # - # Use the following directive to enable TLS on replication links. - # - # tls-replication yes - - # By default, the Redis Cluster bus uses a plain TCP connection. To enable - # TLS for the bus protocol, use the following directive: - # - # tls-cluster yes - - # By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended - # that older formally deprecated versions are kept disabled to reduce the attack surface. - # You can explicitly specify TLS versions to support. - # Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", - # "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. - # To enable only TLSv1.2 and TLSv1.3, use: - # - # tls-protocols "TLSv1.2 TLSv1.3" - - # Configure allowed ciphers. See the ciphers(1ssl) manpage for more information - # about the syntax of this string. - # - # Note: this configuration applies only to <= TLSv1.2. - # - # tls-ciphers DEFAULT:!MEDIUM - - # Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more - # information about the syntax of this string, and specifically for TLSv1.3 - # ciphersuites. - # - # tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 - - # When choosing a cipher, use the server's preference instead of the client - # preference. By default, the server follows the client's preference. - # - # tls-prefer-server-ciphers yes - - # By default, TLS session caching is enabled to allow faster and less expensive - # reconnections by clients that support it. Use the following directive to disable - # caching. - # - # tls-session-caching no - - # Change the default number of TLS sessions cached. A zero value sets the cache - # to unlimited size. The default size is 20480. - # - # tls-session-cache-size 5000 - - # Change the default timeout of cached TLS sessions. The default timeout is 300 - # seconds. - # - # tls-session-cache-timeout 60 - - ################################# GENERAL ##################################### - - # By default Redis does not run as a daemon. Use 'yes' if you need it. - # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. - # When Redis is supervised by upstart or systemd, this parameter has no impact. - daemonize no - - # If you run Redis from upstart or systemd, Redis can interact with your - # supervision tree. Options: - # supervised no - no supervision interaction - # supervised upstart - signal upstart by putting Redis into SIGSTOP mode - # requires "expect stop" in your upstart job config - # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET - # on startup, and updating Redis status on a regular - # basis. - # supervised auto - detect upstart or systemd method based on - # UPSTART_JOB or NOTIFY_SOCKET environment variables - # Note: these supervision methods only signal "process is ready." - # They do not enable continuous pings back to your supervisor. - # - # The default is "no". To run under upstart/systemd, you can simply uncomment - # the line below: - # - # supervised auto - - # If a pid file is specified, Redis writes it where specified at startup - # and removes it at exit. - # - # When the server runs non daemonized, no pid file is created if none is - # specified in the configuration. When the server is daemonized, the pid file - # is used even if not specified, defaulting to "/var/run/redis.pid". - # - # Creating a pid file is best effort: if Redis is not able to create it - # nothing bad happens, the server will start and run normally. - # - # Note that on modern Linux systems "/run/redis.pid" is more conforming - # and should be used instead. - pidfile /var/run/redis_6379.pid - - # Specify the server verbosity level. - # This can be one of: - # debug (a lot of information, useful for development/testing) - # verbose (many rarely useful info, but not a mess like the debug level) - # notice (moderately verbose, what you want in production probably) - # warning (only very important / critical messages are logged) - loglevel notice - - # Specify the log file name. Also the empty string can be used to force - # Redis to log on the standard output. Note that if you use standard - # output for logging but daemonize, logs will be sent to /dev/null - logfile "" - - # To enable logging to the system logger, just set 'syslog-enabled' to yes, - # and optionally update the other syslog parameters to suit your needs. - # syslog-enabled no - - # Specify the syslog identity. - # syslog-ident redis - - # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. - # syslog-facility local0 - - # To disable the built in crash log, which will possibly produce cleaner core - # dumps when they are needed, uncomment the following: - # - # crash-log-enabled no - - # To disable the fast memory check that's run as part of the crash log, which - # will possibly let redis terminate sooner, uncomment the following: - # - # crash-memcheck-enabled no - - # Set the number of databases. The default database is DB 0, you can select - # a different one on a per-connection basis using SELECT where - # dbid is a number between 0 and 'databases'-1 - databases 16 - - # By default Redis shows an ASCII art logo only when started to log to the - # standard output and if the standard output is a TTY and syslog logging is - # disabled. Basically this means that normally a logo is displayed only in - # interactive sessions. - # - # However it is possible to force the pre-4.0 behavior and always show a - # ASCII art logo in startup logs by setting the following option to yes. - always-show-logo no - - # By default, Redis modifies the process title (as seen in 'top' and 'ps') to - # provide some runtime information. It is possible to disable this and leave - # the process name as executed by setting the following to no. - set-proc-title yes - - # When changing the process title, Redis uses the following template to construct - # the modified title. - # - # Template variables are specified in curly brackets. The following variables are - # supported: - # - # {title} Name of process as executed if parent, or type of child process. - # {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or - # Unix socket if only that's available. - # {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". - # {port} TCP port listening on, or 0. - # {tls-port} TLS port listening on, or 0. - # {unixsocket} Unix domain socket listening on, or "". - # {config-file} Name of configuration file used. - # - proc-title-template "{title} {listen-addr} {server-mode}" - - ################################ SNAPSHOTTING ################################ - - # Save the DB to disk. - # - # save - # - # Redis will save the DB if both the given number of seconds and the given - # number of write operations against the DB occurred. - # - # Snapshotting can be completely disabled with a single empty string argument - # as in following example: - # - # save "" - # - # Unless specified otherwise, by default Redis will save the DB: - # * After 3600 seconds (an hour) if at least 1 key changed - # * After 300 seconds (5 minutes) if at least 100 keys changed - # * After 60 seconds if at least 10000 keys changed - # - # You can set these explicitly by uncommenting the three following lines. - # - # save 3600 1 - # save 300 100 - # save 60 10000 - - # By default Redis will stop accepting writes if RDB snapshots are enabled - # (at least one save point) and the latest background save failed. - # This will make the user aware (in a hard way) that data is not persisting - # on disk properly, otherwise chances are that no one will notice and some - # disaster will happen. - # - # If the background saving process will start working again Redis will - # automatically allow writes again. - # - # However if you have setup your proper monitoring of the Redis server - # and persistence, you may want to disable this feature so that Redis will - # continue to work as usual even if there are problems with disk, - # permissions, and so forth. - stop-writes-on-bgsave-error yes - - # Compress string objects using LZF when dump .rdb databases? - # By default compression is enabled as it's almost always a win. - # If you want to save some CPU in the saving child set it to 'no' but - # the dataset will likely be bigger if you have compressible values or keys. - rdbcompression yes - - # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. - # This makes the format more resistant to corruption but there is a performance - # hit to pay (around 10%) when saving and loading RDB files, so you can disable it - # for maximum performances. - # - # RDB files created with checksum disabled have a checksum of zero that will - # tell the loading code to skip the check. - rdbchecksum yes - - # Enables or disables full sanitation checks for ziplist and listpack etc when - # loading an RDB or RESTORE payload. This reduces the chances of a assertion or - # crash later on while processing commands. - # Options: - # no - Never perform full sanitation - # yes - Always perform full sanitation - # clients - Perform full sanitation only for user connections. - # Excludes: RDB files, RESTORE commands received from the master - # connection, and client connections which have the - # skip-sanitize-payload ACL flag. - # The default should be 'clients' but since it currently affects cluster - # resharding via MIGRATE, it is temporarily set to 'no' by default. - # - # sanitize-dump-payload no - - # The filename where to dump the DB - dbfilename dump.rdb - - # Remove RDB files used by replication in instances without persistence - # enabled. By default this option is disabled, however there are environments - # where for regulations or other security concerns, RDB files persisted on - # disk by masters in order to feed replicas, or stored on disk by replicas - # in order to load them for the initial synchronization, should be deleted - # ASAP. Note that this option ONLY WORKS in instances that have both AOF - # and RDB persistence disabled, otherwise is completely ignored. - # - # An alternative (and sometimes better) way to obtain the same effect is - # to use diskless replication on both master and replicas instances. However - # in the case of replicas, diskless is not always an option. - rdb-del-sync-files no - - # The working directory. - # - # The DB will be written inside this directory, with the filename specified - # above using the 'dbfilename' configuration directive. - # - # The Append Only File will also be created inside this directory. - # - # Note that you must specify a directory here, not a file name. - dir ./ - - ################################# REPLICATION ################################# - - # Master-Replica replication. Use replicaof to make a Redis instance a copy of - # another Redis server. A few things to understand ASAP about Redis replication. - # - # +------------------+ +---------------+ - # | Master | ---> | Replica | - # | (receive writes) | | (exact copy) | - # +------------------+ +---------------+ - # - # 1) Redis replication is asynchronous, but you can configure a master to - # stop accepting writes if it appears to be not connected with at least - # a given number of replicas. - # 2) Redis replicas are able to perform a partial resynchronization with the - # master if the replication link is lost for a relatively small amount of - # time. You may want to configure the replication backlog size (see the next - # sections of this file) with a sensible value depending on your needs. - # 3) Replication is automatic and does not need user intervention. After a - # network partition replicas automatically try to reconnect to masters - # and resynchronize with them. - # - # replicaof - - # If the master is password protected (using the "requirepass" configuration - # directive below) it is possible to tell the replica to authenticate before - # starting the replication synchronization process, otherwise the master will - # refuse the replica request. - # - # masterauth - # - # However this is not enough if you are using Redis ACLs (for Redis version - # 6 or greater), and the default user is not capable of running the PSYNC - # command and/or other commands needed for replication. In this case it's - # better to configure a special user to use with replication, and specify the - # masteruser configuration as such: - # - # masteruser - # - # When masteruser is specified, the replica will authenticate against its - # master using the new AUTH form: AUTH . - - # When a replica loses its connection with the master, or when the replication - # is still in progress, the replica can act in two different ways: - # - # 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will - # still reply to client requests, possibly with out of date data, or the - # data set may just be empty if this is the first synchronization. - # - # 2) If replica-serve-stale-data is set to 'no' the replica will reply with - # an error "SYNC with master in progress" to all commands except: - # INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, - # UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, - # HOST and LATENCY. - # - replica-serve-stale-data yes - - # You can configure a replica instance to accept writes or not. Writing against - # a replica instance may be useful to store some ephemeral data (because data - # written on a replica will be easily deleted after resync with the master) but - # may also cause problems if clients are writing to it because of a - # misconfiguration. - # - # Since Redis 2.6 by default replicas are read-only. - # - # Note: read only replicas are not designed to be exposed to untrusted clients - # on the internet. It's just a protection layer against misuse of the instance. - # Still a read only replica exports by default all the administrative commands - # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve - # security of read only replicas using 'rename-command' to shadow all the - # administrative / dangerous commands. - replica-read-only yes - - # Replication SYNC strategy: disk or socket. - # - # New replicas and reconnecting replicas that are not able to continue the - # replication process just receiving differences, need to do what is called a - # "full synchronization". An RDB file is transmitted from the master to the - # replicas. - # - # The transmission can happen in two different ways: - # - # 1) Disk-backed: The Redis master creates a new process that writes the RDB - # file on disk. Later the file is transferred by the parent - # process to the replicas incrementally. - # 2) Diskless: The Redis master creates a new process that directly writes the - # RDB file to replica sockets, without touching the disk at all. - # - # With disk-backed replication, while the RDB file is generated, more replicas - # can be queued and served with the RDB file as soon as the current child - # producing the RDB file finishes its work. With diskless replication instead - # once the transfer starts, new replicas arriving will be queued and a new - # transfer will start when the current one terminates. - # - # When diskless replication is used, the master waits a configurable amount of - # time (in seconds) before starting the transfer in the hope that multiple - # replicas will arrive and the transfer can be parallelized. - # - # With slow disks and fast (large bandwidth) networks, diskless replication - # works better. - repl-diskless-sync no - - # When diskless replication is enabled, it is possible to configure the delay - # the server waits in order to spawn the child that transfers the RDB via socket - # to the replicas. - # - # This is important since once the transfer starts, it is not possible to serve - # new replicas arriving, that will be queued for the next RDB transfer, so the - # server waits a delay in order to let more replicas arrive. - # - # The delay is specified in seconds, and by default is 5 seconds. To disable - # it entirely just set it to 0 seconds and the transfer will start ASAP. - repl-diskless-sync-delay 5 - - # ----------------------------------------------------------------------------- - # WARNING: RDB diskless load is experimental. Since in this setup the replica - # does not immediately store an RDB on disk, it may cause data loss during - # failovers. RDB diskless load + Redis modules not handling I/O reads may also - # cause Redis to abort in case of I/O errors during the initial synchronization - # stage with the master. Use only if you know what you are doing. - # ----------------------------------------------------------------------------- - # - # Replica can load the RDB it reads from the replication link directly from the - # socket, or store the RDB to a file and read that file after it was completely - # received from the master. - # - # In many cases the disk is slower than the network, and storing and loading - # the RDB file may increase replication time (and even increase the master's - # Copy on Write memory and salve buffers). - # However, parsing the RDB file directly from the socket may mean that we have - # to flush the contents of the current database before the full rdb was - # received. For this reason we have the following options: - # - # "disabled" - Don't use diskless load (store the rdb file to the disk first) - # "on-empty-db" - Use diskless load only when it is completely safe. - # "swapdb" - Keep a copy of the current db contents in RAM while parsing - # the data directly from the socket. note that this requires - # sufficient memory, if you don't have it, you risk an OOM kill. - repl-diskless-load disabled - - # Replicas send PINGs to server in a predefined interval. It's possible to - # change this interval with the repl_ping_replica_period option. The default - # value is 10 seconds. - # - # repl-ping-replica-period 10 - - # The following option sets the replication timeout for: - # - # 1) Bulk transfer I/O during SYNC, from the point of view of replica. - # 2) Master timeout from the point of view of replicas (data, pings). - # 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). - # - # It is important to make sure that this value is greater than the value - # specified for repl-ping-replica-period otherwise a timeout will be detected - # every time there is low traffic between the master and the replica. The default - # value is 60 seconds. - # - # repl-timeout 60 - - # Disable TCP_NODELAY on the replica socket after SYNC? - # - # If you select "yes" Redis will use a smaller number of TCP packets and - # less bandwidth to send data to replicas. But this can add a delay for - # the data to appear on the replica side, up to 40 milliseconds with - # Linux kernels using a default configuration. - # - # If you select "no" the delay for data to appear on the replica side will - # be reduced but more bandwidth will be used for replication. - # - # By default we optimize for low latency, but in very high traffic conditions - # or when the master and replicas are many hops away, turning this to "yes" may - # be a good idea. - repl-disable-tcp-nodelay no - - # Set the replication backlog size. The backlog is a buffer that accumulates - # replica data when replicas are disconnected for some time, so that when a - # replica wants to reconnect again, often a full resync is not needed, but a - # partial resync is enough, just passing the portion of data the replica - # missed while disconnected. - # - # The bigger the replication backlog, the longer the replica can endure the - # disconnect and later be able to perform a partial resynchronization. - # - # The backlog is only allocated if there is at least one replica connected. - # - # repl-backlog-size 1mb - - # After a master has no connected replicas for some time, the backlog will be - # freed. The following option configures the amount of seconds that need to - # elapse, starting from the time the last replica disconnected, for the backlog - # buffer to be freed. - # - # Note that replicas never free the backlog for timeout, since they may be - # promoted to masters later, and should be able to correctly "partially - # resynchronize" with other replicas: hence they should always accumulate backlog. - # - # A value of 0 means to never release the backlog. - # - # repl-backlog-ttl 3600 - - # The replica priority is an integer number published by Redis in the INFO - # output. It is used by Redis Sentinel in order to select a replica to promote - # into a master if the master is no longer working correctly. - # - # A replica with a low priority number is considered better for promotion, so - # for instance if there are three replicas with priority 10, 100, 25 Sentinel - # will pick the one with priority 10, that is the lowest. - # - # However a special priority of 0 marks the replica as not able to perform the - # role of master, so a replica with priority of 0 will never be selected by - # Redis Sentinel for promotion. - # - # By default the priority is 100. - replica-priority 100 - - # ----------------------------------------------------------------------------- - # By default, Redis Sentinel includes all replicas in its reports. A replica - # can be excluded from Redis Sentinel's announcements. An unannounced replica - # will be ignored by the 'sentinel replicas ' command and won't be - # exposed to Redis Sentinel's clients. - # - # This option does not change the behavior of replica-priority. Even with - # replica-announced set to 'no', the replica can be promoted to master. To - # prevent this behavior, set replica-priority to 0. - # - # replica-announced yes - - # It is possible for a master to stop accepting writes if there are less than - # N replicas connected, having a lag less or equal than M seconds. - # - # The N replicas need to be in "online" state. - # - # The lag in seconds, that must be <= the specified value, is calculated from - # the last ping received from the replica, that is usually sent every second. - # - # This option does not GUARANTEE that N replicas will accept the write, but - # will limit the window of exposure for lost writes in case not enough replicas - # are available, to the specified number of seconds. - # - # For example to require at least 3 replicas with a lag <= 10 seconds use: - # - # min-replicas-to-write 3 - # min-replicas-max-lag 10 - # - # Setting one or the other to 0 disables the feature. - # - # By default min-replicas-to-write is set to 0 (feature disabled) and - # min-replicas-max-lag is set to 10. - - # A Redis master is able to list the address and port of the attached - # replicas in different ways. For example the "INFO replication" section - # offers this information, which is used, among other tools, by - # Redis Sentinel in order to discover replica instances. - # Another place where this info is available is in the output of the - # "ROLE" command of a master. - # - # The listed IP address and port normally reported by a replica is - # obtained in the following way: - # - # IP: The address is auto detected by checking the peer address - # of the socket used by the replica to connect with the master. - # - # Port: The port is communicated by the replica during the replication - # handshake, and is normally the port that the replica is using to - # listen for connections. - # - # However when port forwarding or Network Address Translation (NAT) is - # used, the replica may actually be reachable via different IP and port - # pairs. The following two options can be used by a replica in order to - # report to its master a specific set of IP and port, so that both INFO - # and ROLE will report those values. - # - # There is no need to use both the options if you need to override just - # the port or the IP address. - # - # replica-announce-ip 5.5.5.5 - # replica-announce-port 1234 - - ############################### KEYS TRACKING ################################# - - # Redis implements server assisted support for client side caching of values. - # This is implemented using an invalidation table that remembers, using - # a radix key indexed by key name, what clients have which keys. In turn - # this is used in order to send invalidation messages to clients. Please - # check this page to understand more about the feature: - # - # https://redis.io/topics/client-side-caching - # - # When tracking is enabled for a client, all the read only queries are assumed - # to be cached: this will force Redis to store information in the invalidation - # table. When keys are modified, such information is flushed away, and - # invalidation messages are sent to the clients. However if the workload is - # heavily dominated by reads, Redis could use more and more memory in order - # to track the keys fetched by many clients. - # - # For this reason it is possible to configure a maximum fill value for the - # invalidation table. By default it is set to 1M of keys, and once this limit - # is reached, Redis will start to evict keys in the invalidation table - # even if they were not modified, just to reclaim memory: this will in turn - # force the clients to invalidate the cached values. Basically the table - # maximum size is a trade off between the memory you want to spend server - # side to track information about who cached what, and the ability of clients - # to retain cached objects in memory. - # - # If you set the value to 0, it means there are no limits, and Redis will - # retain as many keys as needed in the invalidation table. - # In the "stats" INFO section, you can find information about the number of - # keys in the invalidation table at every given moment. - # - # Note: when key tracking is used in broadcasting mode, no memory is used - # in the server side so this setting is useless. - # - # tracking-table-max-keys 1000000 - - ################################## SECURITY ################################### - - # Warning: since Redis is pretty fast, an outside user can try up to - # 1 million passwords per second against a modern box. This means that you - # should use very strong passwords, otherwise they will be very easy to break. - # Note that because the password is really a shared secret between the client - # and the server, and should not be memorized by any human, the password - # can be easily a long string from /dev/urandom or whatever, so by using a - # long and unguessable password no brute force attack will be possible. - - # Redis ACL users are defined in the following format: - # - # user ... acl rules ... - # - # For example: - # - # user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 - # - # The special username "default" is used for new connections. If this user - # has the "nopass" rule, then new connections will be immediately authenticated - # as the "default" user without the need of any password provided via the - # AUTH command. Otherwise if the "default" user is not flagged with "nopass" - # the connections will start in not authenticated state, and will require - # AUTH (or the HELLO command AUTH option) in order to be authenticated and - # start to work. - # - # The ACL rules that describe what a user can do are the following: - # - # on Enable the user: it is possible to authenticate as this user. - # off Disable the user: it's no longer possible to authenticate - # with this user, however the already authenticated connections - # will still work. - # skip-sanitize-payload RESTORE dump-payload sanitation is skipped. - # sanitize-payload RESTORE dump-payload is sanitized (default). - # + Allow the execution of that command - # - Disallow the execution of that command - # +@ Allow the execution of all the commands in such category - # with valid categories are like @admin, @set, @sortedset, ... - # and so forth, see the full list in the server.c file where - # the Redis command table is described and defined. - # The special category @all means all the commands, but currently - # present in the server, and that will be loaded in the future - # via modules. - # +|subcommand Allow a specific subcommand of an otherwise - # disabled command. Note that this form is not - # allowed as negative like -DEBUG|SEGFAULT, but - # only additive starting with "+". - # allcommands Alias for +@all. Note that it implies the ability to execute - # all the future commands loaded via the modules system. - # nocommands Alias for -@all. - # ~ Add a pattern of keys that can be mentioned as part of - # commands. For instance ~* allows all the keys. The pattern - # is a glob-style pattern like the one of KEYS. - # It is possible to specify multiple patterns. - # allkeys Alias for ~* - # resetkeys Flush the list of allowed keys patterns. - # & Add a glob-style pattern of Pub/Sub channels that can be - # accessed by the user. It is possible to specify multiple channel - # patterns. - # allchannels Alias for &* - # resetchannels Flush the list of allowed channel patterns. - # > Add this password to the list of valid password for the user. - # For example >mypass will add "mypass" to the list. - # This directive clears the "nopass" flag (see later). - # < Remove this password from the list of valid passwords. - # nopass All the set passwords of the user are removed, and the user - # is flagged as requiring no password: it means that every - # password will work against this user. If this directive is - # used for the default user, every new connection will be - # immediately authenticated with the default user without - # any explicit AUTH command required. Note that the "resetpass" - # directive will clear this condition. - # resetpass Flush the list of allowed passwords. Moreover removes the - # "nopass" status. After "resetpass" the user has no associated - # passwords and there is no way to authenticate without adding - # some password (or setting it as "nopass" later). - # reset Performs the following actions: resetpass, resetkeys, off, - # -@all. The user returns to the same state it has immediately - # after its creation. - # - # ACL rules can be specified in any order: for instance you can start with - # passwords, then flags, or key patterns. However note that the additive - # and subtractive rules will CHANGE MEANING depending on the ordering. - # For instance see the following example: - # - # user alice on +@all -DEBUG ~* >somepassword - # - # This will allow "alice" to use all the commands with the exception of the - # DEBUG command, since +@all added all the commands to the set of the commands - # alice can use, and later DEBUG was removed. However if we invert the order - # of two ACL rules the result will be different: - # - # user alice on -DEBUG +@all ~* >somepassword - # - # Now DEBUG was removed when alice had yet no commands in the set of allowed - # commands, later all the commands are added, so the user will be able to - # execute everything. - # - # Basically ACL rules are processed left-to-right. - # - # For more information about ACL configuration please refer to - # the Redis web site at https://redis.io/topics/acl - - # ACL LOG - # - # The ACL Log tracks failed commands and authentication events associated - # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked - # by ACLs. The ACL Log is stored in memory. You can reclaim memory with - # ACL LOG RESET. Define the maximum entry length of the ACL Log below. - acllog-max-len 128 - - # Using an external ACL file - # - # Instead of configuring users here in this file, it is possible to use - # a stand-alone file just listing users. The two methods cannot be mixed: - # if you configure users here and at the same time you activate the external - # ACL file, the server will refuse to start. - # - # The format of the external ACL user file is exactly the same as the - # format that is used inside redis.conf to describe users. - # - # aclfile /etc/redis/users.acl - - # IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility - # layer on top of the new ACL system. The option effect will be just setting - # the password for the default user. Clients will still authenticate using - # AUTH as usually, or more explicitly with AUTH default - # if they follow the new protocol: both will work. - # - # The requirepass is not compatable with aclfile option and the ACL LOAD - # command, these will cause requirepass to be ignored. - # - # requirepass foobared - - # New users are initialized with restrictive permissions by default, via the - # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it - # is possible to manage access to Pub/Sub channels with ACL rules as well. The - # default Pub/Sub channels permission if new users is controlled by the - # acl-pubsub-default configuration directive, which accepts one of these values: - # - # allchannels: grants access to all Pub/Sub channels - # resetchannels: revokes access to all Pub/Sub channels - # - # To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default - # defaults to the 'allchannels' permission. - # - # Future compatibility note: it is very likely that in a future version of Redis - # the directive's default of 'allchannels' will be changed to 'resetchannels' in - # order to provide better out-of-the-box Pub/Sub security. Therefore, it is - # recommended that you explicitly define Pub/Sub permissions for all users - # rather then rely on implicit default values. Once you've set explicit - # Pub/Sub for all existing users, you should uncomment the following line. - # - # acl-pubsub-default resetchannels - - # Command renaming (DEPRECATED). - # - # ------------------------------------------------------------------------ - # WARNING: avoid using this option if possible. Instead use ACLs to remove - # commands from the default user, and put them only in some admin user you - # create for administrative purposes. - # ------------------------------------------------------------------------ - # - # It is possible to change the name of dangerous commands in a shared - # environment. For instance the CONFIG command may be renamed into something - # hard to guess so that it will still be available for internal-use tools - # but not available for general clients. - # - # Example: - # - # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 - # - # It is also possible to completely kill a command by renaming it into - # an empty string: - # - # rename-command CONFIG "" - # - # Please note that changing the name of commands that are logged into the - # AOF file or transmitted to replicas may cause problems. - - ################################### CLIENTS #################################### - - # Set the max number of connected clients at the same time. By default - # this limit is set to 10000 clients, however if the Redis server is not - # able to configure the process file limit to allow for the specified limit - # the max number of allowed clients is set to the current file limit - # minus 32 (as Redis reserves a few file descriptors for internal uses). - # - # Once the limit is reached Redis will close all the new connections sending - # an error 'max number of clients reached'. - # - # IMPORTANT: When Redis Cluster is used, the max number of connections is also - # shared with the cluster bus: every node in the cluster will use two - # connections, one incoming and another outgoing. It is important to size the - # limit accordingly in case of very large clusters. - # - # maxclients 10000 - - ############################## MEMORY MANAGEMENT ################################ - - # Set a memory usage limit to the specified amount of bytes. - # When the memory limit is reached Redis will try to remove keys - # according to the eviction policy selected (see maxmemory-policy). - # - # If Redis can't remove keys according to the policy, or if the policy is - # set to 'noeviction', Redis will start to reply with errors to commands - # that would use more memory, like SET, LPUSH, and so on, and will continue - # to reply to read-only commands like GET. - # - # This option is usually useful when using Redis as an LRU or LFU cache, or to - # set a hard memory limit for an instance (using the 'noeviction' policy). - # - # WARNING: If you have replicas attached to an instance with maxmemory on, - # the size of the output buffers needed to feed the replicas are subtracted - # from the used memory count, so that network problems / resyncs will - # not trigger a loop where keys are evicted, and in turn the output - # buffer of replicas is full with DELs of keys evicted triggering the deletion - # of more keys, and so forth until the database is completely emptied. - # - # In short... if you have replicas attached it is suggested that you set a lower - # limit for maxmemory so that there is some free RAM on the system for replica - # output buffers (but this is not needed if the policy is 'noeviction'). - # - # maxmemory - - # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory - # is reached. You can select one from the following behaviors: - # - # volatile-lru -> Evict using approximated LRU, only keys with an expire set. - # allkeys-lru -> Evict any key using approximated LRU. - # volatile-lfu -> Evict using approximated LFU, only keys with an expire set. - # allkeys-lfu -> Evict any key using approximated LFU. - # volatile-random -> Remove a random key having an expire set. - # allkeys-random -> Remove a random key, any key. - # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) - # noeviction -> Don't evict anything, just return an error on write operations. - # - # LRU means Least Recently Used - # LFU means Least Frequently Used - # - # Both LRU, LFU and volatile-ttl are implemented using approximated - # randomized algorithms. - # - # Note: with any of the above policies, when there are no suitable keys for - # eviction, Redis will return an error on write operations that require - # more memory. These are usually commands that create new keys, add data or - # modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, - # SORT (due to the STORE argument), and EXEC (if the transaction includes any - # command that requires memory). - # - # The default is: - # - # maxmemory-policy noeviction - - # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated - # algorithms (in order to save memory), so you can tune it for speed or - # accuracy. By default Redis will check five keys and pick the one that was - # used least recently, you can change the sample size using the following - # configuration directive. - # - # The default of 5 produces good enough results. 10 Approximates very closely - # true LRU but costs more CPU. 3 is faster but not very accurate. - # - # maxmemory-samples 5 - - # Eviction processing is designed to function well with the default setting. - # If there is an unusually large amount of write traffic, this value may need to - # be increased. Decreasing this value may reduce latency at the risk of - # eviction processing effectiveness - # 0 = minimum latency, 10 = default, 100 = process without regard to latency - # - # maxmemory-eviction-tenacity 10 - - # Starting from Redis 5, by default a replica will ignore its maxmemory setting - # (unless it is promoted to master after a failover or manually). It means - # that the eviction of keys will be just handled by the master, sending the - # DEL commands to the replica as keys evict in the master side. - # - # This behavior ensures that masters and replicas stay consistent, and is usually - # what you want, however if your replica is writable, or you want the replica - # to have a different memory setting, and you are sure all the writes performed - # to the replica are idempotent, then you may change this default (but be sure - # to understand what you are doing). - # - # Note that since the replica by default does not evict, it may end using more - # memory than the one set via maxmemory (there are certain buffers that may - # be larger on the replica, or data structures may sometimes take more memory - # and so forth). So make sure you monitor your replicas and make sure they - # have enough memory to never hit a real out-of-memory condition before the - # master hits the configured maxmemory setting. - # - # replica-ignore-maxmemory yes - - # Redis reclaims expired keys in two ways: upon access when those keys are - # found to be expired, and also in background, in what is called the - # "active expire key". The key space is slowly and interactively scanned - # looking for expired keys to reclaim, so that it is possible to free memory - # of keys that are expired and will never be accessed again in a short time. - # - # The default effort of the expire cycle will try to avoid having more than - # ten percent of expired keys still in memory, and will try to avoid consuming - # more than 25% of total memory and to add latency to the system. However - # it is possible to increase the expire "effort" that is normally set to - # "1", to a greater value, up to the value "10". At its maximum value the - # system will use more CPU, longer cycles (and technically may introduce - # more latency), and will tolerate less already expired keys still present - # in the system. It's a tradeoff between memory, CPU and latency. - # - # active-expire-effort 1 - - ############################# LAZY FREEING #################################### - - # Redis has two primitives to delete keys. One is called DEL and is a blocking - # deletion of the object. It means that the server stops processing new commands - # in order to reclaim all the memory associated with an object in a synchronous - # way. If the key deleted is associated with a small object, the time needed - # in order to execute the DEL command is very small and comparable to most other - # O(1) or O(log_N) commands in Redis. However if the key is associated with an - # aggregated value containing millions of elements, the server can block for - # a long time (even seconds) in order to complete the operation. - # - # For the above reasons Redis also offers non blocking deletion primitives - # such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and - # FLUSHDB commands, in order to reclaim memory in background. Those commands - # are executed in constant time. Another thread will incrementally free the - # object in the background as fast as possible. - # - # DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. - # It's up to the design of the application to understand when it is a good - # idea to use one or the other. However the Redis server sometimes has to - # delete keys or flush the whole database as a side effect of other operations. - # Specifically Redis deletes objects independently of a user call in the - # following scenarios: - # - # 1) On eviction, because of the maxmemory and maxmemory policy configurations, - # in order to make room for new data, without going over the specified - # memory limit. - # 2) Because of expire: when a key with an associated time to live (see the - # EXPIRE command) must be deleted from memory. - # 3) Because of a side effect of a command that stores data on a key that may - # already exist. For example the RENAME command may delete the old key - # content when it is replaced with another one. Similarly SUNIONSTORE - # or SORT with STORE option may delete existing keys. The SET command - # itself removes any old content of the specified key in order to replace - # it with the specified string. - # 4) During replication, when a replica performs a full resynchronization with - # its master, the content of the whole database is removed in order to - # load the RDB file just transferred. - # - # In all the above cases the default is to delete objects in a blocking way, - # like if DEL was called. However you can configure each case specifically - # in order to instead release memory in a non-blocking way like if UNLINK - # was called, using the following configuration directives. - - lazyfree-lazy-eviction no - lazyfree-lazy-expire no - lazyfree-lazy-server-del no - replica-lazy-flush no - - # It is also possible, for the case when to replace the user code DEL calls - # with UNLINK calls is not easy, to modify the default behavior of the DEL - # command to act exactly like UNLINK, using the following configuration - # directive: - - lazyfree-lazy-user-del no - - # FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous - # deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the - # commands. When neither flag is passed, this directive will be used to determine - # if the data should be deleted asynchronously. - - lazyfree-lazy-user-flush no - - ################################ THREADED I/O ################################# - - # Redis is mostly single threaded, however there are certain threaded - # operations such as UNLINK, slow I/O accesses and other things that are - # performed on side threads. - # - # Now it is also possible to handle Redis clients socket reads and writes - # in different I/O threads. Since especially writing is so slow, normally - # Redis users use pipelining in order to speed up the Redis performances per - # core, and spawn multiple instances in order to scale more. Using I/O - # threads it is possible to easily speedup two times Redis without resorting - # to pipelining nor sharding of the instance. - # - # By default threading is disabled, we suggest enabling it only in machines - # that have at least 4 or more cores, leaving at least one spare core. - # Using more than 8 threads is unlikely to help much. We also recommend using - # threaded I/O only if you actually have performance problems, with Redis - # instances being able to use a quite big percentage of CPU time, otherwise - # there is no point in using this feature. - # - # So for instance if you have a four cores boxes, try to use 2 or 3 I/O - # threads, if you have a 8 cores, try to use 6 threads. In order to - # enable I/O threads use the following configuration directive: - # - # io-threads 4 - # - # Setting io-threads to 1 will just use the main thread as usual. - # When I/O threads are enabled, we only use threads for writes, that is - # to thread the write(2) syscall and transfer the client buffers to the - # socket. However it is also possible to enable threading of reads and - # protocol parsing using the following configuration directive, by setting - # it to yes: - # - # io-threads-do-reads no - # - # Usually threading reads doesn't help much. - # - # NOTE 1: This configuration directive cannot be changed at runtime via - # CONFIG SET. Aso this feature currently does not work when SSL is - # enabled. - # - # NOTE 2: If you want to test the Redis speedup using redis-benchmark, make - # sure you also run the benchmark itself in threaded mode, using the - # --threads option to match the number of Redis threads, otherwise you'll not - # be able to notice the improvements. - - ############################ KERNEL OOM CONTROL ############################## - - # On Linux, it is possible to hint the kernel OOM killer on what processes - # should be killed first when out of memory. - # - # Enabling this feature makes Redis actively control the oom_score_adj value - # for all its processes, depending on their role. The default scores will - # attempt to have background child processes killed before all others, and - # replicas killed before masters. - # - # Redis supports three options: - # - # no: Don't make changes to oom-score-adj (default). - # yes: Alias to "relative" see below. - # absolute: Values in oom-score-adj-values are written as is to the kernel. - # relative: Values are used relative to the initial value of oom_score_adj when - # the server starts and are then clamped to a range of -1000 to 1000. - # Because typically the initial value is 0, they will often match the - # absolute values. - oom-score-adj no - - # When oom-score-adj is used, this directive controls the specific values used - # for master, replica and background child processes. Values range -2000 to - # 2000 (higher means more likely to be killed). - # - # Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) - # can freely increase their value, but not decrease it below its initial - # settings. This means that setting oom-score-adj to "relative" and setting the - # oom-score-adj-values to positive values will always succeed. - oom-score-adj-values 0 200 800 - - - #################### KERNEL transparent hugepage CONTROL ###################### - - # Usually the kernel Transparent Huge Pages control is set to "madvise" or - # or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which - # case this config has no effect. On systems in which it is set to "always", - # redis will attempt to disable it specifically for the redis process in order - # to avoid latency problems specifically with fork(2) and CoW. - # If for some reason you prefer to keep it enabled, you can set this config to - # "no" and the kernel global to "always". - - disable-thp yes - - ############################## APPEND ONLY MODE ############################### - - # By default Redis asynchronously dumps the dataset on disk. This mode is - # good enough in many applications, but an issue with the Redis process or - # a power outage may result into a few minutes of writes lost (depending on - # the configured save points). - # - # The Append Only File is an alternative persistence mode that provides - # much better durability. For instance using the default data fsync policy - # (see later in the config file) Redis can lose just one second of writes in a - # dramatic event like a server power outage, or a single write if something - # wrong with the Redis process itself happens, but the operating system is - # still running correctly. - # - # AOF and RDB persistence can be enabled at the same time without problems. - # If the AOF is enabled on startup Redis will load the AOF, that is the file - # with the better durability guarantees. - # - # Please check https://redis.io/topics/persistence for more information. - - appendonly no - - # The name of the append only file (default: "appendonly.aof") - - appendfilename "appendonly.aof" - - # The fsync() call tells the Operating System to actually write data on disk - # instead of waiting for more data in the output buffer. Some OS will really flush - # data on disk, some other OS will just try to do it ASAP. - # - # Redis supports three different modes: - # - # no: don't fsync, just let the OS flush the data when it wants. Faster. - # always: fsync after every write to the append only log. Slow, Safest. - # everysec: fsync only one time every second. Compromise. - # - # The default is "everysec", as that's usually the right compromise between - # speed and data safety. It's up to you to understand if you can relax this to - # "no" that will let the operating system flush the output buffer when - # it wants, for better performances (but if you can live with the idea of - # some data loss consider the default persistence mode that's snapshotting), - # or on the contrary, use "always" that's very slow but a bit safer than - # everysec. - # - # More details please check the following article: - # http://antirez.com/post/redis-persistence-demystified.html - # - # If unsure, use "everysec". - - # appendfsync always - appendfsync everysec - # appendfsync no - - # When the AOF fsync policy is set to always or everysec, and a background - # saving process (a background save or AOF log background rewriting) is - # performing a lot of I/O against the disk, in some Linux configurations - # Redis may block too long on the fsync() call. Note that there is no fix for - # this currently, as even performing fsync in a different thread will block - # our synchronous write(2) call. - # - # In order to mitigate this problem it's possible to use the following option - # that will prevent fsync() from being called in the main process while a - # BGSAVE or BGREWRITEAOF is in progress. - # - # This means that while another child is saving, the durability of Redis is - # the same as "appendfsync none". In practical terms, this means that it is - # possible to lose up to 30 seconds of log in the worst scenario (with the - # default Linux settings). - # - # If you have latency problems turn this to "yes". Otherwise leave it as - # "no" that is the safest pick from the point of view of durability. - - no-appendfsync-on-rewrite no - - # Automatic rewrite of the append only file. - # Redis is able to automatically rewrite the log file implicitly calling - # BGREWRITEAOF when the AOF log size grows by the specified percentage. - # - # This is how it works: Redis remembers the size of the AOF file after the - # latest rewrite (if no rewrite has happened since the restart, the size of - # the AOF at startup is used). - # - # This base size is compared to the current size. If the current size is - # bigger than the specified percentage, the rewrite is triggered. Also - # you need to specify a minimal size for the AOF file to be rewritten, this - # is useful to avoid rewriting the AOF file even if the percentage increase - # is reached but it is still pretty small. - # - # Specify a percentage of zero in order to disable the automatic AOF - # rewrite feature. - - auto-aof-rewrite-percentage 100 - auto-aof-rewrite-min-size 64mb - - # An AOF file may be found to be truncated at the end during the Redis - # startup process, when the AOF data gets loaded back into memory. - # This may happen when the system where Redis is running - # crashes, especially when an ext4 filesystem is mounted without the - # data=ordered option (however this can't happen when Redis itself - # crashes or aborts but the operating system still works correctly). - # - # Redis can either exit with an error when this happens, or load as much - # data as possible (the default now) and start if the AOF file is found - # to be truncated at the end. The following option controls this behavior. - # - # If aof-load-truncated is set to yes, a truncated AOF file is loaded and - # the Redis server starts emitting a log to inform the user of the event. - # Otherwise if the option is set to no, the server aborts with an error - # and refuses to start. When the option is set to no, the user requires - # to fix the AOF file using the "redis-check-aof" utility before to restart - # the server. - # - # Note that if the AOF file will be found to be corrupted in the middle - # the server will still exit with an error. This option only applies when - # Redis will try to read more data from the AOF file but not enough bytes - # will be found. - aof-load-truncated yes - - # When rewriting the AOF file, Redis is able to use an RDB preamble in the - # AOF file for faster rewrites and recoveries. When this option is turned - # on the rewritten AOF file is composed of two different stanzas: - # - # [RDB file][AOF tail] - # - # When loading, Redis recognizes that the AOF file starts with the "REDIS" - # string and loads the prefixed RDB file, then continues loading the AOF - # tail. - aof-use-rdb-preamble yes - - ################################ LUA SCRIPTING ############################### - - # Max execution time of a Lua script in milliseconds. - # - # If the maximum execution time is reached Redis will log that a script is - # still in execution after the maximum allowed time and will start to - # reply to queries with an error. - # - # When a long running script exceeds the maximum execution time only the - # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be - # used to stop a script that did not yet call any write commands. The second - # is the only way to shut down the server in the case a write command was - # already issued by the script but the user doesn't want to wait for the natural - # termination of the script. - # - # Set it to 0 or a negative value for unlimited execution without warnings. - lua-time-limit 5000 - - ################################ REDIS CLUSTER ############################### - - # Normal Redis instances can't be part of a Redis Cluster; only nodes that are - # started as cluster nodes can. In order to start a Redis instance as a - # cluster node enable the cluster support uncommenting the following: - # - # cluster-enabled yes - - # Every cluster node has a cluster configuration file. This file is not - # intended to be edited by hand. It is created and updated by Redis nodes. - # Every Redis Cluster node requires a different cluster configuration file. - # Make sure that instances running in the same system do not have - # overlapping cluster configuration file names. - # - # cluster-config-file nodes-6379.conf - - # Cluster node timeout is the amount of milliseconds a node must be unreachable - # for it to be considered in failure state. - # Most other internal time limits are a multiple of the node timeout. - # - # cluster-node-timeout 15000 - - # A replica of a failing master will avoid to start a failover if its data - # looks too old. - # - # There is no simple way for a replica to actually have an exact measure of - # its "data age", so the following two checks are performed: - # - # 1) If there are multiple replicas able to failover, they exchange messages - # in order to try to give an advantage to the replica with the best - # replication offset (more data from the master processed). - # Replicas will try to get their rank by offset, and apply to the start - # of the failover a delay proportional to their rank. - # - # 2) Every single replica computes the time of the last interaction with - # its master. This can be the last ping or command received (if the master - # is still in the "connected" state), or the time that elapsed since the - # disconnection with the master (if the replication link is currently down). - # If the last interaction is too old, the replica will not try to failover - # at all. - # - # The point "2" can be tuned by user. Specifically a replica will not perform - # the failover if, since the last interaction with the master, the time - # elapsed is greater than: - # - # (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period - # - # So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor - # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the - # replica will not try to failover if it was not able to talk with the master - # for longer than 310 seconds. - # - # A large cluster-replica-validity-factor may allow replicas with too old data to failover - # a master, while a too small value may prevent the cluster from being able to - # elect a replica at all. - # - # For maximum availability, it is possible to set the cluster-replica-validity-factor - # to a value of 0, which means, that replicas will always try to failover the - # master regardless of the last time they interacted with the master. - # (However they'll always try to apply a delay proportional to their - # offset rank). - # - # Zero is the only value able to guarantee that when all the partitions heal - # the cluster will always be able to continue. - # - # cluster-replica-validity-factor 10 - - # Cluster replicas are able to migrate to orphaned masters, that are masters - # that are left without working replicas. This improves the cluster ability - # to resist to failures as otherwise an orphaned master can't be failed over - # in case of failure if it has no working replicas. - # - # Replicas migrate to orphaned masters only if there are still at least a - # given number of other working replicas for their old master. This number - # is the "migration barrier". A migration barrier of 1 means that a replica - # will migrate only if there is at least 1 other working replica for its master - # and so forth. It usually reflects the number of replicas you want for every - # master in your cluster. - # - # Default is 1 (replicas migrate only if their masters remain with at least - # one replica). To disable migration just set it to a very large value or - # set cluster-allow-replica-migration to 'no'. - # A value of 0 can be set but is useful only for debugging and dangerous - # in production. - # - # cluster-migration-barrier 1 - - # Turning off this option allows to use less automatic cluster configuration. - # It both disables migration to orphaned masters and migration from masters - # that became empty. - # - # Default is 'yes' (allow automatic migrations). - # - # cluster-allow-replica-migration yes - - # By default Redis Cluster nodes stop accepting queries if they detect there - # is at least a hash slot uncovered (no available node is serving it). - # This way if the cluster is partially down (for example a range of hash slots - # are no longer covered) all the cluster becomes, eventually, unavailable. - # It automatically returns available as soon as all the slots are covered again. - # - # However sometimes you want the subset of the cluster which is working, - # to continue to accept queries for the part of the key space that is still - # covered. In order to do so, just set the cluster-require-full-coverage - # option to no. - # - # cluster-require-full-coverage yes - - # This option, when set to yes, prevents replicas from trying to failover its - # master during master failures. However the replica can still perform a - # manual failover, if forced to do so. - # - # This is useful in different scenarios, especially in the case of multiple - # data center operations, where we want one side to never be promoted if not - # in the case of a total DC failure. - # - # cluster-replica-no-failover no - - # This option, when set to yes, allows nodes to serve read traffic while the - # the cluster is in a down state, as long as it believes it owns the slots. - # - # This is useful for two cases. The first case is for when an application - # doesn't require consistency of data during node failures or network partitions. - # One example of this is a cache, where as long as the node has the data it - # should be able to serve it. - # - # The second use case is for configurations that don't meet the recommended - # three shards but want to enable cluster mode and scale later. A - # master outage in a 1 or 2 shard configuration causes a read/write outage to the - # entire cluster without this option set, with it set there is only a write outage. - # Without a quorum of masters, slot ownership will not change automatically. - # - # cluster-allow-reads-when-down no - - # In order to setup your cluster make sure to read the documentation - # available at https://redis.io web site. - - ########################## CLUSTER DOCKER/NAT support ######################## - - # In certain deployments, Redis Cluster nodes address discovery fails, because - # addresses are NAT-ted or because ports are forwarded (the typical case is - # Docker and other containers). - # - # In order to make Redis Cluster working in such environments, a static - # configuration where each node knows its public address is needed. The - # following four options are used for this scope, and are: - # - # * cluster-announce-ip - # * cluster-announce-port - # * cluster-announce-tls-port - # * cluster-announce-bus-port - # - # Each instructs the node about its address, client ports (for connections - # without and with TLS) and cluster message bus port. The information is then - # published in the header of the bus packets so that other nodes will be able to - # correctly map the address of the node publishing the information. - # - # If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set - # to zero, then cluster-announce-port refers to the TLS port. Note also that - # cluster-announce-tls-port has no effect if cluster-tls is set to no. - # - # If the above options are not used, the normal Redis Cluster auto-detection - # will be used instead. - # - # Note that when remapped, the bus port may not be at the fixed offset of - # clients port + 10000, so you can specify any port and bus-port depending - # on how they get remapped. If the bus-port is not set, a fixed offset of - # 10000 will be used as usual. - # - # Example: - # - # cluster-announce-ip 10.1.1.5 - # cluster-announce-tls-port 6379 - # cluster-announce-port 0 - # cluster-announce-bus-port 6380 - - ################################## SLOW LOG ################################### - - # The Redis Slow Log is a system to log queries that exceeded a specified - # execution time. The execution time does not include the I/O operations - # like talking with the client, sending the reply and so forth, - # but just the time needed to actually execute the command (this is the only - # stage of command execution where the thread is blocked and can not serve - # other requests in the meantime). - # - # You can configure the slow log with two parameters: one tells Redis - # what is the execution time, in microseconds, to exceed in order for the - # command to get logged, and the other parameter is the length of the - # slow log. When a new command is logged the oldest one is removed from the - # queue of logged commands. - - # The following time is expressed in microseconds, so 1000000 is equivalent - # to one second. Note that a negative number disables the slow log, while - # a value of zero forces the logging of every command. - slowlog-log-slower-than 10000 - - # There is no limit to this length. Just be aware that it will consume memory. - # You can reclaim memory used by the slow log with SLOWLOG RESET. - slowlog-max-len 128 - - ################################ LATENCY MONITOR ############################## - - # The Redis latency monitoring subsystem samples different operations - # at runtime in order to collect data related to possible sources of - # latency of a Redis instance. - # - # Via the LATENCY command this information is available to the user that can - # print graphs and obtain reports. - # - # The system only logs operations that were performed in a time equal or - # greater than the amount of milliseconds specified via the - # latency-monitor-threshold configuration directive. When its value is set - # to zero, the latency monitor is turned off. - # - # By default latency monitoring is disabled since it is mostly not needed - # if you don't have latency issues, and collecting data has a performance - # impact, that while very small, can be measured under big load. Latency - # monitoring can easily be enabled at runtime using the command - # "CONFIG SET latency-monitor-threshold " if needed. - latency-monitor-threshold 0 - - ############################# EVENT NOTIFICATION ############################## - - # Redis can notify Pub/Sub clients about events happening in the key space. - # This feature is documented at https://redis.io/topics/notifications - # - # For instance if keyspace events notification is enabled, and a client - # performs a DEL operation on key "foo" stored in the Database 0, two - # messages will be published via Pub/Sub: - # - # PUBLISH __keyspace@0__:foo del - # PUBLISH __keyevent@0__:del foo - # - # It is possible to select the events that Redis will notify among a set - # of classes. Every class is identified by a single character: - # - # K Keyspace events, published with __keyspace@__ prefix. - # E Keyevent events, published with __keyevent@__ prefix. - # g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... - # $ String commands - # l List commands - # s Set commands - # h Hash commands - # z Sorted set commands - # x Expired events (events generated every time a key expires) - # e Evicted events (events generated when a key is evicted for maxmemory) - # t Stream commands - # d Module key type events - # m Key-miss events (Note: It is not included in the 'A' class) - # A Alias for g$lshzxetd, so that the "AKE" string means all the events - # (Except key-miss events which are excluded from 'A' due to their - # unique nature). - # - # The "notify-keyspace-events" takes as argument a string that is composed - # of zero or multiple characters. The empty string means that notifications - # are disabled. - # - # Example: to enable list and generic events, from the point of view of the - # event name, use: - # - # notify-keyspace-events Elg - # - # Example 2: to get the stream of the expired keys subscribing to channel - # name __keyevent@0__:expired use: - # - # notify-keyspace-events Ex - # - # By default all notifications are disabled because most users don't need - # this feature and the feature has some overhead. Note that if you don't - # specify at least one of K or E, no events will be delivered. - notify-keyspace-events "" - - ############################### GOPHER SERVER ################################# - - # Redis contains an implementation of the Gopher protocol, as specified in - # the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). - # - # The Gopher protocol was very popular in the late '90s. It is an alternative - # to the web, and the implementation both server and client side is so simple - # that the Redis server has just 100 lines of code in order to implement this - # support. - # - # What do you do with Gopher nowadays? Well Gopher never *really* died, and - # lately there is a movement in order for the Gopher more hierarchical content - # composed of just plain text documents to be resurrected. Some want a simpler - # internet, others believe that the mainstream internet became too much - # controlled, and it's cool to create an alternative space for people that - # want a bit of fresh air. - # - # Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol - # as a gift. - # - # --- HOW IT WORKS? --- - # - # The Redis Gopher support uses the inline protocol of Redis, and specifically - # two kind of inline requests that were anyway illegal: an empty request - # or any request that starts with "/" (there are no Redis commands starting - # with such a slash). Normal RESP2/RESP3 requests are completely out of the - # path of the Gopher protocol implementation and are served as usual as well. - # - # If you open a connection to Redis when Gopher is enabled and send it - # a string like "/foo", if there is a key named "/foo" it is served via the - # Gopher protocol. - # - # In order to create a real Gopher "hole" (the name of a Gopher site in Gopher - # talking), you likely need a script like the following: - # - # https://github.com/antirez/gopher2redis - # - # --- SECURITY WARNING --- - # - # If you plan to put Redis on the internet in a publicly accessible address - # to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. - # Once a password is set: - # - # 1. The Gopher server (when enabled, not by default) will still serve - # content via Gopher. - # 2. However other commands cannot be called before the client will - # authenticate. - # - # So use the 'requirepass' option to protect your instance. - # - # Note that Gopher is not currently supported when 'io-threads-do-reads' - # is enabled. - # - # To enable Gopher support, uncomment the following line and set the option - # from no (the default) to yes. - # - # gopher-enabled no - - ############################### ADVANCED CONFIG ############################### - - # Hashes are encoded using a memory efficient data structure when they have a - # small number of entries, and the biggest entry does not exceed a given - # threshold. These thresholds can be configured using the following directives. - hash-max-ziplist-entries 512 - hash-max-ziplist-value 64 - - # Lists are also encoded in a special way to save a lot of space. - # The number of entries allowed per internal list node can be specified - # as a fixed maximum size or a maximum number of elements. - # For a fixed maximum size, use -5 through -1, meaning: - # -5: max size: 64 Kb <-- not recommended for normal workloads - # -4: max size: 32 Kb <-- not recommended - # -3: max size: 16 Kb <-- probably not recommended - # -2: max size: 8 Kb <-- good - # -1: max size: 4 Kb <-- good - # Positive numbers mean store up to _exactly_ that number of elements - # per list node. - # The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), - # but if your use case is unique, adjust the settings as necessary. - list-max-ziplist-size -2 - - # Lists may also be compressed. - # Compress depth is the number of quicklist ziplist nodes from *each* side of - # the list to *exclude* from compression. The head and tail of the list - # are always uncompressed for fast push/pop operations. Settings are: - # 0: disable all list compression - # 1: depth 1 means "don't start compressing until after 1 node into the list, - # going from either the head or tail" - # So: [head]->node->node->...->node->[tail] - # [head], [tail] will always be uncompressed; inner nodes will compress. - # 2: [head]->[next]->node->node->...->node->[prev]->[tail] - # 2 here means: don't compress head or head->next or tail->prev or tail, - # but compress all nodes between them. - # 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] - # etc. - list-compress-depth 0 - - # Sets have a special encoding in just one case: when a set is composed - # of just strings that happen to be integers in radix 10 in the range - # of 64 bit signed integers. - # The following configuration setting sets the limit in the size of the - # set in order to use this special memory saving encoding. - set-max-intset-entries 512 - - # Similarly to hashes and lists, sorted sets are also specially encoded in - # order to save a lot of space. This encoding is only used when the length and - # elements of a sorted set are below the following limits: - zset-max-ziplist-entries 128 - zset-max-ziplist-value 64 - - # HyperLogLog sparse representation bytes limit. The limit includes the - # 16 bytes header. When an HyperLogLog using the sparse representation crosses - # this limit, it is converted into the dense representation. - # - # A value greater than 16000 is totally useless, since at that point the - # dense representation is more memory efficient. - # - # The suggested value is ~ 3000 in order to have the benefits of - # the space efficient encoding without slowing down too much PFADD, - # which is O(N) with the sparse encoding. The value can be raised to - # ~ 10000 when CPU is not a concern, but space is, and the data set is - # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. - hll-sparse-max-bytes 3000 - - # Streams macro node max size / items. The stream data structure is a radix - # tree of big nodes that encode multiple items inside. Using this configuration - # it is possible to configure how big a single node can be in bytes, and the - # maximum number of items it may contain before switching to a new node when - # appending new stream entries. If any of the following settings are set to - # zero, the limit is ignored, so for instance it is possible to set just a - # max entries limit by setting max-bytes to 0 and max-entries to the desired - # value. - stream-node-max-bytes 4096 - stream-node-max-entries 100 - - # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in - # order to help rehashing the main Redis hash table (the one mapping top-level - # keys to values). The hash table implementation Redis uses (see dict.c) - # performs a lazy rehashing: the more operation you run into a hash table - # that is rehashing, the more rehashing "steps" are performed, so if the - # server is idle the rehashing is never complete and some more memory is used - # by the hash table. - # - # The default is to use this millisecond 10 times every second in order to - # actively rehash the main dictionaries, freeing memory when possible. - # - # If unsure: - # use "activerehashing no" if you have hard latency requirements and it is - # not a good thing in your environment that Redis can reply from time to time - # to queries with 2 milliseconds delay. - # - # use "activerehashing yes" if you don't have such hard requirements but - # want to free memory asap when possible. - activerehashing yes - - # The client output buffer limits can be used to force disconnection of clients - # that are not reading data from the server fast enough for some reason (a - # common reason is that a Pub/Sub client can't consume messages as fast as the - # publisher can produce them). - # - # The limit can be set differently for the three different classes of clients: - # - # normal -> normal clients including MONITOR clients - # replica -> replica clients - # pubsub -> clients subscribed to at least one pubsub channel or pattern - # - # The syntax of every client-output-buffer-limit directive is the following: - # - # client-output-buffer-limit - # - # A client is immediately disconnected once the hard limit is reached, or if - # the soft limit is reached and remains reached for the specified number of - # seconds (continuously). - # So for instance if the hard limit is 32 megabytes and the soft limit is - # 16 megabytes / 10 seconds, the client will get disconnected immediately - # if the size of the output buffers reach 32 megabytes, but will also get - # disconnected if the client reaches 16 megabytes and continuously overcomes - # the limit for 10 seconds. - # - # By default normal clients are not limited because they don't receive data - # without asking (in a push way), but just after a request, so only - # asynchronous clients may create a scenario where data is requested faster - # than it can read. - # - # Instead there is a default limit for pubsub and replica clients, since - # subscribers and replicas receive data in a push fashion. - # - # Both the hard or the soft limit can be disabled by setting them to zero. - client-output-buffer-limit normal 0 0 0 - client-output-buffer-limit replica 256mb 64mb 60 - client-output-buffer-limit pubsub 32mb 8mb 60 - - # Client query buffers accumulate new commands. They are limited to a fixed - # amount by default in order to avoid that a protocol desynchronization (for - # instance due to a bug in the client) will lead to unbound memory usage in - # the query buffer. However you can configure it here if you have very special - # needs, such us huge multi/exec requests or alike. - # - # client-query-buffer-limit 1gb - - # In the Redis protocol, bulk requests, that are, elements representing single - # strings, are normally limited to 512 mb. However you can change this limit - # here, but must be 1mb or greater - # - # proto-max-bulk-len 512mb - - # Redis calls an internal function to perform many background tasks, like - # closing connections of clients in timeout, purging expired keys that are - # never requested, and so forth. - # - # Not all tasks are performed with the same frequency, but Redis checks for - # tasks to perform according to the specified "hz" value. - # - # By default "hz" is set to 10. Raising the value will use more CPU when - # Redis is idle, but at the same time will make Redis more responsive when - # there are many keys expiring at the same time, and timeouts may be - # handled with more precision. - # - # The range is between 1 and 500, however a value over 100 is usually not - # a good idea. Most users should use the default of 10 and raise this up to - # 100 only in environments where very low latency is required. - hz 10 - - # Normally it is useful to have an HZ value which is proportional to the - # number of clients connected. This is useful in order, for instance, to - # avoid too many clients are processed for each background task invocation - # in order to avoid latency spikes. - # - # Since the default HZ value by default is conservatively set to 10, Redis - # offers, and enables by default, the ability to use an adaptive HZ value - # which will temporarily raise when there are many connected clients. - # - # When dynamic HZ is enabled, the actual configured HZ will be used - # as a baseline, but multiples of the configured HZ value will be actually - # used as needed once more clients are connected. In this way an idle - # instance will use very little CPU time while a busy instance will be - # more responsive. - dynamic-hz yes - - # When a child rewrites the AOF file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - aof-rewrite-incremental-fsync yes - - # When redis saves RDB file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - rdb-save-incremental-fsync yes - - # Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good - # idea to start with the default settings and only change them after investigating - # how to improve the performances and how the keys LFU change over time, which - # is possible to inspect via the OBJECT FREQ command. - # - # There are two tunable parameters in the Redis LFU implementation: the - # counter logarithm factor and the counter decay time. It is important to - # understand what the two parameters mean before changing them. - # - # The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis - # uses a probabilistic increment with logarithmic behavior. Given the value - # of the old counter, when a key is accessed, the counter is incremented in - # this way: - # - # 1. A random number R between 0 and 1 is extracted. - # 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). - # 3. The counter is incremented only if R < P. - # - # The default lfu-log-factor is 10. This is a table of how the frequency - # counter changes with a different number of accesses with different - # logarithmic factors: - # - # +--------+------------+------------+------------+------------+------------+ - # | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | - # +--------+------------+------------+------------+------------+------------+ - # | 0 | 104 | 255 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 1 | 18 | 49 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 10 | 10 | 18 | 142 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 100 | 8 | 11 | 49 | 143 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # - # NOTE: The above table was obtained by running the following commands: - # - # redis-benchmark -n 1000000 incr foo - # redis-cli object freq foo - # - # NOTE 2: The counter initial value is 5 in order to give new objects a chance - # to accumulate hits. - # - # The counter decay time is the time, in minutes, that must elapse in order - # for the key counter to be divided by two (or decremented if it has a value - # less <= 10). - # - # The default value for the lfu-decay-time is 1. A special value of 0 means to - # decay the counter every time it happens to be scanned. - # - # lfu-log-factor 10 - # lfu-decay-time 1 - - ########################### ACTIVE DEFRAGMENTATION ####################### - # - # What is active defragmentation? - # ------------------------------- - # - # Active (online) defragmentation allows a Redis server to compact the - # spaces left between small allocations and deallocations of data in memory, - # thus allowing to reclaim back memory. - # - # Fragmentation is a natural process that happens with every allocator (but - # less so with Jemalloc, fortunately) and certain workloads. Normally a server - # restart is needed in order to lower the fragmentation, or at least to flush - # away all the data and create it again. However thanks to this feature - # implemented by Oran Agra for Redis 4.0 this process can happen at runtime - # in a "hot" way, while the server is running. - # - # Basically when the fragmentation is over a certain level (see the - # configuration options below) Redis will start to create new copies of the - # values in contiguous memory regions by exploiting certain specific Jemalloc - # features (in order to understand if an allocation is causing fragmentation - # and to allocate it in a better place), and at the same time, will release the - # old copies of the data. This process, repeated incrementally for all the keys - # will cause the fragmentation to drop back to normal values. - # - # Important things to understand: - # - # 1. This feature is disabled by default, and only works if you compiled Redis - # to use the copy of Jemalloc we ship with the source code of Redis. - # This is the default with Linux builds. - # - # 2. You never need to enable this feature if you don't have fragmentation - # issues. - # - # 3. Once you experience fragmentation, you can enable this feature when - # needed with the command "CONFIG SET activedefrag yes". - # - # The configuration parameters are able to fine tune the behavior of the - # defragmentation process. If you are not sure about what they mean it is - # a good idea to leave the defaults untouched. - - # Enabled active defragmentation - # activedefrag no - - # Minimum amount of fragmentation waste to start active defrag - # active-defrag-ignore-bytes 100mb - - # Minimum percentage of fragmentation to start active defrag - # active-defrag-threshold-lower 10 - - # Maximum percentage of fragmentation at which we use maximum effort - # active-defrag-threshold-upper 100 - - # Minimal effort for defrag in CPU percentage, to be used when the lower - # threshold is reached - # active-defrag-cycle-min 1 - - # Maximal effort for defrag in CPU percentage, to be used when the upper - # threshold is reached - # active-defrag-cycle-max 25 - - # Maximum number of set/hash/zset/list fields that will be processed from - # the main dictionary scan - # active-defrag-max-scan-fields 1000 - - # Jemalloc background thread for purging will be enabled by default - jemalloc-bg-thread yes - - # It is possible to pin different threads and processes of Redis to specific - # CPUs in your system, in order to maximize the performances of the server. - # This is useful both in order to pin different Redis threads in different - # CPUs, but also in order to make sure that multiple Redis instances running - # in the same host will be pinned to different CPUs. - # - # Normally you can do this using the "taskset" command, however it is also - # possible to this via Redis configuration directly, both in Linux and FreeBSD. - # - # You can pin the server/IO threads, bio threads, aof rewrite child process, and - # the bgsave child process. The syntax to specify the cpu list is the same as - # the taskset command: - # - # Set redis server/io threads to cpu affinity 0,2,4,6: - # server_cpulist 0-7:2 - # - # Set bio threads to cpu affinity 1,3: - # bio_cpulist 1,3 - # - # Set aof rewrite child process to cpu affinity 8,9,10,11: - # aof_rewrite_cpulist 8-11 - # - # Set bgsave child process to cpu affinity 1,10,11 - # bgsave_cpulist 1,10-11 - - # In some cases redis will emit warnings and even refuse to start if it detects - # that the system is in bad state, it is possible to suppress these warnings - # by setting the following config which takes a space delimited list of warnings - # to suppress - # - # ignore-warnings ARM64-COW-BUG -kind: ConfigMap -metadata: - name: redis-config - namespace: default - ---- -apiVersion: v1 -data: - filer.toml: | - [redis] - enabled = true - address = "redis:6379" - password = "" - database = 15 -kind: ConfigMap -metadata: - name: seaweed-config-filer - namespace: default - ---- -apiVersion: v1 -data: - s3config.json: | - { - "identities": [ - { - "name": "admin", - "credentials": [ - { - "accessKey": "admin", - "secretKey": "admin" - } - ], - "actions": ["Admin", "Read", "List", "Tagging", "Write"] - } - ] - } -kind: ConfigMap -metadata: - name: seaweed-config-s3 - namespace: default - ---- - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "tailscale-headless" - labels: - "app.kubernetes.io/name": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "tailscale" - labels: - "app.kubernetes.io/name": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: proxy - port: 80 - targetPort: 80 - protocol: "TCP" - - name: api - port: 4000 - targetPort: 4000 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "tailscale" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "tailscale-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "traefik:v2.5" - name: "container-0" - command: - args: - env: - - name: SERVICE_NAME - value: proxy - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/etc/traefik" - name: "traefik-conf" - readOnly: false - - image: "openmined/grid-vpn-tailscale:latest" - name: "container-1" - command: - args: - env: - - name: HOSTNAME - value: node - - name: STACK_API_KEY - value: 3YlCOFidhRasLBByTZ2dH8N1ZNXPPxRycyq4T4fh2PXcvXZV - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/var/lib/tailscale" - name: "tailscale-data" - subPath: "tailscale-data" - readOnly: false - initContainers: - volumes: - - name: "traefik-conf" - configMap: - name: traefik-main-config - volumeClaimTemplates: - - metadata: - name: "tailscale-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "100Mi" - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "queue" - labels: - "app.kubernetes.io/name": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: ampq - port: 5672 - targetPort: 5672 - protocol: "TCP" - - name: management - port: 15672 - targetPort: 15672 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "queue" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "rabbitmq:3" - name: "container-0" - command: - args: - env: null - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "redis-headless" - labels: - "app.kubernetes.io/name": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "redis" - labels: - "app.kubernetes.io/name": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 6379 - targetPort: 6379 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "redis" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "redis-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "redis:6.2" - name: "container-0" - command: - args: - env: - - name: SERVICE_NAME - value: redis - - name: RELEASE - value: production - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/data" - name: "app-redis-data" - subPath: "" - readOnly: false - - mountPath: "/usr/local/etc/redis" - name: "redis-conf" - readOnly: false - initContainers: - volumes: - - name: "redis-conf" - configMap: - name: redis-config - volumeClaimTemplates: - - metadata: - name: "app-redis-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "5Gi" - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "seaweedfs-headless" - labels: - "app.kubernetes.io/name": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "seaweedfs" - labels: - "app.kubernetes.io/name": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 8333 - targetPort: 8333 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "seaweedfs" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "seaweedfs-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "chrislusf/seaweedfs:latest" - name: "container-0" - command: - - "weed" - args: - - "server" - - "-s3" - - "-s3.config=/etc/s3config.json" - - "-master.volumeSizeLimitMB=1024" - env: - - name: SERVICE_NAME - value: seaweedfs - - name: RELEASE - value: production - - name: S3_VOLUME_SIZE_MB - value: "1024" - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/data" - name: "seaweedfs-data" - subPath: "" - readOnly: false - - mountPath: "/etc/s3config.json" - name: "seaweed-conf-s3" - subPath: "s3config.json" - readOnly: false - - mountPath: "/etc/seaweedfs/filer.toml" - name: "seaweed-conf-filer" - subPath: "filer.toml" - readOnly: false - initContainers: - volumes: - - name: "seaweed-conf-s3" - configMap: - name: seaweed-config-s3 - - name: "seaweed-conf-filer" - configMap: - name: seaweed-config-filer - volumeClaimTemplates: - - metadata: - name: "seaweedfs-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "seaweedfs" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "20Gi" - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "db-headless" - labels: - "app.kubernetes.io/name": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "db" - labels: - "app.kubernetes.io/name": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 5432 - targetPort: 5432 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "db" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "db-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "postgres:12" - name: "container-0" - command: - args: - env: - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/var/lib/postgresql/data" - name: "app-db-data" - subPath: "pgdata" - readOnly: false - initContainers: - volumes: - volumeClaimTemplates: - - metadata: - name: "app-db-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "5Gi" - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "backend" - labels: - "app.kubernetes.io/name": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 80 - targetPort: 80 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - "/start.sh" - args: - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: LOG_LEVEL - value: debug - - name: NODE_TYPE - value: domain - - name: STACK_API_KEY - value: 3YlCOFidhRasLBByTZ2dH8N1ZNXPPxRycyq4T4fh2PXcvXZV - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: USE_BLOB_STORAGE - value: "True" - - name: SERVICE_NAME - value: backend - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "backend-stream" - labels: - "app.kubernetes.io/name": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 80 - targetPort: 80 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend-stream" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - /start.sh - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: LOG_LEVEL - value: debug - - name: STREAM_QUEUE - value: "1" - - name: NODE_TYPE - value: domain - - name: STACK_API_KEY - value: 3YlCOFidhRasLBByTZ2dH8N1ZNXPPxRycyq4T4fh2PXcvXZV - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: USE_BLOB_STORAGE - value: "True" - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend-worker" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - /worker-start.sh - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: CELERY_WORKER - value: "true" - - name: RUN - value: celery -A grid.worker worker -l info -Q main-queue --pool=gevent -c 500 - - name: NODE_TYPE - value: domain - - name: C_FORCE_ROOT - value: "1" - - name: STACK_API_KEY - value: 3YlCOFidhRasLBByTZ2dH8N1ZNXPPxRycyq4T4fh2PXcvXZV - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: NETWORK_CHECK_INTERVAL - value: "60" - - name: DOMAIN_CHECK_INTERVAL - value: "60" - - name: USE_BLOB_STORAGE - value: "True" - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "frontend" - labels: - "app.kubernetes.io/name": "frontend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 80 - targetPort: 80 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "frontend" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "frontend" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "frontend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "frontend" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "frontend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-frontend:latest" - name: "container-0" - command: - args: - env: - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: NODE_TYPE - value: domain - - name: VITE_PUBLIC_API_BASE_URL - value: /api/v1 - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "vpn-iptables" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - hostNetwork: true - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-vpn-iptables:latest" - name: "container-0" - command: - - "sh" - - "-c" - - "/iptables.sh" - args: - env: null - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - resources: - requests: - cpu: "5m" - memory: "10Mi" - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - ingress.kubernetes.io/rewrite-target: / - name: grid-stack-ingress - namespace: default -spec: - rules: - - http: - paths: - - backend: - service: - name: tailscale - port: - number: 80 - path: / - pathType: Prefix - ---- - diff --git a/packages/grid/k8s/rendered/network.yaml b/packages/grid/k8s/rendered/network.yaml deleted file mode 100644 index 47c84f79161..00000000000 --- a/packages/grid/k8s/rendered/network.yaml +++ /dev/null @@ -1,3339 +0,0 @@ -apiVersion: v1 -data: - dynamic.yml: | - http: - services: - frontend: - loadBalancer: - servers: - - url: "http://frontend" - backend: - loadBalancer: - servers: - - url: "http://backend" - backend-stream: - loadBalancer: - servers: - - url: "http://backend-stream" - routers: - frontend: - rule: "PathPrefix(`/`)" - entryPoints: - - "web" - service: "frontend" - backend: - rule: "PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend" - backend-stream: - rule: "PathPrefix(`/api`) && PathPrefix(`/api/v1/syft/stream`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" - entryPoints: - - "web" - service: "backend-stream" - - traefik.yml: | - global: - checkNewVersion: false - sendAnonymousUsage: false - - log: - level: INFO - - entryPoints: - web: - address: :80 - ping: - address: :8082 - - ping: - entryPoint: "ping" - - providers: - file: - filename: /etc/traefik/dynamic.yml -kind: ConfigMap -metadata: - name: traefik-main-config - namespace: default - ---- -apiVersion: v1 -data: - redis.conf: | - # Redis configuration file example. - # - # Note that in order to read the configuration file, Redis must be - # started with the file path as first argument: - # - # ./redis-server /path/to/redis.conf - - # Note on units: when memory size is needed, it is possible to specify - # it in the usual form of 1k 5GB 4M and so forth: - # - # 1k => 1000 bytes - # 1kb => 1024 bytes - # 1m => 1000000 bytes - # 1mb => 1024*1024 bytes - # 1g => 1000000000 bytes - # 1gb => 1024*1024*1024 bytes - # - # units are case insensitive so 1GB 1Gb 1gB are all the same. - - ################################## INCLUDES ################################### - - # Include one or more other config files here. This is useful if you - # have a standard template that goes to all Redis servers but also need - # to customize a few per-server settings. Include files can include - # other files, so use this wisely. - # - # Note that option "include" won't be rewritten by command "CONFIG REWRITE" - # from admin or Redis Sentinel. Since Redis always uses the last processed - # line as value of a configuration directive, you'd better put includes - # at the beginning of this file to avoid overwriting config change at runtime. - # - # If instead you are interested in using includes to override configuration - # options, it is better to use include as the last line. - # - # include /path/to/local.conf - # include /path/to/other.conf - - ################################## MODULES ##################################### - - # Load modules at startup. If the server is not able to load modules - # it will abort. It is possible to use multiple loadmodule directives. - # - # loadmodule /path/to/my_module.so - # loadmodule /path/to/other_module.so - - ################################## NETWORK ##################################### - - # By default, if no "bind" configuration directive is specified, Redis listens - # for connections from all available network interfaces on the host machine. - # It is possible to listen to just one or multiple selected interfaces using - # the "bind" configuration directive, followed by one or more IP addresses. - # Each address can be prefixed by "-", which means that redis will not fail to - # start if the address is not available. Being not available only refers to - # addresses that does not correspond to any network interfece. Addresses that - # are already in use will always fail, and unsupported protocols will always BE - # silently skipped. - # - # Examples: - # - # bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses - # bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 - # bind * -::* # like the default, all available interfaces - # - # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the - # internet, binding to all the interfaces is dangerous and will expose the - # instance to everybody on the internet. So by default we uncomment the - # following bind directive, that will force Redis to listen only on the - # IPv4 and IPv6 (if available) loopback interface addresses (this means Redis - # will only be able to accept client connections from the same host that it is - # running on). - # - # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES - # JUST COMMENT OUT THE FOLLOWING LINE. - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - bind 127.0.0.1 -::1 - - # Protected mode is a layer of security protection, in order to avoid that - # Redis instances left open on the internet are accessed and exploited. - # - # When protected mode is on and if: - # - # 1) The server is not binding explicitly to a set of addresses using the - # "bind" directive. - # 2) No password is configured. - # - # The server only accepts connections from clients connecting from the - # IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain - # sockets. - # - # By default protected mode is enabled. You should disable it only if - # you are sure you want clients from other hosts to connect to Redis - # even if no authentication is configured, nor a specific set of interfaces - # are explicitly listed using the "bind" directive. - protected-mode yes - - # Accept connections on the specified port, default is 6379 (IANA #815344). - # If port 0 is specified Redis will not listen on a TCP socket. - port 6379 - - # TCP listen() backlog. - # - # In high requests-per-second environments you need a high backlog in order - # to avoid slow clients connection issues. Note that the Linux kernel - # will silently truncate it to the value of /proc/sys/net/core/somaxconn so - # make sure to raise both the value of somaxconn and tcp_max_syn_backlog - # in order to get the desired effect. - tcp-backlog 511 - - # Unix socket. - # - # Specify the path for the Unix socket that will be used to listen for - # incoming connections. There is no default, so Redis will not listen - # on a unix socket when not specified. - # - # unixsocket /run/redis.sock - # unixsocketperm 700 - - # Close the connection after a client is idle for N seconds (0 to disable) - timeout 0 - - # TCP keepalive. - # - # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence - # of communication. This is useful for two reasons: - # - # 1) Detect dead peers. - # 2) Force network equipment in the middle to consider the connection to be - # alive. - # - # On Linux, the specified value (in seconds) is the period used to send ACKs. - # Note that to close the connection the double of the time is needed. - # On other kernels the period depends on the kernel configuration. - # - # A reasonable value for this option is 300 seconds, which is the new - # Redis default starting with Redis 3.2.1. - tcp-keepalive 300 - - ################################# TLS/SSL ##################################### - - # By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration - # directive can be used to define TLS-listening ports. To enable TLS on the - # default port, use: - # - # port 0 - # tls-port 6379 - - # Configure a X.509 certificate and private key to use for authenticating the - # server to connected clients, masters or cluster peers. These files should be - # PEM formatted. - # - # tls-cert-file redis.crt - # tls-key-file redis.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-key-file-pass secret - - # Normally Redis uses the same certificate for both server functions (accepting - # connections) and client functions (replicating from a master, establishing - # cluster bus connections, etc.). - # - # Sometimes certificates are issued with attributes that designate them as - # client-only or server-only certificates. In that case it may be desired to use - # different certificates for incoming (server) and outgoing (client) - # connections. To do that, use the following directives: - # - # tls-client-cert-file client.crt - # tls-client-key-file client.key - # - # If the key file is encrypted using a passphrase, it can be included here - # as well. - # - # tls-client-key-file-pass secret - - # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: - # - # tls-dh-params-file redis.dh - - # Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL - # clients and peers. Redis requires an explicit configuration of at least one - # of these, and will not implicitly use the system wide configuration. - # - # tls-ca-cert-file ca.crt - # tls-ca-cert-dir /etc/ssl/certs - - # By default, clients (including replica servers) on a TLS port are required - # to authenticate using valid client side certificates. - # - # If "no" is specified, client certificates are not required and not accepted. - # If "optional" is specified, client certificates are accepted and must be - # valid if provided, but are not required. - # - # tls-auth-clients no - # tls-auth-clients optional - - # By default, a Redis replica does not attempt to establish a TLS connection - # with its master. - # - # Use the following directive to enable TLS on replication links. - # - # tls-replication yes - - # By default, the Redis Cluster bus uses a plain TCP connection. To enable - # TLS for the bus protocol, use the following directive: - # - # tls-cluster yes - - # By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended - # that older formally deprecated versions are kept disabled to reduce the attack surface. - # You can explicitly specify TLS versions to support. - # Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", - # "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. - # To enable only TLSv1.2 and TLSv1.3, use: - # - # tls-protocols "TLSv1.2 TLSv1.3" - - # Configure allowed ciphers. See the ciphers(1ssl) manpage for more information - # about the syntax of this string. - # - # Note: this configuration applies only to <= TLSv1.2. - # - # tls-ciphers DEFAULT:!MEDIUM - - # Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more - # information about the syntax of this string, and specifically for TLSv1.3 - # ciphersuites. - # - # tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 - - # When choosing a cipher, use the server's preference instead of the client - # preference. By default, the server follows the client's preference. - # - # tls-prefer-server-ciphers yes - - # By default, TLS session caching is enabled to allow faster and less expensive - # reconnections by clients that support it. Use the following directive to disable - # caching. - # - # tls-session-caching no - - # Change the default number of TLS sessions cached. A zero value sets the cache - # to unlimited size. The default size is 20480. - # - # tls-session-cache-size 5000 - - # Change the default timeout of cached TLS sessions. The default timeout is 300 - # seconds. - # - # tls-session-cache-timeout 60 - - ################################# GENERAL ##################################### - - # By default Redis does not run as a daemon. Use 'yes' if you need it. - # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. - # When Redis is supervised by upstart or systemd, this parameter has no impact. - daemonize no - - # If you run Redis from upstart or systemd, Redis can interact with your - # supervision tree. Options: - # supervised no - no supervision interaction - # supervised upstart - signal upstart by putting Redis into SIGSTOP mode - # requires "expect stop" in your upstart job config - # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET - # on startup, and updating Redis status on a regular - # basis. - # supervised auto - detect upstart or systemd method based on - # UPSTART_JOB or NOTIFY_SOCKET environment variables - # Note: these supervision methods only signal "process is ready." - # They do not enable continuous pings back to your supervisor. - # - # The default is "no". To run under upstart/systemd, you can simply uncomment - # the line below: - # - # supervised auto - - # If a pid file is specified, Redis writes it where specified at startup - # and removes it at exit. - # - # When the server runs non daemonized, no pid file is created if none is - # specified in the configuration. When the server is daemonized, the pid file - # is used even if not specified, defaulting to "/var/run/redis.pid". - # - # Creating a pid file is best effort: if Redis is not able to create it - # nothing bad happens, the server will start and run normally. - # - # Note that on modern Linux systems "/run/redis.pid" is more conforming - # and should be used instead. - pidfile /var/run/redis_6379.pid - - # Specify the server verbosity level. - # This can be one of: - # debug (a lot of information, useful for development/testing) - # verbose (many rarely useful info, but not a mess like the debug level) - # notice (moderately verbose, what you want in production probably) - # warning (only very important / critical messages are logged) - loglevel notice - - # Specify the log file name. Also the empty string can be used to force - # Redis to log on the standard output. Note that if you use standard - # output for logging but daemonize, logs will be sent to /dev/null - logfile "" - - # To enable logging to the system logger, just set 'syslog-enabled' to yes, - # and optionally update the other syslog parameters to suit your needs. - # syslog-enabled no - - # Specify the syslog identity. - # syslog-ident redis - - # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. - # syslog-facility local0 - - # To disable the built in crash log, which will possibly produce cleaner core - # dumps when they are needed, uncomment the following: - # - # crash-log-enabled no - - # To disable the fast memory check that's run as part of the crash log, which - # will possibly let redis terminate sooner, uncomment the following: - # - # crash-memcheck-enabled no - - # Set the number of databases. The default database is DB 0, you can select - # a different one on a per-connection basis using SELECT where - # dbid is a number between 0 and 'databases'-1 - databases 16 - - # By default Redis shows an ASCII art logo only when started to log to the - # standard output and if the standard output is a TTY and syslog logging is - # disabled. Basically this means that normally a logo is displayed only in - # interactive sessions. - # - # However it is possible to force the pre-4.0 behavior and always show a - # ASCII art logo in startup logs by setting the following option to yes. - always-show-logo no - - # By default, Redis modifies the process title (as seen in 'top' and 'ps') to - # provide some runtime information. It is possible to disable this and leave - # the process name as executed by setting the following to no. - set-proc-title yes - - # When changing the process title, Redis uses the following template to construct - # the modified title. - # - # Template variables are specified in curly brackets. The following variables are - # supported: - # - # {title} Name of process as executed if parent, or type of child process. - # {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or - # Unix socket if only that's available. - # {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". - # {port} TCP port listening on, or 0. - # {tls-port} TLS port listening on, or 0. - # {unixsocket} Unix domain socket listening on, or "". - # {config-file} Name of configuration file used. - # - proc-title-template "{title} {listen-addr} {server-mode}" - - ################################ SNAPSHOTTING ################################ - - # Save the DB to disk. - # - # save - # - # Redis will save the DB if both the given number of seconds and the given - # number of write operations against the DB occurred. - # - # Snapshotting can be completely disabled with a single empty string argument - # as in following example: - # - # save "" - # - # Unless specified otherwise, by default Redis will save the DB: - # * After 3600 seconds (an hour) if at least 1 key changed - # * After 300 seconds (5 minutes) if at least 100 keys changed - # * After 60 seconds if at least 10000 keys changed - # - # You can set these explicitly by uncommenting the three following lines. - # - # save 3600 1 - # save 300 100 - # save 60 10000 - - # By default Redis will stop accepting writes if RDB snapshots are enabled - # (at least one save point) and the latest background save failed. - # This will make the user aware (in a hard way) that data is not persisting - # on disk properly, otherwise chances are that no one will notice and some - # disaster will happen. - # - # If the background saving process will start working again Redis will - # automatically allow writes again. - # - # However if you have setup your proper monitoring of the Redis server - # and persistence, you may want to disable this feature so that Redis will - # continue to work as usual even if there are problems with disk, - # permissions, and so forth. - stop-writes-on-bgsave-error yes - - # Compress string objects using LZF when dump .rdb databases? - # By default compression is enabled as it's almost always a win. - # If you want to save some CPU in the saving child set it to 'no' but - # the dataset will likely be bigger if you have compressible values or keys. - rdbcompression yes - - # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. - # This makes the format more resistant to corruption but there is a performance - # hit to pay (around 10%) when saving and loading RDB files, so you can disable it - # for maximum performances. - # - # RDB files created with checksum disabled have a checksum of zero that will - # tell the loading code to skip the check. - rdbchecksum yes - - # Enables or disables full sanitation checks for ziplist and listpack etc when - # loading an RDB or RESTORE payload. This reduces the chances of a assertion or - # crash later on while processing commands. - # Options: - # no - Never perform full sanitation - # yes - Always perform full sanitation - # clients - Perform full sanitation only for user connections. - # Excludes: RDB files, RESTORE commands received from the master - # connection, and client connections which have the - # skip-sanitize-payload ACL flag. - # The default should be 'clients' but since it currently affects cluster - # resharding via MIGRATE, it is temporarily set to 'no' by default. - # - # sanitize-dump-payload no - - # The filename where to dump the DB - dbfilename dump.rdb - - # Remove RDB files used by replication in instances without persistence - # enabled. By default this option is disabled, however there are environments - # where for regulations or other security concerns, RDB files persisted on - # disk by masters in order to feed replicas, or stored on disk by replicas - # in order to load them for the initial synchronization, should be deleted - # ASAP. Note that this option ONLY WORKS in instances that have both AOF - # and RDB persistence disabled, otherwise is completely ignored. - # - # An alternative (and sometimes better) way to obtain the same effect is - # to use diskless replication on both master and replicas instances. However - # in the case of replicas, diskless is not always an option. - rdb-del-sync-files no - - # The working directory. - # - # The DB will be written inside this directory, with the filename specified - # above using the 'dbfilename' configuration directive. - # - # The Append Only File will also be created inside this directory. - # - # Note that you must specify a directory here, not a file name. - dir ./ - - ################################# REPLICATION ################################# - - # Master-Replica replication. Use replicaof to make a Redis instance a copy of - # another Redis server. A few things to understand ASAP about Redis replication. - # - # +------------------+ +---------------+ - # | Master | ---> | Replica | - # | (receive writes) | | (exact copy) | - # +------------------+ +---------------+ - # - # 1) Redis replication is asynchronous, but you can configure a master to - # stop accepting writes if it appears to be not connected with at least - # a given number of replicas. - # 2) Redis replicas are able to perform a partial resynchronization with the - # master if the replication link is lost for a relatively small amount of - # time. You may want to configure the replication backlog size (see the next - # sections of this file) with a sensible value depending on your needs. - # 3) Replication is automatic and does not need user intervention. After a - # network partition replicas automatically try to reconnect to masters - # and resynchronize with them. - # - # replicaof - - # If the master is password protected (using the "requirepass" configuration - # directive below) it is possible to tell the replica to authenticate before - # starting the replication synchronization process, otherwise the master will - # refuse the replica request. - # - # masterauth - # - # However this is not enough if you are using Redis ACLs (for Redis version - # 6 or greater), and the default user is not capable of running the PSYNC - # command and/or other commands needed for replication. In this case it's - # better to configure a special user to use with replication, and specify the - # masteruser configuration as such: - # - # masteruser - # - # When masteruser is specified, the replica will authenticate against its - # master using the new AUTH form: AUTH . - - # When a replica loses its connection with the master, or when the replication - # is still in progress, the replica can act in two different ways: - # - # 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will - # still reply to client requests, possibly with out of date data, or the - # data set may just be empty if this is the first synchronization. - # - # 2) If replica-serve-stale-data is set to 'no' the replica will reply with - # an error "SYNC with master in progress" to all commands except: - # INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, - # UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, - # HOST and LATENCY. - # - replica-serve-stale-data yes - - # You can configure a replica instance to accept writes or not. Writing against - # a replica instance may be useful to store some ephemeral data (because data - # written on a replica will be easily deleted after resync with the master) but - # may also cause problems if clients are writing to it because of a - # misconfiguration. - # - # Since Redis 2.6 by default replicas are read-only. - # - # Note: read only replicas are not designed to be exposed to untrusted clients - # on the internet. It's just a protection layer against misuse of the instance. - # Still a read only replica exports by default all the administrative commands - # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve - # security of read only replicas using 'rename-command' to shadow all the - # administrative / dangerous commands. - replica-read-only yes - - # Replication SYNC strategy: disk or socket. - # - # New replicas and reconnecting replicas that are not able to continue the - # replication process just receiving differences, need to do what is called a - # "full synchronization". An RDB file is transmitted from the master to the - # replicas. - # - # The transmission can happen in two different ways: - # - # 1) Disk-backed: The Redis master creates a new process that writes the RDB - # file on disk. Later the file is transferred by the parent - # process to the replicas incrementally. - # 2) Diskless: The Redis master creates a new process that directly writes the - # RDB file to replica sockets, without touching the disk at all. - # - # With disk-backed replication, while the RDB file is generated, more replicas - # can be queued and served with the RDB file as soon as the current child - # producing the RDB file finishes its work. With diskless replication instead - # once the transfer starts, new replicas arriving will be queued and a new - # transfer will start when the current one terminates. - # - # When diskless replication is used, the master waits a configurable amount of - # time (in seconds) before starting the transfer in the hope that multiple - # replicas will arrive and the transfer can be parallelized. - # - # With slow disks and fast (large bandwidth) networks, diskless replication - # works better. - repl-diskless-sync no - - # When diskless replication is enabled, it is possible to configure the delay - # the server waits in order to spawn the child that transfers the RDB via socket - # to the replicas. - # - # This is important since once the transfer starts, it is not possible to serve - # new replicas arriving, that will be queued for the next RDB transfer, so the - # server waits a delay in order to let more replicas arrive. - # - # The delay is specified in seconds, and by default is 5 seconds. To disable - # it entirely just set it to 0 seconds and the transfer will start ASAP. - repl-diskless-sync-delay 5 - - # ----------------------------------------------------------------------------- - # WARNING: RDB diskless load is experimental. Since in this setup the replica - # does not immediately store an RDB on disk, it may cause data loss during - # failovers. RDB diskless load + Redis modules not handling I/O reads may also - # cause Redis to abort in case of I/O errors during the initial synchronization - # stage with the master. Use only if you know what you are doing. - # ----------------------------------------------------------------------------- - # - # Replica can load the RDB it reads from the replication link directly from the - # socket, or store the RDB to a file and read that file after it was completely - # received from the master. - # - # In many cases the disk is slower than the network, and storing and loading - # the RDB file may increase replication time (and even increase the master's - # Copy on Write memory and salve buffers). - # However, parsing the RDB file directly from the socket may mean that we have - # to flush the contents of the current database before the full rdb was - # received. For this reason we have the following options: - # - # "disabled" - Don't use diskless load (store the rdb file to the disk first) - # "on-empty-db" - Use diskless load only when it is completely safe. - # "swapdb" - Keep a copy of the current db contents in RAM while parsing - # the data directly from the socket. note that this requires - # sufficient memory, if you don't have it, you risk an OOM kill. - repl-diskless-load disabled - - # Replicas send PINGs to server in a predefined interval. It's possible to - # change this interval with the repl_ping_replica_period option. The default - # value is 10 seconds. - # - # repl-ping-replica-period 10 - - # The following option sets the replication timeout for: - # - # 1) Bulk transfer I/O during SYNC, from the point of view of replica. - # 2) Master timeout from the point of view of replicas (data, pings). - # 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). - # - # It is important to make sure that this value is greater than the value - # specified for repl-ping-replica-period otherwise a timeout will be detected - # every time there is low traffic between the master and the replica. The default - # value is 60 seconds. - # - # repl-timeout 60 - - # Disable TCP_NODELAY on the replica socket after SYNC? - # - # If you select "yes" Redis will use a smaller number of TCP packets and - # less bandwidth to send data to replicas. But this can add a delay for - # the data to appear on the replica side, up to 40 milliseconds with - # Linux kernels using a default configuration. - # - # If you select "no" the delay for data to appear on the replica side will - # be reduced but more bandwidth will be used for replication. - # - # By default we optimize for low latency, but in very high traffic conditions - # or when the master and replicas are many hops away, turning this to "yes" may - # be a good idea. - repl-disable-tcp-nodelay no - - # Set the replication backlog size. The backlog is a buffer that accumulates - # replica data when replicas are disconnected for some time, so that when a - # replica wants to reconnect again, often a full resync is not needed, but a - # partial resync is enough, just passing the portion of data the replica - # missed while disconnected. - # - # The bigger the replication backlog, the longer the replica can endure the - # disconnect and later be able to perform a partial resynchronization. - # - # The backlog is only allocated if there is at least one replica connected. - # - # repl-backlog-size 1mb - - # After a master has no connected replicas for some time, the backlog will be - # freed. The following option configures the amount of seconds that need to - # elapse, starting from the time the last replica disconnected, for the backlog - # buffer to be freed. - # - # Note that replicas never free the backlog for timeout, since they may be - # promoted to masters later, and should be able to correctly "partially - # resynchronize" with other replicas: hence they should always accumulate backlog. - # - # A value of 0 means to never release the backlog. - # - # repl-backlog-ttl 3600 - - # The replica priority is an integer number published by Redis in the INFO - # output. It is used by Redis Sentinel in order to select a replica to promote - # into a master if the master is no longer working correctly. - # - # A replica with a low priority number is considered better for promotion, so - # for instance if there are three replicas with priority 10, 100, 25 Sentinel - # will pick the one with priority 10, that is the lowest. - # - # However a special priority of 0 marks the replica as not able to perform the - # role of master, so a replica with priority of 0 will never be selected by - # Redis Sentinel for promotion. - # - # By default the priority is 100. - replica-priority 100 - - # ----------------------------------------------------------------------------- - # By default, Redis Sentinel includes all replicas in its reports. A replica - # can be excluded from Redis Sentinel's announcements. An unannounced replica - # will be ignored by the 'sentinel replicas ' command and won't be - # exposed to Redis Sentinel's clients. - # - # This option does not change the behavior of replica-priority. Even with - # replica-announced set to 'no', the replica can be promoted to master. To - # prevent this behavior, set replica-priority to 0. - # - # replica-announced yes - - # It is possible for a master to stop accepting writes if there are less than - # N replicas connected, having a lag less or equal than M seconds. - # - # The N replicas need to be in "online" state. - # - # The lag in seconds, that must be <= the specified value, is calculated from - # the last ping received from the replica, that is usually sent every second. - # - # This option does not GUARANTEE that N replicas will accept the write, but - # will limit the window of exposure for lost writes in case not enough replicas - # are available, to the specified number of seconds. - # - # For example to require at least 3 replicas with a lag <= 10 seconds use: - # - # min-replicas-to-write 3 - # min-replicas-max-lag 10 - # - # Setting one or the other to 0 disables the feature. - # - # By default min-replicas-to-write is set to 0 (feature disabled) and - # min-replicas-max-lag is set to 10. - - # A Redis master is able to list the address and port of the attached - # replicas in different ways. For example the "INFO replication" section - # offers this information, which is used, among other tools, by - # Redis Sentinel in order to discover replica instances. - # Another place where this info is available is in the output of the - # "ROLE" command of a master. - # - # The listed IP address and port normally reported by a replica is - # obtained in the following way: - # - # IP: The address is auto detected by checking the peer address - # of the socket used by the replica to connect with the master. - # - # Port: The port is communicated by the replica during the replication - # handshake, and is normally the port that the replica is using to - # listen for connections. - # - # However when port forwarding or Network Address Translation (NAT) is - # used, the replica may actually be reachable via different IP and port - # pairs. The following two options can be used by a replica in order to - # report to its master a specific set of IP and port, so that both INFO - # and ROLE will report those values. - # - # There is no need to use both the options if you need to override just - # the port or the IP address. - # - # replica-announce-ip 5.5.5.5 - # replica-announce-port 1234 - - ############################### KEYS TRACKING ################################# - - # Redis implements server assisted support for client side caching of values. - # This is implemented using an invalidation table that remembers, using - # a radix key indexed by key name, what clients have which keys. In turn - # this is used in order to send invalidation messages to clients. Please - # check this page to understand more about the feature: - # - # https://redis.io/topics/client-side-caching - # - # When tracking is enabled for a client, all the read only queries are assumed - # to be cached: this will force Redis to store information in the invalidation - # table. When keys are modified, such information is flushed away, and - # invalidation messages are sent to the clients. However if the workload is - # heavily dominated by reads, Redis could use more and more memory in order - # to track the keys fetched by many clients. - # - # For this reason it is possible to configure a maximum fill value for the - # invalidation table. By default it is set to 1M of keys, and once this limit - # is reached, Redis will start to evict keys in the invalidation table - # even if they were not modified, just to reclaim memory: this will in turn - # force the clients to invalidate the cached values. Basically the table - # maximum size is a trade off between the memory you want to spend server - # side to track information about who cached what, and the ability of clients - # to retain cached objects in memory. - # - # If you set the value to 0, it means there are no limits, and Redis will - # retain as many keys as needed in the invalidation table. - # In the "stats" INFO section, you can find information about the number of - # keys in the invalidation table at every given moment. - # - # Note: when key tracking is used in broadcasting mode, no memory is used - # in the server side so this setting is useless. - # - # tracking-table-max-keys 1000000 - - ################################## SECURITY ################################### - - # Warning: since Redis is pretty fast, an outside user can try up to - # 1 million passwords per second against a modern box. This means that you - # should use very strong passwords, otherwise they will be very easy to break. - # Note that because the password is really a shared secret between the client - # and the server, and should not be memorized by any human, the password - # can be easily a long string from /dev/urandom or whatever, so by using a - # long and unguessable password no brute force attack will be possible. - - # Redis ACL users are defined in the following format: - # - # user ... acl rules ... - # - # For example: - # - # user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 - # - # The special username "default" is used for new connections. If this user - # has the "nopass" rule, then new connections will be immediately authenticated - # as the "default" user without the need of any password provided via the - # AUTH command. Otherwise if the "default" user is not flagged with "nopass" - # the connections will start in not authenticated state, and will require - # AUTH (or the HELLO command AUTH option) in order to be authenticated and - # start to work. - # - # The ACL rules that describe what a user can do are the following: - # - # on Enable the user: it is possible to authenticate as this user. - # off Disable the user: it's no longer possible to authenticate - # with this user, however the already authenticated connections - # will still work. - # skip-sanitize-payload RESTORE dump-payload sanitation is skipped. - # sanitize-payload RESTORE dump-payload is sanitized (default). - # + Allow the execution of that command - # - Disallow the execution of that command - # +@ Allow the execution of all the commands in such category - # with valid categories are like @admin, @set, @sortedset, ... - # and so forth, see the full list in the server.c file where - # the Redis command table is described and defined. - # The special category @all means all the commands, but currently - # present in the server, and that will be loaded in the future - # via modules. - # +|subcommand Allow a specific subcommand of an otherwise - # disabled command. Note that this form is not - # allowed as negative like -DEBUG|SEGFAULT, but - # only additive starting with "+". - # allcommands Alias for +@all. Note that it implies the ability to execute - # all the future commands loaded via the modules system. - # nocommands Alias for -@all. - # ~ Add a pattern of keys that can be mentioned as part of - # commands. For instance ~* allows all the keys. The pattern - # is a glob-style pattern like the one of KEYS. - # It is possible to specify multiple patterns. - # allkeys Alias for ~* - # resetkeys Flush the list of allowed keys patterns. - # & Add a glob-style pattern of Pub/Sub channels that can be - # accessed by the user. It is possible to specify multiple channel - # patterns. - # allchannels Alias for &* - # resetchannels Flush the list of allowed channel patterns. - # > Add this password to the list of valid password for the user. - # For example >mypass will add "mypass" to the list. - # This directive clears the "nopass" flag (see later). - # < Remove this password from the list of valid passwords. - # nopass All the set passwords of the user are removed, and the user - # is flagged as requiring no password: it means that every - # password will work against this user. If this directive is - # used for the default user, every new connection will be - # immediately authenticated with the default user without - # any explicit AUTH command required. Note that the "resetpass" - # directive will clear this condition. - # resetpass Flush the list of allowed passwords. Moreover removes the - # "nopass" status. After "resetpass" the user has no associated - # passwords and there is no way to authenticate without adding - # some password (or setting it as "nopass" later). - # reset Performs the following actions: resetpass, resetkeys, off, - # -@all. The user returns to the same state it has immediately - # after its creation. - # - # ACL rules can be specified in any order: for instance you can start with - # passwords, then flags, or key patterns. However note that the additive - # and subtractive rules will CHANGE MEANING depending on the ordering. - # For instance see the following example: - # - # user alice on +@all -DEBUG ~* >somepassword - # - # This will allow "alice" to use all the commands with the exception of the - # DEBUG command, since +@all added all the commands to the set of the commands - # alice can use, and later DEBUG was removed. However if we invert the order - # of two ACL rules the result will be different: - # - # user alice on -DEBUG +@all ~* >somepassword - # - # Now DEBUG was removed when alice had yet no commands in the set of allowed - # commands, later all the commands are added, so the user will be able to - # execute everything. - # - # Basically ACL rules are processed left-to-right. - # - # For more information about ACL configuration please refer to - # the Redis web site at https://redis.io/topics/acl - - # ACL LOG - # - # The ACL Log tracks failed commands and authentication events associated - # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked - # by ACLs. The ACL Log is stored in memory. You can reclaim memory with - # ACL LOG RESET. Define the maximum entry length of the ACL Log below. - acllog-max-len 128 - - # Using an external ACL file - # - # Instead of configuring users here in this file, it is possible to use - # a stand-alone file just listing users. The two methods cannot be mixed: - # if you configure users here and at the same time you activate the external - # ACL file, the server will refuse to start. - # - # The format of the external ACL user file is exactly the same as the - # format that is used inside redis.conf to describe users. - # - # aclfile /etc/redis/users.acl - - # IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility - # layer on top of the new ACL system. The option effect will be just setting - # the password for the default user. Clients will still authenticate using - # AUTH as usually, or more explicitly with AUTH default - # if they follow the new protocol: both will work. - # - # The requirepass is not compatable with aclfile option and the ACL LOAD - # command, these will cause requirepass to be ignored. - # - # requirepass foobared - - # New users are initialized with restrictive permissions by default, via the - # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it - # is possible to manage access to Pub/Sub channels with ACL rules as well. The - # default Pub/Sub channels permission if new users is controlled by the - # acl-pubsub-default configuration directive, which accepts one of these values: - # - # allchannels: grants access to all Pub/Sub channels - # resetchannels: revokes access to all Pub/Sub channels - # - # To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default - # defaults to the 'allchannels' permission. - # - # Future compatibility note: it is very likely that in a future version of Redis - # the directive's default of 'allchannels' will be changed to 'resetchannels' in - # order to provide better out-of-the-box Pub/Sub security. Therefore, it is - # recommended that you explicitly define Pub/Sub permissions for all users - # rather then rely on implicit default values. Once you've set explicit - # Pub/Sub for all existing users, you should uncomment the following line. - # - # acl-pubsub-default resetchannels - - # Command renaming (DEPRECATED). - # - # ------------------------------------------------------------------------ - # WARNING: avoid using this option if possible. Instead use ACLs to remove - # commands from the default user, and put them only in some admin user you - # create for administrative purposes. - # ------------------------------------------------------------------------ - # - # It is possible to change the name of dangerous commands in a shared - # environment. For instance the CONFIG command may be renamed into something - # hard to guess so that it will still be available for internal-use tools - # but not available for general clients. - # - # Example: - # - # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 - # - # It is also possible to completely kill a command by renaming it into - # an empty string: - # - # rename-command CONFIG "" - # - # Please note that changing the name of commands that are logged into the - # AOF file or transmitted to replicas may cause problems. - - ################################### CLIENTS #################################### - - # Set the max number of connected clients at the same time. By default - # this limit is set to 10000 clients, however if the Redis server is not - # able to configure the process file limit to allow for the specified limit - # the max number of allowed clients is set to the current file limit - # minus 32 (as Redis reserves a few file descriptors for internal uses). - # - # Once the limit is reached Redis will close all the new connections sending - # an error 'max number of clients reached'. - # - # IMPORTANT: When Redis Cluster is used, the max number of connections is also - # shared with the cluster bus: every node in the cluster will use two - # connections, one incoming and another outgoing. It is important to size the - # limit accordingly in case of very large clusters. - # - # maxclients 10000 - - ############################## MEMORY MANAGEMENT ################################ - - # Set a memory usage limit to the specified amount of bytes. - # When the memory limit is reached Redis will try to remove keys - # according to the eviction policy selected (see maxmemory-policy). - # - # If Redis can't remove keys according to the policy, or if the policy is - # set to 'noeviction', Redis will start to reply with errors to commands - # that would use more memory, like SET, LPUSH, and so on, and will continue - # to reply to read-only commands like GET. - # - # This option is usually useful when using Redis as an LRU or LFU cache, or to - # set a hard memory limit for an instance (using the 'noeviction' policy). - # - # WARNING: If you have replicas attached to an instance with maxmemory on, - # the size of the output buffers needed to feed the replicas are subtracted - # from the used memory count, so that network problems / resyncs will - # not trigger a loop where keys are evicted, and in turn the output - # buffer of replicas is full with DELs of keys evicted triggering the deletion - # of more keys, and so forth until the database is completely emptied. - # - # In short... if you have replicas attached it is suggested that you set a lower - # limit for maxmemory so that there is some free RAM on the system for replica - # output buffers (but this is not needed if the policy is 'noeviction'). - # - # maxmemory - - # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory - # is reached. You can select one from the following behaviors: - # - # volatile-lru -> Evict using approximated LRU, only keys with an expire set. - # allkeys-lru -> Evict any key using approximated LRU. - # volatile-lfu -> Evict using approximated LFU, only keys with an expire set. - # allkeys-lfu -> Evict any key using approximated LFU. - # volatile-random -> Remove a random key having an expire set. - # allkeys-random -> Remove a random key, any key. - # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) - # noeviction -> Don't evict anything, just return an error on write operations. - # - # LRU means Least Recently Used - # LFU means Least Frequently Used - # - # Both LRU, LFU and volatile-ttl are implemented using approximated - # randomized algorithms. - # - # Note: with any of the above policies, when there are no suitable keys for - # eviction, Redis will return an error on write operations that require - # more memory. These are usually commands that create new keys, add data or - # modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, - # SORT (due to the STORE argument), and EXEC (if the transaction includes any - # command that requires memory). - # - # The default is: - # - # maxmemory-policy noeviction - - # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated - # algorithms (in order to save memory), so you can tune it for speed or - # accuracy. By default Redis will check five keys and pick the one that was - # used least recently, you can change the sample size using the following - # configuration directive. - # - # The default of 5 produces good enough results. 10 Approximates very closely - # true LRU but costs more CPU. 3 is faster but not very accurate. - # - # maxmemory-samples 5 - - # Eviction processing is designed to function well with the default setting. - # If there is an unusually large amount of write traffic, this value may need to - # be increased. Decreasing this value may reduce latency at the risk of - # eviction processing effectiveness - # 0 = minimum latency, 10 = default, 100 = process without regard to latency - # - # maxmemory-eviction-tenacity 10 - - # Starting from Redis 5, by default a replica will ignore its maxmemory setting - # (unless it is promoted to master after a failover or manually). It means - # that the eviction of keys will be just handled by the master, sending the - # DEL commands to the replica as keys evict in the master side. - # - # This behavior ensures that masters and replicas stay consistent, and is usually - # what you want, however if your replica is writable, or you want the replica - # to have a different memory setting, and you are sure all the writes performed - # to the replica are idempotent, then you may change this default (but be sure - # to understand what you are doing). - # - # Note that since the replica by default does not evict, it may end using more - # memory than the one set via maxmemory (there are certain buffers that may - # be larger on the replica, or data structures may sometimes take more memory - # and so forth). So make sure you monitor your replicas and make sure they - # have enough memory to never hit a real out-of-memory condition before the - # master hits the configured maxmemory setting. - # - # replica-ignore-maxmemory yes - - # Redis reclaims expired keys in two ways: upon access when those keys are - # found to be expired, and also in background, in what is called the - # "active expire key". The key space is slowly and interactively scanned - # looking for expired keys to reclaim, so that it is possible to free memory - # of keys that are expired and will never be accessed again in a short time. - # - # The default effort of the expire cycle will try to avoid having more than - # ten percent of expired keys still in memory, and will try to avoid consuming - # more than 25% of total memory and to add latency to the system. However - # it is possible to increase the expire "effort" that is normally set to - # "1", to a greater value, up to the value "10". At its maximum value the - # system will use more CPU, longer cycles (and technically may introduce - # more latency), and will tolerate less already expired keys still present - # in the system. It's a tradeoff between memory, CPU and latency. - # - # active-expire-effort 1 - - ############################# LAZY FREEING #################################### - - # Redis has two primitives to delete keys. One is called DEL and is a blocking - # deletion of the object. It means that the server stops processing new commands - # in order to reclaim all the memory associated with an object in a synchronous - # way. If the key deleted is associated with a small object, the time needed - # in order to execute the DEL command is very small and comparable to most other - # O(1) or O(log_N) commands in Redis. However if the key is associated with an - # aggregated value containing millions of elements, the server can block for - # a long time (even seconds) in order to complete the operation. - # - # For the above reasons Redis also offers non blocking deletion primitives - # such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and - # FLUSHDB commands, in order to reclaim memory in background. Those commands - # are executed in constant time. Another thread will incrementally free the - # object in the background as fast as possible. - # - # DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. - # It's up to the design of the application to understand when it is a good - # idea to use one or the other. However the Redis server sometimes has to - # delete keys or flush the whole database as a side effect of other operations. - # Specifically Redis deletes objects independently of a user call in the - # following scenarios: - # - # 1) On eviction, because of the maxmemory and maxmemory policy configurations, - # in order to make room for new data, without going over the specified - # memory limit. - # 2) Because of expire: when a key with an associated time to live (see the - # EXPIRE command) must be deleted from memory. - # 3) Because of a side effect of a command that stores data on a key that may - # already exist. For example the RENAME command may delete the old key - # content when it is replaced with another one. Similarly SUNIONSTORE - # or SORT with STORE option may delete existing keys. The SET command - # itself removes any old content of the specified key in order to replace - # it with the specified string. - # 4) During replication, when a replica performs a full resynchronization with - # its master, the content of the whole database is removed in order to - # load the RDB file just transferred. - # - # In all the above cases the default is to delete objects in a blocking way, - # like if DEL was called. However you can configure each case specifically - # in order to instead release memory in a non-blocking way like if UNLINK - # was called, using the following configuration directives. - - lazyfree-lazy-eviction no - lazyfree-lazy-expire no - lazyfree-lazy-server-del no - replica-lazy-flush no - - # It is also possible, for the case when to replace the user code DEL calls - # with UNLINK calls is not easy, to modify the default behavior of the DEL - # command to act exactly like UNLINK, using the following configuration - # directive: - - lazyfree-lazy-user-del no - - # FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous - # deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the - # commands. When neither flag is passed, this directive will be used to determine - # if the data should be deleted asynchronously. - - lazyfree-lazy-user-flush no - - ################################ THREADED I/O ################################# - - # Redis is mostly single threaded, however there are certain threaded - # operations such as UNLINK, slow I/O accesses and other things that are - # performed on side threads. - # - # Now it is also possible to handle Redis clients socket reads and writes - # in different I/O threads. Since especially writing is so slow, normally - # Redis users use pipelining in order to speed up the Redis performances per - # core, and spawn multiple instances in order to scale more. Using I/O - # threads it is possible to easily speedup two times Redis without resorting - # to pipelining nor sharding of the instance. - # - # By default threading is disabled, we suggest enabling it only in machines - # that have at least 4 or more cores, leaving at least one spare core. - # Using more than 8 threads is unlikely to help much. We also recommend using - # threaded I/O only if you actually have performance problems, with Redis - # instances being able to use a quite big percentage of CPU time, otherwise - # there is no point in using this feature. - # - # So for instance if you have a four cores boxes, try to use 2 or 3 I/O - # threads, if you have a 8 cores, try to use 6 threads. In order to - # enable I/O threads use the following configuration directive: - # - # io-threads 4 - # - # Setting io-threads to 1 will just use the main thread as usual. - # When I/O threads are enabled, we only use threads for writes, that is - # to thread the write(2) syscall and transfer the client buffers to the - # socket. However it is also possible to enable threading of reads and - # protocol parsing using the following configuration directive, by setting - # it to yes: - # - # io-threads-do-reads no - # - # Usually threading reads doesn't help much. - # - # NOTE 1: This configuration directive cannot be changed at runtime via - # CONFIG SET. Aso this feature currently does not work when SSL is - # enabled. - # - # NOTE 2: If you want to test the Redis speedup using redis-benchmark, make - # sure you also run the benchmark itself in threaded mode, using the - # --threads option to match the number of Redis threads, otherwise you'll not - # be able to notice the improvements. - - ############################ KERNEL OOM CONTROL ############################## - - # On Linux, it is possible to hint the kernel OOM killer on what processes - # should be killed first when out of memory. - # - # Enabling this feature makes Redis actively control the oom_score_adj value - # for all its processes, depending on their role. The default scores will - # attempt to have background child processes killed before all others, and - # replicas killed before masters. - # - # Redis supports three options: - # - # no: Don't make changes to oom-score-adj (default). - # yes: Alias to "relative" see below. - # absolute: Values in oom-score-adj-values are written as is to the kernel. - # relative: Values are used relative to the initial value of oom_score_adj when - # the server starts and are then clamped to a range of -1000 to 1000. - # Because typically the initial value is 0, they will often match the - # absolute values. - oom-score-adj no - - # When oom-score-adj is used, this directive controls the specific values used - # for master, replica and background child processes. Values range -2000 to - # 2000 (higher means more likely to be killed). - # - # Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) - # can freely increase their value, but not decrease it below its initial - # settings. This means that setting oom-score-adj to "relative" and setting the - # oom-score-adj-values to positive values will always succeed. - oom-score-adj-values 0 200 800 - - - #################### KERNEL transparent hugepage CONTROL ###################### - - # Usually the kernel Transparent Huge Pages control is set to "madvise" or - # or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which - # case this config has no effect. On systems in which it is set to "always", - # redis will attempt to disable it specifically for the redis process in order - # to avoid latency problems specifically with fork(2) and CoW. - # If for some reason you prefer to keep it enabled, you can set this config to - # "no" and the kernel global to "always". - - disable-thp yes - - ############################## APPEND ONLY MODE ############################### - - # By default Redis asynchronously dumps the dataset on disk. This mode is - # good enough in many applications, but an issue with the Redis process or - # a power outage may result into a few minutes of writes lost (depending on - # the configured save points). - # - # The Append Only File is an alternative persistence mode that provides - # much better durability. For instance using the default data fsync policy - # (see later in the config file) Redis can lose just one second of writes in a - # dramatic event like a server power outage, or a single write if something - # wrong with the Redis process itself happens, but the operating system is - # still running correctly. - # - # AOF and RDB persistence can be enabled at the same time without problems. - # If the AOF is enabled on startup Redis will load the AOF, that is the file - # with the better durability guarantees. - # - # Please check https://redis.io/topics/persistence for more information. - - appendonly no - - # The name of the append only file (default: "appendonly.aof") - - appendfilename "appendonly.aof" - - # The fsync() call tells the Operating System to actually write data on disk - # instead of waiting for more data in the output buffer. Some OS will really flush - # data on disk, some other OS will just try to do it ASAP. - # - # Redis supports three different modes: - # - # no: don't fsync, just let the OS flush the data when it wants. Faster. - # always: fsync after every write to the append only log. Slow, Safest. - # everysec: fsync only one time every second. Compromise. - # - # The default is "everysec", as that's usually the right compromise between - # speed and data safety. It's up to you to understand if you can relax this to - # "no" that will let the operating system flush the output buffer when - # it wants, for better performances (but if you can live with the idea of - # some data loss consider the default persistence mode that's snapshotting), - # or on the contrary, use "always" that's very slow but a bit safer than - # everysec. - # - # More details please check the following article: - # http://antirez.com/post/redis-persistence-demystified.html - # - # If unsure, use "everysec". - - # appendfsync always - appendfsync everysec - # appendfsync no - - # When the AOF fsync policy is set to always or everysec, and a background - # saving process (a background save or AOF log background rewriting) is - # performing a lot of I/O against the disk, in some Linux configurations - # Redis may block too long on the fsync() call. Note that there is no fix for - # this currently, as even performing fsync in a different thread will block - # our synchronous write(2) call. - # - # In order to mitigate this problem it's possible to use the following option - # that will prevent fsync() from being called in the main process while a - # BGSAVE or BGREWRITEAOF is in progress. - # - # This means that while another child is saving, the durability of Redis is - # the same as "appendfsync none". In practical terms, this means that it is - # possible to lose up to 30 seconds of log in the worst scenario (with the - # default Linux settings). - # - # If you have latency problems turn this to "yes". Otherwise leave it as - # "no" that is the safest pick from the point of view of durability. - - no-appendfsync-on-rewrite no - - # Automatic rewrite of the append only file. - # Redis is able to automatically rewrite the log file implicitly calling - # BGREWRITEAOF when the AOF log size grows by the specified percentage. - # - # This is how it works: Redis remembers the size of the AOF file after the - # latest rewrite (if no rewrite has happened since the restart, the size of - # the AOF at startup is used). - # - # This base size is compared to the current size. If the current size is - # bigger than the specified percentage, the rewrite is triggered. Also - # you need to specify a minimal size for the AOF file to be rewritten, this - # is useful to avoid rewriting the AOF file even if the percentage increase - # is reached but it is still pretty small. - # - # Specify a percentage of zero in order to disable the automatic AOF - # rewrite feature. - - auto-aof-rewrite-percentage 100 - auto-aof-rewrite-min-size 64mb - - # An AOF file may be found to be truncated at the end during the Redis - # startup process, when the AOF data gets loaded back into memory. - # This may happen when the system where Redis is running - # crashes, especially when an ext4 filesystem is mounted without the - # data=ordered option (however this can't happen when Redis itself - # crashes or aborts but the operating system still works correctly). - # - # Redis can either exit with an error when this happens, or load as much - # data as possible (the default now) and start if the AOF file is found - # to be truncated at the end. The following option controls this behavior. - # - # If aof-load-truncated is set to yes, a truncated AOF file is loaded and - # the Redis server starts emitting a log to inform the user of the event. - # Otherwise if the option is set to no, the server aborts with an error - # and refuses to start. When the option is set to no, the user requires - # to fix the AOF file using the "redis-check-aof" utility before to restart - # the server. - # - # Note that if the AOF file will be found to be corrupted in the middle - # the server will still exit with an error. This option only applies when - # Redis will try to read more data from the AOF file but not enough bytes - # will be found. - aof-load-truncated yes - - # When rewriting the AOF file, Redis is able to use an RDB preamble in the - # AOF file for faster rewrites and recoveries. When this option is turned - # on the rewritten AOF file is composed of two different stanzas: - # - # [RDB file][AOF tail] - # - # When loading, Redis recognizes that the AOF file starts with the "REDIS" - # string and loads the prefixed RDB file, then continues loading the AOF - # tail. - aof-use-rdb-preamble yes - - ################################ LUA SCRIPTING ############################### - - # Max execution time of a Lua script in milliseconds. - # - # If the maximum execution time is reached Redis will log that a script is - # still in execution after the maximum allowed time and will start to - # reply to queries with an error. - # - # When a long running script exceeds the maximum execution time only the - # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be - # used to stop a script that did not yet call any write commands. The second - # is the only way to shut down the server in the case a write command was - # already issued by the script but the user doesn't want to wait for the natural - # termination of the script. - # - # Set it to 0 or a negative value for unlimited execution without warnings. - lua-time-limit 5000 - - ################################ REDIS CLUSTER ############################### - - # Normal Redis instances can't be part of a Redis Cluster; only nodes that are - # started as cluster nodes can. In order to start a Redis instance as a - # cluster node enable the cluster support uncommenting the following: - # - # cluster-enabled yes - - # Every cluster node has a cluster configuration file. This file is not - # intended to be edited by hand. It is created and updated by Redis nodes. - # Every Redis Cluster node requires a different cluster configuration file. - # Make sure that instances running in the same system do not have - # overlapping cluster configuration file names. - # - # cluster-config-file nodes-6379.conf - - # Cluster node timeout is the amount of milliseconds a node must be unreachable - # for it to be considered in failure state. - # Most other internal time limits are a multiple of the node timeout. - # - # cluster-node-timeout 15000 - - # A replica of a failing master will avoid to start a failover if its data - # looks too old. - # - # There is no simple way for a replica to actually have an exact measure of - # its "data age", so the following two checks are performed: - # - # 1) If there are multiple replicas able to failover, they exchange messages - # in order to try to give an advantage to the replica with the best - # replication offset (more data from the master processed). - # Replicas will try to get their rank by offset, and apply to the start - # of the failover a delay proportional to their rank. - # - # 2) Every single replica computes the time of the last interaction with - # its master. This can be the last ping or command received (if the master - # is still in the "connected" state), or the time that elapsed since the - # disconnection with the master (if the replication link is currently down). - # If the last interaction is too old, the replica will not try to failover - # at all. - # - # The point "2" can be tuned by user. Specifically a replica will not perform - # the failover if, since the last interaction with the master, the time - # elapsed is greater than: - # - # (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period - # - # So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor - # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the - # replica will not try to failover if it was not able to talk with the master - # for longer than 310 seconds. - # - # A large cluster-replica-validity-factor may allow replicas with too old data to failover - # a master, while a too small value may prevent the cluster from being able to - # elect a replica at all. - # - # For maximum availability, it is possible to set the cluster-replica-validity-factor - # to a value of 0, which means, that replicas will always try to failover the - # master regardless of the last time they interacted with the master. - # (However they'll always try to apply a delay proportional to their - # offset rank). - # - # Zero is the only value able to guarantee that when all the partitions heal - # the cluster will always be able to continue. - # - # cluster-replica-validity-factor 10 - - # Cluster replicas are able to migrate to orphaned masters, that are masters - # that are left without working replicas. This improves the cluster ability - # to resist to failures as otherwise an orphaned master can't be failed over - # in case of failure if it has no working replicas. - # - # Replicas migrate to orphaned masters only if there are still at least a - # given number of other working replicas for their old master. This number - # is the "migration barrier". A migration barrier of 1 means that a replica - # will migrate only if there is at least 1 other working replica for its master - # and so forth. It usually reflects the number of replicas you want for every - # master in your cluster. - # - # Default is 1 (replicas migrate only if their masters remain with at least - # one replica). To disable migration just set it to a very large value or - # set cluster-allow-replica-migration to 'no'. - # A value of 0 can be set but is useful only for debugging and dangerous - # in production. - # - # cluster-migration-barrier 1 - - # Turning off this option allows to use less automatic cluster configuration. - # It both disables migration to orphaned masters and migration from masters - # that became empty. - # - # Default is 'yes' (allow automatic migrations). - # - # cluster-allow-replica-migration yes - - # By default Redis Cluster nodes stop accepting queries if they detect there - # is at least a hash slot uncovered (no available node is serving it). - # This way if the cluster is partially down (for example a range of hash slots - # are no longer covered) all the cluster becomes, eventually, unavailable. - # It automatically returns available as soon as all the slots are covered again. - # - # However sometimes you want the subset of the cluster which is working, - # to continue to accept queries for the part of the key space that is still - # covered. In order to do so, just set the cluster-require-full-coverage - # option to no. - # - # cluster-require-full-coverage yes - - # This option, when set to yes, prevents replicas from trying to failover its - # master during master failures. However the replica can still perform a - # manual failover, if forced to do so. - # - # This is useful in different scenarios, especially in the case of multiple - # data center operations, where we want one side to never be promoted if not - # in the case of a total DC failure. - # - # cluster-replica-no-failover no - - # This option, when set to yes, allows nodes to serve read traffic while the - # the cluster is in a down state, as long as it believes it owns the slots. - # - # This is useful for two cases. The first case is for when an application - # doesn't require consistency of data during node failures or network partitions. - # One example of this is a cache, where as long as the node has the data it - # should be able to serve it. - # - # The second use case is for configurations that don't meet the recommended - # three shards but want to enable cluster mode and scale later. A - # master outage in a 1 or 2 shard configuration causes a read/write outage to the - # entire cluster without this option set, with it set there is only a write outage. - # Without a quorum of masters, slot ownership will not change automatically. - # - # cluster-allow-reads-when-down no - - # In order to setup your cluster make sure to read the documentation - # available at https://redis.io web site. - - ########################## CLUSTER DOCKER/NAT support ######################## - - # In certain deployments, Redis Cluster nodes address discovery fails, because - # addresses are NAT-ted or because ports are forwarded (the typical case is - # Docker and other containers). - # - # In order to make Redis Cluster working in such environments, a static - # configuration where each node knows its public address is needed. The - # following four options are used for this scope, and are: - # - # * cluster-announce-ip - # * cluster-announce-port - # * cluster-announce-tls-port - # * cluster-announce-bus-port - # - # Each instructs the node about its address, client ports (for connections - # without and with TLS) and cluster message bus port. The information is then - # published in the header of the bus packets so that other nodes will be able to - # correctly map the address of the node publishing the information. - # - # If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set - # to zero, then cluster-announce-port refers to the TLS port. Note also that - # cluster-announce-tls-port has no effect if cluster-tls is set to no. - # - # If the above options are not used, the normal Redis Cluster auto-detection - # will be used instead. - # - # Note that when remapped, the bus port may not be at the fixed offset of - # clients port + 10000, so you can specify any port and bus-port depending - # on how they get remapped. If the bus-port is not set, a fixed offset of - # 10000 will be used as usual. - # - # Example: - # - # cluster-announce-ip 10.1.1.5 - # cluster-announce-tls-port 6379 - # cluster-announce-port 0 - # cluster-announce-bus-port 6380 - - ################################## SLOW LOG ################################### - - # The Redis Slow Log is a system to log queries that exceeded a specified - # execution time. The execution time does not include the I/O operations - # like talking with the client, sending the reply and so forth, - # but just the time needed to actually execute the command (this is the only - # stage of command execution where the thread is blocked and can not serve - # other requests in the meantime). - # - # You can configure the slow log with two parameters: one tells Redis - # what is the execution time, in microseconds, to exceed in order for the - # command to get logged, and the other parameter is the length of the - # slow log. When a new command is logged the oldest one is removed from the - # queue of logged commands. - - # The following time is expressed in microseconds, so 1000000 is equivalent - # to one second. Note that a negative number disables the slow log, while - # a value of zero forces the logging of every command. - slowlog-log-slower-than 10000 - - # There is no limit to this length. Just be aware that it will consume memory. - # You can reclaim memory used by the slow log with SLOWLOG RESET. - slowlog-max-len 128 - - ################################ LATENCY MONITOR ############################## - - # The Redis latency monitoring subsystem samples different operations - # at runtime in order to collect data related to possible sources of - # latency of a Redis instance. - # - # Via the LATENCY command this information is available to the user that can - # print graphs and obtain reports. - # - # The system only logs operations that were performed in a time equal or - # greater than the amount of milliseconds specified via the - # latency-monitor-threshold configuration directive. When its value is set - # to zero, the latency monitor is turned off. - # - # By default latency monitoring is disabled since it is mostly not needed - # if you don't have latency issues, and collecting data has a performance - # impact, that while very small, can be measured under big load. Latency - # monitoring can easily be enabled at runtime using the command - # "CONFIG SET latency-monitor-threshold " if needed. - latency-monitor-threshold 0 - - ############################# EVENT NOTIFICATION ############################## - - # Redis can notify Pub/Sub clients about events happening in the key space. - # This feature is documented at https://redis.io/topics/notifications - # - # For instance if keyspace events notification is enabled, and a client - # performs a DEL operation on key "foo" stored in the Database 0, two - # messages will be published via Pub/Sub: - # - # PUBLISH __keyspace@0__:foo del - # PUBLISH __keyevent@0__:del foo - # - # It is possible to select the events that Redis will notify among a set - # of classes. Every class is identified by a single character: - # - # K Keyspace events, published with __keyspace@__ prefix. - # E Keyevent events, published with __keyevent@__ prefix. - # g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... - # $ String commands - # l List commands - # s Set commands - # h Hash commands - # z Sorted set commands - # x Expired events (events generated every time a key expires) - # e Evicted events (events generated when a key is evicted for maxmemory) - # t Stream commands - # d Module key type events - # m Key-miss events (Note: It is not included in the 'A' class) - # A Alias for g$lshzxetd, so that the "AKE" string means all the events - # (Except key-miss events which are excluded from 'A' due to their - # unique nature). - # - # The "notify-keyspace-events" takes as argument a string that is composed - # of zero or multiple characters. The empty string means that notifications - # are disabled. - # - # Example: to enable list and generic events, from the point of view of the - # event name, use: - # - # notify-keyspace-events Elg - # - # Example 2: to get the stream of the expired keys subscribing to channel - # name __keyevent@0__:expired use: - # - # notify-keyspace-events Ex - # - # By default all notifications are disabled because most users don't need - # this feature and the feature has some overhead. Note that if you don't - # specify at least one of K or E, no events will be delivered. - notify-keyspace-events "" - - ############################### GOPHER SERVER ################################# - - # Redis contains an implementation of the Gopher protocol, as specified in - # the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). - # - # The Gopher protocol was very popular in the late '90s. It is an alternative - # to the web, and the implementation both server and client side is so simple - # that the Redis server has just 100 lines of code in order to implement this - # support. - # - # What do you do with Gopher nowadays? Well Gopher never *really* died, and - # lately there is a movement in order for the Gopher more hierarchical content - # composed of just plain text documents to be resurrected. Some want a simpler - # internet, others believe that the mainstream internet became too much - # controlled, and it's cool to create an alternative space for people that - # want a bit of fresh air. - # - # Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol - # as a gift. - # - # --- HOW IT WORKS? --- - # - # The Redis Gopher support uses the inline protocol of Redis, and specifically - # two kind of inline requests that were anyway illegal: an empty request - # or any request that starts with "/" (there are no Redis commands starting - # with such a slash). Normal RESP2/RESP3 requests are completely out of the - # path of the Gopher protocol implementation and are served as usual as well. - # - # If you open a connection to Redis when Gopher is enabled and send it - # a string like "/foo", if there is a key named "/foo" it is served via the - # Gopher protocol. - # - # In order to create a real Gopher "hole" (the name of a Gopher site in Gopher - # talking), you likely need a script like the following: - # - # https://github.com/antirez/gopher2redis - # - # --- SECURITY WARNING --- - # - # If you plan to put Redis on the internet in a publicly accessible address - # to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. - # Once a password is set: - # - # 1. The Gopher server (when enabled, not by default) will still serve - # content via Gopher. - # 2. However other commands cannot be called before the client will - # authenticate. - # - # So use the 'requirepass' option to protect your instance. - # - # Note that Gopher is not currently supported when 'io-threads-do-reads' - # is enabled. - # - # To enable Gopher support, uncomment the following line and set the option - # from no (the default) to yes. - # - # gopher-enabled no - - ############################### ADVANCED CONFIG ############################### - - # Hashes are encoded using a memory efficient data structure when they have a - # small number of entries, and the biggest entry does not exceed a given - # threshold. These thresholds can be configured using the following directives. - hash-max-ziplist-entries 512 - hash-max-ziplist-value 64 - - # Lists are also encoded in a special way to save a lot of space. - # The number of entries allowed per internal list node can be specified - # as a fixed maximum size or a maximum number of elements. - # For a fixed maximum size, use -5 through -1, meaning: - # -5: max size: 64 Kb <-- not recommended for normal workloads - # -4: max size: 32 Kb <-- not recommended - # -3: max size: 16 Kb <-- probably not recommended - # -2: max size: 8 Kb <-- good - # -1: max size: 4 Kb <-- good - # Positive numbers mean store up to _exactly_ that number of elements - # per list node. - # The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), - # but if your use case is unique, adjust the settings as necessary. - list-max-ziplist-size -2 - - # Lists may also be compressed. - # Compress depth is the number of quicklist ziplist nodes from *each* side of - # the list to *exclude* from compression. The head and tail of the list - # are always uncompressed for fast push/pop operations. Settings are: - # 0: disable all list compression - # 1: depth 1 means "don't start compressing until after 1 node into the list, - # going from either the head or tail" - # So: [head]->node->node->...->node->[tail] - # [head], [tail] will always be uncompressed; inner nodes will compress. - # 2: [head]->[next]->node->node->...->node->[prev]->[tail] - # 2 here means: don't compress head or head->next or tail->prev or tail, - # but compress all nodes between them. - # 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] - # etc. - list-compress-depth 0 - - # Sets have a special encoding in just one case: when a set is composed - # of just strings that happen to be integers in radix 10 in the range - # of 64 bit signed integers. - # The following configuration setting sets the limit in the size of the - # set in order to use this special memory saving encoding. - set-max-intset-entries 512 - - # Similarly to hashes and lists, sorted sets are also specially encoded in - # order to save a lot of space. This encoding is only used when the length and - # elements of a sorted set are below the following limits: - zset-max-ziplist-entries 128 - zset-max-ziplist-value 64 - - # HyperLogLog sparse representation bytes limit. The limit includes the - # 16 bytes header. When an HyperLogLog using the sparse representation crosses - # this limit, it is converted into the dense representation. - # - # A value greater than 16000 is totally useless, since at that point the - # dense representation is more memory efficient. - # - # The suggested value is ~ 3000 in order to have the benefits of - # the space efficient encoding without slowing down too much PFADD, - # which is O(N) with the sparse encoding. The value can be raised to - # ~ 10000 when CPU is not a concern, but space is, and the data set is - # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. - hll-sparse-max-bytes 3000 - - # Streams macro node max size / items. The stream data structure is a radix - # tree of big nodes that encode multiple items inside. Using this configuration - # it is possible to configure how big a single node can be in bytes, and the - # maximum number of items it may contain before switching to a new node when - # appending new stream entries. If any of the following settings are set to - # zero, the limit is ignored, so for instance it is possible to set just a - # max entries limit by setting max-bytes to 0 and max-entries to the desired - # value. - stream-node-max-bytes 4096 - stream-node-max-entries 100 - - # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in - # order to help rehashing the main Redis hash table (the one mapping top-level - # keys to values). The hash table implementation Redis uses (see dict.c) - # performs a lazy rehashing: the more operation you run into a hash table - # that is rehashing, the more rehashing "steps" are performed, so if the - # server is idle the rehashing is never complete and some more memory is used - # by the hash table. - # - # The default is to use this millisecond 10 times every second in order to - # actively rehash the main dictionaries, freeing memory when possible. - # - # If unsure: - # use "activerehashing no" if you have hard latency requirements and it is - # not a good thing in your environment that Redis can reply from time to time - # to queries with 2 milliseconds delay. - # - # use "activerehashing yes" if you don't have such hard requirements but - # want to free memory asap when possible. - activerehashing yes - - # The client output buffer limits can be used to force disconnection of clients - # that are not reading data from the server fast enough for some reason (a - # common reason is that a Pub/Sub client can't consume messages as fast as the - # publisher can produce them). - # - # The limit can be set differently for the three different classes of clients: - # - # normal -> normal clients including MONITOR clients - # replica -> replica clients - # pubsub -> clients subscribed to at least one pubsub channel or pattern - # - # The syntax of every client-output-buffer-limit directive is the following: - # - # client-output-buffer-limit - # - # A client is immediately disconnected once the hard limit is reached, or if - # the soft limit is reached and remains reached for the specified number of - # seconds (continuously). - # So for instance if the hard limit is 32 megabytes and the soft limit is - # 16 megabytes / 10 seconds, the client will get disconnected immediately - # if the size of the output buffers reach 32 megabytes, but will also get - # disconnected if the client reaches 16 megabytes and continuously overcomes - # the limit for 10 seconds. - # - # By default normal clients are not limited because they don't receive data - # without asking (in a push way), but just after a request, so only - # asynchronous clients may create a scenario where data is requested faster - # than it can read. - # - # Instead there is a default limit for pubsub and replica clients, since - # subscribers and replicas receive data in a push fashion. - # - # Both the hard or the soft limit can be disabled by setting them to zero. - client-output-buffer-limit normal 0 0 0 - client-output-buffer-limit replica 256mb 64mb 60 - client-output-buffer-limit pubsub 32mb 8mb 60 - - # Client query buffers accumulate new commands. They are limited to a fixed - # amount by default in order to avoid that a protocol desynchronization (for - # instance due to a bug in the client) will lead to unbound memory usage in - # the query buffer. However you can configure it here if you have very special - # needs, such us huge multi/exec requests or alike. - # - # client-query-buffer-limit 1gb - - # In the Redis protocol, bulk requests, that are, elements representing single - # strings, are normally limited to 512 mb. However you can change this limit - # here, but must be 1mb or greater - # - # proto-max-bulk-len 512mb - - # Redis calls an internal function to perform many background tasks, like - # closing connections of clients in timeout, purging expired keys that are - # never requested, and so forth. - # - # Not all tasks are performed with the same frequency, but Redis checks for - # tasks to perform according to the specified "hz" value. - # - # By default "hz" is set to 10. Raising the value will use more CPU when - # Redis is idle, but at the same time will make Redis more responsive when - # there are many keys expiring at the same time, and timeouts may be - # handled with more precision. - # - # The range is between 1 and 500, however a value over 100 is usually not - # a good idea. Most users should use the default of 10 and raise this up to - # 100 only in environments where very low latency is required. - hz 10 - - # Normally it is useful to have an HZ value which is proportional to the - # number of clients connected. This is useful in order, for instance, to - # avoid too many clients are processed for each background task invocation - # in order to avoid latency spikes. - # - # Since the default HZ value by default is conservatively set to 10, Redis - # offers, and enables by default, the ability to use an adaptive HZ value - # which will temporarily raise when there are many connected clients. - # - # When dynamic HZ is enabled, the actual configured HZ will be used - # as a baseline, but multiples of the configured HZ value will be actually - # used as needed once more clients are connected. In this way an idle - # instance will use very little CPU time while a busy instance will be - # more responsive. - dynamic-hz yes - - # When a child rewrites the AOF file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - aof-rewrite-incremental-fsync yes - - # When redis saves RDB file, if the following option is enabled - # the file will be fsync-ed every 32 MB of data generated. This is useful - # in order to commit the file to the disk more incrementally and avoid - # big latency spikes. - rdb-save-incremental-fsync yes - - # Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good - # idea to start with the default settings and only change them after investigating - # how to improve the performances and how the keys LFU change over time, which - # is possible to inspect via the OBJECT FREQ command. - # - # There are two tunable parameters in the Redis LFU implementation: the - # counter logarithm factor and the counter decay time. It is important to - # understand what the two parameters mean before changing them. - # - # The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis - # uses a probabilistic increment with logarithmic behavior. Given the value - # of the old counter, when a key is accessed, the counter is incremented in - # this way: - # - # 1. A random number R between 0 and 1 is extracted. - # 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). - # 3. The counter is incremented only if R < P. - # - # The default lfu-log-factor is 10. This is a table of how the frequency - # counter changes with a different number of accesses with different - # logarithmic factors: - # - # +--------+------------+------------+------------+------------+------------+ - # | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | - # +--------+------------+------------+------------+------------+------------+ - # | 0 | 104 | 255 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 1 | 18 | 49 | 255 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 10 | 10 | 18 | 142 | 255 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # | 100 | 8 | 11 | 49 | 143 | 255 | - # +--------+------------+------------+------------+------------+------------+ - # - # NOTE: The above table was obtained by running the following commands: - # - # redis-benchmark -n 1000000 incr foo - # redis-cli object freq foo - # - # NOTE 2: The counter initial value is 5 in order to give new objects a chance - # to accumulate hits. - # - # The counter decay time is the time, in minutes, that must elapse in order - # for the key counter to be divided by two (or decremented if it has a value - # less <= 10). - # - # The default value for the lfu-decay-time is 1. A special value of 0 means to - # decay the counter every time it happens to be scanned. - # - # lfu-log-factor 10 - # lfu-decay-time 1 - - ########################### ACTIVE DEFRAGMENTATION ####################### - # - # What is active defragmentation? - # ------------------------------- - # - # Active (online) defragmentation allows a Redis server to compact the - # spaces left between small allocations and deallocations of data in memory, - # thus allowing to reclaim back memory. - # - # Fragmentation is a natural process that happens with every allocator (but - # less so with Jemalloc, fortunately) and certain workloads. Normally a server - # restart is needed in order to lower the fragmentation, or at least to flush - # away all the data and create it again. However thanks to this feature - # implemented by Oran Agra for Redis 4.0 this process can happen at runtime - # in a "hot" way, while the server is running. - # - # Basically when the fragmentation is over a certain level (see the - # configuration options below) Redis will start to create new copies of the - # values in contiguous memory regions by exploiting certain specific Jemalloc - # features (in order to understand if an allocation is causing fragmentation - # and to allocate it in a better place), and at the same time, will release the - # old copies of the data. This process, repeated incrementally for all the keys - # will cause the fragmentation to drop back to normal values. - # - # Important things to understand: - # - # 1. This feature is disabled by default, and only works if you compiled Redis - # to use the copy of Jemalloc we ship with the source code of Redis. - # This is the default with Linux builds. - # - # 2. You never need to enable this feature if you don't have fragmentation - # issues. - # - # 3. Once you experience fragmentation, you can enable this feature when - # needed with the command "CONFIG SET activedefrag yes". - # - # The configuration parameters are able to fine tune the behavior of the - # defragmentation process. If you are not sure about what they mean it is - # a good idea to leave the defaults untouched. - - # Enabled active defragmentation - # activedefrag no - - # Minimum amount of fragmentation waste to start active defrag - # active-defrag-ignore-bytes 100mb - - # Minimum percentage of fragmentation to start active defrag - # active-defrag-threshold-lower 10 - - # Maximum percentage of fragmentation at which we use maximum effort - # active-defrag-threshold-upper 100 - - # Minimal effort for defrag in CPU percentage, to be used when the lower - # threshold is reached - # active-defrag-cycle-min 1 - - # Maximal effort for defrag in CPU percentage, to be used when the upper - # threshold is reached - # active-defrag-cycle-max 25 - - # Maximum number of set/hash/zset/list fields that will be processed from - # the main dictionary scan - # active-defrag-max-scan-fields 1000 - - # Jemalloc background thread for purging will be enabled by default - jemalloc-bg-thread yes - - # It is possible to pin different threads and processes of Redis to specific - # CPUs in your system, in order to maximize the performances of the server. - # This is useful both in order to pin different Redis threads in different - # CPUs, but also in order to make sure that multiple Redis instances running - # in the same host will be pinned to different CPUs. - # - # Normally you can do this using the "taskset" command, however it is also - # possible to this via Redis configuration directly, both in Linux and FreeBSD. - # - # You can pin the server/IO threads, bio threads, aof rewrite child process, and - # the bgsave child process. The syntax to specify the cpu list is the same as - # the taskset command: - # - # Set redis server/io threads to cpu affinity 0,2,4,6: - # server_cpulist 0-7:2 - # - # Set bio threads to cpu affinity 1,3: - # bio_cpulist 1,3 - # - # Set aof rewrite child process to cpu affinity 8,9,10,11: - # aof_rewrite_cpulist 8-11 - # - # Set bgsave child process to cpu affinity 1,10,11 - # bgsave_cpulist 1,10-11 - - # In some cases redis will emit warnings and even refuse to start if it detects - # that the system is in bad state, it is possible to suppress these warnings - # by setting the following config which takes a space delimited list of warnings - # to suppress - # - # ignore-warnings ARM64-COW-BUG -kind: ConfigMap -metadata: - name: redis-config - namespace: default - ---- -apiVersion: v1 -data: - filer.toml: | - [redis] - enabled = true - address = "redis:6379" - password = "" - database = 15 -kind: ConfigMap -metadata: - name: seaweed-config-filer - namespace: default - ---- -apiVersion: v1 -data: - s3config.json: | - { - "identities": [ - { - "name": "admin", - "credentials": [ - { - "accessKey": "admin", - "secretKey": "admin" - } - ], - "actions": ["Admin", "Read", "List", "Tagging", "Write"] - } - ] - } -kind: ConfigMap -metadata: - name: seaweed-config-s3 - namespace: default - ---- - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "tailscale-headless" - labels: - "app.kubernetes.io/name": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "tailscale" - labels: - "app.kubernetes.io/name": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: proxy - port: 80 - targetPort: 80 - protocol: "TCP" - - name: api - port: 4000 - targetPort: 4000 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "tailscale" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "tailscale-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "traefik:v2.5" - name: "container-0" - command: - args: - env: - - name: SERVICE_NAME - value: proxy - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/etc/traefik" - name: "traefik-conf" - readOnly: false - - image: "openmined/grid-vpn-tailscale:latest" - name: "container-1" - command: - args: - env: - - name: HOSTNAME - value: node - - name: STACK_API_KEY - value: w9N59fxaSrb6Vl64mVHR3WVRTMZZQ7XYYTfiJ9GEUkPviQTq - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/var/lib/tailscale" - name: "tailscale-data" - subPath: "tailscale-data" - readOnly: false - initContainers: - volumes: - - name: "traefik-conf" - configMap: - name: traefik-main-config - volumeClaimTemplates: - - metadata: - name: "tailscale-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "tailscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "100Mi" - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "queue" - labels: - "app.kubernetes.io/name": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: ampq - port: 5672 - targetPort: 5672 - protocol: "TCP" - - name: management - port: 15672 - targetPort: 15672 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "queue" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "queue" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "rabbitmq:3" - name: "container-0" - command: - args: - env: null - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "redis-headless" - labels: - "app.kubernetes.io/name": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "redis" - labels: - "app.kubernetes.io/name": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 6379 - targetPort: 6379 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "redis" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "redis-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "redis:6.2" - name: "container-0" - command: - args: - env: - - name: SERVICE_NAME - value: redis - - name: RELEASE - value: production - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/data" - name: "app-redis-data" - subPath: "" - readOnly: false - - mountPath: "/usr/local/etc/redis" - name: "redis-conf" - readOnly: false - initContainers: - volumes: - - name: "redis-conf" - configMap: - name: redis-config - volumeClaimTemplates: - - metadata: - name: "app-redis-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "redis" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "5Gi" - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "db-headless" - labels: - "app.kubernetes.io/name": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "db" - labels: - "app.kubernetes.io/name": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 5432 - targetPort: 5432 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "db" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "db-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "postgres:12" - name: "container-0" - command: - args: - env: - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/var/lib/postgresql/data" - name: "app-db-data" - subPath: "pgdata" - readOnly: false - initContainers: - volumes: - volumeClaimTemplates: - - metadata: - name: "app-db-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "db" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "5Gi" - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "backend" - labels: - "app.kubernetes.io/name": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 80 - targetPort: 80 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - "/start.sh" - args: - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: LOG_LEVEL - value: debug - - name: NODE_TYPE - value: network - - name: STACK_API_KEY - value: w9N59fxaSrb6Vl64mVHR3WVRTMZZQ7XYYTfiJ9GEUkPviQTq - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: USE_BLOB_STORAGE - value: "False" - - name: SERVICE_NAME - value: backend - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "backend-stream" - labels: - "app.kubernetes.io/name": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: "port-0" - port: 80 - targetPort: 80 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend-stream" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-stream" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - /start.sh - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: LOG_LEVEL - value: debug - - name: STREAM_QUEUE - value: "1" - - name: NODE_TYPE - value: network - - name: STACK_API_KEY - value: w9N59fxaSrb6Vl64mVHR3WVRTMZZQ7XYYTfiJ9GEUkPviQTq - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: USE_BLOB_STORAGE - value: "False" - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /api/v1/status - port: 80 - initialDelaySeconds: 5 - periodSeconds: 60 - successThreshold: 1 - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "backend-worker" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "backend-worker" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-backend:latest" - name: "container-0" - command: - - /worker-start.sh - env: - - name: NODE_NAME - value: node - - name: POSTGRES_SERVER - value: db - - name: POSTGRES_DB - value: app - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: changethis - - name: VERSION - value: 0.7.0-beta.26 - - name: VERSION_HASH - value: 7c7332c307fe3121d6cc0e46c4242975d384ccc1 - - name: SERVER_HOST - value: https://localhost - - name: CELERY_WORKER - value: "true" - - name: RUN - value: celery -A grid.worker worker -l info -Q main-queue --pool=gevent -c 500 - - name: NODE_TYPE - value: network - - name: C_FORCE_ROOT - value: "1" - - name: STACK_API_KEY - value: w9N59fxaSrb6Vl64mVHR3WVRTMZZQ7XYYTfiJ9GEUkPviQTq - - name: CONTAINER_HOST - value: kubernetes - - name: JAX_ENABLE_X64 - value: "True" - - name: USE_NEW_SERVICE - value: "True" - - name: REDIS_STORE_DB_ID - value: "0" - - name: REDIS_LEDGER_DB_ID - value: "1" - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: "6379" - - name: S3_ENDPOINT - value: seaweedfs - - name: S3_PORT - value: "8333" - - name: S3_ROOT_USER - value: admin - - name: S3_ROOT_PWD - value: admin - - name: S3_REGION - value: us-east-1 - - name: S3_PRESIGNED_TIMEOUT_SECS - value: "1800" - - name: RELEASE - value: production - - name: NETWORK_CHECK_INTERVAL - value: "60" - - name: USE_BLOB_STORAGE - value: "False" - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet -apiVersion: v1 -kind: Service -metadata: - name: "headscale-headless" - labels: - "app.kubernetes.io/name": "headscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - ports: - # At least one port is required to have Kubernetes define an endpoint which is required for DNS to work - - port: 80 - name: web - clusterIP: None - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - "app.kubernetes.io/managed-by": "Helm" ---- -# Source: component-chart/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: "headscale" - labels: - "app.kubernetes.io/name": "headscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - externalIPs: - ports: - - name: vpn - port: 8080 - targetPort: 8080 - protocol: "TCP" - - name: api - port: 4000 - targetPort: 4000 - protocol: "TCP" - selector: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - type: "ClusterIP" ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: "headscale" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - podManagementPolicy: "OrderedReady" - serviceName: "headscale-headless" - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-vpn-headscale:latest" - name: "container-0" - command: - args: - env: - - name: NETWORK_NAME - value: omnet - - name: STACK_API_KEY - value: w9N59fxaSrb6Vl64mVHR3WVRTMZZQ7XYYTfiJ9GEUkPviQTq - envFrom: null - securityContext: null - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - - mountPath: "/headscale/data" - name: "headscale-data" - subPath: "headscale-data" - readOnly: false - initContainers: - volumes: - volumeClaimTemplates: - - metadata: - name: "headscale-data" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "headscale" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "100Mi" - ---- -# Source: component-chart/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "vpn-iptables" - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - template: - metadata: - labels: - "app.kubernetes.io/name": "devspace-app" - "app.kubernetes.io/component": "vpn-iptables" - "app.kubernetes.io/managed-by": "Helm" - annotations: - "helm.sh/chart": "component-chart-0.8.4" - spec: - imagePullSecrets: - nodeSelector: null - nodeName: null - affinity: null - tolerations: null - dnsConfig: null - hostAliases: null - overhead: null - readinessGates: null - securityContext: null - topologySpreadConstraints: null - hostNetwork: true - terminationGracePeriodSeconds: 5 - ephemeralContainers: null - containers: - - image: "openmined/grid-vpn-iptables:latest" - name: "container-0" - command: - - "sh" - - "-c" - - "/iptables.sh" - args: - env: null - envFrom: null - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - lifecycle: null - livenessProbe: null - readinessProbe: null - startupProbe: null - volumeDevices: null - volumeMounts: - resources: - requests: - cpu: "5m" - memory: "10Mi" - initContainers: - volumes: - volumeClaimTemplates: ---- -# Source: component-chart/templates/deployment.yaml -# Create headless service for StatefulSet - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - ingress.kubernetes.io/rewrite-target: / - name: grid-stack-ingress - namespace: default -spec: - rules: - - http: - paths: - - backend: - service: - name: tailscale - port: - number: 80 - path: / - pathType: Prefix - ---- - From da9a9ef94394282dc7df04da609d8fe1b83973fc Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Sun, 10 Dec 2023 20:15:22 +0530 Subject: [PATCH 47/47] updated values.yaml to new syft beta version --- packages/grid/helm/syft/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index e58186c5eb5..622ce322fcb 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -35,7 +35,7 @@ seaweedfs: container: registry: "docker.io" - syft_version: 0.8.4-beta.1 + syft_version: 0.8.4-beta.2 node: settings: