Merge pull request #3536 from SwissDataScienceCenter/release/v2.6.0

chore: release v2.6.0

Author: Panaetius

CHANGES.rst

@@ -18,6 +18,24 @@
 Changes
 =======
 
+`2.6.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.5.0...v2.6.0>`__ (2023-06-20)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **service:** fixes importing private datasets in deployments with
+   external gitlab
+   (`#3523 <https://github.com/SwissDataScienceCenter/renku-python/issues/3523>`__)
+   (`d64c179 <https://github.com/SwissDataScienceCenter/renku-python/commit/d64c179857d0f1faa08c11a0cf0149eec97b4e46>`__)
+
+Features
+~~~~~~~~
+
+-  **service:** add support for horizontal scaling
+   (`#3178 <https://github.com/SwissDataScienceCenter/renku-python/issues/3178>`__)
+   (`fab2b58 <https://github.com/SwissDataScienceCenter/renku-python/commit/fab2b583f3c36fa179b6388ca7a28fa68b2aad8b>`__)
+
 `2.5.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.4.1...v2.5.0>`__ (2023-06-02)
 -------------------------------------------------------------------------------------------------------
 

Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.9-slim as base
 
 # hadolint ignore=DL3008,DL3009,DL3013
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y git git-lfs=2.* python3-dev tini bash curl && \
+    apt-get install --no-install-recommends -y git git-lfs=3.* python3-dev tini bash curl && \
     pip install --no-cache-dir --upgrade pip
 
 FROM base as builder

chartpress.yaml

@@ -15,3 +15,4 @@ charts:
         buildArgs:
           CLEAN_INSTALL: "1"
           BUILD_CORE_SERVICE: "1"
+

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 2.5.0
+version: 2.6.0

helm-chart/renku-core/templates/configmap.yaml

@@ -1,54 +1,9 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "renku-core.fullname" . }}-nginx
+  name: {{ include "renku-core.fullname" . }}-metadata-versions
 data:
-  nginx-server-blocks.conf: |
-    {{- range $version := .Values.versions }}
-    upstream {{ $version.name }} {
-      server {{ include "renku-core.fullname" $ }}-{{ $version.name }};
-      keepalive 32;
-      keepalive_timeout 60s;
-    }
-    {{ end }}
-
-    server {
-      listen 8080;
-
-      server_name {{ include "renku-core.fullname" . }};
-
-      sendfile on;
-      tcp_nopush on;
-      client_max_body_size 0; # Required for uploading large files
-
-      location /renku/version {
-        root /;
-        add_header Content-Type application/json;
-        try_files /usr/share/nginx/html/version.json =404;
-      }
-
-      {{- range $version := .Values.versions }}
-      location /renku/{{ $version.prefix }} {
-        rewrite /renku/{{ $version.prefix }}/(.*) /renku/$1  break;
-        proxy_set_header Host $host;
-        proxy_pass http://{{ $version.name }};
-        proxy_send_timeout {{ $.Values.requestTimeout }}s;
-        proxy_read_timeout {{ $.Values.requestTimeout }}s;
-        proxy_http_version 1.1;
-        proxy_set_header "Connection" "";
-      }
-      {{- end }}
-
-      location /renku {
-        proxy_set_header Host $host;
-        proxy_pass http://{{ .Values.versions.latest.name }};
-        proxy_send_timeout {{ $.Values.requestTimeout }}s;
-        proxy_read_timeout {{ $.Values.requestTimeout }}s;
-        proxy_http_version 1.1;
-        proxy_set_header "Connection" "";
-      }
-    }
-  version.json: |
+  metadata-versions.json: |
     {
       "name": "renku-core",
       "versions": [

helm-chart/renku-core/templates/deployment-nginx.yaml

@@ -17,7 +17,10 @@ metadata:
 spec:
   replicas: {{ $.Values.replicaCount }}
   strategy:
-    type: Recreate
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "renku-core.name" $ }}
@@ -44,6 +47,9 @@ spec:
       volumes:
         - name: shared-volume
           emptyDir: {}
+        - name: metadata-versions
+          configMap:
+            name: {{ include "renku-core.fullname" $ }}-metadata-versions
         {{- include "certificates.volumes" $ | nindent 8 }}
       initContainers:
         {{- include "certificates.initContainer" $ | nindent 8 }}
@@ -139,10 +145,14 @@ spec:
               value: {{ $.Values.global.renku.domain }}
             - name: RENKU_PROJECT_DEFAULT_CLI_VERSION
               value: {{ $.Values.global.renku.cli_version | default "" | quote }}
+            - name: METADATA_VERSIONS_LIST
+              value: /svc/config/metadata-versions/metadata-versions.json
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
+            - name: metadata-versions
+              mountPath: /svc/config/metadata-versions
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
           ports:
             - name: http
@@ -157,7 +167,7 @@ spec:
               path: /health
               port: http
           resources:
-            {{- toYaml $.Values.resources | nindent 12 }}
+            {{- toYaml $.Values.resources.core | nindent 12 }}
         - name: {{ $.Chart.Name }}-datasets-workers
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
           imagePullPolicy: {{ $version.image.pullPolicy }}
@@ -213,6 +223,8 @@ spec:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.datasetsWorkers | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-management-workers
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -267,6 +279,8 @@ spec:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.managementWorkers | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-scheduler
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -307,6 +321,8 @@ spec:
             {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
+          resources:
+            {{- toYaml $.Values.resources.scheduler | nindent 12 }}
       {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm-chart/renku-core/templates/deployment.yaml

@@ -0,0 +1,29 @@
+{{- range $version := .Values.versions }}
+---
+{{- if $.Capabilities.APIVersions.Has "autoscaling/v2" }}
+apiVersion: autoscaling/v2
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta2" }}
+apiVersion: autoscaling/v2beta2
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta1" }}
+apiVersion: autoscaling/v2beta1
+{{- else }}
+  {{- fail "ERROR: You must have at least autoscaling/v2beta1 to use HorizontalPodAutoscaler" }}
+{{- end }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+  minReplicas: {{ $.Values.horizontalPodAutoscaling.minReplicas }}
+  maxReplicas: {{ $.Values.horizontalPodAutoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ $.Values.horizontalPodAutoscaling.averageMemoryUtilization }}
+{{ end }}

helm-chart/renku-core/templates/hpa.yaml

@@ -0,0 +1,12 @@
+{{- range $version := .Values.versions }}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-{{ $version.name }}
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/deploymentVersion: {{ $version.name }}
+{{ end }}