fix(service): fix side-by-side deployments using same job queues

Author: Panaetius

CHANGES.rst

@@ -18,6 +18,16 @@
 Changes
 =======
 
+`0.16.4 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.16.3...v0.16.4>`__ (2022-01-25)
+----------------------------------------------------------------------------------------------------------
+
+This is a backport release that contains changes necessary for 1.0.0 multi-service compatibility.
+
+Bug Fixes
+~~~~~~~~~
+
+- **core:** Fix different versions of core-service using same Job queues
+
 `0.16.3 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.16.2...v0.16.3>`__ (2022-01-21)
 ----------------------------------------------------------------------------------------------------------
 

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 0.16.3
+version: 0.16.4

renku/service/cache/base.py

@@ -23,7 +23,7 @@
 from redis import RedisError
 from walrus import Database
 
-from renku.service.cache.config import REDIS_DATABASE, REDIS_HOST, REDIS_PASSWORD, REDIS_PORT
+from renku.service.cache.config import REDIS_DATABASE, REDIS_HOST, REDIS_NAMESPACE, REDIS_PASSWORD, REDIS_PORT
 
 
 class BaseCache:
@@ -40,6 +40,7 @@ class BaseCache:
 
     cache = redis.Redis(**config_)
     model_db = Database(**config_)
+    namespace = REDIS_NAMESPACE
 
     def set_record(self, name, key, value):
         """Insert a record to hash set."""

renku/service/cache/config.py

@@ -22,3 +22,8 @@
 REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
 REDIS_DATABASE = int(os.getenv("REDIS_DATABASE", 0))
 REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
+
+REDIS_NAMESPACE = os.getenv("REDIS_NAMESPACE", None)
+
+if REDIS_NAMESPACE:
+    REDIS_NAMESPACE = f"{REDIS_NAMESPACE}."

renku/service/controllers/api/mixins.py

@@ -29,6 +29,7 @@
 from renku.core.management.config import RENKU_HOME
 from renku.core.management.repository import RepositoryApiMixin
 from renku.core.utils.contexts import click_context
+from renku.service.cache.config import REDIS_NAMESPACE
 from renku.service.cache.models.job import Job
 from renku.service.cache.models.project import Project
 from renku.service.cache.models.user import User
@@ -137,7 +138,7 @@ def execute_op(self):
 
             job = self.cache.make_job(self.user, job_data={"ctrl_context": {**self.context, **ctrl_cls}})
 
-            with enqueue_retry(f"delayed.ctrl.{ctrl_cls['renku_ctrl']}") as queue:
+            with enqueue_retry(f"{REDIS_NAMESPACE}delayed.ctrl.{ctrl_cls['renku_ctrl']}") as queue:
                 queue.enqueue(delayed_ctrl_job, self.context, self.user_data, job.job_id, **ctrl_cls)
 
             return job

renku/service/jobs/queues.py

@@ -21,22 +21,23 @@
 from rq import Queue
 
 from renku.service.cache.base import BaseCache
+from renku.service.cache.config import REDIS_NAMESPACE
 
-CLEANUP_QUEUE_FILES = "cache.cleanup.files"
-CLEANUP_QUEUE_PROJECTS = "cache.cleanup.projects"
+CLEANUP_QUEUE_FILES = f"{REDIS_NAMESPACE}cache.cleanup.files"
+CLEANUP_QUEUE_PROJECTS = f"{REDIS_NAMESPACE}cache.cleanup.projects"
 
-DATASETS_JOB_QUEUE = "datasets.jobs"
-MIGRATIONS_JOB_QUEUE = "project.migrations"
-GRAPH_JOB_QUEUE = "graph.jobs"
+DATASETS_JOB_QUEUE = f"{REDIS_NAMESPACE}datasets.jobs"
+MIGRATIONS_JOB_QUEUE = f"{REDIS_NAMESPACE}project.migrations"
+GRAPH_JOB_QUEUE = f"{REDIS_NAMESPACE}graph.jobs"
 
-DELAYED_CTRL_DATASETS_CREATE = "delayed.ctrl.DatasetsCreateCtrl"
-DELAYED_CTRL_DATASETS_ADD = "delayed.ctrl.DatasetsAddFileCtrl"
-DELAYED_CTRL_DATASETS_REMOVE = "delayed.ctrl.DatasetsRemoveCtrl"
-DELAYED_CTRL_DATASETS_IMPORT = "delayed.ctrl.DatasetsImportCtrl"
-DELAYED_CTRL_DATASETS_EDIT = "delayed.ctrl.DatasetsEditCtrl"
-DELAYED_CTRL_DATASETS_UNLINK = "delayed.ctrl.DatasetsUnlinkCtrl"
-DELAYED_CTRL_PROJECT_MIGRATE = "delayed.ctrl.MigrateProjectCtrl"
-DELAYED_CTRL_CONFIG_SET = "delayed.ctrl.SetConfigCtrl"
+DELAYED_CTRL_DATASETS_CREATE = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsCreateCtrl"
+DELAYED_CTRL_DATASETS_ADD = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsAddFileCtrl"
+DELAYED_CTRL_DATASETS_REMOVE = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsRemoveCtrl"
+DELAYED_CTRL_DATASETS_IMPORT = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsImportCtrl"
+DELAYED_CTRL_DATASETS_EDIT = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsEditCtrl"
+DELAYED_CTRL_DATASETS_UNLINK = f"{REDIS_NAMESPACE}delayed.ctrl.DatasetsUnlinkCtrl"
+DELAYED_CTRL_PROJECT_MIGRATE = f"{REDIS_NAMESPACE}delayed.ctrl.MigrateProjectCtrl"
+DELAYED_CTRL_CONFIG_SET = f"{REDIS_NAMESPACE}delayed.ctrl.SetConfigCtrl"
 
 
 QUEUES = [