From 06602d1c2c382937be84d53277de8bbc0bd2bd12 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 5 Sep 2024 16:09:05 +0530 Subject: [PATCH 1/9] feat: get taskmaster values from config --- deployment/config.yaml | 17 ++++++ tesk/custom_config.py | 73 ++++++++++++++++++++++- tesk/utils.py | 127 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 1 deletion(-) diff --git a/deployment/config.yaml b/deployment/config.yaml index 0006f11..aea3f70 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -72,6 +72,23 @@ custom: tesResources_backend_parameters: - VmSize - ParamToRecogniseDataComingFromConfig + taskmaster: + imageName: docker.io/elixircloud/tesk-core-taskmaster + imageVersion: v0.10.2 + filerImageName: docker.io/elixircloud/tesk-core-filer + filerImageVersion: v0.10.2 + ftp: + # Name of the secret with FTP account credentials + secretName: account-secret + # If FTP account enabled (based on non-emptiness of secretName) + enabled: true + # If verbose (debug) mode of taskmaster is on (passes additional flag to taskmaster and sets image pull policy to Always) + debug: false + # Environment variables, that will be passed to taskmaster + environment: + key: value + # Service Account name for taskmaster + serviceAccountName: taskmaster # Logging configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig diff --git a/tesk/custom_config.py b/tesk/custom_config.py index ae1cbaf..7acab14 100644 --- a/tesk/custom_config.py +++ b/tesk/custom_config.py @@ -1,8 +1,78 @@ """Custom configuration model for the FOCA app.""" -from pydantic import BaseModel +from typing import Dict, Optional + +from pydantic import BaseModel, Field from tesk.api.ga4gh.tes.models import Service +from tesk.constants import tesk_constants + + +class FtpConfig(BaseModel): + """Ftp configuration model for the TESK.""" + + secretName: Optional[str] = Field( + default=None, description="Name of the secret with FTP account credentials" + ) + enabled: bool = Field( + default=False, + description="If FTP account enabled (based on non-emptiness of secretName)", + ) + + +class ExecutorSecret(BaseModel): + """Executor secret configuration.""" + + name: Optional[str] = Field( + default=None, + description=( + "Name of a secret that will be mounted as volume to each executor. The same" + " name will be used for the secret and the volume" + ), + ) + mountPath: Optional[str] = Field( + default=None, + alias="mountPath", + description="The path where the secret will be mounted to executors", + ) + enabled: bool = Field( + default=False, description="Indicates whether the secret is enabled" + ) + + +class Taskmaster(BaseModel): + """Taskmaster environment properties model for the TESK.""" + + imageName: str = Field( + default=tesk_constants.TASKMASTER_IMAGE_NAME, + description="Taskmaster image name", + ) + imageVersion: str = Field( + default=tesk_constants.TASKMASTER_IMAGE_VERSION, + description="Taskmaster image version", + ) + filerImageName: str = Field( + default=tesk_constants.FILER_IMAGE_NAME, description="Filer image name" + ) + filerImageVersion: str = Field( + default=tesk_constants.FILER_IMAGE_VERSION, description="Filer image version" + ) + ftp: FtpConfig = Field(default=None, description="Test FTP account settings") + debug: bool = Field( + default=False, + description="If verbose (debug) mode of taskmaster is on (passes additional " + "flag to taskmaster and sets image pull policy to Always)", + ) + environment: Optional[Dict[str, str]] = Field( + default=None, + description="Environment variables, that will be passed to taskmaster", + ) + serviceAccountName: str = Field( + default="default", description="Service Account name for taskmaster" + ) + executorSecret: Optional[ExecutorSecret] = Field( + default=None, description="Executor secret configuration" + ) class CustomConfig(BaseModel): @@ -10,3 +80,4 @@ class CustomConfig(BaseModel): # Define custom configuration fields here service_info: Service + taskmaster: Taskmaster diff --git a/tesk/utils.py b/tesk/utils.py index 214ffd6..cdbbb7f 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -3,6 +3,28 @@ import os from pathlib import Path +from foca import Foca +from kubernetes.client.models import ( + V1Container, + V1EnvVar, + V1EnvVarSource, + V1Job, + V1JobSpec, + V1ObjectMeta, + V1PodSpec, + V1PodTemplateSpec, + V1SecretKeySelector, + V1VolumeMount, +) + +from tesk.constants import TeskConstants +from tesk.custom_config import ( + CustomConfig, + TaskmasterEnvProperties, +) +from tesk.exceptions import ConfigNotFoundError +from tesk.k8s.constants import TeskK8sConstants + def get_config_path() -> Path: """Get the configuration path. @@ -15,3 +37,108 @@ def get_config_path() -> Path: return Path(config_path_env).resolve() else: return (Path(__file__).parents[1] / "deployment" / "config.yaml").resolve() + + +def get_custom_config() -> CustomConfig: + """Get the custom configuration. + + Returns: + The custom configuration. + """ + conf = Foca(config_file=get_config_path()).conf + try: + return CustomConfig(**conf.custom) + except AttributeError: + raise ConfigNotFoundError( + "Custom configuration not found in config file." + ) from None + + +def get_taskmaster_template() -> V1Job: + """Get the taskmaster template from the custom configuration. + + Returns: + The taskmaster template. + """ + job = V1Job( + api_version="batch/v1", + kind="Job", + metadata=V1ObjectMeta( + name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, + labels={"app": TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM}, + ), + spec=V1JobSpec( + template=V1PodTemplateSpec( + metadata=V1ObjectMeta( + name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM + ), + spec=V1PodSpec( + service_account_name="default", + containers=[ + V1Container( + name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, + image=f"{TeskConstants.TASKMASTER_IMAGE_NAME}:{TeskConstants.TASKMASTER_IMAGE_VERSION}", + args=[ + "-f", + f"/jsoninput/{TeskK8sConstants.job_constants.TASKMASTER_INPUT}.gz", + ], + env=[ + V1EnvVar( + name=TeskK8sConstants.ftp_constants.FTP_SECRET_USERNAME_ENV, + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name="ftp-secret", + key="username", + optional=True, + ) + ), + ), + V1EnvVar( + name=TeskK8sConstants.ftp_constants.FTP_SECRET_PASSWORD_ENV, + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name="ftp-secret", + key="password", + optional=True, + ) + ), + ), + ], + volume_mounts=[ + V1VolumeMount( + name="podinfo", + mount_path="/podinfo", + read_only=True, + ), + V1VolumeMount( + name="jsoninput", + mount_path="/jsoninput", + read_only=True, + ), + ], + ) + ], + volumes=[], + restart_policy=TeskK8sConstants.k8s_constants.JOB_RESTART_POLICY, + ), + ) + ), + ) + return job + + +def get_taskmaster_env_property() -> TaskmasterEnvProperties: + """Get the taskmaster env property from the custom configuration. + + Returns: + The taskmaster env property. + """ + custom_conf = get_custom_config() + try: + return custom_conf.taskmaster_env_properties + except AttributeError: + raise ConfigNotFoundError( + "Custom configuration doesn't seem to have taskmaster_env_properties in " + "config file." + f"Custom config:\n{custom_conf}" + ) from None From 3fa2eca05dc4fb6fe49df8369572199658e1fb2a Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 5 Sep 2024 23:48:42 +0530 Subject: [PATCH 2/9] fix: add podinfo volumes --- tesk/utils.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tesk/utils.py b/tesk/utils.py index cdbbb7f..9e68aec 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -6,14 +6,18 @@ from foca import Foca from kubernetes.client.models import ( V1Container, + V1DownwardAPIVolumeFile, + V1DownwardAPIVolumeSource, V1EnvVar, V1EnvVarSource, V1Job, V1JobSpec, + V1ObjectFieldSelector, V1ObjectMeta, V1PodSpec, V1PodTemplateSpec, V1SecretKeySelector, + V1Volume, V1VolumeMount, ) @@ -118,7 +122,27 @@ def get_taskmaster_template() -> V1Job: ], ) ], - volumes=[], + volumes=[ + V1Volume( + name="podinfo", + downward_api=V1DownwardAPIVolumeSource( + items=[ + V1DownwardAPIVolumeFile( + path="labels", + field_ref=V1ObjectFieldSelector( + field_path="metadata.labels" + ), + ), + V1DownwardAPIVolumeFile( + path="annotations", + field_ref=V1ObjectFieldSelector( + field_path="metadata.annotations" + ), + ), + ] + ), + ), + ], restart_policy=TeskK8sConstants.k8s_constants.JOB_RESTART_POLICY, ), ) From ffecf225d28579d2d8eefefbb7312fe41ede27e6 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Fri, 6 Sep 2024 17:40:33 +0530 Subject: [PATCH 3/9] minoe --- tesk/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tesk/utils.py b/tesk/utils.py index 9e68aec..4e788da 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -24,7 +24,7 @@ from tesk.constants import TeskConstants from tesk.custom_config import ( CustomConfig, - TaskmasterEnvProperties, + Taskmaster, ) from tesk.exceptions import ConfigNotFoundError from tesk.k8s.constants import TeskK8sConstants @@ -151,7 +151,7 @@ def get_taskmaster_template() -> V1Job: return job -def get_taskmaster_env_property() -> TaskmasterEnvProperties: +def get_taskmaster_env_property() -> Taskmaster: """Get the taskmaster env property from the custom configuration. Returns: @@ -159,7 +159,7 @@ def get_taskmaster_env_property() -> TaskmasterEnvProperties: """ custom_conf = get_custom_config() try: - return custom_conf.taskmaster_env_properties + return custom_conf.taskmaster except AttributeError: raise ConfigNotFoundError( "Custom configuration doesn't seem to have taskmaster_env_properties in " From 0acd4b0df9b9d03ab6b3ee183a62bf6a21e117f1 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Mon, 9 Sep 2024 17:27:02 +0530 Subject: [PATCH 4/9] minor --- tesk/utils.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tesk/utils.py b/tesk/utils.py index 4e788da..55cd8e4 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -21,13 +21,13 @@ V1VolumeMount, ) -from tesk.constants import TeskConstants +from tesk.constants import tesk_constants from tesk.custom_config import ( CustomConfig, Taskmaster, ) from tesk.exceptions import ConfigNotFoundError -from tesk.k8s.constants import TeskK8sConstants +from tesk.k8s.constants import tesk_k8s_constants def get_config_path() -> Path: @@ -68,27 +68,26 @@ def get_taskmaster_template() -> V1Job: api_version="batch/v1", kind="Job", metadata=V1ObjectMeta( - name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, - labels={"app": TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM}, + name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, ), spec=V1JobSpec( template=V1PodTemplateSpec( metadata=V1ObjectMeta( - name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM + name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM ), spec=V1PodSpec( - service_account_name="default", + service_account_name=tesk_constants.TASKMASTER_SERVICE_ACCOUNT_NAME, containers=[ V1Container( - name=TeskK8sConstants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, - image=f"{TeskConstants.TASKMASTER_IMAGE_NAME}:{TeskConstants.TASKMASTER_IMAGE_VERSION}", + name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, + image=f"{tesk_constants.TASKMASTER_IMAGE_NAME}:{tesk_constants.TASKMASTER_IMAGE_VERSION}", args=[ "-f", - f"/jsoninput/{TeskK8sConstants.job_constants.TASKMASTER_INPUT}.gz", + f"/jsoninput/{tesk_k8s_constants.job_constants.TASKMASTER_INPUT}.gz", ], env=[ V1EnvVar( - name=TeskK8sConstants.ftp_constants.FTP_SECRET_USERNAME_ENV, + name=tesk_k8s_constants.ftp_constants.FTP_SECRET_USERNAME_ENV, value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name="ftp-secret", @@ -98,7 +97,7 @@ def get_taskmaster_template() -> V1Job: ), ), V1EnvVar( - name=TeskK8sConstants.ftp_constants.FTP_SECRET_PASSWORD_ENV, + name=tesk_k8s_constants.ftp_constants.FTP_SECRET_PASSWORD_ENV, value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name="ftp-secret", @@ -133,17 +132,11 @@ def get_taskmaster_template() -> V1Job: field_path="metadata.labels" ), ), - V1DownwardAPIVolumeFile( - path="annotations", - field_ref=V1ObjectFieldSelector( - field_path="metadata.annotations" - ), - ), ] ), ), ], - restart_policy=TeskK8sConstants.k8s_constants.JOB_RESTART_POLICY, + restart_policy=tesk_k8s_constants.k8s_constants.JOB_RESTART_POLICY, ), ) ), From 2948508b6d9702a192422a25838ea6a5922bd67f Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 11 Sep 2024 00:00:04 +0530 Subject: [PATCH 5/9] review changes --- tesk/custom_config.py | 112 ++++++++++++++++++++---------------------- tesk/exceptions.py | 4 ++ tesk/utils.py | 54 ++++++++++---------- 3 files changed, 85 insertions(+), 85 deletions(-) diff --git a/tesk/custom_config.py b/tesk/custom_config.py index 7acab14..7680b21 100644 --- a/tesk/custom_config.py +++ b/tesk/custom_config.py @@ -2,82 +2,74 @@ from typing import Dict, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel from tesk.api.ga4gh.tes.models import Service from tesk.constants import tesk_constants class FtpConfig(BaseModel): - """Ftp configuration model for the TESK.""" + """Ftp configuration model for the TESK. - secretName: Optional[str] = Field( - default=None, description="Name of the secret with FTP account credentials" - ) - enabled: bool = Field( - default=False, - description="If FTP account enabled (based on non-emptiness of secretName)", - ) + Args: + secretName: Name of the secret with FTP account credentials. + enabled: If FTP account enabled (based on non-emptiness of secretName). + """ + + secretName: Optional[str] = None + enabled: bool = False class ExecutorSecret(BaseModel): - """Executor secret configuration.""" - - name: Optional[str] = Field( - default=None, - description=( - "Name of a secret that will be mounted as volume to each executor. The same" - " name will be used for the secret and the volume" - ), - ) - mountPath: Optional[str] = Field( - default=None, - alias="mountPath", - description="The path where the secret will be mounted to executors", - ) - enabled: bool = Field( - default=False, description="Indicates whether the secret is enabled" - ) + """Executor secret configuration. + + Args: + name: Name of a secret that will be mounted as volume to each executor. The same + name will be used for the secret and the volume. + mountPath: The path where the secret will be mounted to executors. + enabled: Indicates whether the secret is enabled. + """ + + name: Optional[str] = None + mountPath: Optional[str] = None + enabled: bool = False class Taskmaster(BaseModel): - """Taskmaster environment properties model for the TESK.""" - - imageName: str = Field( - default=tesk_constants.TASKMASTER_IMAGE_NAME, - description="Taskmaster image name", - ) - imageVersion: str = Field( - default=tesk_constants.TASKMASTER_IMAGE_VERSION, - description="Taskmaster image version", - ) - filerImageName: str = Field( - default=tesk_constants.FILER_IMAGE_NAME, description="Filer image name" - ) - filerImageVersion: str = Field( - default=tesk_constants.FILER_IMAGE_VERSION, description="Filer image version" - ) - ftp: FtpConfig = Field(default=None, description="Test FTP account settings") - debug: bool = Field( - default=False, - description="If verbose (debug) mode of taskmaster is on (passes additional " - "flag to taskmaster and sets image pull policy to Always)", - ) - environment: Optional[Dict[str, str]] = Field( - default=None, - description="Environment variables, that will be passed to taskmaster", - ) - serviceAccountName: str = Field( - default="default", description="Service Account name for taskmaster" - ) - executorSecret: Optional[ExecutorSecret] = Field( - default=None, description="Executor secret configuration" - ) + """Taskmaster's configuration model for the TESK. + + Args: + imageName: Taskmaster image name. + imageVersion: Taskmaster image version. + filerImageName: Filer image name. + filerImageVersion: Filer image version. + ftp: FTP account settings. + debug: If verbose (debug) mode of taskmaster is on (passes additional flag to + taskmaster and sets image pull policy to Always). + environment: Environment variables, that will be passed to taskmaster. + serviceAccountName: Service Account name for taskmaster. + executorSecret: Executor secret configuration + """ + + imageName: str = tesk_constants.TASKMASTER_IMAGE_NAME + imageVersion: str = tesk_constants.TASKMASTER_IMAGE_VERSION + filerImageName: str = tesk_constants.FILER_IMAGE_NAME + filerImageVersion: str = tesk_constants.FILER_IMAGE_VERSION + ftp: FtpConfig = FtpConfig() + debug: bool = False + environment: Optional[Dict[str, str]] = None + serviceAccountName: str = tesk_constants.TASKMASTER_SERVICE_ACCOUNT_NAME + executorSecret: Optional[ExecutorSecret] = None class CustomConfig(BaseModel): - """Custom configuration model for the FOCA app.""" + """Custom configuration model for the FOCA app. + + Args: + service_info: Service information. + taskmaster: Taskmaster environment. + """ # Define custom configuration fields here service_info: Service - taskmaster: Taskmaster + taskmaster: Taskmaster = Taskmaster() diff --git a/tesk/exceptions.py b/tesk/exceptions.py index d0aa727..d270678 100644 --- a/tesk/exceptions.py +++ b/tesk/exceptions.py @@ -19,6 +19,10 @@ class ConfigNotFoundError(FileNotFoundError): """Configuration file not found error.""" +class ConfigInvalidError(ValueError): + """Configuration file is invalid.""" + + class KubernetesError(ApiException): """Kubernetes error.""" diff --git a/tesk/utils.py b/tesk/utils.py index 55cd8e4..c040643 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -21,12 +21,11 @@ V1VolumeMount, ) -from tesk.constants import tesk_constants from tesk.custom_config import ( CustomConfig, Taskmaster, ) -from tesk.exceptions import ConfigNotFoundError +from tesk.exceptions import ConfigInvalidError from tesk.k8s.constants import tesk_k8s_constants @@ -34,7 +33,7 @@ def get_config_path() -> Path: """Get the configuration path. Returns: - The path of the config file. + The path of the config file. """ # Determine the configuration path if config_path_env := os.getenv("TESK_FOCA_CONFIG_PATH"): @@ -47,23 +46,45 @@ def get_custom_config() -> CustomConfig: """Get the custom configuration. Returns: - The custom configuration. + The custom configuration. """ conf = Foca(config_file=get_config_path()).conf try: return CustomConfig(**conf.custom) except AttributeError: - raise ConfigNotFoundError( + raise ConfigInvalidError( "Custom configuration not found in config file." ) from None +def get_taskmaster_config() -> Taskmaster: + """Get the taskmaster env property from the custom configuration. + + Returns: + The taskmaster env property. + """ + custom_conf = get_custom_config() + try: + return custom_conf.taskmaster + except AttributeError: + raise ConfigInvalidError( + "Custom configuration doesn't seem to have taskmaster_env_properties in " + "config file." + f"Custom config:\n{custom_conf}" + ) from None + + def get_taskmaster_template() -> V1Job: """Get the taskmaster template from the custom configuration. + This will be used to create the taskmaster job, API will inject values + into the template, depending upon the type of job and request. + Returns: - The taskmaster template. + The taskmaster template. """ + taskmaster_conf: Taskmaster = get_taskmaster_config() + job = V1Job( api_version="batch/v1", kind="Job", @@ -76,11 +97,11 @@ def get_taskmaster_template() -> V1Job: name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM ), spec=V1PodSpec( - service_account_name=tesk_constants.TASKMASTER_SERVICE_ACCOUNT_NAME, + service_account_name=taskmaster_conf.serviceAccountName, containers=[ V1Container( name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, - image=f"{tesk_constants.TASKMASTER_IMAGE_NAME}:{tesk_constants.TASKMASTER_IMAGE_VERSION}", + image=f"{taskmaster_conf.imageName}:{taskmaster_conf.imageVersion}", args=[ "-f", f"/jsoninput/{tesk_k8s_constants.job_constants.TASKMASTER_INPUT}.gz", @@ -142,20 +163,3 @@ def get_taskmaster_template() -> V1Job: ), ) return job - - -def get_taskmaster_env_property() -> Taskmaster: - """Get the taskmaster env property from the custom configuration. - - Returns: - The taskmaster env property. - """ - custom_conf = get_custom_config() - try: - return custom_conf.taskmaster - except AttributeError: - raise ConfigNotFoundError( - "Custom configuration doesn't seem to have taskmaster_env_properties in " - "config file." - f"Custom config:\n{custom_conf}" - ) from None From 7f21d1f9d57e19d6a4a1ed5d748b71ea0548f228 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 25 Sep 2024 22:58:02 +0530 Subject: [PATCH 6/9] remove getting env from os --- deployment/config.yaml | 2 ++ tesk/constants.py | 53 ++++++------------------------------------ tesk/custom_config.py | 2 ++ 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/deployment/config.yaml b/deployment/config.yaml index aea3f70..09027d9 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -89,6 +89,8 @@ custom: key: value # Service Account name for taskmaster serviceAccountName: taskmaster + filerBackoffLimit: 2 + executorBackoffLimit: 2 # Logging configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig diff --git a/tesk/constants.py b/tesk/constants.py index 5508eb2..a301504 100644 --- a/tesk/constants.py +++ b/tesk/constants.py @@ -19,55 +19,16 @@ class TeskConstants(BaseModel): TASKMASTER_ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT: Backoff limit for taskmaster env FILER_BACKOFF_LIMIT: Backoff limit got filer job EXECUTOR_BACKOFF_LIMIT: Backoff limit for executor job - - Note: - Below are the mentioned environment variable with which these constants can be - configured, otherwise mentioned default will be assigned. - - variable: - ENV_VARIABLE = default - - FILER_IMAGE_NAME: - TESK_API_TASKMASTER_FILER_IMAGE_NAME = docker.io/elixircloud/tesk-core-filer - FILER_IMAGE_VERSION: - TESK_API_TASKMASTER_FILER_IMAGE_VERSION = latest - TASKMASTER_IMAGE_NAME: - TESK_API_TASKMASTER_IMAGE_NAME = docker.io/elixircloud/tesk-core-taskmaster - TASKMASTER_IMAGE_VERSION: - TESK_API_TASKMASTER_IMAGE_VERSION = latest - TESK_NAMESPACE: - TESK_API_K8S_NAMESPACE = tesk - TASKMASTER_SERVICE_ACCOUNT_NAME: - TESK_API_TASKMASTER_SERVICE_ACCOUNT_NAME = taskmaster - TASKMASTER_ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT: - ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT = 6 - FILER_BACKOFF_LIMIT: - FILER_BACKOFF_LIMIT = 2 - EXECUTOR_BACKOFF_LIMIT: - EXECUTOR_BACKOFF_LIMIT = 2 """ - FILER_IMAGE_NAME: str = os.getenv( - "TESK_API_TASKMASTER_FILER_IMAGE_NAME", "docker.io/elixircloud/tesk-core-filer" - ) - FILER_IMAGE_VERSION: str = os.getenv( - "TESK_API_TASKMASTER_FILER_IMAGE_VERSION", "latest" - ) - TASKMASTER_IMAGE_NAME: str = os.getenv( - "TESK_API_TASKMASTER_IMAGE_NAME", "docker.io/elixircloud/tesk-core-taskmaster" - ) - TASKMASTER_IMAGE_VERSION: str = os.getenv( - "TESK_API_TASKMASTER_IMAGE_VERSION", "latest" - ) TESK_NAMESPACE: str = os.getenv("TESK_API_K8S_NAMESPACE", "tesk") - TASKMASTER_SERVICE_ACCOUNT_NAME: str = os.getenv( - "TESK_API_TASKMASTER_SERVICE_ACCOUNT_NAME", "taskmaster" - ) - TASKMASTER_ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT: str = os.getenv( - "ENVIRONMENT_EXECUTOR_BACKOFF_LIMIT", "6" - ) - FILER_BACKOFF_LIMIT: str = os.getenv("FILER_BACKOFF_LIMIT", "2") - EXECUTOR_BACKOFF_LIMIT: str = os.getenv("EXECUTOR_BACKOFF_LIMIT", "2") + FILER_IMAGE_NAME: str = "docker.io/elixircloud/tesk-core-filer" + FILER_IMAGE_VERSION: str = "latest" + TASKMASTER_IMAGE_NAME: str = "docker.io/elixircloud/tesk-core-taskmaster" + TASKMASTER_IMAGE_VERSION: str = "latest" + TASKMASTER_SERVICE_ACCOUNT_NAME: str = "taskmaster" + FILER_BACKOFF_LIMIT: str = "2" + EXECUTOR_BACKOFF_LIMIT: str = "2" class Config: """Configuration for class.""" diff --git a/tesk/custom_config.py b/tesk/custom_config.py index 7680b21..5adaccf 100644 --- a/tesk/custom_config.py +++ b/tesk/custom_config.py @@ -60,6 +60,8 @@ class Taskmaster(BaseModel): environment: Optional[Dict[str, str]] = None serviceAccountName: str = tesk_constants.TASKMASTER_SERVICE_ACCOUNT_NAME executorSecret: Optional[ExecutorSecret] = None + filerBackoffLimit: int = tesk_constants.FILER_BACKOFF_LIMIT + executorBackoffLimit: int = tesk_constants.EXECUTOR_BACKOFF_LIMIT class CustomConfig(BaseModel): From 71d9be1230e8ed40d7957a0d87d0423cde7095dd Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 25 Sep 2024 23:00:50 +0530 Subject: [PATCH 7/9] mypy --- tesk/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tesk/constants.py b/tesk/constants.py index a301504..ff856bc 100644 --- a/tesk/constants.py +++ b/tesk/constants.py @@ -27,8 +27,8 @@ class TeskConstants(BaseModel): TASKMASTER_IMAGE_NAME: str = "docker.io/elixircloud/tesk-core-taskmaster" TASKMASTER_IMAGE_VERSION: str = "latest" TASKMASTER_SERVICE_ACCOUNT_NAME: str = "taskmaster" - FILER_BACKOFF_LIMIT: str = "2" - EXECUTOR_BACKOFF_LIMIT: str = "2" + FILER_BACKOFF_LIMIT: int = 2 + EXECUTOR_BACKOFF_LIMIT: int = 2 class Config: """Configuration for class.""" From 41d8d19bb9d7ec190cadba958f864ff0e3fe2636 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Fri, 4 Oct 2024 17:39:22 +0530 Subject: [PATCH 8/9] remove inline typing and use constants --- tesk/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tesk/utils.py b/tesk/utils.py index c040643..960549f 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -85,9 +85,9 @@ def get_taskmaster_template() -> V1Job: """ taskmaster_conf: Taskmaster = get_taskmaster_config() - job = V1Job( - api_version="batch/v1", - kind="Job", + return V1Job( + api_version=tesk_k8s_constants.k8s_constants.K8S_BATCH_API_VERSION, + kind=tesk_k8s_constants.k8s_constants.K8S_BATCH_API_JOB_TYPE, metadata=V1ObjectMeta( name=tesk_k8s_constants.label_constants.LABEL_JOBTYPE_VALUE_TASKM, ), @@ -162,4 +162,3 @@ def get_taskmaster_template() -> V1Job: ) ), ) - return job From 2ac6959d39ce239aee0274656079b7e579d23606 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Fri, 4 Oct 2024 18:12:55 +0530 Subject: [PATCH 9/9] change int to str to make it json marshalable --- tesk/constants.py | 4 ++-- tesk/custom_config.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tesk/constants.py b/tesk/constants.py index ff856bc..a301504 100644 --- a/tesk/constants.py +++ b/tesk/constants.py @@ -27,8 +27,8 @@ class TeskConstants(BaseModel): TASKMASTER_IMAGE_NAME: str = "docker.io/elixircloud/tesk-core-taskmaster" TASKMASTER_IMAGE_VERSION: str = "latest" TASKMASTER_SERVICE_ACCOUNT_NAME: str = "taskmaster" - FILER_BACKOFF_LIMIT: int = 2 - EXECUTOR_BACKOFF_LIMIT: int = 2 + FILER_BACKOFF_LIMIT: str = "2" + EXECUTOR_BACKOFF_LIMIT: str = "2" class Config: """Configuration for class.""" diff --git a/tesk/custom_config.py b/tesk/custom_config.py index 5adaccf..4b584b1 100644 --- a/tesk/custom_config.py +++ b/tesk/custom_config.py @@ -60,8 +60,8 @@ class Taskmaster(BaseModel): environment: Optional[Dict[str, str]] = None serviceAccountName: str = tesk_constants.TASKMASTER_SERVICE_ACCOUNT_NAME executorSecret: Optional[ExecutorSecret] = None - filerBackoffLimit: int = tesk_constants.FILER_BACKOFF_LIMIT - executorBackoffLimit: int = tesk_constants.EXECUTOR_BACKOFF_LIMIT + filerBackoffLimit: str = tesk_constants.FILER_BACKOFF_LIMIT + executorBackoffLimit: str = tesk_constants.EXECUTOR_BACKOFF_LIMIT class CustomConfig(BaseModel):