From e005825483338d3e4a9614cd212f16a869d9d7f6 Mon Sep 17 00:00:00 2001 From: mahdikhashan Date: Fri, 17 Jan 2025 14:30:34 +0100 Subject: [PATCH] fix lint Signed-off-by: mahdikhashan --- .../kubeflow/katib/api/katib_client.py | 366 +++++++++--------- .../v1beta1/kubeflow/katib/utils/utils.py | 6 +- 2 files changed, 189 insertions(+), 183 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index e3cb1b12fdb..8f2947856af 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -33,11 +33,11 @@ class KatibClient(object): def __init__( - self, - config_file: Optional[str] = None, - context: Optional[str] = None, - client_configuration: Optional[client.Configuration] = None, - namespace: str = utils.get_default_target_namespace(), + self, + config_file: Optional[str] = None, + context: Optional[str] = None, + client_configuration: Optional[client.Configuration] = None, + namespace: str = utils.get_default_target_namespace(), ): """KatibClient constructor. Configure logging in your application as follows to see detailed information from the KatibClient APIs: @@ -89,9 +89,9 @@ def _is_ipython(self): return True def create_experiment( - self, - experiment: models.V1beta1Experiment, - namespace: Optional[str] = None, + self, + experiment: models.V1beta1Experiment, + namespace: Optional[str] = None, ): """Create the Katib Experiment. @@ -164,46 +164,46 @@ def create_experiment( ) def tune( - self, - # TODO (andreyvelich): How to be consistent with other APIs (name) ? - name: str, - model_provider_parameters: Optional[ - "HuggingFaceModelParams" # noqa: F821 - ] = None, - dataset_provider_parameters: Optional[ - Union["HuggingFaceDatasetParams", "S3DatasetParams"] # noqa: F821 - ] = None, - trainer_parameters: Optional["HuggingFaceTrainerParams"] = None, # noqa: F821 - storage_config: Optional[Dict[str, Optional[Union[str, List[str]]]]] = { - "size": constants.PVC_DEFAULT_SIZE, - "storage_class": None, - "access_modes": constants.PVC_DEFAULT_ACCESS_MODES, - }, - objective: Optional[Callable] = None, - base_image: Optional[str] = constants.BASE_IMAGE_TENSORFLOW, - parameters: Optional[Dict[str, Any]] = None, - namespace: Optional[str] = None, - env_per_trial: Optional[ - Union[Dict[str, str], List[Union[client.V1EnvVar, client.V1EnvFromSource]]] - ] = None, - algorithm_name: str = "random", - algorithm_settings: Union[ - dict, List[models.V1beta1AlgorithmSetting], None - ] = None, - objective_metric_name: str = None, - additional_metric_names: List[str] = [], - objective_type: str = "maximize", - objective_goal: float = None, - max_trial_count: int = None, - parallel_trial_count: int = None, - max_failed_trial_count: int = None, - resources_per_trial: Optional[ - Union[dict, client.V1ResourceRequirements, TrainerResources] - ] = None, - retain_trials: bool = False, - packages_to_install: List[str] = None, - pip_index_url: str = "https://pypi.org/simple", - metrics_collector_config: Dict[str, Any] = {"kind": "StdOut"}, + self, + # TODO (andreyvelich): How to be consistent with other APIs (name) ? + name: str, + model_provider_parameters: Optional[ + "HuggingFaceModelParams" # noqa: F821 + ] = None, + dataset_provider_parameters: Optional[ + Union["HuggingFaceDatasetParams", "S3DatasetParams"] # noqa: F821 + ] = None, + trainer_parameters: Optional["HuggingFaceTrainerParams"] = None, # noqa: F821 + storage_config: Optional[Dict[str, Optional[Union[str, List[str]]]]] = { + "size": constants.PVC_DEFAULT_SIZE, + "storage_class": None, + "access_modes": constants.PVC_DEFAULT_ACCESS_MODES, + }, + objective: Optional[Callable] = None, + base_image: Optional[str] = constants.BASE_IMAGE_TENSORFLOW, + parameters: Optional[Dict[str, Any]] = None, + namespace: Optional[str] = None, + env_per_trial: Optional[ + Union[Dict[str, str], List[Union[client.V1EnvVar, client.V1EnvFromSource]]] + ] = None, + algorithm_name: str = "random", + algorithm_settings: Union[ + dict, List[models.V1beta1AlgorithmSetting], None + ] = None, + objective_metric_name: str = None, + additional_metric_names: List[str] = [], + objective_type: str = "maximize", + objective_goal: float = None, + max_trial_count: int = None, + parallel_trial_count: int = None, + max_failed_trial_count: int = None, + resources_per_trial: Optional[ + Union[dict, client.V1ResourceRequirements, TrainerResources] + ] = None, + retain_trials: bool = False, + packages_to_install: List[str] = None, + pip_index_url: str = "https://pypi.org/simple", + metrics_collector_config: Dict[str, Any] = {"kind": "StdOut"}, ): """ Create HyperParameter Tuning Katib Experiment using one of the following @@ -349,19 +349,19 @@ class name in this argument. """ if ( - ( - model_provider_parameters is not None - or dataset_provider_parameters is not None - or trainer_parameters is not None - ) - and (objective is not None or parameters is not None) + ( + model_provider_parameters is not None + or dataset_provider_parameters is not None + or trainer_parameters is not None + ) + and (objective is not None or parameters is not None) ) or ( - ( - model_provider_parameters is None - and dataset_provider_parameters is None - and trainer_parameters is None - ) - and (objective is None and parameters is None) + ( + model_provider_parameters is None + and dataset_provider_parameters is None + and trainer_parameters is None + ) + and (objective is None and parameters is None) ): raise ValueError( "Invalid configuration for creating a Katib Experiment for hyperparameter " @@ -504,9 +504,9 @@ class name in this argument. # If users choose to use external models and datasets. else: if ( - not model_provider_parameters - or not dataset_provider_parameters - or not trainer_parameters + not model_provider_parameters + or not dataset_provider_parameters + or not trainer_parameters ): raise ValueError("One of the required parameters is None") @@ -558,7 +558,8 @@ class name in this argument. # TODO (helenxie-bit): PVC Creation should be part of Katib Controller. try: if not utils.is_valid_pvc_name(name): - raise ValueError(f""" + raise ValueError( + f""" Invalid PVC name '{name}'. It must comply with RFC 1123. A lowercase RFC 1123 subdomain must consist of lowercase alphanumeric characters, '-' or '.', @@ -566,7 +567,8 @@ class name in this argument. For example, 'example.com' is valid. The regex used for validation is: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - """) + """ + ) self.core_api.create_namespaced_persistent_volume_claim( namespace=namespace, @@ -726,10 +728,10 @@ class name in this argument. self.create_experiment(experiment, namespace) def get_experiment( - self, - name: str, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the Katib Experiment. @@ -768,9 +770,9 @@ def get_experiment( raise RuntimeError(f"Failed to get Katib Experiment: {namespace}/{name}") def list_experiments( - self, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """List of all Katib Experiments in namespace. @@ -817,11 +819,11 @@ def list_experiments( return result def get_experiment_conditions( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the Experiment conditions. Experiment is in the condition when `status` is True for the appropriate condition `type`. @@ -850,20 +852,20 @@ def get_experiment_conditions( experiment = self.get_experiment(name, namespace, timeout) if ( - experiment.status - and experiment.status.conditions - and len(experiment.status.conditions) > 0 + experiment.status + and experiment.status.conditions + and len(experiment.status.conditions) > 0 ): return experiment.status.conditions return [] def is_experiment_created( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Check if Experiment is Created. @@ -890,11 +892,11 @@ def is_experiment_created( ) def is_experiment_running( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Check if Experiment is Running. @@ -921,11 +923,11 @@ def is_experiment_running( ) def is_experiment_restarting( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Check if Experiment is Restarting. Args: @@ -951,11 +953,11 @@ def is_experiment_restarting( ) def is_experiment_succeeded( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Check if Experiment is Succeeded. Args: @@ -981,11 +983,11 @@ def is_experiment_succeeded( ) def is_experiment_failed( - self, - name: str, - namespace: Optional[str] = None, - experiment: models.V1beta1Experiment = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + experiment: models.V1beta1Experiment = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Check if Experiment is Failed. Args: @@ -1011,13 +1013,13 @@ def is_experiment_failed( ) def wait_for_experiment_condition( - self, - name: str, - namespace: Optional[str] = None, - expected_condition: str = constants.EXPERIMENT_CONDITION_SUCCEEDED, - timeout: int = 600, - polling_interval: int = 15, - apiserver_timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + expected_condition: str = constants.EXPERIMENT_CONDITION_SUCCEEDED, + timeout: int = 600, + polling_interval: int = 15, + apiserver_timeout: int = constants.DEFAULT_TIMEOUT, ): """Wait until Experiment reaches specific condition. By default it waits for the Succeeded condition. @@ -1049,10 +1051,10 @@ def wait_for_experiment_condition( # Wait for Failed condition. if ( - expected_condition == constants.EXPERIMENT_CONDITION_FAILED - and self.is_experiment_failed( - name, namespace, experiment, apiserver_timeout - ) + expected_condition == constants.EXPERIMENT_CONDITION_FAILED + and self.is_experiment_failed( + name, namespace, experiment, apiserver_timeout + ) ): utils.print_experiment_status(experiment) logger.debug( @@ -1062,7 +1064,7 @@ def wait_for_experiment_condition( # Raise exception if Experiment is Failed. elif self.is_experiment_failed( - name, namespace, experiment, apiserver_timeout + name, namespace, experiment, apiserver_timeout ): raise RuntimeError( f"Experiment: {namespace}/{name} is Failed. " @@ -1071,10 +1073,10 @@ def wait_for_experiment_condition( # Check if Experiment reaches Created condition. elif ( - expected_condition == constants.EXPERIMENT_CONDITION_CREATED - and self.is_experiment_created( - name, namespace, experiment, apiserver_timeout - ) + expected_condition == constants.EXPERIMENT_CONDITION_CREATED + and self.is_experiment_created( + name, namespace, experiment, apiserver_timeout + ) ): utils.print_experiment_status(experiment) logger.debug( @@ -1084,10 +1086,10 @@ def wait_for_experiment_condition( # Check if Experiment reaches Running condition. elif ( - expected_condition == constants.EXPERIMENT_CONDITION_RUNNING - and self.is_experiment_running( - name, namespace, experiment, apiserver_timeout - ) + expected_condition == constants.EXPERIMENT_CONDITION_RUNNING + and self.is_experiment_running( + name, namespace, experiment, apiserver_timeout + ) ): utils.print_experiment_status(experiment) logger.debug( @@ -1097,10 +1099,10 @@ def wait_for_experiment_condition( # Check if Experiment reaches Restarting condition. elif ( - expected_condition == constants.EXPERIMENT_CONDITION_RESTARTING - and self.is_experiment_restarting( - name, namespace, experiment, apiserver_timeout - ) + expected_condition == constants.EXPERIMENT_CONDITION_RESTARTING + and self.is_experiment_restarting( + name, namespace, experiment, apiserver_timeout + ) ): utils.print_experiment_status(experiment) logger.debug( @@ -1110,10 +1112,10 @@ def wait_for_experiment_condition( # Check if Experiment reaches Succeeded condition. elif ( - expected_condition == constants.EXPERIMENT_CONDITION_SUCCEEDED - and self.is_experiment_succeeded( - name, namespace, experiment, apiserver_timeout - ) + expected_condition == constants.EXPERIMENT_CONDITION_SUCCEEDED + and self.is_experiment_succeeded( + name, namespace, experiment, apiserver_timeout + ) ): utils.print_experiment_status(experiment) @@ -1137,13 +1139,13 @@ def wait_for_experiment_condition( ) def edit_experiment_budget( - self, - name: str, - namespace: Optional[str] = None, - max_trial_count: int = None, - parallel_trial_count: int = None, - max_failed_trial_count: int = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + max_trial_count: int = None, + parallel_trial_count: int = None, + max_failed_trial_count: int = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Update Experiment budget for the running Trials. You can modify Trial budget to resume Succeeded Experiments with `LongRunning` and `FromVolume` @@ -1173,9 +1175,9 @@ def edit_experiment_budget( # The new Trial budget must be set. if ( - max_trial_count is None - and parallel_trial_count is None - and max_failed_trial_count is None + max_trial_count is None + and parallel_trial_count is None + and max_failed_trial_count is None ): raise ValueError( "Invalid input arguments. " @@ -1210,10 +1212,10 @@ def edit_experiment_budget( logger.debug(f"Experiment {namespace}/{name} has been updated") def delete_experiment( - self, - name: str, - namespace: Optional[str] = None, - delete_options: client.V1DeleteOptions = None, + self, + name: str, + namespace: Optional[str] = None, + delete_options: client.V1DeleteOptions = None, ): """Delete the Katib Experiment. @@ -1249,10 +1251,10 @@ def delete_experiment( logger.debug(f"Experiment {namespace}/{name} has been deleted") def get_suggestion( - self, - name: str, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the Katib Suggestion. @@ -1291,9 +1293,9 @@ def get_suggestion( raise RuntimeError(f"Failed to get Katib Suggestion: {namespace}/{name}") def list_suggestions( - self, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """List of all Katib Suggestion in namespace. @@ -1340,10 +1342,10 @@ def list_suggestions( return result def get_trial( - self, - name: str, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the Katib Trial. @@ -1382,10 +1384,10 @@ def get_trial( raise RuntimeError(f"Failed to get Katib Trial: {namespace}/{name}") def list_trials( - self, - experiment_name: str = None, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + experiment_name: str = None, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """List of all Trials in namespace. If Experiment name is set, it returns all Trials belong to the Experiment. @@ -1442,10 +1444,10 @@ def list_trials( return result def get_success_trial_details( - self, - experiment_name: str = None, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + experiment_name: str = None, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the Succeeded Trial details. If Experiment name is set, it returns Succeeded Trials details belong to the Experiment. @@ -1492,12 +1494,12 @@ def get_success_trial_details( utils.FakeResponse(item), models.V1beta1Trial ) if ( - trial.status - and trial.status.conditions - and len(trial.status.conditions) > 0 + trial.status + and trial.status.conditions + and len(trial.status.conditions) > 0 ): if utils.has_condition( - trial.status.conditions, constants.TRIAL_CONDITION_SUCCEEDED + trial.status.conditions, constants.TRIAL_CONDITION_SUCCEEDED ): output = {} output["name"] = trial.metadata.name @@ -1515,10 +1517,10 @@ def get_success_trial_details( return result def get_optimal_hyperparameters( - self, - name: str, - namespace: Optional[str] = None, - timeout: int = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + timeout: int = constants.DEFAULT_TIMEOUT, ): """Get the current optimal Trial from the Experiment. @@ -1541,20 +1543,20 @@ def get_optimal_hyperparameters( experiment = self.get_experiment(name, namespace, timeout) if ( - experiment.status - and experiment.status.current_optimal_trial - and experiment.status.current_optimal_trial.observation.metrics + experiment.status + and experiment.status.current_optimal_trial + and experiment.status.current_optimal_trial.observation.metrics ): return experiment.status.current_optimal_trial else: return None def get_trial_metrics( - self, - name: str, - namespace: Optional[str] = None, - db_manager_address: str = constants.DEFAULT_DB_MANAGER_ADDRESS, - timeout: str = constants.DEFAULT_TIMEOUT, + self, + name: str, + namespace: Optional[str] = None, + db_manager_address: str = constants.DEFAULT_DB_MANAGER_ADDRESS, + timeout: str = constants.DEFAULT_TIMEOUT, ): """Get the Trial Metric Results from the Katib DB. Katib DB Manager service should be accessible while calling this API. diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index c740b46fdf4..4a50bb76b24 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -272,4 +272,8 @@ def get_exec_script_from_objective( def is_valid_pvc_name(name: str) -> bool: # RFC 1123 regex for valid PVC names: lowercase alphanumeric, '-', or '.'. - return bool(re.match(r'^[a-z0-9]([a-z0-9\-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9\-]*[a-z0-9])?)*$', name)) + return bool( + re.match( + r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9\-]*[a-z0-9])?)*$", name + ) + )