Skip to content

Commit

Permalink
Refactoring code related to MLCube CLI options. (#350)
Browse files Browse the repository at this point in the history 
- Removing deprecated option definitions from `__main__.py` (new options are defined in `Options` class).
- Moving new option definitions (network, security etc.) to `Options` class.
- Removing unused and incorrect code related to parsing options accepting multiple values from the `__main__.py` file since this functionality is implemented in `cli` module.

In addition, updating the changed files to comply with python style guidelines.
  • Loading branch information
@sergey-serebryakov
sergey-serebryakov authored Dec 11, 2023
1 parent 4938fe6 commit 78199b5
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 174 deletions.
125 changes: 7 additions & 118 deletions mlcube/mlcube/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,7 @@
import coloredlogs
from omegaconf import OmegaConf

from mlcube.cli import (
MLCubeCommand,
MultiValueOption,
Options,
UsageExamples,
parse_cli_args,
)
from mlcube.config import MountType
from mlcube.cli import MLCubeCommand, MultiValueOption, Options, UsageExamples, parse_cli_args
from mlcube.errors import ExecutionError, IllegalParameterValueError, MLCubeError
from mlcube.parser import CliParser
from mlcube.shell import Shell
Expand All @@ -29,110 +22,6 @@
"""Width of a user terminal. MLCube overrides default (80) character width to make usage examples look better."""


def add_to_parser(self, parser: click.parser.OptionParser, ctx: click.core.Context):
def parser_process(value: str, state: click.parser.ParsingState):
values: t.List[str] = [value]
prefixes: t.Tuple[str] = tuple(self._eat_all_parser.prefixes)
while state.rargs:
if state.rargs[0].startswith(prefixes):
break
values.append(state.rargs.pop(0))
self._previous_parser_process(tuple(values), state)

super(MultiValueOption, self).add_to_parser(parser, ctx)
for opt_name in self.opts:
our_parser: t.Optional[click.parser.Option] = parser._long_opt.get(opt_name) or parser._short_opt.get(opt_name)
if our_parser:
self._eat_all_parser = our_parser
self._previous_parser_process = our_parser.process
our_parser.process = parser_process
break


log_level_option = click.option(
"--log-level",
"--log_level",
required=False,
type=str,
default="warning",
help="Log level to set, default is to do nothing.",
)
mlcube_option = click.option(
"--mlcube",
required=False,
type=str,
default=os.getcwd(),
help="Path to MLCube. This can be either a directory path that becomes MLCube's root directory, or path to MLCube"
"definition file (.yaml). In the latter case the MLCube's root directory becomes parent directory of the yaml"
"file. Default is current directory.",
)
platform_option = click.option(
"--platform",
required=False,
type=str,
default="docker",
help="Platform to run MLCube, default is 'docker' (that also supports podman).",
)
task_option = click.option(
"--task",
required=False,
type=str,
default=None,
help="MLCube task name(s) to run, default is `main`. This parameter can take a list value, in which case task names"
"are separated with ','.",
)
workspace_option = click.option(
"--workspace",
required=False,
type=str,
default=None,
help="Workspace location that is used to store input/output artifacts of MLCube tasks.",
)
network_option = click.option(
"--network",
required=False,
type=str,
default=None,
help="Networking options defined during MLCube container execution.",
)
security_option = click.option(
"--security",
required=False,
type=str,
default=None,
help="Security options defined during MLCube container execution.",
)
gpus_option = click.option(
"--gpus",
required=False,
type=str,
default=None,
help="GPU usage options defined during MLCube container execution.",
)
memory_option = click.option(
"--memory",
required=False,
type=str,
default=None,
help="Memory RAM options defined during MLCube container execution.",
)
cpu_option = click.option(
"--cpu",
required=False,
type=str,
default=None,
help="CPU options defined during MLCube container execution.",
)
mount_option = click.option(
"--mount",
required=False,
type=click.Choice([MountType.RW, MountType.RO]),
default=None,
help="Mount options for all input parameters. These mount options override any other mount options defined for "
"each input parameters. A typical use case is to ensure that inputs are mounted in read-only (ro) mode.",
)


@click.group(name="mlcube", add_help_option=False)
@Options.loglevel
@Options.help
Expand Down Expand Up @@ -280,12 +169,12 @@ def configure(mlcube: t.Optional[str], platform: str, p: t.Tuple[str]) -> None:
@Options.platform
@Options.task
@Options.workspace
@network_option
@security_option
@gpus_option
@memory_option
@cpu_option
@mount_option
@Options.network
@Options.security
@Options.gpus
@Options.memory
@Options.cpu
@Options.mount
@Options.parameter
@Options.help
@click.pass_context
Expand Down
102 changes: 60 additions & 42 deletions mlcube/mlcube/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from markdown import Markdown
from omegaconf import DictConfig

from mlcube.config import MLCubeConfig
from mlcube.config import MLCubeConfig, MountType
from mlcube.parser import CliParser, MLCubeDirectory
from mlcube.platform import Platform
from mlcube.runner import Runner
Expand Down Expand Up @@ -54,15 +54,11 @@ def parse_cli_args(
mlcube_inst: MLCubeDirectory = CliParser.parse_mlcube_arg(parsed_args["mlcube"])
Validate.validate_type(mlcube_inst, MLCubeDirectory)

mlcube_cli_args, task_cli_args = CliParser.parse_extra_arg(
unparsed_args, parsed_args
)
mlcube_cli_args, task_cli_args = CliParser.parse_extra_arg(unparsed_args, parsed_args)

if parsed_args.get("platform", None) is not None:
system_settings = SystemSettings()
runner_config: t.Optional[DictConfig] = system_settings.get_platform(
parsed_args["platform"]
)
runner_config: t.Optional[DictConfig] = system_settings.get_platform(parsed_args["platform"])
runner_cls: t.Optional[t.Type[Runner]] = Platform.get_runner(
system_settings.runners.get(runner_config.runner, None)
)
Expand Down Expand Up @@ -147,9 +143,9 @@ def parser_process(value: str, state: click.parser.ParsingState):

super(MultiValueOption, self).add_to_parser(parser, ctx)
for opt_name in self.opts:
our_parser: t.Optional[click.parser.Option] = parser._long_opt.get(
our_parser: t.Optional[click.parser.Option] = parser._long_opt.get(opt_name) or parser._short_opt.get(
opt_name
) or parser._short_opt.get(opt_name)
)
if our_parser:
self._eat_all_parser = our_parser
self._previous_parser_process = our_parser.process
Expand All @@ -171,9 +167,7 @@ class HelpEpilog(object):
class Example:
"""Context manager for helping with formatting epilogues when users invoke help on a command line."""

def __init__(
self, formatter: click.formatting.HelpFormatter, title: str
) -> None:
def __init__(self, formatter: click.formatting.HelpFormatter, title: str) -> None:
self.formatter = formatter
self.title = title

Expand All @@ -191,9 +185,7 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None:
def __init__(self, examples: t.List[t.Tuple[str, t.List[str]]]) -> None:
self.examples = examples

def format_epilog(
self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter
) -> None:
def format_epilog(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None:
if not self.examples:
return
formatter.write_heading("\nExamples")
Expand Down Expand Up @@ -233,7 +225,7 @@ def format_help_text(self, ctx, formatter):
if start_idx >= 0:
end_idx: int = text.find("</long>", start_idx)
if end_idx >= 0:
text = text[:start_idx] + text[end_idx + 7:]
text = text[:start_idx] + text[end_idx + 7 :]
# This is supported in default implementation, so need to reimplement it here. All that goes after the
# `\f` character is not rendered (e.g., description of parameters, TODOs, etc.).
text = text.split("\f")[0]
Expand All @@ -244,9 +236,7 @@ def format_help_text(self, ctx, formatter):
if len(paragraphs) == 1:
brief_description, long_description = paragraphs[0], ""
else:
brief_description, long_description = paragraphs[0], "\n".join(
paragraphs[1:]
)
brief_description, long_description = paragraphs[0], "\n".join(paragraphs[1:])

formatter.write_heading("\nBrief description")
with formatter.indentation():
Expand All @@ -265,9 +255,7 @@ def format_help_text(self, ctx, formatter):
with formatter.indentation():
formatter.write_text(DEPRECATED_HELP_NOTICE)

def format_options(
self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter
) -> None:
def format_options(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None:
"""Writes all the options into the formatter if they exist.
This implementation removes Markdown format from the options' help messages should they exist. Any errors
Expand All @@ -283,9 +271,7 @@ def format_options(
with formatter.section("Options"):
formatter.write_dl(opts)

def format_epilog(
self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter
) -> None:
def format_epilog(self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter) -> None:
"""Format epilog if its type `mlcube.EpilogWithExamples`, else fallback to default implementation."""
if isinstance(self.epilog, HelpEpilog):
self.epilog.format_epilog(ctx, formatter)
Expand Down Expand Up @@ -332,9 +318,7 @@ class Options:
type=click.Choice(["critical", "error", "warning", "info", "debug"]),
help="Logging level is a lower-case string value for Python's logging library (see "
"[Logging Levels]({log_level}) for more details). Only messages with this logging level or higher are "
"logged.".format(
log_level="https://docs.python.org/3/library/logging.html#logging-levels"
),
"logged.".format(log_level="https://docs.python.org/3/library/logging.html#logging-levels"),
)

mlcube = click.option(
Expand Down Expand Up @@ -382,9 +366,7 @@ class Options:
type=str,
default=None,
help="MLCube [task]({task}) name(s) to run, default is `main`. This parameter can take a list of values, in "
"which case task names are separated with comma (,).".format(
task=OnlineDocs.concept_url("task")
),
"which case task names are separated with comma (,).".format(task=OnlineDocs.concept_url("task")),
)

workspace = click.option(
Expand Down Expand Up @@ -434,6 +416,50 @@ class Options:
),
)

network = click.option(
"--network",
required=False,
type=str,
default=None,
help="Networking options defined during MLCube container execution.",
)
security = click.option(
"--security",
required=False,
type=str,
default=None,
help="Security options defined during MLCube container execution.",
)
gpus = click.option(
"--gpus",
required=False,
type=str,
default=None,
help="GPU usage options defined during MLCube container execution.",
)
memory = click.option(
"--memory",
required=False,
type=str,
default=None,
help="Memory RAM options defined during MLCube container execution.",
)
cpu = click.option(
"--cpu",
required=False,
type=str,
default=None,
help="CPU options defined during MLCube container execution.",
)
mount = click.option(
"--mount",
required=False,
type=click.Choice([MountType.RW, MountType.RO]),
default=None,
help="Mount options for all input parameters. These mount options override any other mount options defined for "
"each input parameters. A typical use case is to ensure that inputs are mounted in read-only (ro) mode.",
)


def _mnist(steps: t.List[str]) -> t.List[str]:
return [
Expand All @@ -451,9 +477,7 @@ class UsageExamples:
),
(
"Show effective MLCube configuration overriding parameters on a command line",
_mnist(
["mlcube show_config --mlcube=mnist -Pdocker.build_strategy=auto"]
),
_mnist(["mlcube show_config --mlcube=mnist -Pdocker.build_strategy=auto"]),
),
]
)
Expand All @@ -473,19 +497,13 @@ class UsageExamples:
[
(
"Run MNIST MLCube project",
_mnist(
[
"mlcube run --mlcube=mnist --platform=docker --task=download,train"
]
),
_mnist(["mlcube run --mlcube=mnist --platform=docker --task=download,train"]),
)
]
)
"""Usage examples for `mlcube run` command."""

describe = HelpEpilog(
[("Run MNIST MLCube project", _mnist(["mlcube describe --mlcube=mnist"]))]
)
describe = HelpEpilog([("Run MNIST MLCube project", _mnist(["mlcube describe --mlcube=mnist"]))])
"""Usage examples for `mlcube describe` command."""

create = HelpEpilog([("Create a new empty MLCube project", ["mlcube create"])])
Expand Down
Loading

0 comments on commit 78199b5

Please sign in to comment.