feat(core): add shell_complete implementation for workflows and datasets (#2512)

* add shell_complete implementation for workflows
* add how-to guide for enabling renku cli auto-completion
* add dataset autocomplete functionality
* add sphinx-tabs for better documentation shell integration
* remove click_completion

Author: vigsterkr

docs/conf.py

@@ -47,6 +47,7 @@
     "sphinx.ext.intersphinx",
     "sphinx.ext.viewcode",
     "sphinxcontrib.spelling",
+    "sphinx_tabs.tabs",
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/how-to-guides/hpc.rst

@@ -12,7 +12,7 @@ By default all workflows are executed by the ``cwltool`` provider, that
 exports the workflow to CWL and then uses `cwltool <https://github.com/common-workflow-language/cwltool>`_
 to execute the given CWL.
 
-The workflow backend can be changed by using the ``-p/--provider <PROVIDER>`` 
+The workflow backend can be changed by using the ``-p/--provider <PROVIDER>``
 command line option. A backend's default configuration can be overridden by
 providing the  ``-c/--config <config.yaml>`` command line parameter.
 The following ``renku`` commands support the above mentioned workflow provider
@@ -27,7 +27,7 @@ simply would run the following command:
 
 .. code-block:: console
 
-   $ renku execute -p toil my_plan
+   $ renku workflow execute -p toil my_plan
 
 Using ``toil`` as a workflow provider has the advantage that it supports running
 the workflows on various `high-performance computing <https://toil.readthedocs.io/en/latest/running/hpcEnvironments.html>`_
@@ -57,7 +57,7 @@ Taking the example above, the following command line will execute ``my_plan`` on
 
 .. code-block:: console
 
-   $ TOIL_SLURM_ARGS="-A my_account --export=ALL" renku execute -p toil -c provider.yaml my_plan
+   $ TOIL_SLURM_ARGS="-A my_account --export=ALL" renku workflow execute -p toil -c provider.yaml my_plan
 
 where
 

docs/how-to-guides/index.rst

@@ -13,3 +13,4 @@ aimed at active users of Renku CLI and target specific use-cases or common issue
 
    hpc
    provider
+   shell-integration

docs/how-to-guides/shell-integration.rst

@@ -0,0 +1,46 @@
+.. _shell-integration:
+
+Shell integration of Renku CLI
+==============================
+
+Renku CLI supports shell auto-completion for Renku commands and their arguments like datasets and workflows.
+
+To activate tab completion for your supported shell run the following command after installing Renku CLI:
+
+.. tabs::
+
+  .. tab:: bash
+
+    .. code-block:: console
+
+      $ eval "$(_RENKU_COMPLETE=bash_source renku)"
+
+  .. tab:: fish
+
+    .. code-block:: console
+
+      $ eval (env _RENKU_COMPLETE=fish_source renku)
+
+  .. tab:: zsh
+
+    .. code-block:: console
+
+      $ eval "$(_RENKU_COMPLETE=zsh_source renku)"
+
+After this not only sub-commands of ``renku`` will be auto-completed using tab, but for example
+in case of ``renku workflow execute`` the available ``Plans`` are going to be listed.
+
+.. code-block:: console
+
+        $ renku workflow execute run<TAB>
+        run1   run10  run11  run12  run13  run14  run2   run3   run4   run7   run8
+
+.. note::
+
+   Tab completion of available ``Plans`` only works if the user is executing the command
+   within a Renku project.
+
+
+For more information on how to set up shell auto-completion, see documentation for the Click library,
+which used under the hood by Renku CLI:
+`shell completion <https://click.palletsprojects.com/en/8.0.x/shell-completion/>`_

docs/spelling_wordlist.txt

@@ -249,4 +249,5 @@ worktrees
 yaml
 yml
 Zenodo
+zsh
 連句

poetry.lock

@@ -64,7 +64,6 @@ calamus = "<0.4,>=0.3.13"
 check-manifest = { version = "<0.48,>=0.37", optional = true }
 circus = { version = "==0.17.1", optional = true }
 click = "<8.0.2,>=7.0"
-click-completion = "<=0.5.3,>=0.5.0"
 click-option-group = "<0.6.0,>=0.5.2"
 click-plugins = "==1.1.1"
 coverage = { version = "<6.2,>=4.5.3", optional = true }
@@ -135,6 +134,7 @@ rq-scheduler = { version = "==0.11.0", optional = true }
 sentry-sdk = { version = "<1.4.4,>=0.7.4", extras = ["flask"], optional = true }
 sphinxcontrib-spelling = { version = "7.*", optional = true }
 sphinx-rtd-theme = { version = "<1.1,>=0.5.0", optional = true }
+sphinx-tabs = { version = "==3.2.0", optional = true }
 tabulate = "<0.8.10,>=0.7.7"
 toil = { version = "==5.5.0", optional = true }
 tqdm = "<4.62.4,>=4.48.1"
@@ -189,7 +189,7 @@ tests = [
     "responses",
 ]
 toil = ["toil"]
-docs = ["renku-sphinx-theme", "sphinx-rtd-theme", "sphinxcontrib-spelling"]
+docs = ["renku-sphinx-theme", "sphinx-rtd-theme", "sphinxcontrib-spelling", "sphinx-tabs"]
 all = [
     "apispec",
     "apispec-webframeworks",
@@ -230,6 +230,7 @@ all = [
     "sentry-sdk",
     "sphinxcontrib-spelling",
     "sphinx-rtd-theme",
+    "sphinx-tabs",
     "toil",
     "walrus",
 ]

pyproject.toml

@@ -34,7 +34,6 @@
     Options:
       --version                       Print version number.
       --global-config-path            Print global application's config path.
-      --install-completion            Install completion for the current shell.
       --path <path>                   Location of a Renku repository.
                                       [default: (dynamic)]
       --external-storage / -S, --no-external-storage
@@ -66,7 +65,6 @@
 from pathlib import Path
 
 import click
-import click_completion
 import yaml
 from click_plugins import with_plugins
 
@@ -94,7 +92,7 @@
 from renku.cli.update import update
 from renku.cli.workflow import workflow
 from renku.core.commands.echo import WARNING
-from renku.core.commands.options import install_completion, option_external_storage_requested
+from renku.core.commands.options import option_external_storage_requested
 from renku.core.commands.version import check_version, print_version
 from renku.core.errors import UsageError
 from renku.core.utils.git import default_path
@@ -116,9 +114,6 @@ def get_entry_points(name: str):
         return all_entry_points.get(name, [])
 
 
-#: Monkeypatch Click application.
-click_completion.init()
-
 WARNING_UNPROTECTED_COMMANDS = ["clone", "init", "help", "login", "logout", "service", "credentials"]
 
 
@@ -160,14 +155,6 @@ def is_allowed_command(ctx):
     is_eager=True,
     help=print_global_config_path.__doc__,
 )
-@click.option(
-    "--install-completion",
-    is_flag=True,
-    callback=install_completion,
-    expose_value=False,
-    is_eager=True,
-    help=install_completion.__doc__,
-)
 @click.option(
     "--path", show_default=True, metavar="<path>", default=default_path, help="Location of a Renku repository."
 )

renku/cli/__init__.py

@@ -431,6 +431,16 @@
 from renku.core.commands.format.datasets import DATASETS_COLUMNS, DATASETS_FORMATS
 
 
+def _complete_datasets(ctx, param, incomplete):
+    from renku.core.commands.dataset import search_datasets
+
+    try:
+        result = search_datasets().build().execute(name=incomplete)
+        return result.output
+    except Exception:
+        return []
+
+
 @click.group()
 def dataset():
     """Dataset commands."""
@@ -508,7 +518,7 @@ def create(name, title, description, creators, metadata, keyword):
 
 
 @dataset.command()
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @click.option("-t", "--title", default=None, type=click.STRING, help="Title of the dataset.")
 @click.option("-d", "--description", default=None, type=click.STRING, help="Dataset's description.")
 @click.option(
@@ -570,7 +580,7 @@ def edit(name, title, description, creators, metadata, keyword):
 
 
 @dataset.command("show")
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 def show(name):
     """Show metadata of a dataset."""
     from renku.core.commands.dataset import show_dataset
@@ -607,8 +617,8 @@ def show(name):
 
 
 @dataset.command()
-@click.argument("name")
-@click.argument("urls", nargs=-1)
+@click.argument("name", shell_complete=_complete_datasets)
+@click.argument("urls", type=click.Path(), nargs=-1)
 @click.option("-e", "--external", is_flag=True, help="Creates a link to external data.")
 @click.option("--force", is_flag=True, help="Allow adding otherwise ignored files.")
 @click.option("-o", "--overwrite", is_flag=True, help="Overwrite existing files.")
@@ -640,7 +650,7 @@ def add(name, urls, external, force, overwrite, create, sources, destination, re
 
 
 @dataset.command("ls-files")
-@click.argument("names", nargs=-1)
+@click.argument("names", nargs=-1, shell_complete=_complete_datasets)
 @click.option(
     "--creators",
     help="Filter files which where authored by specific creators. Multiple creators are specified by comma.",
@@ -671,7 +681,7 @@ def ls_files(names, creators, include, exclude, format, columns):
 
 
 @dataset.command()
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @click.option("-I", "--include", multiple=True, help="Include files matching given pattern.")
 @click.option("-X", "--exclude", multiple=True, help="Exclude files matching given pattern.")
 @click.option("-y", "--yes", is_flag=True, help="Confirm unlinking of all files.")
@@ -695,7 +705,7 @@ def remove(name):
 
 
 @dataset.command("tag")
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @click.argument("tag")
 @click.option("-d", "--description", default="", help="A description for this tag")
 @click.option("--force", is_flag=True, help="Allow overwriting existing tags.")
@@ -708,7 +718,7 @@ def tag(name, tag, description, force):
 
 
 @dataset.command("rm-tags")
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @click.argument("tags", nargs=-1)
 def remove_tags(name, tags):
     """Remove tags from a dataset."""
@@ -719,7 +729,7 @@ def remove_tags(name, tags):
 
 
 @dataset.command("ls-tags")
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @click.option("--format", type=click.Choice(DATASET_TAGS_FORMATS), default="tabular", help="Choose an output format.")
 def ls_tags(name, format):
     """List all tags of a dataset."""
@@ -776,7 +786,7 @@ def wrapper(f):
 
 
 @dataset.command("export")
-@click.argument("name")
+@click.argument("name", shell_complete=_complete_datasets)
 @export_provider_argument()
 @click.option("-p", "--publish", is_flag=True, help="Automatically publish exported dataset.")
 @click.option("-t", "--tag", help="Dataset tag to export")
@@ -817,7 +827,7 @@ def import_(uri, name, extract, yes):
 
 
 @dataset.command("update")
-@click.argument("names", nargs=-1)
+@click.argument("names", nargs=-1, shell_complete=_complete_datasets)
 @click.option(
     "--creators",
     help="Filter files which where authored by specific creators. Multiple creators are specified by comma.",