From 633dc945ed62ac61f68bbe48af3f22cfe2d2e6a5 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 29 Apr 2024 12:40:53 +0200 Subject: [PATCH] CLI to build static website of imputation comparison (#60) * :art: ignore jupyter_execute folder + sphinx-new-tab-link - open links in new tab - myst write to jupyter_execute folder locally (add to excluded path, otherwise a tree is created) * :sparkles: Add setup for static website for imputation comparison * :bug: add init file - fix module load error * :bug: fix wrong path * :bug: do not turn warnings into errors * :bug: raise error on if READTHEDOCS execution fails * :bug: add missing import (njab) --- .github/workflows/ci.yaml | 8 + docs/conf.py | 6 +- project/04_1_train_pimms_models.ipynb | 6 +- setup.cfg | 5 + tests/test_imports.py | 6 + vaep/analyzers/__init__.py | 2 + vaep/cmd_interface/__init__.py | 0 vaep/cmd_interface/setup_imp_cp_website.py | 205 +++++++++++++++++++++ 8 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 tests/test_imports.py create mode 100644 vaep/cmd_interface/__init__.py create mode 100644 vaep/cmd_interface/setup_imp_cp_website.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 514eb4416..d42d9808a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,6 +72,14 @@ jobs: cd project snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml -n snakemake -p -c1 -k --configfile config/single_dev_dataset/example/config.yaml + - name: Install website dependencies + run: | + pip install .[docs] + - name: Build imputation comparison website + run: | + pimms-setup-imputation-comparison -f project/runs/example/ + cd project/runs/example/ + sphinx-build -n --keep-going -b html ./ ./_build/ - name: Archive results uses: actions/upload-artifact@v3 with: diff --git a/docs/conf.py b/docs/conf.py index f7981f7d9..0d827e29d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,6 @@ # import os from importlib import metadata -# import sys -# sys.path.insert(0, os.path.abspath('../.')) - # -- Project information ----------------------------------------------------- @@ -39,6 +36,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'myst_nb', + 'sphinx_new_tab_link', ] myst_enable_extensions = [ @@ -72,7 +70,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', - # 'README.md', + 'jupyter_execute', # avoid local re-execution of written nbs during development ] # Intersphinx options diff --git a/project/04_1_train_pimms_models.ipynb b/project/04_1_train_pimms_models.ipynb index f0a9d6538..14cf6618c 100644 --- a/project/04_1_train_pimms_models.ipynb +++ b/project/04_1_train_pimms_models.ipynb @@ -680,9 +680,9 @@ } ], "metadata": { - "execution": { - "allow_errors": false, - "timeout": 120 + "mystnb": { + "execution_raise_on_error": true, + "execution_timeout": 120 }, "jupytext": { "cell_metadata_filter": "-all", diff --git a/setup.cfg b/setup.cfg index eb5b835d9..c3dfb14c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifiers = packages = find: include_package_data = True install_requires = + njab numpy matplotlib pandas @@ -39,6 +40,7 @@ docs = sphinx sphinx-book-theme myst-nb + sphinx-new-tab-link!=0.2.2 [options.packages.find] @@ -46,6 +48,9 @@ docs = exclude = test* +[options.entry_points] +console_scripts = + pimms-setup-imputation-comparison = vaep.cmd_interface.setup_imp_cp_website:main ###################### # Tool configuration # diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 000000000..72c899208 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,6 @@ + +def test_imports(): + import vaep.analyzers + import vaep.sklearn + print(vaep.analyzers.__doc__) + print(vaep.sklearn.__doc__) diff --git a/vaep/analyzers/__init__.py b/vaep/analyzers/__init__.py index c50b18b25..952822b09 100644 --- a/vaep/analyzers/__init__.py +++ b/vaep/analyzers/__init__.py @@ -1,3 +1,5 @@ +"""General classes formalizing an experiment. +""" from types import SimpleNamespace from . import diff_analysis diff --git a/vaep/cmd_interface/__init__.py b/vaep/cmd_interface/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vaep/cmd_interface/setup_imp_cp_website.py b/vaep/cmd_interface/setup_imp_cp_website.py new file mode 100644 index 000000000..481940053 --- /dev/null +++ b/vaep/cmd_interface/setup_imp_cp_website.py @@ -0,0 +1,205 @@ +"""Console script to create index.rst and conf.py for static website of +the imputation comparison workflow.""" +import argparse +import textwrap +from collections import defaultdict +from pathlib import Path + + +def split_nb_name(nb: str) -> list: + return nb.split('.')[0].split('_') + + +INDEX_RST = textwrap.dedent("""\ + Comparison Workflow Notebooks + ================================ + + Inspect the notebooks associated with the imputation workflow. + + .. toctree:: + :maxdepth: 2 + :caption: Split creation and data handling + + {nb_0} + + .. toctree:: + :maxdepth: 2 + :caption: PIMMS models + + {nb_1_PIMMS} + + .. toctree:: + :maxdepth: 2 + :caption: R models + + {nb_1_NAGuideR} + + .. toctree:: + :maxdepth: 2 + :caption: Imputation model comparison + + {nb_2} + """) + +CONF_PY = textwrap.dedent("""\ + # Configuration file for the Sphinx documentation builder. + # (Build by PIMMS workflow for the imputation comparison study) + # This file only contains a selection of the most common options. For a full + # list see the documentation: + # https://www.sphinx-doc.org/en/master/usage/configuration.html + + # -- Project information ----------------------------------------------------- + from importlib import metadata + + PACKAGE_VERSION = metadata.version('pimms-learn') + + project = 'pimms_workflow' + copyright = '2023, Henry Webel' + author = 'PIMMS' + version = PACKAGE_VERSION + release = PACKAGE_VERSION + + + # -- General configuration --------------------------------------------------- + + # Add any Sphinx extension module names here, as strings. They can be + # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom + # ones. + extensions = [ + 'myst_nb', + 'sphinx_new_tab_link', + ] + + # https://myst-nb.readthedocs.io/en/latest/computation/execute.html + nb_execution_mode = "off" + + myst_enable_extensions = ["dollarmath", "amsmath"] + + # Plolty support through require javascript library + # https://myst-nb.readthedocs.io/en/latest/render/interactive.html#plotly + html_js_files = ["https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"] + + # https://myst-nb.readthedocs.io/en/latest/configuration.html + # Execution + nb_execution_raise_on_error = True + # Rendering + nb_merge_streams = True + + # Add any paths that contain templates here, relative to this directory. + templates_path = ['_templates'] + + # List of patterns, relative to source directory, that match files and + # directories to ignore when looking for source files. + # This pattern also affects html_static_path and html_extra_path. + exclude_patterns = ['_build', 'jupyter_execute', 'diff_analysis', 'figures', + 'Thumbs.db', '.DS_Store'] + + # -- Options for HTML output ------------------------------------------------- + + # The theme to use for HTML and HTML Help pages. See the documentation for + # a list of builtin themes. + # See: + # https://github.com/executablebooks/MyST-NB/blob/master/docs/conf.py + # html_title = "" + html_theme = "sphinx_book_theme" + # html_logo = "_static/logo-wide.svg" + # html_favicon = "_static/logo-square.svg" + html_theme_options = { + "home_page_in_toc": True, + "use_download_button": True, + "launch_buttons": { + }, + "navigation_with_keys": False, + } + """) + + +def main(): + parser = argparse.ArgumentParser( + description='Create index.rst and conf.py for static website ' + 'of the imputation comparison workflow.') + parser.add_argument('--folder', '-f', + type=str, + help='Path to the folder', + required=True) + args = parser.parse_args() + + folder_experiment = args.folder + + folder_experiment = Path(folder_experiment) + nbs = [_f.name for _f in folder_experiment.iterdir() if _f.suffix == '.ipynb'] + nbs + + groups = defaultdict(list) + for nb in nbs: + _group = nb.split('_')[1] + groups[_group].append(nb) + groups = dict(groups) + groups + + # Parse notebooks present in imputation workflow + + nb_0 = '' + for nb in groups['0']: + nb_0 += " " * 4 + f"{nb}\n" + + nb_1_PIMMS = '' + for nb in groups['1']: + if '_NAGuideR_' not in nb: + nb_1_PIMMS += " " * 4 + split_nb_name(nb)[-1] + f" <{nb}>\n" + + nb_1_NAGuideR = '' + for nb in groups['1']: + if '_NAGuideR_' in nb: + _model = split_nb_name(nb)[-1] + if _model.isupper(): + nb_1_NAGuideR += " " * 4 + _model + f" <{nb}>\n" + else: + nb_1_NAGuideR += " " * 4 + ' '.join(split_nb_name(nb[5:])) + f" <{nb}>\n" + + nb_2 = '' + for nb in groups['2']: + nb_2 += " " * 4 + ' '.join(split_nb_name(nb[5:])) + f" <{nb}>\n" + + index_rst = INDEX_RST.format(nb_0=nb_0, + nb_1_PIMMS=nb_1_PIMMS, + nb_1_NAGuideR=nb_1_NAGuideR, + nb_2=nb_2) + # write to file and print further instructions + with open(folder_experiment / 'index.rst', 'w') as f: + f.write(index_rst) + with open(folder_experiment / 'conf.py', 'w') as f: + f.write(CONF_PY) + + msg = f"""\ + The index.rst file has been created in {folder_experiment}: + {folder_experiment / 'index.rst'} + The conf.py file has been created in {folder_experiment}: + {folder_experiment / 'conf.py'} + + The dependencies for the website can be installed using pip + + pip install pimms-learn[docs] + + To create the html website run the following command in the terminal: + + cd {folder_experiment} + sphinx-build -n -W --keep-going -b html ./ ./_build/ + + This will build a website in the _build folder in the {folder_experiment} directory. + + Open the index.html file in the _build folder to view the website. + + Find these instructions in the README.md file in the {folder_experiment} directory: + {folder_experiment / 'README.md'} + """ + + msg = textwrap.dedent(msg) + print(msg) + + with open(folder_experiment / 'README.md', 'w') as f: + f.write(msg) + + +if __name__ == '__main__': + main()