From f086312e4178007b43848dd446e35db6dba9977b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20K=C3=BChbach?= Date: Wed, 11 Sep 2024 17:49:27 +0200 Subject: [PATCH] Prepare v0.2.1 (#23) * Repo readme edits (#20) * Readme updated * pyproject.toml updated * Switched to source-based layout * precommit and ci updated * precommit and devrequirements created fresh --------- Co-authored-by: mkuehbach * Fixed linting (#21) Co-authored-by: mkuehbach * Relocate files and fix #18 (#22) * Relocate files and fix #18 * Adding FAIRmat docs corporate to fix #16 * Relocated mkdocs * Cleaned unnecessary readme.md --------- Co-authored-by: mkuehbach * Reverted unintentional change of build docs * Added programmatic tests (#24) * Added programmatic tests * Add debug configuration * Various fixes * Fixed unknown enum for service * if map_to_bool gets a value that is itself a bool the if src_val will do nothing * Bumped pynxtools to 0.7.0 --------- Co-authored-by: mkuehbach --------- Co-authored-by: mkuehbach --- .github/workflows/build_docs.yml | 30 +- .github/workflows/publish.yml | 37 ++- .github/workflows/pylint.yml | 37 +-- .github/workflows/pytest.yml | 23 +- ...mit-config.yaml => .pre-commit-config.yaml | 2 +- .vscode/launch.json | 25 ++ README.md | 31 +- dev-requirements.txt | 271 ++++++++++-------- {tests => dev}/batch_queue.04.sh | 0 docs/assets/favicon.png | Bin 0 -> 4056 bytes docs/assets/nomad-logo.png | Bin 0 -> 11384 bytes docs/theme/partials/header.html | 86 ++++++ examples/eln_data.yaml | 12 +- .../nxapm.schema.archive.yaml | 2 +- mkdocs.yaml | 39 ++- pynxtools_apm/README.md | 45 --- pyproject.toml | 35 ++- .../pynxtools_apm}/__init__.py | 0 .../pynxtools_apm}/concepts/__init__.py | 0 .../concepts/mapping_functors.py | 28 +- .../pynxtools_apm}/config/eln_cfg.py | 10 +- .../pynxtools_apm}/config/oasis_cfg.py | 0 docs/macros.py => src/pynxtools_apm/mkdocs.py | 0 .../pynxtools_apm}/reader.py | 132 ++++----- .../pynxtools_apm}/utils/__init__.py | 0 .../utils/create_nx_default_plots.py | 0 .../utils/generate_synthetic_data.py | 0 .../pynxtools_apm}/utils/get_file_checksum.py | 0 .../utils/get_gitrepo_commit.py | 0 .../pynxtools_apm}/utils/interpret_boolean.py | 0 .../pynxtools_apm}/utils/io_case_logic.py | 50 ++-- .../pynxtools_apm}/utils/load_ranging.py | 2 +- .../utils/load_reconstruction.py | 11 +- .../utils/oasis_config_reader.py | 0 .../pynxtools_apm}/utils/oasis_eln_reader.py | 0 .../utils/parse_composition_table.py | 0 .../utils/string_conversions.py | 0 .../pynxtools_apm}/utils/versioning.py | 0 tests/data/config_file.json | 1 - tests/run_tests.ipynb | 153 ++++++++++ 40 files changed, 713 insertions(+), 349 deletions(-) rename .precommit-config.yaml => .pre-commit-config.yaml (92%) create mode 100644 .vscode/launch.json rename {tests => dev}/batch_queue.04.sh (100%) create mode 100644 docs/assets/favicon.png create mode 100644 docs/assets/nomad-logo.png create mode 100644 docs/theme/partials/header.html rename {tests/data/nomad_oasis_eln_schema_for_nx_apm => examples}/nxapm.schema.archive.yaml (99%) delete mode 100644 pynxtools_apm/README.md rename {pynxtools_apm => src/pynxtools_apm}/__init__.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/concepts/__init__.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/concepts/mapping_functors.py (94%) rename {pynxtools_apm => src/pynxtools_apm}/config/eln_cfg.py (94%) rename {pynxtools_apm => src/pynxtools_apm}/config/oasis_cfg.py (100%) rename docs/macros.py => src/pynxtools_apm/mkdocs.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/reader.py (60%) rename {pynxtools_apm => src/pynxtools_apm}/utils/__init__.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/create_nx_default_plots.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/generate_synthetic_data.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/get_file_checksum.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/get_gitrepo_commit.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/interpret_boolean.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/io_case_logic.py (84%) rename {pynxtools_apm => src/pynxtools_apm}/utils/load_ranging.py (99%) rename {pynxtools_apm => src/pynxtools_apm}/utils/load_reconstruction.py (99%) rename {pynxtools_apm => src/pynxtools_apm}/utils/oasis_config_reader.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/oasis_eln_reader.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/parse_composition_table.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/string_conversions.py (100%) rename {pynxtools_apm => src/pynxtools_apm}/utils/versioning.py (100%) delete mode 100644 tests/data/config_file.json create mode 100644 tests/run_tests.ipynb diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 116e8e5..3974fb8 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -1,27 +1,41 @@ name: build_docs on: push: - branches: [main] + branches: + - main # Triggers deployment on push to the main branch + +env: + UV_SYSTEM_PYTHON: true permissions: contents: write jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout Repository + uses: actions/checkout@v4 - name: Configure Git Credentials run: | git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: 3.x - - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v3 + python-version: '3.x' + + - name: Cache mkdocs-material enviroment + uses: actions/cache@v3 with: key: mkdocs-material-${{ env.cache_id }} path: .cache restore-keys: | mkdocs-material- - - run: pip install ".[docs]" - - run: mkdocs gh-deploy --force \ No newline at end of file + + - name: Install Dependencies + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install ".[docs]" + + - name: Build and Deploy + run: | + mkdocs gh-deploy --force --remote-branch gh-pages \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e57c43f..4937573 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,30 +1,45 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. name: Upload Python Package on: workflow_dispatch # release: # types: [published] +env: + UV_SYSTEM_PYTHON: true jobs: deploy: + name: Upload release to PyPI runs-on: ubuntu-latest - + environment: + name: pypi + url: https://pypi.org/p/pynxtools-apm + permissions: + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive + # submodules: recursive - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install build + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install build - name: Build package run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + # 27b31702a0e7fc50959f5ad993c78deac1bdfc29 + # with: + # user: __token__ + # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3d1cfe7..9a159bd 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -2,32 +2,35 @@ name: linting on: [push] +env: + UV_SYSTEM_PYTHON: true jobs: linting: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ matrix.python_version }} - name: Install dependencies run: | - git submodule sync --recursive - git submodule update --init --recursive --jobs=4 - python -m pip install --upgrade pip - - name: Install package - run: | - python -m pip install --no-deps . - - name: Install requirements - run: | - python -m pip install -r dev-requirements.txt - - name: ruff + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install ".[dev,docs]" + - name: ruff check run: | - ruff pynxtools_apm tests + ruff check src/pynxtools_apm tests - name: ruff formatting run: | - ruff format --check pynxtools_apm tests + ruff format --check src/pynxtools_apm tests - name: mypy run: | - mypy pynxtools_apm tests + mypy src/pynxtools_apm tests diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index afda63b..13f1e7e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,3 +1,6 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + name: pytest on: @@ -6,6 +9,9 @@ on: pull_request: branches: [main] +env: + UV_SYSTEM_PYTHON: true + jobs: pytest: runs-on: ubuntu-latest @@ -15,20 +21,27 @@ jobs: python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} - name: Install dependencies run: | - python -m pip install --upgrade pip + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install coverage coveralls - name: Install package run: | - pip install ".[dev]" + uv pip install ".[dev]" - name: Test with pytest run: | - pytest tests + coverage run -m pytest -sv --show-capture=no tests + - name: Submit to coveralls + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls --service=github diff --git a/.precommit-config.yaml b/.pre-commit-config.yaml similarity index 92% rename from .precommit-config.yaml rename to .pre-commit-config.yaml index 774bf8c..81a6893 100644 --- a/.precommit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.3.4 + rev: v0.5.5 hooks: # Run the linter. - id: ruff diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..dff93ec --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "pynx-apm debug", + "type": "python", + "request": "launch", + "cwd": "${workspaceFolder}", + "program": "../.py3.12.4/bin/dataconverter", + "args": [//"convert", + "tests/data/eln/eln_data.yaml", + "tests/data/eln/apm.oasis.specific.yaml", + "tests/data/apt/Si.apt", + "tests/data/rng/87D_1.rng", + "--reader", + "apm", + "--nxdl", + "NXapm", + "--output=tests/prod/apt.Si.apt.nxs"] + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 3ecf2c7..ed8f47a 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ ![](https://img.shields.io/pypi/v/pynxtools-apm) ![](https://coveralls.io/repos/github/FAIRmat-NFDI/pynxtools-apm/badge.svg?branch=main) -# A parser and normalizer for atom probe tomography and field-ion microscopy data +# Parse and normalize atom probe tomography and field-ion microscopy data -# Installation +## Installation It is recommended to use python 3.11 with a dedicated virtual environment for this package. Learn how to manage [python versions](https://github.com/pyenv/pyenv) and [virtual environments](https://realpython.com/python-virtual-environments-a-primer/). @@ -19,9 +19,11 @@ This package is a reader plugin for [`pynxtools`](https://github.com/FAIRmat-NFD pip install pynxtools[apm] ``` -for the latest development version. +for the latest release version from [pypi](https://pypi.org/project/pynxtools-em/). -# Purpose +If you are interested in the newest version, we recommend to work with a development installation instead. + +## Purpose This reader plugin for [`pynxtools`](https://github.com/FAIRmat-NFDI/pynxtools) is used to translate diverse file formats from the scientific community and technology partners within the field of atom probe tomography and field-ion microscopy into a standardized representation using the [NeXus](https://www.nexusformat.org/) application definition [NXapm](https://fairmat-nfdi.github.io/nexus_definitions/classes/contributed_definitions/NXapm.html#nxapm). @@ -30,14 +32,12 @@ within the field of atom probe tomography and field-ion microscopy into a standa This plugin supports the majority of the file formats that are currently used for atom probe. A detailed summary is available in the [reference section of the documentation](https://fairmat-nfdi.github.io/pynxtools-apm). -# Getting started +## Getting started [A getting started tutorial](https://github.com/FAIRmat-NFDI/pynxtools-apm/tree/main/examples) is offered that guides you -how to use the apm reader for converting your data to NeXus using a Jupyter notebook. That notebook details also the commands how to convert data via command line calls. Note that not every combination of input from a supported file format and other, typically electronic lab notebook, input for the parser allows filling the required and recommended fields and attributes of the NXapm application definition. -Therefore, you may need to provide an ELN file that contains the missing values in order for the -validation step of the APM reader to pass. +on how to use the apm reader for converting your data to NeXus using a Jupyter notebook or command line calls. Note that not every combination of input from a supported file format and other input, such as from an electronic lab notebook, allows filling the required and recommended fields and their attributes of the NXapm application definition. Therefore, you may need to provide an ELN file that contains the missing values in order for the validation step of the APM reader to pass. -# Contributing -We are continously working on adding parsers for other data formats, technology partners, and atom probers. +## Contributing +We are continously working on improving the collection of parsers and their functionalities. If you would like to implement a parser for your data, feel free to get in contact. ## Development install @@ -47,19 +47,12 @@ Install the package with its dependencies: git clone https://github.com/FAIRmat-NFDI/pynxtools-apm.git --branch main --recursive pynxtools_apm cd pynxtools_apm python -m pip install --upgrade pip -python -m pip install -e . python -m pip install -e ".[dev,docs]" -``` - - +The last line installs a [pre-commit hook](https://pre-commit.com/#intro) which +automatically formats (linting) and type checks the code before committing. ## Test this software Especially relevant for developers, there exists a basic test framework written in diff --git a/dev-requirements.txt b/dev-requirements.txt index 55f7385..1de3684 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,20 +1,18 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml -# -anyio==4.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --extra=dev --extra=docs -o dev-requirements.txt +anyio==4.4.0 # via # httpx # jupyter-server +anytree==2.12.1 + # via pynxtools argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 # via isoduration -ase==3.22.1 +ase==3.23.0 # via # ifes-apt-tc-data-modeling # pynxtools @@ -22,11 +20,11 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -babel==2.14.0 +babel==2.16.0 # via # jupyterlab-server # mkdocs-material @@ -34,23 +32,26 @@ beautifulsoup4==4.12.3 # via nbconvert bleach==6.1.0 # via nbconvert -blosc2==2.5.1 +blosc2==2.7.1 # via tables -build==1.1.1 +build==1.2.1 # via pip-tools -certifi==2024.2.2 +certifi==2024.8.30 # via # httpcore # httpx # requests -cffi==1.16.0 +cffi==1.17.1 # via argon2-cffi-bindings +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 # via requests click==8.1.7 # via # click-default-group # mkdocs + # mkdocs-click # pip-tools # pynxtools click-default-group==1.2.4 @@ -61,23 +62,29 @@ comm==0.2.2 # via # ipykernel # ipywidgets -contourpy==1.2.0 +contourpy==1.3.0 # via matplotlib +coverage==7.6.1 + # via pytest-cov cycler==0.12.1 # via matplotlib -debugpy==1.8.1 +debugpy==1.8.5 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -executing==2.0.1 +distlib==0.3.8 + # via virtualenv +executing==2.1.0 # via stack-data -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat +filelock==3.15.4 + # via virtualenv flatdict==4.0.1 # via ifes-apt-tc-data-modeling -fonttools==4.49.0 +fonttools==4.53.1 # via matplotlib fqdn==1.5.1 # via jsonschema @@ -85,9 +92,9 @@ ghp-import==2.1.0 # via mkdocs h11==0.14.0 # via httpcore -h5grove==2.1.0 +h5grove==2.2.0 # via jupyterlab-h5web -h5py==3.10.0 +h5py==3.11.0 # via # h5grove # ifes-apt-tc-data-modeling @@ -95,9 +102,11 @@ h5py==3.10.0 # pynxtools httpcore==1.0.5 # via httpx -httpx==0.27.0 +httpx==0.27.2 # via jupyterlab -idna==3.7 +identify==2.6.0 + # via pre-commit +idna==3.8 # via # anyio # httpx @@ -105,28 +114,27 @@ idna==3.7 # requests ifes-apt-tc-data-modeling==0.2.2 # via pynxtools-apm (pyproject.toml) -importlib-metadata==7.0.2 +importlib-metadata==8.4.0 # via pynxtools iniconfig==2.0.0 # via pytest -ipykernel==6.29.4 +ipykernel==6.29.5 # via # jupyter # jupyter-console # jupyterlab - # qtconsole -ipython==8.24.0 +ipython==8.27.0 # via # ipykernel # ipywidgets # jupyter-console -ipywidgets==8.1.2 +ipywidgets==8.1.5 # via jupyter isoduration==20.11.0 # via jsonschema jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via # jupyter-server # jupyterlab @@ -137,26 +145,25 @@ jinja2==3.1.3 # nbconvert json5==0.9.25 # via jupyterlab-server -jsonpointer==2.4 +jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema==4.23.0 # via # jupyter-events # jupyterlab-server # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter==1.0.0 +jupyter==1.1.1 # via - # ifes-apt-tc-data-modeling # pynxtools-apm (pyproject.toml) -jupyter-client==8.6.1 + # ifes-apt-tc-data-modeling +jupyter-client==8.6.2 # via # ipykernel # jupyter-console # jupyter-server # nbclient - # qtconsole jupyter-console==6.6.3 # via jupyter jupyter-core==5.7.2 @@ -169,12 +176,11 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat - # qtconsole jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.2 # via # jupyter-lsp # jupyterlab @@ -184,38 +190,43 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.7 +jupyterlab==4.2.5 # via + # pynxtools-apm (pyproject.toml) # ifes-apt-tc-data-modeling + # jupyter # notebook - # pynxtools-apm (pyproject.toml) -jupyterlab-h5web==12.1.0 +jupyterlab-h5web==12.2.0 # via - # ifes-apt-tc-data-modeling # pynxtools-apm (pyproject.toml) + # ifes-apt-tc-data-modeling jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.3 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.13 # via ipywidgets -kiwisolver==1.4.5 +kiwisolver==1.4.7 # via matplotlib -lxml==5.1.0 +lxml==5.3.0 # via pynxtools -markdown==3.6 +markdown==3.7 # via + # markdown-include # mkdocs + # mkdocs-click # mkdocs-material # pymdown-extensions +markdown-include==0.8.1 + # via pynxtools-apm (pyproject.toml) markupsafe==2.1.5 # via # jinja2 # mkdocs # nbconvert -matplotlib==3.8.3 +matplotlib==3.9.2 # via # ase # radioactivedecay @@ -226,33 +237,38 @@ matplotlib-inline==0.1.7 mergedeep==1.3.4 # via # mkdocs + # mkdocs-get-deps # pynxtools mistune==3.0.2 # via nbconvert -mkdocs==1.5.3 +mkdocs==1.6.1 # via + # pynxtools-apm (pyproject.toml) # mkdocs-macros-plugin # mkdocs-material - # pynxtools-apm (pyproject.toml) +mkdocs-click==0.8.1 + # via pynxtools-apm (pyproject.toml) +mkdocs-get-deps==0.2.0 + # via mkdocs mkdocs-macros-plugin==1.0.5 # via pynxtools-apm (pyproject.toml) -mkdocs-material==9.5.18 +mkdocs-material==9.5.34 # via pynxtools-apm (pyproject.toml) mkdocs-material-extensions==1.3.1 # via - # mkdocs-material # pynxtools-apm (pyproject.toml) + # mkdocs-material mpmath==1.3.0 # via sympy msgpack==1.0.8 # via blosc2 -mypy==1.9.0 +mypy==1.11.2 # via pynxtools-apm (pyproject.toml) mypy-extensions==1.0.0 # via mypy nbclient==0.10.0 # via nbconvert -nbconvert==7.16.3 +nbconvert==7.16.4 # via # jupyter # jupyter-server @@ -265,17 +281,21 @@ ndindex==1.8 # via blosc2 nest-asyncio==1.6.0 # via ipykernel -networkx==3.2.1 +networkx==3.3 # via radioactivedecay -notebook==7.1.3 +nodeenv==1.9.1 + # via pre-commit +notebook==7.2.2 # via jupyter notebook-shim==0.2.4 # via # jupyterlab # notebook -numexpr==2.9.0 - # via tables -numpy==1.26.4 +numexpr==2.10.1 + # via + # blosc2 + # tables +numpy==2.1.1 # via # ase # blosc2 @@ -292,11 +312,11 @@ numpy==1.26.4 # tables # tifffile # xarray -orjson==3.10.1 +orjson==3.10.7 # via h5grove overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==24.1 # via # build # ipykernel @@ -306,17 +326,17 @@ packaging==24.0 # matplotlib # mkdocs # nbconvert + # pint # pytest - # qtconsole - # qtpy # tables # xarray -paginate==0.5.6 +paginate==0.5.7 # via mkdocs-material -pandas==2.2.1 +pandas==2.2.2 # via # ifes-apt-tc-data-modeling # pynxtools + # radioactivedecay # xarray pandocfilters==1.5.1 # via nbconvert @@ -326,29 +346,36 @@ pathspec==0.12.1 # via mkdocs pexpect==4.9.0 # via ipython -pillow==10.0.1 +pillow==10.4.0 # via matplotlib +pint==0.17 + # via pynxtools-apm (pyproject.toml) +pip==24.2 + # via pip-tools pip-tools==7.4.1 # via pynxtools-apm (pyproject.toml) -platformdirs==4.2.0 +platformdirs==4.2.2 # via # jupyter-core - # mkdocs -pluggy==1.4.0 + # mkdocs-get-deps + # virtualenv +pluggy==1.5.0 # via pytest +pre-commit==3.8.0 + # via pynxtools-apm (pyproject.toml) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via # ipython # jupyter-console -psutil==5.9.8 +psutil==6.0.0 # via ipykernel ptyprocess==0.7.0 # via # pexpect # terminado -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data py-cpuinfo==9.0.0 # via @@ -356,24 +383,30 @@ py-cpuinfo==9.0.0 # tables pycparser==2.22 # via cffi -pygments==2.17.2 +pygments==2.18.0 # via # ipython # jupyter-console # mkdocs-material # nbconvert - # qtconsole -pymdown-extensions==10.8 +pymdown-extensions==10.9 # via mkdocs-material -pynxtools==0.2.1 +pynxtools==0.7.0 # via pynxtools-apm (pyproject.toml) -pyparsing==3.1.2 +pyparsing==3.1.4 # via matplotlib -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.1.1 +pytest==8.3.2 + # via + # pynxtools-apm (pyproject.toml) + # pytest-cov + # pytest-timeout +pytest-cov==5.0.0 + # via pynxtools-apm (pyproject.toml) +pytest-timeout==2.3.1 # via pynxtools-apm (pyproject.toml) python-dateutil==2.9.0.post0 # via @@ -387,37 +420,34 @@ python-json-logger==2.0.7 # via jupyter-events pytz==2024.1 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # jupyter-events # mkdocs + # mkdocs-get-deps # mkdocs-macros-plugin + # pre-commit # pymdown-extensions # pynxtools # pyyaml-env-tag pyyaml-env-tag==0.1 # via mkdocs -pyzmq==26.0.2 +pyzmq==26.2.0 # via # ipykernel # jupyter-client # jupyter-console # jupyter-server - # qtconsole -qtconsole==5.5.1 - # via jupyter -qtpy==2.4.1 - # via qtconsole -radioactivedecay==0.4.22 +radioactivedecay==0.5.1 # via ifes-apt-tc-data-modeling -referencing==0.35.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.16 +regex==2024.7.24 # via mkdocs-material -requests==2.31.0 +requests==2.32.3 # via # jupyterlab-server # mkdocs-material @@ -429,20 +459,26 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -ruff==0.3.4 +ruff==0.5.5 # via pynxtools-apm (pyproject.toml) -scipy==1.12.0 +scipy==1.14.1 # via # ase # radioactivedecay send2trash==1.8.3 # via jupyter-server +setuptools==74.1.2 + # via + # jupyterlab + # pip-tools + # radioactivedecay six==1.16.0 # via + # anytree # asttokens # bleach # python-dateutil @@ -451,13 +487,15 @@ sniffio==1.3.1 # via # anyio # httpx -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 stack-data==0.6.3 # via ipython -sympy==1.12 +structlog==24.4.0 + # via pynxtools-apm (pyproject.toml) +sympy==1.13.2 # via radioactivedecay -tables==3.9.2 +tables==3.10.1 # via ifes-apt-tc-data-modeling termcolor==2.4.0 # via mkdocs-macros-plugin @@ -465,11 +503,11 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -tifffile==2024.4.24 +tifffile==2024.8.30 # via h5grove tinycss2==1.3.0 # via nbconvert -tornado==6.4 +tornado==6.4.1 # via # ipykernel # jupyter-client @@ -493,27 +531,36 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat - # qtconsole -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20240906 # via arrow -types-pyyaml==6.0.12.20240311 +types-pytz==2024.1.0.20240417 + # via pynxtools-apm (pyproject.toml) +types-pyyaml==6.0.12.20240808 # via pynxtools-apm (pyproject.toml) -typing-extensions==4.10.0 +types-requests==2.32.0.20240905 + # via pynxtools-apm (pyproject.toml) +typing-extensions==4.12.2 # via # h5grove - # ipython # mypy + # tables tzdata==2024.1 # via pandas uri-template==1.3.0 # via jsonschema -urllib3==2.2.1 - # via requests -watchdog==4.0.0 +urllib3==2.2.2 + # via + # requests + # types-requests +uv==0.4.6 + # via pynxtools-apm (pyproject.toml) +virtualenv==20.26.3 + # via pre-commit +watchdog==5.0.2 # via mkdocs wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 +webcolors==24.8.0 # via jsonschema webencodings==0.5.1 # via @@ -521,17 +568,13 @@ webencodings==0.5.1 # tinycss2 websocket-client==1.8.0 # via jupyter-server -wheel==0.43.0 +wheel==0.44.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.13 # via ipywidgets -xarray==2024.2.0 +xarray==2024.7.0 # via pynxtools xmltodict==0.13.0 # via ifes-apt-tc-data-modeling -zipp==3.18.1 +zipp==3.20.1 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/tests/batch_queue.04.sh b/dev/batch_queue.04.sh similarity index 100% rename from tests/batch_queue.04.sh rename to dev/batch_queue.04.sh diff --git a/docs/assets/favicon.png b/docs/assets/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..8c344efd1b06fdb3ea6bf14fcfe5d231387c8995 GIT binary patch literal 4056 zcmV;}4=3=6P)Wv! z88ts%?>$Mj%65tp8^0F(1#FeIQbbC&tceYJ&NFHxb8hFJ&;8zePWPS*N-U#gt;%l5 z=4>-M5)RJ9XM^5H6uboRQLSp0F*G?jPFZ`qp~8(Mr1!Y4ea8Kx~f zDm=by8!m4O2eh=hAbc#`Vaf%CuU$P+^>e6Cp$o12Tu`q?ayPQ6|6pJAZ<_p4qfl57C8D-w>b4eqcN{@y>PQq}r1eoteT+q_wY64swbokQ@l~r< zM{T8QwE`*v8bs^+F;zrbSjI!G)mRY&sMzul6fd`boa}XXZ}uje$H`_lpP8L|cQ-lb zm*l(WeSSa4XluTtDkI&n`UAs&p+L_0I|SGStOo3gziWl5eXhzjRpj!;zQ73NeJHW1>wSn#0J>rI1`YxYU_6j-;PmytUx5X{LSbraqPI{DxqNXXFu{2p z4)ku|^#1{m0rP={cB!yFYWpG=fNoghfYX7afn6g=vJ+U0ya~4nQ~NlAG-D-~FAf8y zA|HAq!taT#z;nPuz&&=U@M#o@MkWB=unq>U0>(v^rVIZCt^;lsrnWK0d;*cn7Y70t z0zU(GiZQEqfJ=dycB!x}OhTarpc_^mxDq%vY+4Pl7AdwngsDaHO9(?QU)&Lx23!K{ zqA=@az(v3lcB#-AB!R#J&<(2(a1B!J${Le3SimG(JbZxLa{1!nUI%%W0J>of z0A2?4D6CS*>B7|RiYl>OzW9A)^_M0-0uHoGg}3V+(4zpH;oP~X?c+zqXTVrtYJUrx zP%d9Q7&!{JL)f$uKmdo-8%Fl30xlq(4g+wF^}-&bK|r~A$^f!^g);o42sZ}Bssxp%T!UX1o#!{t@GTNsvDNp z$O*Z8aW~{VM=Nn%E?=x__;(Y4Zdk*C)0CN{hV6h~GMhQ8A>;L|7*E8n;YAg$d#FNOj8`#CU|FAfI|jMtpRs1kOS0CdCpGH|3) zbJSw^Jnpzytw@$JxqNX%`AuyJa3rl)?^WWrbergXNUE7o{-;&|$0#;LHDkQS1f044 zBh;9a7+s}&tpHp{i&_k?QN6*)UPCK0y!=nC0CI{ANdm8H9HzvIWa+xbS^*4IY)BG# zU1K{4V5n2oeIWqduy#Q0lIEoFDglOjeBTQ6MpkuIB|tm;`XQ?V$_#^Q0km@WuM)58 zY{gMfH~K)e09Geth-$p9^L-^&B+n|S7QlL7OER`7B@=*LI=%)f1RzYU6S+6P)$zK< zb`ZeYvfqK)ZeS*W_7FhVHP#B?J;jD3!77jM*8rimT&IA#0(efX9m=tRfAx4ksjwAz zMU6R$@nZR(S^=1f4N;9LOwG@)dv5HOBum%zRS7_t+Dag?z4i+5EAaWTT9GU(>{6kt z6m(k=)T*Vx7T_sA2im2=o51Vwnv)pwD&M#X;88_}sOA}AYM(T4{5ULKmyDND(g~{pk3{E*0JY?u^Hr#JI55tC;J? z5MgRh0Z%J2Neve}!fuqyk)BLjfoG8(RI2D{Z(!OthL%|%Ozov0C)%Zg4O}0wJxOt) zn=84x0uZM5D)4ZuCdE%D;f4HLoQabZGt({=T-$YC{sDeMx~GY+5T;fNlYm_+Y(Q=i z59Elk39ylV9$e{B0K(K(A$O7_?u|15bAT(NNXRY~=9A7UaSPISVBHBWPbX2r)D{6} zMroA-mH|_I4*UA$8q((@XiB}Jzl<+0VZzkzBRtIkS_r^6lkz7od8>rzJvc0N8)y=umxEqHv4?Z?f9(zd<(#F!Fn>%>r}HcHE5U5 zFLtT02D#7v!8r2?NC)s^V47by@H3NkP4ZPeQh`n}wyo0ZGIe%H@j_kd4Fss`IX3IdFVW4k7E&NE@J%xxk`;=f=TB zNd=j5x`=xNw*sndSc5F`M+bTM5H4WESfHWeCMKRodIU{;peC0u?uD!pPK-ILjmTcv z@9a_`ROp;gkv??8S`3T{lSTxL7pC@D!ZuWq%NIuj=ObOphw@Zf!&^wF-8a~!!rCYj z>=W|*M@Uuxn`n+ws0zDOSOT1q%NMsphKehPjoUpya*L4nc|IV5#s;cK*b?BIz&|3g zB?$9`sinf5i_>e^7tr`3U}X94KxBPxBbRKK{|aYDGB(1N0Lzdr$)ku57=+ZUS#W;2 zf+ibB3EB$)x>gUZjL4QCyqjR&EkP&&I3zh;+zwmWVkHUzB)M23l9q5g6NLc!MUX}s z^KJ>E5I~Zi#oEEES*&CNXmdh!19ih11nd!!EkXDe>071Kg>G2AkO3(MAQ!T%S3RU1 zb|C0f;Lj1+5`?#ay_;bkpeJ<0+77vf;22~Or=iYk2-QxhHY0mgtC2mNSCETvR>XCU zVFfT7_-;hD1Z7`gYX3~whHmJFH5mE8$0LJThKaxQ4sz4(0^nI;YLP9{4Ko|~72s`> zk0X4oFtzCk+t7t`UoHlwAw$+{670CdAT2zY_!srTz)0u$p{BP!QvzIzT-ca4_5ohh4eMUrunq{4fNof$bisE?#2XR zYLB>mQC9#?5a~C8NbAxjVGHuwHX@gnZv#dmSM0W8$hdcbeca{{dYTSCkJfznAj8J& zm$VTI*&UdQY`|9GS_0^XH4v$qFf&*g&Bcenp0#5n^%{X^kv;rYV-Rw?S{2<&0Nt>* zM^5vmsoR&~kBz`cVQL?hKX5AnP9l5wtx7*CB8P*!0H&$2B_kn@t$b5e1voQF8wkH= zFr#_d1Pl_UwgtME0C}>9-|F-QjwrvW6~Jhvreq{VS2d6cpgrjt;?4xno^S_c! z^W7WC+5qi92e5-MwT{XXpfB0OZx?zaM+U2^fOd!+p78}a$OO>N42J6P@!NoIizG5y zmXD!(FCZ%g+J`mJT>zUCFeD=x)*M7AESghEP zjAU3|{wEVad(svAsa60>Q6KamLoJ=a4L1)X@?om&!`hixUK*kFO;peNn{D~MjQ!z z6&b8)N9VN%aQJ7lb&@zsXq$$XfBW>{s-T}r`ZU(Kz zmzy}ww9^)FKNisT2o~I#7 zR|%>5NF$00Q(Fy;L%NDf1E+aCd}!QL=sw60re6zcRm?nuyzo!~mA%jnYbx-2nsXWb z7T^qFYWMp&IEb_8fxvx8hm6faC-6&TeXk^Xh;CSiBLz2DMSeb6iS&}{dtpydck#WD zjiO6}oF6e8fK!C2J*qlm-LQ5C<|1PkCC!V#35_;*!-kBR4BSp~PLr1d->V(pR*B<| z^;Y1A37N3X$^Qa3--lj=5&(4d5YBgA{lli`lSravM2T)#`vX@Z;{>VVA>gtwKY-8z zDC2~Xn+}|V99fHlRmiI6t|;Q=#DU|VxJbP2br%Ac3RC-g*n}bzKpEYzh5~0000< KMNUMnLSTY{rem7` literal 0 HcmV?d00001 diff --git a/docs/assets/nomad-logo.png b/docs/assets/nomad-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..3a1282f59431a26c51dfe4a09df354cb7861de28 GIT binary patch literal 11384 zcmY+qbyU;u|2{q#CEYD4;V5Y)B?6;Cx*KG4BP}RBYQR8RN(3du5t0)aARsW6?nY@r zQsMi2y+7yo$L~I8+jD0-_qpQ{*L6J}$xrk(C`egJK_C!?mZq8!2!!tqJo}Om0q@;5 z=S3h8UW~h{>XSetjmKPC>Z&s0P#GB!2~lwnNDvu}92ZhEq#smXa^Q#MSXx-<_L5#2 z(y_Vnm)%Pd&XjuFl1e1_b-?*(6l-IdS%?boIQzZ=t9_ZNOhl6d@z%bA>xTa5}Vk+@TtCu(WP z-*YYh{6*)6hXmVuhbGMwDVe+FqFTYWPDVktumpzI+vnFFeAnTK$$yS-!~Kf#WR83= zZlC>^&hKuYfE3RCa)max&tKn~o<7#{T#UbxmV=79CyU70CccqZm7etd!drhY`zVqi z*dN{*SMbScJo*gU89!4SIZ5bs_^6|(q4XAirrNFQ_+(B5xRN`5nwEhe5Egu0JaR&7x#oddh)s^?8eiH!XKf>$ zz?i}i&#xxF$g1{LAi(o@*Lae#vO#VAwwO(fE2i{UGx9Q8`u*GgjUBgeiMh(A5pXG< z65;y|mC`T_1LhcpC`kMh{uCa1+KfSB1Y%7UM6N-8tVjRCy8&|LDQ)O!Q zzF`JqXdGM{SfN{BCa?gD$N^Pjv<~U`Tw+YNzvDRdUluvawx-{mrY8Rzy0Af5p-G7H z$~#QK&7@_VdF(Bi-5VWx|Ahi5AwlCAQMZbl0nct7@g)#i!;H&Vi(1XY` z$kE8%FM3E9z~&c&E9bD9m^j+eh$V}`s*bSsWn)zS%nkiPZkmyE8O^$08X%tpetxpP zYja~PW#SvINc1C(QH=jd2XJ^!uiZ!BLOw^!U`mpAW0eZ{d%oy4#r`$m%ydB8_?%Q9 zq6FCC0A}hf7?S99nf!k|NgheShzcI z8Pf#y1POibzGpGT_N6i^AAf=ydig<*LP~efFy{UP)|tewP0n6X@20QL!w>?zLzW~p zdv7pyvDnC2I5HS7*(Aia2q{B1lPp)Vo)~M{7Wrd0AUdP6>3a-S^;AXjd@7@C6|E?0 zk^EJO_6;+KxRp(gRq}D>uxNr%cVU4Q6YZk&ygH-TMtKViV$brB7S&<~b z>CTFq(QLC~-7}seN(0Z!4RqNmj->{*6U$rK+lF2mek_*R>LJ@e+bB3dJAo4j@`MQ>*Gf?Cn1<1b8fio^0|Mamw+s~gh z1vxusuL*($)ugN*-byjbq*`FA=i|s?#z!mGoo8N#Pv-vsViYyr~ zE`evG8it=);L0ZXH)iF_nZxx$s4kLi%X#T$C2t9a*6c2ONQAcGg2{&m5&T&l5j>yh zkX6-skPf9re-Fh1i-gW^;a4b>ALV<_74kNFZP(=@?R<67^j?Ngj$t&}iV_k9+X8MvGBNBB(WA2v1y(;ggP$agwL-i*o& zO*&>)#G-wh@9PhI#wZ%iHZa-Bls8UrT0^c`7iQi_e4x9#_arevqxwm=%{%wDi*QS5 z*KpTPdN17{)1ugQ=WbsgFLE0(RFr{-x&(Jit2bNt;p}$dUTP*!o_pet@YKFv7;9r8 zaf~n}dpd%>hz~^}J{8D4V8@=AZT{G!BQ>OPeg+k=g%{}o4@JM%T4)&jBU8Hrs@V%@ z%awPf%xLY)!ce8ywAyqDe@OC1Z!)sRD=2zLWS5CsW>ygUO<0i1v0fCKoJn#YIy)=J zX~faezxwJgKM*1(Rmp<~*o$&At>O{1X3wJrB}is8I}L87?l$z(FCwwIhZ6BzO`{6Q z9u}oBv`dFsrwvao?k@oy7Yu*yCNpCiv*D ztIS>#=Mw7c)v2)vT7~^hSGcOfZaJ(xtl6urVbJOiz_f@HFq9Rw_GeCVTZYZ0B>L9! zAVX%*~N^eq4%DS>?9?zcFU|a)Y zhDm#Gt=P{s94O~|6&4?^x*38C2M@l0N01I?KN=rQ zIQC*s_s028B;6cau_v5f;y2o`zR}rKy+E0hApMtXYyxgdOlv&5Sfi)NS((`zWDmGS zsZ}z$Tx?{Y3Dh5Lau)MLTg@c{Q+yC8GZ*k1jru}8_Yt0HkCb)Oa_u-vp-in#yAcmD zV`t^qPgMIejW_tljE4^hIc-ZW4Ez3NCLw*m1WpV#w_1lT58G?=g%!MwEb+VwtTHGq zayxJ98Pe}#utu}>1MxDcEh`mFJmQw}8PVF`_VNOuMvy{JJH00M*lM*%3ohfDU4Z@qGzn(iEZ#bbHKFbp4oN|B;)Q)H4v8*@9vQ`HEbcLUhtSI?ue z9?XU*u<9Wi6$~5FI&-Y#)Chv?P^Za=yL$G4ydR& zirFi&$`G}nJDMA9<`MiJD=B{&pMWzvZ_uIT2X-Z{dyT#D+8TM;Brn|Q#ar`-xsMLB z-`2wG21;DtqJB;&+qeW0nxCN7ReoiIrD4_&lB*s7(fQSnI<^N3cGo@upx%KSIVVKjRmhPW)55>k!GuN*Hc zZMskUYzN`TPhS5CT|6P+DO|d>eILIH<Ku#7q1jFrd6l;h!SgC^dXGpyt~8=eY9pOYqdbI za1RwlHb$){K79^k#bQ$1D^#;TRc=JM?ORQiA;nUPZxM4L1E%NkYlJI#m=65mS9mtm ze*bJjzrc?lvkfXzDUdIG`}=jRAe8OMe?d{$Rr4=F&0ahVvCS|4HcX=eEJR<>!zyy% z>p!|_D0OGBC)U$gh$fS+vdQ&4E?vivJ17z#H;xy}XgTWbDEauH`nMfDEh;ew>6`bFrB@m2AuTFvD&;T3(X5 zg;*DRl<&MElGXwMbEVC*7L?_A$Yq;J$jcwv7nQHXSm#I1s2gA6*2XOG(-@ZtDEasL z;0jlZ^DQ4uLfol#bFgq77LLW#qopAs#9C>l<3H>j<(?KXajQ-zv>cOLr!uGI-zM}> zgq;JLA-R5;q|T7*fY8V1Wacp5!Nn@e?UI_ns+?p88 zre6PajFnm#@mb%tD{VN1cP^h4X|jShR$=8U7-8w~mZ9Y6mg&h9DMlY}nv*1zkqRu2 zFV4$!C}^c7(hJ}t+56^VjY$0n=1EW9^VPAB!bKJhY5nA~n;1q+gptr{?AKURo-g6_ z=KpfoE3og{7)3u=kWF8-+Rs`*I?Skg4(H)ymVQ~o-&=fF{26v1~4+&%}k(gPwe^UD&Au)OU*1zMZp6n9y>n&3u@x;xZAM^#v8 zm{}{ZzA=HkDJJVtfeQ%W?(X}UJqZrqR$5V93?XfphCKxn)AoF;quT;qPkZvZl5SqP z=&ii{p?#|PeqtxMZMN)_KlzdFJpAOdXOv41OG z5*s2(9Z=CM^MKv&pUQgZYg1(ld$cc_Ar)kvfU9yV!F$Rqax*!?128^{w$C%r-x6aE z*AX7-w1MwULIj&>7r9Wjcr&6|ZO)464}T}%oMe1RqR3^2-a=1;@un8KMmiOj%W#&b z_Mt%r4h!r&2A#cfhGjfW!NHF{JuyXwSG?+`7Se5xH3>o1gXxw)DITM$F+B-CYRR_1 z#B3s+JEj0Pd?~XP#kX=4?!D1kgBpsN#yLoc$!z3 zDP7X);tUi`bP>BP-QlX_r`shH;0OTPwUb^g-~$?@@WlIR_QX-yK?# z-yDUR@a%tg!2!|@@naUf<=^Z(ggQL!l3Dx5t}{k&LzemN-Vet10mx%P)$g17^S^^e zT96xPRg(2L#uZ;%^Y+DHi1uyiLE8E6EnuHGJ57G6xfJ>r%fz{*#@%kpeq{ay2M0DA z)}pWL{xcee<3iW;P0DT7a)&bVH(GU*a?UIyK_=32cwp z3j|`9)iS>eeMoI7%Bojo_MX>q&D&qOXo`o$5*C(N#Eo6HPs5KNA)FtZBh9#tGU!b) z%4u94cN|)`e1n=Qpv1DJQVgXJtOyQG%^K3+HpCm$$^Gl%oZB9YwRUWmf%o!VH}{fR zDW48`AUFOt?Ih<)!w?Z9*8VdNn@&@n0V$1*e>(SQ(a>+T%(l%4w@brR8J>7eUWz1N z0XLB#yZL&+Zt+K$r9_kOx%F+`qP@`!@pnEX+R3&p!2PX-9f4gbwv}i`(1t`}8)Ci! zD$F_H9o^Zv*T}S(j7qa|Jv%0?u}$R01y@3=H>A;s0J@(gq1JvkTEqM+q0cD*-1-4@ zg`b3xm=hm57|`p!gU#wM9+e4=p8KUJ0-Wm**>dxw`xVJy4MoB zkOJ#NsVgbbTX&-rbg4&7WU}DRQ}L}=&j1h9zFBzpuynXZ01i;4OQwm% zN8`-B2sf$KH_zmEIDZ{G3GC7qP^_p?uAiuz@_#|vI9`0B8-W+z)k>#w`pN3)Wg#Ee zHYN+%a%m~gx%49Zpx|AmT4By2V&{G`GZE%}t?9}{CG0$rB%Tq(T~Sx4{F+Grs6r>= zcYwRwlZG^fB({aw}2fEuy4JGDLOM<@PqHWJ&Rbcjh1`@;92{Wvgmet7mP6OO=Hm0uT z&rr@oKYl-t*MIT|^aq>Ezo3|5^UgFb7cTnuptH>0gww0b@WQcsFg9n%n8FeJaqm3! z;CGpWS6msRgPS6)@wm0Hn!P_5yHyh>A{1^bo7r30%=gA%kO^k^i}l2jW?ir5Xgu#u zp~bamjhDXSrvW1tZ)YF}P$_V#nE1e~lN#3!L~3245@EhOT%+4BnW5fgft^0mRzX${ zW)zuXOyUaO-wQhUgE|V6%iyGz3?+h3(eY!7@&fB@Ph8Py2C3BU`cUtca>8zk5_jDP z-)bu>-VM(if=DvNgt<}l|8&xQH)zLxP^Eb0` z9npeEN!k`I)_-(n0-L{Z*>o3DNaZ%V&o420Pjq}rAj%cJy43OHlQYFbnK>X;(sZk; znyUXw?!9KOu$_L<+2SLXI%&%^_br1|cuTj&>;2zh{Uk@Z{$ZMvFY7ce z0NH%m_HQoaTzFd?6KeO70f=}LrWLoZBi4mNlcbKWUcyqZ{l-gLUcsz&fsKk=Ej*eb zYTU)Yz3cbSCye_Ad1;!lGfi!ITRwAGU@>jE{d@>r(5G__R=p>mIfeJj;1M+KBxQN^ z#K6RLl}(d@lbqsL38HP)!8VEz6hpsZ)8{Q0CF+hL?)(JF2lm)~-O?{rrdmYo;62r6 zlf(OdohE0GE7Zxw^gBkmBkinNx+cUq7@<*=tPAOd?t6hbPn;_!UnEAp#+iy+M@f<7 zziSt148Z&;lkMA;SGLJ+scw<3SDyE1V}ncH<_;?*ynEeA^NeQME)~osn96vr<=R6ODUq49>0gTE#jjhY z5VzSv;bfF%VXTE!k#U8sqhROkIQaJGKmwK;e2rdYWp#ju}j97>MqZs9)yk8v}~Crg=?L! z;0<|jkRu?9V4r7s%11to)vatoXt^$vEs)ue>}mKyq3`e$SBKkuQNsswAG7#A!;cA$ z;$wIi!DE1>R-2Txs3*I+Jl%E|A~8&DM4E)`zq!rpn$&ptq;QTR_7Avj5tH~iSh_e( zdoQO-^F;Vi)`{8sW1(@~!^N+W$k3OOg;1V1t$;|a4w0vmy?1jhmyz$=LR6nUY$t?@ z@{IWtxbZxO7{9)OWI3Y7)+sb%_$eBz9X&P)myC1&-XtGmau%#R(NRYXxN})=g*Zf@ zVJ{f30&9%%RoIbWDeLQh`G~%v?o!#?o;luJaFR3Z{>mvcw>qG~%c<^5IAl-!_FI1% zEr7cF^E|T#x5l?JgK>Ji6&v)JywCB|+KCmT6N|THf(BwSAc@+&uVX7BXwt?;$%pju zBAKdk-+xqe&%JJf`&*Q??&MqeS1TG9VKmNj1fD%y54NFID?SSoJNl4`PwKmrhKYDQ z%#wH?*=?VSSWejt0Yr^T*o#hda$CAApGpp-dCsmQ}L!n_(?-dLno+(TAcF18L z@ZTbI)anwU7Z+-A9DaNr7f#Rz7Y8)Zd;Wm^f#bdZHLKY>5JTO|7{)hFVsrt7obKl)_F%*!Iy_{%@1*W!5o zM(BGeJ~ljcZ->5kTNB6mW&Ndvy6%0|@O>kqyS`FZokdT&53iSeE_@#@h?$sH-Zk2| z92_FHaY@*3`>(hoVza6;hT=2!Cz|dvn*CVSwA*||ShFW#H2X}`Rl0tK>)ecu`1^AG zuH=xsd#8DQ%00Ip&4BiD&8!Spi#Oyyn@1XA7c>^CWQ3!((WVJs`b`e?w%O^m#u%Ij z9YwRgn~i?Ey0iJSrJKi(-Y>`GjJxRkeS*$PBu8HVCl1j>7lC0}m+PRKy-66N!&qpI ztpfYeXqGOxzMT(zvPxQW*`jTn`DBvaIs7 zG`-PbCuoMkgqiHf;rE$y&OB#{6cSI zye<)kX)Mp|2(lYn5L8UP3)LLAVahHZS;4+BN^N;p` z`*B`>jTa@a*_* zHJ_o?sMD~9cL6O{l3n8+45z6kA%rW1H0q+Xtnup4B%^*F@6d%1T!mr&y2Xaophz&n zYsJ$7l}&xXTz!fDrW4|!si)jsE%$2?fyLxmFPSC~|5vH_kkNcqC%ImD`=L|rlELrxSBg9XIlEboQZx{ntLqg+DAMz2e z!-b!Jt*mKn828(B^(_6>RR9R|G??e=^kEse-t_aATZ^N3aV&R@R;f^ZU?`WsT5a@o znnj&5##D3rf%_V{4ey@C2KBeyoF5DY>qvxtcZoWR`z;6G)XwT z7oD}GJi$mqU$c|e98=V5ZzpawN~CX4rvd6F9!(QGlsB3M^9)w>>Wf0@ ztB*;)BmJ3<8h&V}mieZ(I2jF5as;u^n+e=5RpA5!{zl`ovsj{Cv)&q?#M20!SC-3O zHUH6g*|5_7=$rzEAdcl!qyKd8$hgK!x331bevH4;_9`GYFqN_DWP(*>OG01k`Qt4- zS%5xZG}Ih@Yj+H*0e=M6U}|&qI3?!;oj@T>f&tK-6>f-`&Ev`%-bEk?_Q$KHctO>4 ztzrQb$s+thsTe3(V^m}my^#xMX<&Cvje4rYQL~M^IelTU>DwdpB-QY3u z;}WHUs`tz(gP9CM9@O$h>)va1hS9|bZ z=1_Y4)qhHU34WDPf~G407F zmyxTnrI`b2al>#SD%BbWk~ExCiOHEJaAO%?1l`{Rv)|d{Ln+c-AaQF9oCroJCntV7 z(bU<8r-=%Be+e9y7l;*f7tHYKklDDYU4Cctm1ZS+mPb!lzL{L*@?16Yi1vnpe8$dq zv*pL89)dumS^xC{KnPp$dH@-(4<4asVGB`cN-+uf^2KTCUkq*yC_hx48nOZUM`>kK zUN*niM{ytBuQfn`i#-gIe}~qrzF;0>)`fEHETr~^**c@#wJLI@0UeaJYdi)S_vHaB z9kIelE1HZV*qlvabL>eh9bl2fH=qS7g$GMR^cD`4b>^K*`0VNPRc|&}RyzA1UN|xS zOlw)#Y6-c^A!39Fsi=zx$r;uc3fJ8lBSx6dewu>g%nXhC;GV8EW*gW?CTc7q9GSy* zkN#%uIsh_6-msnCRb-)Q%z9xDH>wUywUXL~jg?&^Z#J#)oGv&?V>GBT>!q8nHjC+e zTrn}P=%Yi#m3EH{u?q2h{>lo8w`#d!yw9=F_5fmF{pnS)RculujbLAT1D!sf51)Gn z`RbKb7iTC{BSjdimjT}P8?#;}7(xRt+ahftwZ)EiZu24#BbHrPCh9%|; z5Gni9`tJr$>H3rz?kFj+R)Dw@^;F64);gv8E!>H0EB3&B&?RJg(aXnazA^4y!=TwO z=10}i4WL$o6)udk>xjJ8&VD`bU51s3E>mHG&-2H;w$j2-LI>&rWh0!h&+JH|CSb{;Frh^f) z!8w}M8S2Jd_uWmpW=h%}D-sH#jhMbJV55c5!$nPoUGIIe0I-cj=` z3W9$k6()^I1r{W_MroIUHTQRZ4H?)LtiW ztt&zY2oV5P0Y8do#;vqN$vUxA8`SA3DmXjtr~S@K6J|_j1VQv1sA19M6Lh?bnf`um zD&shHw9-i-0AUDnafb^;os&RyAOkya@U(m18JEJfK=t=tGi|;fW0ZxuC;CFZQPI8E z<&!_-k$#1`eVFMy*Vgh)dTO1?coK1P z1Umv#JH;6$L1pf67QMVQ8=u<3Lw=n6*g;*1xmNAZ2OE+fMnW_%%EcCm@60PmYLpP3 z$yh~7HuK!M?f>J6G~Kq-qDHd};E^4H5~4G4t0Fp{ zW@jWBP=O#`@$4p4->_TpSc$+~Vlw=|`5!x09&SxS0b=ulN!k*2jzP5gsQ z)kVvszo(PRxSThvlzL76KGUl7(E{G+QU1AMz|++N(Zr3dqO4zoW!XpY)E0X2fWh`n z(iP)a*U`$9Gu?s7HfZ^?ct)uMpEqysEpiS;S?t|yn@s~MaNSDC&qQPjOwVPkiee>~ zln8V~Bl&;CP*G_AvB_iCYd=;td|+jh;TbUNJGq#)&|eIB>CNp#&%43S8b9_wDeIOX zn$rR%wm_i3X)$1@9`A$g{YyDB3IE_m{*r?K(}NWK{Y31Oa^`j>(51aCVAMuzL-vF*LtfGo+|OTXAc?H4OLf2!O`{R!0OwEU)^MT!YW~%&OU< z3CY<+lswQo9W5eCf(M@5UET(uXT&zk#QWiMXB4&ZyxWm2M5enLN1Z9Vo)vEfS5g zhRBgvVBsI$;nsR{N~UvR*2ggyw4$-fRkF^EqU@|23HHn{=Bc|Q!zB)gTV$*@icSlJ zw(+2p-INIckf^{;$H?MWXKi0DBSttEd*R64p-T2UmMR4iaQeAW8GyNBq$er9_)7RZ z>WtwLkP6bDoWuAzut5Ff{YV=b9=M<{0H|=ws3?^!Y4~S{hgqaPXV@o=t%|suwP<0kP&CRc#VWm06f+<-^}KOqhVb6!Ig~;m(iU<+Ffo zt(MaU#>}pFZh@W;DS89Vb3ld6-`BD2E)_doCx4RJ?W)~Mpt_(S>Hjk7^-8aPDX|%a z@NfSWsg>uC*{by7WZI46cSS#&(kJqde4&r`KOI~Pt_8^5f?fzwqmUz~pR`T)U@%5n zJs@bz!r@=c?)X{b00*XtW6F}GBK}X~sGI*%(vwm5^;f$j*RlkH#-=BEe$LsCf7=ib zmS}`V&26%*Bo~os&sFVS1RXz;qnZ8`CGN`#y-D9Cy6sC!807(^B(PFihl|v*MIpa{oaR3;FCUW@VROK-VpQ~j(bGmx9UBMC3^H?e zbUrX;lo{JP?GtVOuhVig^c}(dT5TEPv(G5U9v1jy>fxOtg!Ov(s*63L04fT6z|PPJ zZq!BChjCm`9I!XMoeTZHc1M#S)yCh8y-9@$WIa;755$g2i?5wseQPa?)|-U)%3^Ng oGJjzLO?g7p|NjFEI=aOp3DM&3?}X0*?W-Uyb$zw^#||(4AE@uP0ssI2 literal 0 HcmV?d00001 diff --git a/docs/theme/partials/header.html b/docs/theme/partials/header.html new file mode 100644 index 0000000..5b091f3 --- /dev/null +++ b/docs/theme/partials/header.html @@ -0,0 +1,86 @@ +{#- + This file was automatically generated - do not edit +-#} +{% set class = "md-header" %} +{% if "navigation.tabs.sticky" in features %} + {% set class = class ~ " md-header--lifted" %} +{% endif %} +
+ + {% if "navigation.tabs.sticky" in features %} + {% if "navigation.tabs" in features %} + {% include "partials/tabs.html" %} + {% endif %} + {% endif %} +
\ No newline at end of file diff --git a/examples/eln_data.yaml b/examples/eln_data.yaml index 1a08a04..e14ce1b 100644 --- a/examples/eln_data.yaml +++ b/examples/eln_data.yaml @@ -79,8 +79,12 @@ sample: - Al 12 +- 3 - B 50 ppm +- 12 - C 3.6 - method: experiment + type: experiment alias: Si + identifier: + service: undefined + identifier: 'Si in sample' + # is_persistent: false grain_diameter: unit: µm value: 200. @@ -101,8 +105,12 @@ sample: value: 10. description: '

nothing

' specimen: - method: experiment + type: experiment alias: usa_denton_smith_si + identifier: + service: undefined + identifier: 'Si in specimen' + # is_persistent: false description: '

normal

bold

diff --git a/tests/data/nomad_oasis_eln_schema_for_nx_apm/nxapm.schema.archive.yaml b/examples/nxapm.schema.archive.yaml similarity index 99% rename from tests/data/nomad_oasis_eln_schema_for_nx_apm/nxapm.schema.archive.yaml rename to examples/nxapm.schema.archive.yaml index ba4a00b..1cc7c31 100644 --- a/tests/data/nomad_oasis_eln_schema_for_nx_apm/nxapm.schema.archive.yaml +++ b/examples/nxapm.schema.archive.yaml @@ -7,7 +7,7 @@ definitions: sections: # section definitions what were back in the old days msections AtomProbeMicroscopy: # the actual schema base_sections: - - 'nomad.datamodel.metainfo.eln.NexusDataConverter' + - 'pynxtools.nomad.dataconverter.NexusDataConverter' - 'nomad.datamodel.data.EntryData' m_annotations: # Here you can set your default values for the reader and nxdl. diff --git a/mkdocs.yaml b/mkdocs.yaml index fb2d8ae..c2211e7 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -1,7 +1,10 @@ site_name: pynxtools-apm site_description: | The documentation for the pynxtools-apm plugin -site_author: The FAIRmat authors +site_author: The NOMAD authors +repo_url: https://github.com/FAIRmat-NFDI/pynxtools-apm +edit_uri: "" + nav: - Home: index.md - Tutorials: @@ -29,11 +32,27 @@ nav: - reference/camecaroot.md - reference/mqanalysis.md - reference/atomtypes.md + +plugins: + - search + - macros: + module_name: src/pynxtools_apm/mkdocs + theme: name: material + palette: + primary: '#2A4CDF' + accent: '#008A67' + font: + text: 'Titillium Web' + logo: assets/nomad-logo.png + favicon: assets/favicon.png features: + - navigation.instant - content.code.copy - content.code.annotate + custom_dir: docs/theme + markdown_extensions: - attr_list - md_in_html @@ -48,16 +67,24 @@ markdown_extensions: - pymdownx.arithmatex: generic: true - pymdownx.emoji + - mkdocs-click - pymdownx.extra - pymdownx.superfences: custom_fences: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + +extra: + generator: false + homepage: https://nomad-lab.eu + use_directory_urls: false -plugins: - - search - - macros: - module_name: docs/macros + extra_css: - - stylesheets/extra.css \ No newline at end of file + - stylesheets/extra.css + +extra_javascript: + - javascript.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js diff --git a/pynxtools_apm/README.md b/pynxtools_apm/README.md deleted file mode 100644 index 657dc39..0000000 --- a/pynxtools_apm/README.md +++ /dev/null @@ -1,45 +0,0 @@ - diff --git a/pyproject.toml b/pyproject.toml index c1618cd..1a1198f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "pynxtools>=0.2.1", + "pint==0.17", + "pynxtools>=0.7.0", "ifes_apt_tc_data_modeling>=0.2.2", ] @@ -31,32 +32,48 @@ dependencies = [ "Bug Tracker" = "https://github.com/FAIRmat-NFDI/pynxtools-apm/issues" [project.optional-dependencies] -docs = [ - "mkdocs", - "mkdocs-material", - "mkdocs-material-extensions", - "mkdocs-macros-plugin", -] dev = [ "mypy", - "ruff==0.3.4", + "ruff==0.5.5", "pytest", + "pytest-cov", + "pytest-timeout", + "pytest-cov", + "structlog", "types-pyyaml", + "types-pytz", + "types-requests", "pip-tools", + "uv", + "pre-commit", "jupyter", "jupyterlab", "jupyterlab-h5web" ] +docs = [ + "mkdocs", + "mkdocs-material", + "mkdocs-material-extensions", + "mkdocs-macros-plugin", + "mkdocs-click", + "markdown-include" +] [project.entry-points."pynxtools.reader"] apm = "pynxtools_apm.reader:APMReader" +[tool.setuptools.packages.find] +where = [ + "src", +] +exclude = ["dev/*"] + [tool.setuptools_scm] version_scheme = "no-guess-dev" local_scheme = "node-and-date" [tool.ruff] -include = ["pynxtools_apm/*.py", "tests/*.py"] +include = ["src/*.py", "tests/*.py"] line-length = 88 indent-width = 4 diff --git a/pynxtools_apm/__init__.py b/src/pynxtools_apm/__init__.py similarity index 100% rename from pynxtools_apm/__init__.py rename to src/pynxtools_apm/__init__.py diff --git a/pynxtools_apm/concepts/__init__.py b/src/pynxtools_apm/concepts/__init__.py similarity index 100% rename from pynxtools_apm/concepts/__init__.py rename to src/pynxtools_apm/concepts/__init__.py diff --git a/pynxtools_apm/concepts/mapping_functors.py b/src/pynxtools_apm/concepts/mapping_functors.py similarity index 94% rename from pynxtools_apm/concepts/mapping_functors.py rename to src/pynxtools_apm/concepts/mapping_functors.py index a60868e..0df17d6 100644 --- a/pynxtools_apm/concepts/mapping_functors.py +++ b/src/pynxtools_apm/concepts/mapping_functors.py @@ -18,13 +18,14 @@ """Utilities for working with NeXus concepts encoded as Python dicts in the concepts dir.""" from datetime import datetime -import pytz + import flatdict as fd import numpy as np +import pytz -from pynxtools_apm.utils.string_conversions import string_to_number, rchop -from pynxtools_apm.utils.interpret_boolean import try_interpret_as_boolean from pynxtools_apm.utils.get_file_checksum import get_sha256_of_file_content +from pynxtools_apm.utils.interpret_boolean import try_interpret_as_boolean +from pynxtools_apm.utils.string_conversions import rchop, string_to_number def variadic_path_to_specific_path(path: str, instance_identifier: list): @@ -333,13 +334,18 @@ def add_specific_metadata( trg = variadic_path_to_specific_path( f"{variadic_prefix_trg}/{entry[0]}", identifier ) - with open(orgmeta[f"{prefix_src}{entry[1]}"], "rb") as fp: - template[f"{rchop(trg, 'checksum')}checksum"] = ( - get_sha256_of_file_content(fp) + try: + with open(orgmeta[f"{prefix_src}{entry[1]}"], "rb") as fp: + template[f"{rchop(trg, 'checksum')}checksum"] = ( + get_sha256_of_file_content(fp) + ) + template[f"{rchop(trg, 'checksum')}type"] = "file" + template[f"{rchop(trg, 'checksum')}path"] = orgmeta[ + f"{prefix_src}{entry[1]}" + ] + template[f"{rchop(trg, 'checksum')}algorithm"] = "sha256" + except (FileNotFoundError, IOError): + print( + f"File {orgmeta[f'''{prefix_src}{entry[1]}''']} not found !" ) - template[f"{rchop(trg, 'checksum')}type"] = "file" - template[f"{rchop(trg, 'checksum')}path"] = orgmeta[ - f"{prefix_src}{entry[1]}" - ] - template[f"{rchop(trg, 'checksum')}algorithm"] = "sha256" return template diff --git a/pynxtools_apm/config/eln_cfg.py b/src/pynxtools_apm/config/eln_cfg.py similarity index 94% rename from pynxtools_apm/config/eln_cfg.py rename to src/pynxtools_apm/config/eln_cfg.py index 2566051..73c6f91 100644 --- a/pynxtools_apm/config/eln_cfg.py +++ b/src/pynxtools_apm/config/eln_cfg.py @@ -48,7 +48,10 @@ "map_to_str": [ "alias", "description", - "method", + "type", + ("identifier/identifier", "identifier/identifier"), + ("identifier/service", "identifier/service"), + ("identifier/is_persistent", "identifier/is_persistent"), ("grain_diameter/@units", "grain_diameter/unit"), ("grain_diameter_error/@units", "grain_diameter/unit"), ("heat_treatment_temperature/@units", "heat_treatment_temperature/unit"), @@ -77,7 +80,10 @@ "alias", "preparation_date", "description", - "method", + "type", + ("identifier/identifier", "identifier/identifier"), + ("identifier/service", "identifier/service"), + ("identifier/is_persistent", "identifier/is_persistent"), ("initial_radius/@units", "initial_radius/unit"), ("shank_angle/@units", "shank_angle/unit"), ], diff --git a/pynxtools_apm/config/oasis_cfg.py b/src/pynxtools_apm/config/oasis_cfg.py similarity index 100% rename from pynxtools_apm/config/oasis_cfg.py rename to src/pynxtools_apm/config/oasis_cfg.py diff --git a/docs/macros.py b/src/pynxtools_apm/mkdocs.py similarity index 100% rename from docs/macros.py rename to src/pynxtools_apm/mkdocs.py diff --git a/pynxtools_apm/reader.py b/src/pynxtools_apm/reader.py similarity index 60% rename from pynxtools_apm/reader.py rename to src/pynxtools_apm/reader.py index fca91c5..c33f7ff 100644 --- a/pynxtools_apm/reader.py +++ b/src/pynxtools_apm/reader.py @@ -18,28 +18,30 @@ """Generic parser for loading atom probe microscopy data into NXapm.""" from time import perf_counter_ns -from typing import Tuple, Any -import numpy as np +from typing import Any, Tuple +import numpy as np from pynxtools.dataconverter.readers.base.reader import BaseReader + +from pynxtools_apm.utils.create_nx_default_plots import ( + apm_default_plot_generator, +) from pynxtools_apm.utils.io_case_logic import ( ApmUseCaseSelector, ) -from pynxtools_apm.utils.oasis_config_reader import ( - NxApmNomadOasisConfigurationParser, -) -from pynxtools_apm.utils.oasis_eln_reader import ( - NxApmNomadOasisElnSchemaParser, +from pynxtools_apm.utils.load_ranging import ( + ApmRangingDefinitionsParser, ) from pynxtools_apm.utils.load_reconstruction import ( ApmReconstructionParser, ) -from pynxtools_apm.utils.load_ranging import ( - ApmRangingDefinitionsParser, +from pynxtools_apm.utils.oasis_config_reader import ( + NxApmNomadOasisConfigurationParser, ) -from pynxtools_apm.utils.create_nx_default_plots import ( - apm_default_plot_generator, +from pynxtools_apm.utils.oasis_eln_reader import ( + NxApmNomadOasisElnSchemaParser, ) + # from pynxtools_apm.utils.apm_generate_synthetic_data import ( # ApmCreateExampleData, # ) @@ -79,59 +81,58 @@ def read( template.clear() entry_id = 1 - if len(file_paths) == 1: - pass - """ - # TODO::better make this an option rather than hijack and demand a - # specifically named file to trigger the synthesizer - # the synthesize functionality is currently deactivated, we have enough - # example datasets and the synthesizer is in need for a refactoring. - if file_paths[0].startswith("synthesize"): - synthesis_id = int(file_paths[0].replace("synthesize", "")) - print(f"synthesis_id {synthesis_id}") - else: - synthesis_id = 1 - print("Create one synthetic entry in one NeXus file...") - synthetic = ApmCreateExampleData(synthesis_id) - synthetic.synthesize(template) - """ - else: # eln_data, and ideally recon and ranging definitions from technology partner file - print("Parse ELN and technology partner file(s)...") - case = ApmUseCaseSelector(file_paths) - if not case.is_valid: - print("Such a combination of input-file(s, if any) is not supported !") - return {} - case.report_workflow(template, entry_id) - - if len(case.cfg) == 1: - print("Parse (meta)data coming from a configuration of an RDM...") - nx_apm_cfg = NxApmNomadOasisConfigurationParser( - case.cfg[0], entry_id, False - ) - nx_apm_cfg.parse(template) - else: - print("No input file defined for config data !") - - if len(case.eln) == 1: - print("Parse (meta)data coming from an ELN...") - nx_apm_eln = NxApmNomadOasisElnSchemaParser(case.eln[0], entry_id) - nx_apm_eln.parse(template) - else: - print("No input file defined for eln data !") - - if len(case.reconstruction) == 1: - print("Parse (meta)data from a reconstructed dataset file...") - nx_apm_recon = ApmReconstructionParser(case.reconstruction[0], entry_id) - nx_apm_recon.report(template) - else: - print("No input-file defined for reconstructed dataset!") - - if len(case.ranging) == 1: - print("Parse (meta)data from a ranging definitions file...") - nx_apm_range = ApmRangingDefinitionsParser(case.ranging[0], entry_id) - nx_apm_range.report(template) - else: - print("No input-file defined for ranging definitions!") + # if len(file_paths) == 1: + """ + # TODO::better make this an option rather than hijack and demand a + # specifically named file to trigger the synthesizer + # the synthesize functionality is currently deactivated, we have enough + # example datasets and the synthesizer is in need for a refactoring. + if file_paths[0].startswith("synthesize"): + synthesis_id = int(file_paths[0].replace("synthesize", "")) + print(f"synthesis_id {synthesis_id}") + else: + synthesis_id = 1 + print("Create one synthetic entry in one NeXus file...") + synthetic = ApmCreateExampleData(synthesis_id) + synthetic.synthesize(template) + """ + # eln_data, and ideally recon and ranging definitions from technology partner file + print("Parse ELN and technology partner file(s)...") + case = ApmUseCaseSelector(file_paths) + if not case.is_valid: + print("Such a combination of input-file(s, if any) is not supported !") + return {} + case.report_workflow(template, entry_id) + + if len(case.cfg) == 1: + print("Parse (meta)data coming from a configuration of an RDM...") + nx_apm_cfg = NxApmNomadOasisConfigurationParser( + case.cfg[0], entry_id, False + ) + nx_apm_cfg.parse(template) + else: + print("No input file defined for config data !") + + if len(case.eln) == 1: + print("Parse (meta)data coming from an ELN...") + nx_apm_eln = NxApmNomadOasisElnSchemaParser(case.eln[0], entry_id) + nx_apm_eln.parse(template) + else: + print("No input file defined for eln data !") + + if len(case.reconstruction) == 1: + print("Parse (meta)data from a reconstructed dataset file...") + nx_apm_recon = ApmReconstructionParser(case.reconstruction[0], entry_id) + nx_apm_recon.parse(template) + else: + print("No input-file defined for reconstructed dataset!") + + if len(case.ranging) == 1: + print("Parse (meta)data from a ranging definitions file...") + nx_apm_range = ApmRangingDefinitionsParser(case.ranging[0], entry_id) + nx_apm_range.parse(template) + else: + print("No input-file defined for ranging definitions!") print("Create NeXus default plottable data...") apm_default_plot_generator(template, entry_id) @@ -154,8 +155,7 @@ def read( print("Forward instantiated template to the NXS writer...") toc = perf_counter_ns() - trg = f"/entry{entry_id}/profiling" - template[f"{trg}/@NX_class"] = "NXcs_profiling" + trg = f"/ENTRY[entry{entry_id}]/profiling" template[f"{trg}/template_filling_elapsed_time"] = np.float64( (toc - tic) / 1.0e9 ) diff --git a/pynxtools_apm/utils/__init__.py b/src/pynxtools_apm/utils/__init__.py similarity index 100% rename from pynxtools_apm/utils/__init__.py rename to src/pynxtools_apm/utils/__init__.py diff --git a/pynxtools_apm/utils/create_nx_default_plots.py b/src/pynxtools_apm/utils/create_nx_default_plots.py similarity index 100% rename from pynxtools_apm/utils/create_nx_default_plots.py rename to src/pynxtools_apm/utils/create_nx_default_plots.py diff --git a/pynxtools_apm/utils/generate_synthetic_data.py b/src/pynxtools_apm/utils/generate_synthetic_data.py similarity index 100% rename from pynxtools_apm/utils/generate_synthetic_data.py rename to src/pynxtools_apm/utils/generate_synthetic_data.py diff --git a/pynxtools_apm/utils/get_file_checksum.py b/src/pynxtools_apm/utils/get_file_checksum.py similarity index 100% rename from pynxtools_apm/utils/get_file_checksum.py rename to src/pynxtools_apm/utils/get_file_checksum.py diff --git a/pynxtools_apm/utils/get_gitrepo_commit.py b/src/pynxtools_apm/utils/get_gitrepo_commit.py similarity index 100% rename from pynxtools_apm/utils/get_gitrepo_commit.py rename to src/pynxtools_apm/utils/get_gitrepo_commit.py diff --git a/pynxtools_apm/utils/interpret_boolean.py b/src/pynxtools_apm/utils/interpret_boolean.py similarity index 100% rename from pynxtools_apm/utils/interpret_boolean.py rename to src/pynxtools_apm/utils/interpret_boolean.py diff --git a/pynxtools_apm/utils/io_case_logic.py b/src/pynxtools_apm/utils/io_case_logic.py similarity index 84% rename from pynxtools_apm/utils/io_case_logic.py rename to src/pynxtools_apm/utils/io_case_logic.py index 792ced0..2d6bcba 100644 --- a/pynxtools_apm/utils/io_case_logic.py +++ b/src/pynxtools_apm/utils/io_case_logic.py @@ -17,7 +17,7 @@ # """Utility class to analyze which vendor/community files are passed to apm reader.""" -from typing import Tuple, Dict, List +from typing import Dict, List, Tuple from pynxtools_apm.concepts.mapping_functors import ( variadic_path_to_specific_path, @@ -112,30 +112,30 @@ def check_validity_of_file_combinations(self): recon_input += len(value) # print(f"{recon_input}, {range_input}, {other_input}") - if 1 <= other_input <= 2: # and (recon_input == 1) and (range_input == 1) - self.is_valid = True - self.reconstruction: List[str] = [] - self.ranging: List[str] = [] - for suffix in VALID_FILE_NAME_SUFFIX_RECON: - self.reconstruction += self.case[suffix] - for suffix in VALID_FILE_NAME_SUFFIX_RANGE: - self.ranging += self.case[suffix] - yml: List[str] = [] - for suffix in VALID_FILE_NAME_SUFFIX_CONFIG: - yml += self.case[suffix] - for entry in yml: - if entry.endswith(".oasis.specific.yaml") or entry.endswith( - ".oasis.specific.yml" - ): - self.cfg += [entry] - else: - self.eln += [entry] - print( - f"recon_results: {self.reconstruction}\n" - f"range_results: {self.ranging}\n" - f"Oasis ELN: {self.eln}\n" - f"Oasis local config: {self.cfg}\n" - ) + # if 1 <= other_input <= 2: # and (recon_input == 1) and (range_input == 1) + self.is_valid = True + self.reconstruction: List[str] = [] + self.ranging: List[str] = [] + for suffix in VALID_FILE_NAME_SUFFIX_RECON: + self.reconstruction += self.case[suffix] + for suffix in VALID_FILE_NAME_SUFFIX_RANGE: + self.ranging += self.case[suffix] + yml: List[str] = [] + for suffix in VALID_FILE_NAME_SUFFIX_CONFIG: + yml += self.case[suffix] + for entry in yml: + if entry.endswith(".oasis.specific.yaml") or entry.endswith( + ".oasis.specific.yml" + ): + self.cfg += [entry] + else: + self.eln += [entry] + print( + f"recon_results: {self.reconstruction}\n" + f"range_results: {self.ranging}\n" + f"Oasis ELN: {self.eln}\n" + f"Oasis local config: {self.cfg}\n" + ) def report_workflow(self, template: dict, entry_id: int) -> dict: """Initialize the reporting of the workflow.""" diff --git a/pynxtools_apm/utils/load_ranging.py b/src/pynxtools_apm/utils/load_ranging.py similarity index 99% rename from pynxtools_apm/utils/load_ranging.py rename to src/pynxtools_apm/utils/load_ranging.py index 26b4d11..7ce759d 100644 --- a/pynxtools_apm/utils/load_ranging.py +++ b/src/pynxtools_apm/utils/load_ranging.py @@ -302,7 +302,7 @@ def update_atom_types_ranging_definitions_based(self, template: dict) -> dict: return template - def report(self, template: dict) -> dict: + def parse(self, template: dict) -> dict: """Copy data from self into template the appdef instance. Paths in template are prefixed by prefix and have to be compliant diff --git a/pynxtools_apm/utils/load_reconstruction.py b/src/pynxtools_apm/utils/load_reconstruction.py similarity index 99% rename from pynxtools_apm/utils/load_reconstruction.py rename to src/pynxtools_apm/utils/load_reconstruction.py index 4e86533..22e0c4d 100644 --- a/pynxtools_apm/utils/load_reconstruction.py +++ b/src/pynxtools_apm/utils/load_reconstruction.py @@ -17,17 +17,18 @@ # """Wrapping multiple parsers for vendor files with reconstructed dataset files.""" -from typing import Dict, Any -import numpy as np +from typing import Any, Dict +import numpy as np from ifes_apt_tc_data_modeling.apt.apt6_reader import ReadAptFileFormat -from ifes_apt_tc_data_modeling.epos.epos_reader import ReadEposFileFormat -from ifes_apt_tc_data_modeling.pos.pos_reader import ReadPosFileFormat from ifes_apt_tc_data_modeling.ato.ato_reader import ReadAtoFileFormat from ifes_apt_tc_data_modeling.csv.csv_reader import ReadCsvFileFormat +from ifes_apt_tc_data_modeling.epos.epos_reader import ReadEposFileFormat +from ifes_apt_tc_data_modeling.pos.pos_reader import ReadPosFileFormat from ifes_apt_tc_data_modeling.pyccapt.pyccapt_reader import ( ReadPyccaptCalibrationFileFormat, ) + from pynxtools_apm.utils.io_case_logic import ( VALID_FILE_NAME_SUFFIX_RECON, ) @@ -256,7 +257,7 @@ def __init__(self, file_path: str, entry_id: int): if self.meta["file_format"] is None: raise ValueError(f"{file_path} is not a supported reconstruction file!") - def report(self, template: dict) -> dict: + def parse(self, template: dict) -> dict: """Copy data from self into template the appdef instance. Paths in template are prefixed by prefix and have to be compliant diff --git a/pynxtools_apm/utils/oasis_config_reader.py b/src/pynxtools_apm/utils/oasis_config_reader.py similarity index 100% rename from pynxtools_apm/utils/oasis_config_reader.py rename to src/pynxtools_apm/utils/oasis_config_reader.py diff --git a/pynxtools_apm/utils/oasis_eln_reader.py b/src/pynxtools_apm/utils/oasis_eln_reader.py similarity index 100% rename from pynxtools_apm/utils/oasis_eln_reader.py rename to src/pynxtools_apm/utils/oasis_eln_reader.py diff --git a/pynxtools_apm/utils/parse_composition_table.py b/src/pynxtools_apm/utils/parse_composition_table.py similarity index 100% rename from pynxtools_apm/utils/parse_composition_table.py rename to src/pynxtools_apm/utils/parse_composition_table.py diff --git a/pynxtools_apm/utils/string_conversions.py b/src/pynxtools_apm/utils/string_conversions.py similarity index 100% rename from pynxtools_apm/utils/string_conversions.py rename to src/pynxtools_apm/utils/string_conversions.py diff --git a/pynxtools_apm/utils/versioning.py b/src/pynxtools_apm/utils/versioning.py similarity index 100% rename from pynxtools_apm/utils/versioning.py rename to src/pynxtools_apm/utils/versioning.py diff --git a/tests/data/config_file.json b/tests/data/config_file.json deleted file mode 100644 index 9e26dfe..0000000 --- a/tests/data/config_file.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/tests/run_tests.ipynb b/tests/run_tests.ipynb new file mode 100644 index 0000000..0d278b2 --- /dev/null +++ b/tests/run_tests.ipynb @@ -0,0 +1,153 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 21, + "id": "fc9345cb-46bf-4df3-9dac-1bde062d9020", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/kaiobach/Research/hu_hu_hu/sprint24/pynxtools-apm/pynxtools_apm/tests\n" + ] + } + ], + "source": [ + "import os\n", + "print(os.getcwd())" + ] + }, + { + "cell_type": "markdown", + "id": "e69ad935-80fd-4e70-af69-ac98e0bf2e34", + "metadata": {}, + "source": [ + "## Define tests" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "48f6e922-52d3-412f-8345-0641d33e676e", + "metadata": {}, + "outputs": [], + "source": [ + "tests = {\n", + " # typical full tests with eln_data.yaml apm.oasis.specific.yaml Si.apt 87D_1.rng\n", + " # \"eln\": [(\"eln_data.yaml\", \"apm.oasis.specific.yaml\")],\n", + " # \"apt\": [\"Si.apt\"],\n", + " # \"csv\": [\"Annealed_CoCrNi_100.csv\"],\n", + " # \"env\": [\"ErMnO.env\"],\n", + " # \"epos\": [\"R45_04472-v03.epos\"],\n", + " # \"fig\": [\"Superalloy_MassSpec_ranged.fig.txt\"],\n", + " # \"imago\": [\"default.analysis\"],\n", + " # \"pos\": [\"ErMnO_pole.pos\"],\n", + " # \"pyccapt\": [(\"1748_Al_range_.h5\", \"1748_Al.h5\")],\n", + " # \"rng\": [\"87D_1.rng\"],\n", + " # \"rrng\": [\"VAlN_film_plan-view_700C.rrng\"]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "6ae5057c-2101-4792-9804-3af046003db7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/kaiobach/Research/hu_hu_hu/sprint24/pynxtools-apm/.py3.12.4/bin/python\n" + ] + } + ], + "source": [ + "# ! mkdir -p log\n", + "# ! mkdir -p prod\n", + "# ! which python" + ] + }, + { + "cell_type": "markdown", + "id": "36a351a5-940d-4891-bb7e-1f4ba4100f6d", + "metadata": {}, + "source": [ + "## Run tests" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "809a88fe-5e4f-4429-b668-8a7956210606", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running test apt/Si.apt\n", + "Ran all tests\n" + ] + } + ], + "source": [ + "for parser_type, list_of_tests in tests.items():\n", + " for entry in list_of_tests:\n", + " if isinstance(entry, str): # no sidecar file \n", + " print(f\"Running test {parser_type}/{entry}\")\n", + " curr_dir = f\"{os.getcwd()}/data/{parser_type}\"\n", + " in_one = f\"{curr_dir}/{entry}\"\n", + " out = f\"prod/data.{parser_type}.{entry}.nxs\"\n", + " stdout = f\"log/stdout.{parser_type}.{entry}.txt\"\n", + " stderr = f\"log/stderr.{parser_type}.{entry}.txt\"\n", + " ! dataconverter $in_one --reader apm --nxdl NXapm --output $out 1>$stdout 2>$stderr \n", + " continue\n", + " elif isinstance(entry, tuple): # with sidecar file\n", + " if len(entry) == 2 and all(isinstance(val, str) for val in entry):\n", + " print(f\"Running test {parser_type}/{entry}\")\n", + " curr_dir = f\"{os.getcwd()}/data/{parser_type}\"\n", + " in_one = f\"{curr_dir}/{entry[0]}\"\n", + " in_two = f\"{curr_dir}/{entry[1]}\"\n", + " out = f\"prod/data.{parser_type}.{entry[0]}.nxs\"\n", + " stdout = f\"log/stdout.{parser_type}.{entry[0]}.txt\"\n", + " stderr = f\"log/stderr.{parser_type}.{entry[0]}.txt\"\n", + " ! dataconverter $in_one $in_two --reader apm --nxdl NXapm --output $out 1>$stdout 2>$stderr\n", + " continue\n", + " print(f\"Skipping test {parser_type}/{entry}\")\n", + "print(\"Ran all tests\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "213077cb-6c43-42d0-8f0a-213c0f82696d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}