diff --git a/environment.yml b/environment.yml index be05f5eb17..86eee6f90b 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - circus~=0.18.0 - click-spinner~=0.1.8 - click~=8.1 -- disk-objectstore~=1.1 +- disk-objectstore~=1.2 - docstring_parser - get-annotations~=0.1 - python-graphviz~=0.19 diff --git a/pyproject.toml b/pyproject.toml index 129746e0fc..52b2536deb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ 'circus~=0.18.0', 'click-spinner~=0.1.8', 'click~=8.1', - 'disk-objectstore~=1.1', + 'disk-objectstore~=1.2', 'docstring-parser', 'get-annotations~=0.1;python_version<"3.10"', 'graphviz~=0.19', diff --git a/requirements/requirements-py-3.10.txt b/requirements/requirements-py-3.10.txt index 91b0f013cb..b2408e8087 100644 --- a/requirements/requirements-py-3.10.txt +++ b/requirements/requirements-py-3.10.txt @@ -42,7 +42,7 @@ debugpy==1.6.7 decorator==5.1.1 defusedxml==0.7.1 deprecation==2.1.0 -disk-objectstore==1.1.0 +disk-objectstore==1.2.0 docstring-parser==0.15 docutils==0.20.1 exceptiongroup==1.1.1 diff --git a/requirements/requirements-py-3.11.txt b/requirements/requirements-py-3.11.txt index 422f729d1b..24acc25a6b 100644 --- a/requirements/requirements-py-3.11.txt +++ b/requirements/requirements-py-3.11.txt @@ -42,7 +42,7 @@ debugpy==1.6.7 decorator==5.1.1 defusedxml==0.7.1 deprecation==2.1.0 -disk-objectstore==1.1.0 +disk-objectstore==1.2.0 docstring-parser==0.15 docutils==0.20.1 executing==1.2.0 diff --git a/requirements/requirements-py-3.12.txt b/requirements/requirements-py-3.12.txt index a16c0c82f5..3f5d72ebb6 100644 --- a/requirements/requirements-py-3.12.txt +++ b/requirements/requirements-py-3.12.txt @@ -42,7 +42,7 @@ debugpy==1.8.0 decorator==5.1.1 defusedxml==0.7.1 deprecation==2.1.0 -disk-objectstore==1.1.0 +disk-objectstore==1.2.0 docstring-parser==0.15 docutils==0.20.1 executing==2.0.0 diff --git a/requirements/requirements-py-3.9.txt b/requirements/requirements-py-3.9.txt index 08e6bcf5f8..3087e62844 100644 --- a/requirements/requirements-py-3.9.txt +++ b/requirements/requirements-py-3.9.txt @@ -42,7 +42,7 @@ debugpy==1.6.7 decorator==5.1.1 defusedxml==0.7.1 deprecation==2.1.0 -disk-objectstore==1.1.0 +disk-objectstore==1.2.0 docstring-parser==0.15 docutils==0.20.1 exceptiongroup==1.2.1 diff --git a/src/aiida/cmdline/commands/cmd_storage.py b/src/aiida/cmdline/commands/cmd_storage.py index f6f64b755e..02f82b9c9c 100644 --- a/src/aiida/cmdline/commands/cmd_storage.py +++ b/src/aiida/cmdline/commands/cmd_storage.py @@ -8,6 +8,7 @@ ########################################################################### """`verdi storage` commands.""" +import logging import sys import click @@ -153,7 +154,9 @@ def storage_info(detailed): def storage_maintain(ctx, full, no_repack, force, dry_run, compress): """Performs maintenance tasks on the repository.""" from aiida.common.exceptions import LockingProfileError + from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter from aiida.manage.manager import get_manager + from aiida.storage.log import STORAGE_LOGGER manager = get_manager() profile = ctx.obj.profile @@ -184,6 +187,12 @@ def storage_maintain(ctx, full, no_repack, force, dry_run, compress): if not dry_run and not force and not click.confirm('Are you sure you want continue in this mode?'): return + if STORAGE_LOGGER.level <= logging.REPORT: + # Only keep the first tqdm bar if it is info. To keep the nested bar information one needs report level + set_progress_bar_tqdm(leave=STORAGE_LOGGER.level <= logging.INFO) + else: + set_progress_reporter(None) + try: if full and no_repack: storage.maintain(full=full, dry_run=dry_run, do_repack=False, compress=compress) diff --git a/src/aiida/repository/backend/disk_object_store.py b/src/aiida/repository/backend/disk_object_store.py index b8928be575..f1bd22faaa 100644 --- a/src/aiida/repository/backend/disk_object_store.py +++ b/src/aiida/repository/backend/disk_object_store.py @@ -167,6 +167,8 @@ def maintain( # type: ignore[override] """ from disk_objectstore import CompressMode + from aiida.common.progress_reporter import create_callback, get_progress_reporter + if live and (do_repack or clean_storage or do_vacuum): overrides = {'do_repack': do_repack, 'clean_storage': clean_storage, 'do_vacuum': do_vacuum} keys = ', '.join([key for key, override in overrides.items() if override is True]) # type: ignore @@ -192,14 +194,18 @@ def maintain( # type: ignore[override] files_size = container.get_total_size().total_size_loose * BYTES_TO_MB logger.report(f'Packing all loose files ({files_numb} files occupying {files_size} MB) ...') if not dry_run: - container.pack_all_loose(compress=compress) + with get_progress_reporter()(total=1) as progress: + callback = create_callback(progress) + container.pack_all_loose(compress=compress, callback=callback) if do_repack: files_numb = container.count_objects().packed files_size = container.get_total_size().total_size_packfiles_on_disk * BYTES_TO_MB logger.report(f'Re-packing all pack files ({files_numb} files in packs, occupying {files_size} MB) ...') if not dry_run: - container.repack() + with get_progress_reporter()(total=1) as progress: + callback = create_callback(progress) + container.repack(callback=callback) if clean_storage: logger.report(f'Cleaning the repository database (with `vacuum={do_vacuum}`) ...')