Support for Python 3.13

.github/workflows/ci-code.yml

@@ -21,12 +21,12 @@ jobs:
   tests:
 
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
+    timeout-minutes: 180
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.12']
+        python-version: ['3.9', '3.13']
         database-backend: [psql]
         include:
         - python-version: '3.9'
@@ -67,17 +67,43 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    # uncomment for debug purposes
+    #- name: Setup upterm session
+    #  env:
+    #    AIIDA_TEST_PROFILE: test_aiida
+    #    AIIDA_WARN_v3: 1
+    #  uses: lhotari/action-upterm@v1
+
     - name: Setup environment
       run: .github/workflows/setup.sh
 
+    - name: ulimit
+      run: |
+        ulimit -a
+
     - name: Run test suite
       env:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
       # NOTE1: Python 3.12 has a performance regression when running with code coverage
       # so run code coverage only for python 3.9.
       run: |
-        pytest -n auto --db-backend ${{ matrix.database-backend }} -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
+        pytest --db-backend ${{ matrix.database-backend }} -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
+    - name: Cat module open fds
+      if: ${{ always() }}
+      env:
+        AIIDA_TEST_PROFILE: test_aiida
+        AIIDA_WARN_v3: 1
+      run: |
+        cat module_open_fds.log
+
+    - name: Cat function open fds
+      if: ${{ always() }}
+      env:
+        AIIDA_TEST_PROFILE: test_aiida
+        AIIDA_WARN_v3: 1
+      run: |
+        cat function_open_fds.log
 
     - name: Upload coverage report
       if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core'
@@ -107,6 +133,10 @@ jobs:
     - name: Setup SSH on localhost
       run: .github/workflows/setup_ssh.sh
 
+    # uncomment for debug purposes
+    #- name: Setup upterm session
+    #  uses: lhotari/action-upterm@v1
+
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0

.github/workflows/test-install.yml

@@ -111,7 +111,7 @@ jobs:
       fail-fast: false
       matrix:
 
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
         # Not being able to install with conda on a specific Python version is
         # not sufficient to fail the run, but something we want to be aware of.

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
   'alembic~=1.2',
   'archive-path~=0.4.2',
   "asyncssh~=2.19.0",
-  'circus~=0.18.0',
+  'circus~=0.19.0',
   'click-spinner~=0.1.8',
   'click~=8.1',
   'disk-objectstore~=1.2',
@@ -521,3 +521,11 @@ commands = molecule {posargs:test}
 # .github/actions/install-aiida-core/action.yml
 # .readthedocs.yml
 required-version = ">=0.5.21"
+
+[[tool.uv.index]]
+explicit = true
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+
+[tool.uv.sources]
+circus = {index = "testpypi"}

src/aiida/storage/psql_dos/backend.py

@@ -182,20 +182,66 @@ def get_session(self) -> Session:
         return self._session_factory()
 
     def close(self) -> None:
-        if self._session_factory is None:
-            return  # the instance is already closed, and so this is a no-op
-        # close the connection
+        import os
+
+        import psutil
+
+        # does not create a ref
+        from sqlalchemy.orm.session import _sessions
+
+        # this seems to make the gc active to delte sessnios, wtf?
+        list(_sessions.values())
+
+        def list_open_fds():
+            process = psutil.Process(os.getpid())
+            return process.open_files()
+
+        breakpoint()
+        # sessions = [sess for sess in _sessions.values()]
+        # if self._session_factory is None:
+        #    return  # the instance is already closed, and so this is a no-op
+        ## close the connection
+
+        ## somehow this closes it but it should be equivalent to
+        # engines = []
+        # for sess in sessions:
+        #    engines.append(sess.bind)
+        # for sess in sessions:
+        #    #sess.flush()
+        #    #sess.expunge_all()
+        #    sess.close()
 
+        # somehow it is important to close the session before the engine is first time disposed
         engine = self._session_factory.bind
-        if engine is not None:
-            engine.dispose()  # type: ignore[union-attr]
+        from sqlalchemy.orm.session import close_all_sessions
+
+        close_all_sessions()
+
         self._session_factory.expunge_all()
+        # gc.collect()
+        # self._session_factory.session_factory.close_all()
+        # gc.collect()
+        # self._session_factory.remove()
+        # gc.collect()
         self._session_factory.close()
+        # gc.collect()
+        # self._session_factory.close_all()
+        # gc.collect()
         self._session_factory = None
 
+        # for engine in engines:
+        #    engine.dispose()
+        if engine is not None:
+            engine.dispose()  # type: ignore[union-attr]
+
         # Without this, sqlalchemy keeps a weakref to a session
         # in sqlalchemy.orm.session._sessions
         gc.collect()
+        # from aiida.manage import get_manager
+        # get_manager().reset_profile_storage()
+        # with self.migrator_context(self._profile) as migrator:
+        #    migrator.get_container().close()
+        breakpoint()
 
     def _clear(self) -> None:
         from aiida.storage.psql_dos.models.settings import DbSetting
@@ -277,6 +323,7 @@ def transaction(self) -> Iterator[Session]:
                 with session.begin_nested() as savepoint:
                     yield session
                     savepoint.commit()
+        session.close()
 
     @property
     def in_transaction(self) -> bool:
@@ -367,6 +414,7 @@ def delete(self, delete_database_user: bool = False) -> None:
 
         if repository.exists():
             shutil.rmtree(repository)
+            breakpoint()
             LOGGER.report(f'Deleted repository at `{repository}`.')
 
         if postgres.db_exists(config['database_name']):

src/aiida/storage/psql_dos/migrator.py

@@ -51,6 +51,8 @@
 
 REPOSITORY_UUID_KEY = 'repository|uuid'
 
+_CONTAINERS = {}
+
 
 class PsqlDosMigrator:
     """Class for validating and migrating `psql_dos` storage instances.
@@ -179,7 +181,12 @@ def get_container(self) -> 'Container':
 
         from .backend import get_filepath_container
 
-        return Container(get_filepath_container(self.profile))
+        # TODO need upddate for each profile
+        breakpoint()
+        global _CONTAINER
+        if _CONTAINERS.get(self.profile, None) is None:
+            _CONTAINERS[self.profile] = Container(get_filepath_container(self.profile))
+        return _CONTAINERS[self.profile]
 
     def get_repository_uuid(self) -> str:
         """Return the UUID of the repository.
@@ -206,6 +213,7 @@ def initialise(self, reset: bool = False) -> bool:
             tests having run.
         :returns: ``True`` if the storage was initialised by the function call, ``False`` if it was already initialised.
         """
+        breakpoint()
         if reset:
             self.reset_repository()
             self.reset_database()

src/aiida/storage/psql_dos/utils.py

@@ -50,8 +50,11 @@ def create_sqlalchemy_engine(config: PsqlConfig):
         port=config['database_port'],
         name=config['database_name'],
     )
+    from sqlalchemy.pool import NullPool
+
     return create_engine(
         engine_url,
+        poolclass=NullPool,
         json_serializer=json.dumps,
         json_deserializer=json.loads,
         **config.get('engine_kwargs', {}),

src/aiida/storage/sqlite_dos/backend.py

@@ -54,6 +54,7 @@
 ALEMBIC_REL_PATH = 'migrations'
 
 REPOSITORY_UUID_KEY = 'repository|uuid'
+_CONTAINERS = {}
 
 
 class SqliteDosMigrator(PsqlDosMigrator):
@@ -84,8 +85,12 @@ def get_container(self) -> Container:
 
         :returns: The disk-object store container configured for the repository path of the current profile.
         """
-        filepath_container = Path(self.profile.storage_config['filepath']) / FILENAME_CONTAINER
-        return Container(str(filepath_container))
+        # TODO also is in the migrator, wtf?
+        global _CONTAINERS
+        if _CONTAINERS.get(self.profile, None) is None:
+            filepath_container = Path(self.profile.storage_config['filepath']) / FILENAME_CONTAINER
+            _CONTAINERS[self.profile] = Container(str(filepath_container))
+        return _CONTAINERS[self.profile]
 
     def initialise_database(self) -> None:
         """Initialise the database.

src/aiida/storage/sqlite_zip/utils.py

@@ -81,7 +81,11 @@ def register_json_contains(dbapi_connection, _):
 
 def create_sqla_engine(path: Union[str, Path], *, enforce_foreign_keys: bool = True, **kwargs) -> Engine:
     """Create a new engine instance."""
-    engine = create_engine(f'sqlite:///{path}', json_serializer=json.dumps, json_deserializer=json.loads, **kwargs)
+    from sqlalchemy.pool import NullPool
+
+    engine = create_engine(
+        f'sqlite:///{path}', poolclass=NullPool, json_serializer=json.dumps, json_deserializer=json.loads, **kwargs
+    )
     event.listen(engine, 'connect', sqlite_case_sensitive_like)
     if enforce_foreign_keys:
         event.listen(engine, 'connect', sqlite_enforce_foreign_keys)

src/aiida/tools/dbimporters/baseclasses.py

@@ -96,6 +96,9 @@ def __init__(self, results, increment=1):
             self._position = 0
             self._increment = increment
 
+        def __iter__(self):
+            return self
+
         def __next__(self):
             """Return the next entry in the iterator."""
             pos = self._position

src/aiida/tools/pytest_fixtures/configuration.py

@@ -161,6 +161,7 @@ def reset_storage():
                         pass
 
             manager.get_profile_storage()._clear()
+            breakpoint()
             manager.reset_profile()
 
             User(email=profile.default_user_email or email).store()
@@ -208,8 +209,34 @@ def aiida_profile_clean(aiida_profile):
 
     :returns :class:`~aiida.manage.configuration.profile.Profile`: The loaded temporary profile.
     """
+    import os
+
+    import psutil
+
+    def list_open_fds():
+        process = psutil.Process(os.getpid())
+        return process.open_files()
+
+    # import tracemalloc
+    # tracemalloc.start()
+    ## ... start your application ...
+
+    # snapshot1 = tracemalloc.take_snapshot()
+    ## ... call the function leaking memory ...
+    # for _ in range(100):
+    #    aiida_profile.reset_storage()
+    breakpoint()
     aiida_profile.reset_storage()
-    yield aiida_profile
+    breakpoint()
+    # yield aiida_profile
+
+    # snapshot2 = tracemalloc.take_snapshot()
+
+    # top_stats = snapshot2.compare_to(snapshot1, 'lineno')
+
+    # print("[ Top 10 differences ]")
+    # for stat in top_stats[:10]:
+    #    print(stat)
 
 
 @pytest.fixture(scope='class')

src/aiida/transports/plugins/ssh.py

@@ -496,6 +496,7 @@ def open(self):
             connection_arguments['sock'] = self._proxy
 
         try:
+            connection_arguments['banner_timeout'] = 200
             self._client.connect(self._machine, **connection_arguments)
         except Exception as exc:
             self.logger.error(