From 197c666d362b3a7dd03bec9ccc1e41bb44023e7c Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Fri, 29 Nov 2024 15:03:27 +0100 Subject: [PATCH 1/5] Test refactoring: use tmp path fixture to mock remote and local for transport plugins (#6627) In `tests/transport/test_all_plugins.py`, the tests were first wrote couple years ago before there were tmp_path fixture from pytest. The tmp folders were created and shared between multiple tests. When running tests in parallel, they will failed because of override. This commit refactoring all the tests in this file by using `tmp_path_factory` to create `tmp_path_remote` and `tmp_path_local` respectively to mock the transport behavior from different folder in different location. --------- Co-authored-by: Ali Khosravi --- src/aiida/transports/plugins/local.py | 7 +- tests/transports/test_all_plugins.py | 1012 +++++++++++-------------- 2 files changed, 436 insertions(+), 583 deletions(-) diff --git a/src/aiida/transports/plugins/local.py b/src/aiida/transports/plugins/local.py index 755476a066..8de49838e3 100644 --- a/src/aiida/transports/plugins/local.py +++ b/src/aiida/transports/plugins/local.py @@ -453,8 +453,8 @@ def getfile(self, remotepath, localpath, *args, **kwargs): """Copies a file recursively from 'remote' remotepath to 'local' localpath. - :param remotepath: path to local file - :param localpath: absolute path to remote file + :param remotepath: absolute path to remote file + :param localpath: path to local file :param overwrite: if True overwrites localpath. Default = False @@ -462,6 +462,9 @@ def getfile(self, remotepath, localpath, *args, **kwargs): :raise ValueError: if 'local' localpath is not valid :raise OSError: if unintentionally overwriting """ + if not os.path.isabs(localpath): + raise ValueError('localpath must be an absolute path') + overwrite = kwargs.get('overwrite', args[0] if args else True) if not localpath: raise ValueError('Input localpath to get function must be a non empty string') diff --git a/tests/transports/test_all_plugins.py b/tests/transports/test_all_plugins.py index 0a25add0a7..8aea2529cb 100644 --- a/tests/transports/test_all_plugins.py +++ b/tests/transports/test_all_plugins.py @@ -12,15 +12,12 @@ """ import io -import os -import pathlib -import random import shutil import signal -import string import tempfile import time import uuid +from pathlib import Path import psutil import pytest @@ -33,8 +30,20 @@ # TODO : silly cases of copy/put/get from self to self +@pytest.fixture(scope='function') +def tmp_path_remote(tmp_path_factory): + """Mock the remote tmp path using tmp_path_factory to create folder start with prefix 'remote'""" + return tmp_path_factory.mktemp('remote') + + +@pytest.fixture(scope='function') +def tmp_path_local(tmp_path_factory): + """Mock the local tmp path using tmp_path_factory to create folder start with prefix 'local'""" + return tmp_path_factory.mktemp('local') + + @pytest.fixture(scope='function', params=entry_point.get_entry_point_names('aiida.transports')) -def custom_transport(request, tmp_path, monkeypatch) -> Transport: +def custom_transport(request, tmp_path_factory, monkeypatch) -> Transport: """Fixture that parametrizes over all the registered implementations of the ``CommonRelaxWorkChain``.""" plugin = TransportFactory(request.param) @@ -42,10 +51,12 @@ def custom_transport(request, tmp_path, monkeypatch) -> Transport: kwargs = {'machine': 'localhost', 'timeout': 30, 'load_system_host_keys': True, 'key_policy': 'AutoAddPolicy'} elif request.param == 'core.ssh_auto': kwargs = {'machine': 'localhost'} - filepath_config = tmp_path / 'config' + # The transport config is store in a independent temporary path per test to not mix up + # with the files under operating. + filepath_config = tmp_path_factory.mktemp('transport') / 'config' monkeypatch.setattr(plugin, 'FILEPATH_CONFIG', filepath_config) - if not filepath_config.exists(): - filepath_config.write_text('Host localhost') + + filepath_config.write_text('Host localhost') else: kwargs = {} @@ -62,26 +73,36 @@ def test_is_open(custom_transport): assert not custom_transport.is_open -def test_makedirs(custom_transport): - """Verify the functioning of makedirs command""" +def test_chdir_and_getcwd_deprecated(custom_transport, tmp_path_remote): + """Test to be deprecated ``chdir``/``getcwd`` methods still work.""" with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' + location = str(tmp_path_remote) transport.chdir(location) assert location == transport.getcwd() - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - transport.mkdir(directory) - transport.chdir(directory) + +def test_chdir_to_empty_string_deprecated(custom_transport, tmp_path_remote): + """I check that if I pass an empty string to chdir, the cwd does + not change (this is a paramiko default behavior), but getcwd() + is still correctly defined. + """ + with custom_transport as transport: + new_dir = str(tmp_path_remote) + transport.chdir(new_dir) + transport.chdir('') + assert new_dir == transport.getcwd() + + +def test_makedirs(custom_transport, tmp_path_remote): + """Verify the functioning of makedirs command""" + with custom_transport as transport: # define folder structure - dir_tree = os.path.join('1', '2') + dir_tree = str(tmp_path_remote / '1' / '2') # I create the tree transport.makedirs(dir_tree) # verify the existence - assert transport.isdir('1') + assert transport.isdir(str(tmp_path_remote / '1')) assert dir_tree # try to recreate the same folder @@ -91,93 +112,55 @@ def test_makedirs(custom_transport): # recreate but with ignore flag transport.makedirs(dir_tree, True) - transport.rmdir(dir_tree) - transport.rmdir('1') - - transport.chdir('..') - transport.rmdir(directory) - -def test_rmtree(custom_transport): +def test_rmtree(custom_transport, tmp_path_remote): """Verify the functioning of rmtree command""" with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - transport.chdir(location) - - assert location == transport.getcwd() - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - transport.mkdir(directory) - transport.chdir(directory) - # define folder structure - dir_tree = os.path.join('1', '2') + dir_tree = str(tmp_path_remote / '1' / '2') # I create the tree transport.makedirs(dir_tree) # remove it - transport.rmtree('1') + transport.rmtree(str(tmp_path_remote / '1')) # verify the removal - assert not transport.isdir('1') + assert not transport.isdir(str(tmp_path_remote / '1')) # also tests that it works with a single file # create file local_file_name = 'file.txt' text = 'Viva Verdi\n' - with open(os.path.join(transport.getcwd(), local_file_name), 'w', encoding='utf8') as fhandle: + single_file_path = str(tmp_path_remote / local_file_name) + with open(single_file_path, 'w', encoding='utf8') as fhandle: fhandle.write(text) # remove it - transport.rmtree(local_file_name) + transport.rmtree(single_file_path) # verify the removal - assert not transport.isfile(local_file_name) - - transport.chdir('..') - transport.rmdir(directory) + assert not transport.isfile(single_file_path) -def test_listdir(custom_transport): +def test_listdir(custom_transport, tmp_path_remote): """Create directories, verify listdir, delete a folder with subfolders""" - with custom_transport as trans: - # We cannot use tempfile.mkdtemp because we're on a remote folder - location = trans.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - trans.chdir(location) - - assert location == trans.getcwd() - while trans.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - trans.mkdir(directory) - trans.chdir(directory) + with custom_transport as transport: list_of_dir = ['1', '-f a&', 'as', 'a2', 'a4f'] list_of_files = ['a', 'b'] for this_dir in list_of_dir: - trans.mkdir(this_dir) + transport.mkdir(str(tmp_path_remote / this_dir)) + for fname in list_of_files: with tempfile.NamedTemporaryFile() as tmpf: # Just put an empty file there at the right file name - trans.putfile(tmpf.name, fname) + transport.putfile(tmpf.name, str(tmp_path_remote / fname)) - list_found = trans.listdir('.') + list_found = transport.listdir(str(tmp_path_remote)) assert sorted(list_found) == sorted(list_of_dir + list_of_files) - assert sorted(trans.listdir('.', 'a*')), sorted(['as', 'a2', 'a4f']) - assert sorted(trans.listdir('.', 'a?')), sorted(['as', 'a2']) - assert sorted(trans.listdir('.', 'a[2-4]*')), sorted(['a2', 'a4f']) - - for this_dir in list_of_dir: - trans.rmdir(this_dir) - - for this_file in list_of_files: - trans.remove(this_file) - - trans.chdir('..') - trans.rmdir(directory) + assert sorted(transport.listdir(str(tmp_path_remote), 'a*')), sorted(['as', 'a2', 'a4f']) + assert sorted(transport.listdir(str(tmp_path_remote), 'a?')), sorted(['as', 'a2']) + assert sorted(transport.listdir(str(tmp_path_remote), 'a[2-4]*')), sorted(['a2', 'a4f']) -def test_listdir_withattributes(custom_transport): +def test_listdir_withattributes(custom_transport, tmp_path_remote): """Create directories, verify listdir_withattributes, delete a folder with subfolders""" def simplify_attributes(data): @@ -189,115 +172,79 @@ def simplify_attributes(data): """ return {_['name']: _['isdir'] for _ in data} - with custom_transport as trans: - # We cannot use tempfile.mkdtemp because we're on a remote folder - location = trans.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - trans.chdir(location) - - assert location == trans.getcwd() - while trans.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - trans.mkdir(directory) - trans.chdir(directory) + with custom_transport as transport: list_of_dir = ['1', '-f a&', 'as', 'a2', 'a4f'] list_of_files = ['a', 'b'] for this_dir in list_of_dir: - trans.mkdir(this_dir) + transport.mkdir(str(tmp_path_remote / this_dir)) + for fname in list_of_files: with tempfile.NamedTemporaryFile() as tmpf: # Just put an empty file there at the right file name - trans.putfile(tmpf.name, fname) + transport.putfile(tmpf.name, str(tmp_path_remote / fname)) comparison_list = {k: True for k in list_of_dir} for k in list_of_files: comparison_list[k] = False - assert simplify_attributes(trans.listdir_withattributes('.')), comparison_list - assert simplify_attributes(trans.listdir_withattributes('.', 'a*')), { + assert simplify_attributes(transport.listdir_withattributes(str(tmp_path_remote))), comparison_list + assert simplify_attributes(transport.listdir_withattributes(str(tmp_path_remote), 'a*')), { 'as': True, 'a2': True, 'a4f': True, 'a': False, } - assert simplify_attributes(trans.listdir_withattributes('.', 'a?')), {'as': True, 'a2': True} - assert simplify_attributes(trans.listdir_withattributes('.', 'a[2-4]*')), {'a2': True, 'a4f': True} - - for this_dir in list_of_dir: - trans.rmdir(this_dir) - - for this_file in list_of_files: - trans.remove(this_file) - - trans.chdir('..') - trans.rmdir(directory) + assert simplify_attributes(transport.listdir_withattributes(str(tmp_path_remote), 'a?')), { + 'as': True, + 'a2': True, + } + assert simplify_attributes(transport.listdir_withattributes(str(tmp_path_remote), 'a[2-4]*')), { + 'a2': True, + 'a4f': True, + } -def test_dir_creation_deletion(custom_transport): +def test_dir_creation_deletion(custom_transport, tmp_path_remote): """Test creating and deleting directories.""" with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - transport.chdir(location) - - assert location == transport.getcwd() - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - transport.mkdir(directory) + new_dir = str(tmp_path_remote / 'new') + transport.mkdir(new_dir) with pytest.raises(OSError): # I create twice the same directory - transport.mkdir(directory) + transport.mkdir(new_dir) - transport.isdir(directory) - assert not transport.isfile(directory) - transport.rmdir(directory) + transport.isdir(new_dir) + assert not transport.isfile(new_dir) -def test_dir_copy(custom_transport): +def test_dir_copy(custom_transport, tmp_path_remote): """Verify if in the copy of a directory also the protection bits are carried over """ with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - transport.chdir(location) - - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - transport.mkdir(directory) + # Create a src dir + src_dir = str(tmp_path_remote / 'copy_src') + transport.mkdir(src_dir) - dest_directory = f'{directory}_copy' - transport.copy(directory, dest_directory) + dst_dir = str(tmp_path_remote / 'copy_dst') + transport.copy(src_dir, dst_dir) with pytest.raises(ValueError): - transport.copy(directory, '') + transport.copy(src_dir, '') with pytest.raises(ValueError): - transport.copy('', directory) + transport.copy('', dst_dir) - transport.rmdir(directory) - transport.rmdir(dest_directory) - -def test_dir_permissions_creation_modification(custom_transport): +def test_dir_permissions_creation_modification(custom_transport, tmp_path_remote): """Verify if chmod raises OSError when trying to change bits on a non-existing folder """ with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - transport.chdir(location) + directory = str(tmp_path_remote / 'test') - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - - # create directory with non default permissions - transport.mkdir(directory) + transport.makedirs(directory) # change permissions transport.chmod(directory, 0o777) @@ -318,7 +265,6 @@ def test_dir_permissions_creation_modification(custom_transport): # the new directory modes. To see if we want a higher # level function to ask for the mode, or we just # use get_attribute - transport.chdir(directory) # change permissions of an empty string, non existing folder. fake_dir = '' @@ -328,24 +274,15 @@ def test_dir_permissions_creation_modification(custom_transport): fake_dir = 'pippo' with pytest.raises(OSError): # chmod to a non existing folder - transport.chmod(fake_dir, 0o777) + transport.chmod(str(tmp_path_remote / fake_dir), 0o777) - transport.chdir('..') - transport.rmdir(directory) - -def test_dir_reading_permissions(custom_transport): +def test_dir_reading_permissions(custom_transport, tmp_path_remote): """Try to enter a directory with no read permissions. Verify that the cwd has not changed after failed try. """ with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - directory = 'temp_dir_test' - transport.chdir(location) - - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + directory = str(tmp_path_remote / 'test') # create directory with non default permissions transport.mkdir(directory) @@ -365,278 +302,250 @@ def test_dir_reading_permissions(custom_transport): assert old_cwd == new_cwd - # TODO : the test leaves a directory even if it is successful - # The bug is in paramiko. After lowering the permissions, - # I cannot restore them to higher values - # transport.rmdir(directory) - def test_isfile_isdir_to_empty_string(custom_transport): """I check that isdir or isfile return False when executed on an empty string """ with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - transport.chdir(location) assert not transport.isdir('') assert not transport.isfile('') -def test_isfile_isdir_to_non_existing_string(custom_transport): +def test_isfile_isdir_to_non_existing_string(custom_transport, tmp_path_remote): """I check that isdir or isfile return False when executed on an empty string """ with custom_transport as transport: - location = transport.normalize(os.path.join('/', 'tmp')) - transport.chdir(location) - fake_folder = 'pippo' + fake_folder = str(tmp_path_remote / 'pippo') assert not transport.isfile(fake_folder) assert not transport.isdir(fake_folder) with pytest.raises(OSError): transport.chdir(fake_folder) -def test_chdir_to_empty_string(custom_transport): - """I check that if I pass an empty string to chdir, the cwd does - not change (this is a paramiko default behavior), but getcwd() - is still correctly defined. - """ +def test_put_and_get(custom_transport, tmp_path_remote, tmp_path_local): + """Test putting and getting files.""" + directory = 'tmp_try' + with custom_transport as transport: - new_dir = transport.normalize(os.path.join('/', 'tmp')) - transport.chdir(new_dir) - transport.chdir('') - assert new_dir == transport.getcwd() + (tmp_path_local / directory).mkdir() + transport.mkdir(str(tmp_path_remote / directory)) + + local_file_name = 'file.txt' + retrieved_file_name = 'file_retrieved.txt' + + remote_file_name = 'file_remote.txt' + # here use full path in src and dst + local_file_abs_path = str(tmp_path_local / directory / local_file_name) + retrieved_file_abs_path = str(tmp_path_local / directory / retrieved_file_name) + remote_file_abs_path = str(tmp_path_remote / directory / remote_file_name) + + text = 'Viva Verdi\n' + with open(local_file_abs_path, 'w', encoding='utf8') as fhandle: + fhandle.write(text) + + transport.put(local_file_abs_path, remote_file_abs_path) + transport.get(remote_file_abs_path, retrieved_file_abs_path) + + list_of_files = transport.listdir(str(tmp_path_remote / directory)) + # it is False because local_file_name has the full path, + # while list_of_files has not + assert local_file_name not in list_of_files + assert remote_file_name in list_of_files + assert retrieved_file_name not in list_of_files -def test_put_and_get_file(custom_transport): + +def test_putfile_and_getfile(custom_transport, tmp_path_remote, tmp_path_local): """Test putting and getting files.""" - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote + directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + (local_dir / directory).mkdir() + transport.mkdir(str(remote_dir / directory)) - transport.mkdir(directory) - transport.chdir(directory) + local_file_name = 'file.txt' + retrieved_file_name = 'file_retrieved.txt' - local_file_name = os.path.join(local_dir, directory, 'file.txt') remote_file_name = 'file_remote.txt' - retrieved_file_name = os.path.join(local_dir, directory, 'file_retrieved.txt') + + # here use full path in src and dst + local_file_abs_path = str(local_dir / directory / local_file_name) + retrieved_file_abs_path = str(local_dir / directory / retrieved_file_name) + remote_file_abs_path = str(remote_dir / directory / remote_file_name) text = 'Viva Verdi\n' - with open(local_file_name, 'w', encoding='utf8') as fhandle: + with open(local_file_abs_path, 'w', encoding='utf8') as fhandle: fhandle.write(text) - # here use full path in src and dst - transport.put(local_file_name, remote_file_name) - transport.get(remote_file_name, retrieved_file_name) - transport.putfile(local_file_name, remote_file_name) - transport.getfile(remote_file_name, retrieved_file_name) + transport.putfile(local_file_abs_path, remote_file_abs_path) + transport.getfile(remote_file_abs_path, retrieved_file_abs_path) - list_of_files = transport.listdir('.') + list_of_files = transport.listdir(str(remote_dir / directory)) # it is False because local_file_name has the full path, # while list_of_files has not assert local_file_name not in list_of_files assert remote_file_name in list_of_files assert retrieved_file_name not in list_of_files - os.remove(local_file_name) - transport.remove(remote_file_name) - os.remove(retrieved_file_name) - - transport.chdir('..') - transport.rmdir(directory) - -def test_put_get_abs_path_file(custom_transport): +def test_put_get_abs_path_file(custom_transport, tmp_path_remote, tmp_path_local): """Test of exception for non existing files and abs path""" - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote + directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + (local_dir / directory).mkdir() + transport.mkdir(str(remote_dir / directory)) - transport.mkdir(directory) - transport.chdir(directory) + local_file_name = 'file.txt' + retrieved_file_name = 'file_retrieved.txt' - partial_file_name = 'file.txt' - local_file_name = os.path.join(local_dir, directory, 'file.txt') remote_file_name = 'file_remote.txt' - retrieved_file_name = os.path.join(local_dir, directory, 'file_retrieved.txt') + local_file_rel_path = local_file_name + remote_file_rel_path = remote_file_name - pathlib.Path(local_file_name).touch() + retrieved_file_abs_path = str(local_dir / directory / retrieved_file_name) + remote_file_abs_path = str(remote_dir / directory / remote_file_name) # partial_file_name is not an abs path with pytest.raises(ValueError): - transport.put(partial_file_name, remote_file_name) + transport.put(local_file_rel_path, remote_file_abs_path) with pytest.raises(ValueError): - transport.putfile(partial_file_name, remote_file_name) + transport.putfile(local_file_rel_path, remote_file_abs_path) # retrieved_file_name does not exist with pytest.raises(OSError): - transport.put(retrieved_file_name, remote_file_name) + transport.put(retrieved_file_abs_path, remote_file_abs_path) with pytest.raises(OSError): - transport.putfile(retrieved_file_name, remote_file_name) + transport.putfile(retrieved_file_abs_path, remote_file_abs_path) # remote_file_name does not exist with pytest.raises(OSError): - transport.get(remote_file_name, retrieved_file_name) + transport.get(remote_file_abs_path, retrieved_file_abs_path) with pytest.raises(OSError): - transport.getfile(remote_file_name, retrieved_file_name) - - transport.put(local_file_name, remote_file_name) - transport.putfile(local_file_name, remote_file_name) + transport.getfile(remote_file_abs_path, retrieved_file_abs_path) - # local filename is not an abs path + # remote filename is not an abs path with pytest.raises(ValueError): - transport.get(remote_file_name, 'delete_me.txt') + transport.get(remote_file_rel_path, 'delete_me.txt') with pytest.raises(ValueError): - transport.getfile(remote_file_name, 'delete_me.txt') + transport.getfile(remote_file_rel_path, 'delete_me.txt') - transport.remove(remote_file_name) - os.remove(local_file_name) - transport.chdir('..') - transport.rmdir(directory) - - -def test_put_get_empty_string_file(custom_transport): +def test_put_get_empty_string_file(custom_transport, tmp_path_remote, tmp_path_local): """Test of exception put/get of empty strings""" - # TODO : verify the correctness of \n at the end of a file - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote + directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - while transport.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + (local_dir / directory).mkdir() + transport.mkdir(str(remote_dir / directory)) - transport.mkdir(directory) - transport.chdir(directory) + local_file_name = 'file.txt' + retrieved_file_name = 'file_retrieved.txt' - local_file_name = os.path.join(local_dir, directory, 'file_local.txt') remote_file_name = 'file_remote.txt' - retrieved_file_name = os.path.join(local_dir, directory, 'file_retrieved.txt') + + # here use full path in src and dst + local_file_abs_path = str(local_dir / directory / local_file_name) + retrieved_file_abs_path = str(local_dir / directory / retrieved_file_name) + remote_file_abs_path = str(remote_dir / directory / remote_file_name) text = 'Viva Verdi\n' - with open(local_file_name, 'w', encoding='utf8') as fhandle: + with open(local_file_abs_path, 'w', encoding='utf8') as fhandle: fhandle.write(text) # localpath is an empty string # ValueError because it is not an abs path with pytest.raises(ValueError): - transport.put('', remote_file_name) + transport.put('', remote_file_abs_path) with pytest.raises(ValueError): - transport.putfile('', remote_file_name) + transport.putfile('', remote_file_abs_path) # remote path is an empty string with pytest.raises(OSError): - transport.put(local_file_name, '') + transport.put(local_file_abs_path, '') with pytest.raises(OSError): - transport.putfile(local_file_name, '') + transport.putfile(local_file_abs_path, '') - transport.put(local_file_name, remote_file_name) + transport.put(local_file_abs_path, remote_file_abs_path) # overwrite the remote_file_name - transport.putfile(local_file_name, remote_file_name) + transport.putfile(local_file_abs_path, remote_file_abs_path) # remote path is an empty string with pytest.raises(OSError): - transport.get('', retrieved_file_name) + transport.get('', retrieved_file_abs_path) with pytest.raises(OSError): - transport.getfile('', retrieved_file_name) + transport.getfile('', retrieved_file_abs_path) # local path is an empty string # ValueError because it is not an abs path with pytest.raises(ValueError): - transport.get(remote_file_name, '') + transport.get(remote_file_abs_path, '') with pytest.raises(ValueError): - transport.getfile(remote_file_name, '') + transport.getfile(remote_file_abs_path, '') # TODO : get doesn't retrieve empty files. # Is it what we want? - transport.get(remote_file_name, retrieved_file_name) - # overwrite retrieved_file_name - transport.getfile(remote_file_name, retrieved_file_name) + transport.get(remote_file_abs_path, retrieved_file_abs_path) + assert Path(retrieved_file_abs_path).exists() + t1 = Path(retrieved_file_abs_path).stat().st_mtime_ns - os.remove(local_file_name) - transport.remove(remote_file_name) - # If it couldn't end the copy, it leaves what he did on - # local file - assert 'file_retrieved.txt' in transport.listdir('.') - os.remove(retrieved_file_name) + # overwrite retrieved_file_name in 0.01 s + time.sleep(0.01) + transport.getfile(remote_file_abs_path, retrieved_file_abs_path) + assert Path(retrieved_file_abs_path).exists() + t2 = Path(retrieved_file_abs_path).stat().st_mtime_ns - transport.chdir('..') - transport.rmdir(directory) + # Check st_mtime_ns to sure it is override + assert t2 > t1 -def test_put_and_get_tree(custom_transport): +def test_put_and_get_tree(custom_transport, tmp_path_remote, tmp_path_local): """Test putting and getting files.""" - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote + directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - - local_subfolder = os.path.join(local_dir, directory, 'tmp1') - remote_subfolder = 'tmp2' - retrieved_subfolder = os.path.join(local_dir, directory, 'tmp3') + local_subfolder: Path = local_dir / directory / 'tmp1' + remote_subfolder: Path = remote_dir / 'tmp2' + retrieved_subfolder: Path = local_dir / directory / 'tmp3' - os.mkdir(os.path.join(local_dir, directory)) - os.mkdir(os.path.join(local_dir, directory, local_subfolder)) + local_subfolder.mkdir(parents=True) - transport.chdir(directory) - - local_file_name = os.path.join(local_subfolder, 'file.txt') + local_file = local_subfolder / 'file.txt' text = 'Viva Verdi\n' - with open(local_file_name, 'w', encoding='utf8') as fhandle: + with open(local_file, 'w', encoding='utf8') as fhandle: fhandle.write(text) # here use full path in src and dst - for i in range(2): - if i == 0: - transport.put(local_subfolder, remote_subfolder) - transport.get(remote_subfolder, retrieved_subfolder) - else: - transport.puttree(local_subfolder, remote_subfolder) - transport.gettree(remote_subfolder, retrieved_subfolder) - - # Here I am mixing the local with the remote fold - list_of_dirs = transport.listdir('.') - # # it is False because local_file_name has the full path, - # # while list_of_files has not - assert local_subfolder not in list_of_dirs - assert remote_subfolder in list_of_dirs - assert retrieved_subfolder not in list_of_dirs - assert 'tmp1' in list_of_dirs - assert 'tmp3' in list_of_dirs - - list_pushed_file = transport.listdir('tmp2') - list_retrieved_file = transport.listdir('tmp3') - assert 'file.txt' in list_pushed_file - assert 'file.txt' in list_retrieved_file - - shutil.rmtree(local_subfolder) - shutil.rmtree(retrieved_subfolder) - transport.rmtree(remote_subfolder) - - transport.chdir('..') - transport.rmdir(directory) + transport.puttree(str(local_subfolder), str(remote_subfolder)) + transport.gettree(str(remote_subfolder), str(retrieved_subfolder)) + + list_of_dirs = [p.name for p in (local_dir / directory).iterdir()] + + assert local_subfolder not in list_of_dirs + assert remote_subfolder not in list_of_dirs + assert retrieved_subfolder not in list_of_dirs + assert 'tmp1' in list_of_dirs + assert 'tmp3' in list_of_dirs + + list_pushed_file = transport.listdir(str(remote_subfolder)) + list_retrieved_file = [p.name for p in retrieved_subfolder.iterdir()] + assert 'file.txt' in list_pushed_file + assert 'file.txt' in list_retrieved_file @pytest.mark.parametrize( @@ -710,251 +619,245 @@ def test_put_and_get_overwrite( ) -def test_copy(custom_transport): - """Test copying.""" - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir +def test_copy(custom_transport, tmp_path_remote): + """Test copying from a remote src to remote dst""" + remote_dir = tmp_path_remote + directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) + workdir = remote_dir / directory - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + transport.mkdir(str(workdir)) - transport.mkdir(directory) - transport.chdir(directory) - - local_base_dir = os.path.join(local_dir, directory, 'local') - os.mkdir(local_base_dir) + base_dir = workdir / 'origin' + base_dir.mkdir() - # first test put: I create three files in local - file_1 = os.path.join(local_base_dir, 'a.txt') - file_2 = os.path.join(local_base_dir, 'b.tmp') - file_3 = os.path.join(local_base_dir, 'c.txt') + # first create three files + file_1 = base_dir / 'a.txt' + file_2 = base_dir / 'b.tmp' + file_3 = base_dir / 'c.txt' text = 'Viva Verdi\n' for filename in [file_1, file_2, file_3]: with open(filename, 'w', encoding='utf8') as fhandle: fhandle.write(text) # first test the copy. Copy of two files matching patterns, into a folder - transport.copy(os.path.join('local', '*.txt'), '.') - assert set(['a.txt', 'c.txt', 'local']) == set(transport.listdir('.')) - transport.remove('a.txt') - transport.remove('c.txt') + transport.copy(str(base_dir / '*.txt'), str(workdir)) + assert set(['a.txt', 'c.txt', 'origin']) == set(transport.listdir(str(workdir))) + transport.remove(str(workdir / 'a.txt')) + transport.remove(str(workdir / 'c.txt')) + # second test copy. Copy of two folders - transport.copy('local', 'prova') - assert set(['prova', 'local']) == set(transport.listdir('.')) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir('prova')) - transport.rmtree('prova') + transport.copy(str(base_dir), str(workdir / 'prova')) + assert set(['prova', 'origin']) == set(transport.listdir(str(workdir))) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(str(workdir / 'prova'))) + transport.rmtree(str(workdir / 'prova')) + # third test copy. Can copy one file into a new file - transport.copy(os.path.join('local', '*.tmp'), 'prova') - assert set(['prova', 'local']) == set(transport.listdir('.')) - transport.remove('prova') + transport.copy(str(base_dir / '*.tmp'), str(workdir / 'prova')) + assert transport.isfile(str(workdir / 'prova')) + transport.remove(str(workdir / 'prova')) + # fourth test copy: can't copy more than one file on the same file, # i.e., the destination should be a folder with pytest.raises(OSError): - transport.copy(os.path.join('local', '*.txt'), 'prova') + transport.copy(str(base_dir / '*.txt'), str(workdir / 'prova')) + # fifth test, copying one file into a folder - transport.mkdir('prova') - transport.copy(os.path.join('local', 'a.txt'), 'prova') - assert set(transport.listdir('prova')) == set(['a.txt']) - transport.rmtree('prova') + transport.mkdir(str(workdir / 'prova')) + transport.copy(str(base_dir / 'a.txt'), str(workdir / 'prova')) + assert set(transport.listdir(str(workdir / 'prova'))) == set(['a.txt']) + transport.rmtree(str(workdir / 'prova')) + # sixth test, copying one file into a file - transport.copy(os.path.join('local', 'a.txt'), 'prova') - assert transport.isfile('prova') - transport.remove('prova') + transport.copy(str(base_dir / 'a.txt'), str(workdir / 'prova')) + assert transport.isfile(str(workdir / 'prova')) + transport.remove(str(workdir / 'prova')) # copy of folder into an existing folder # NOTE: the command cp has a different behavior on Mac vs Ubuntu # tests performed locally on a Mac may result in a failure. - transport.mkdir('prova') - transport.copy('local', 'prova') - assert set(['local']) == set(transport.listdir('prova')) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(os.path.join('prova', 'local'))) - transport.rmtree('prova') + transport.mkdir(str(workdir / 'prova')) + transport.copy(str(base_dir), str(workdir / 'prova')) + assert set(['origin']) == set(transport.listdir(str(workdir / 'prova'))) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(str(workdir / 'prova' / 'origin'))) + transport.rmtree(str(workdir / 'prova')) # exit - transport.chdir('..') - transport.rmtree(directory) + transport.rmtree(str(workdir)) -def test_put(custom_transport): - """Test putting files.""" - # exactly the same tests of copy, just with the put function - # and therefore the local path must be absolute - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir +def test_put(custom_transport, tmp_path_remote, tmp_path_local): + """Test putting files. + Those are similar tests of copy, just with the put function which copy from mocked local to mocked remote + and therefore the local path must be absolute + """ + local_dir = tmp_path_local + remote_dir = tmp_path_remote directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) + local_workdir = local_dir / directory + remote_workdir = remote_dir / directory - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + transport.mkdir(str(remote_workdir)) - transport.mkdir(directory) - transport.chdir(directory) - - local_base_dir = os.path.join(local_dir, directory, 'local') - os.mkdir(local_base_dir) + local_base_dir: Path = local_workdir / 'origin' + local_base_dir.mkdir(parents=True) # first test put: I create three files in local - file_1 = os.path.join(local_base_dir, 'a.txt') - file_2 = os.path.join(local_base_dir, 'b.tmp') - file_3 = os.path.join(local_base_dir, 'c.txt') + file_1 = local_base_dir / 'a.txt' + file_2 = local_base_dir / 'b.tmp' + file_3 = local_base_dir / 'c.txt' text = 'Viva Verdi\n' for filename in [file_1, file_2, file_3]: with open(filename, 'w', encoding='utf8') as fhandle: fhandle.write(text) - # first test putransport. Copy of two files matching patterns, into a folder - transport.put(os.path.join(local_base_dir, '*.txt'), '.') - assert set(['a.txt', 'c.txt', 'local']) == set(transport.listdir('.')) - transport.remove('a.txt') - transport.remove('c.txt') - # second. Copy of folder into a non existing folder - transport.put(local_base_dir, 'prova') - assert set(['prova', 'local']) == set(transport.listdir('.')) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir('prova')) - transport.rmtree('prova') - # third. copy of folder into an existing folder - transport.mkdir('prova') - transport.put(local_base_dir, 'prova') - assert set(['prova', 'local']) == set(transport.listdir('.')) - assert set(['local']) == set(transport.listdir('prova')) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(os.path.join('prova', 'local'))) - transport.rmtree('prova') - # third test copy. Can copy one file into a new file - transport.put(os.path.join(local_base_dir, '*.tmp'), 'prova') - assert set(['prova', 'local']) == set(transport.listdir('.')) - transport.remove('prova') - # fourth test copy: can't copy more than one file on the same file, + # first test the put. Copy of two files matching patterns, into a folder + transport.put(str(local_base_dir / '*.txt'), str(remote_workdir)) + assert set(['a.txt', 'c.txt']) == set(transport.listdir(str(remote_workdir))) + transport.remove(str(remote_workdir / 'a.txt')) + transport.remove(str(remote_workdir / 'c.txt')) + + # second test put. Put of two folders + transport.put(str(local_base_dir), str(remote_workdir / 'prova')) + assert set(['prova']) == set(transport.listdir(str(remote_workdir))) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(str(remote_workdir / 'prova'))) + transport.rmtree(str(remote_workdir / 'prova')) + + # third test put. Can copy one file into a new file + transport.put(str(local_base_dir / '*.tmp'), str(remote_workdir / 'prova')) + assert transport.isfile(str(remote_workdir / 'prova')) + transport.remove(str(remote_workdir / 'prova')) + + # fourth test put: can't copy more than one file to the same file, # i.e., the destination should be a folder with pytest.raises(OSError): - transport.put(os.path.join(local_base_dir, '*.txt'), 'prova') - # copy of folder into file - with open(os.path.join(local_dir, directory, 'existing.txt'), 'w', encoding='utf8') as fhandle: + transport.put(str(local_base_dir / '*.txt'), str(remote_workdir / 'prova')) + + # can't copy folder to an exist file + with open(remote_workdir / 'existing.txt', 'w', encoding='utf8') as fhandle: fhandle.write(text) with pytest.raises(OSError): - transport.put(os.path.join(local_base_dir), 'existing.txt') - transport.remove('existing.txt') + transport.put(str(local_base_dir), str(remote_workdir / 'existing.txt')) + transport.remove(str(remote_workdir / 'existing.txt')) + # fifth test, copying one file into a folder - transport.mkdir('prova') - transport.put(os.path.join(local_base_dir, 'a.txt'), 'prova') - assert set(transport.listdir('prova')) == set(['a.txt']) - transport.rmtree('prova') + transport.mkdir(str(remote_workdir / 'prova')) + transport.put(str(local_base_dir / 'a.txt'), str(remote_workdir / 'prova')) + assert set(transport.listdir(str(remote_workdir / 'prova'))) == set(['a.txt']) + transport.rmtree(str(remote_workdir / 'prova')) + # sixth test, copying one file into a file - transport.put(os.path.join(local_base_dir, 'a.txt'), 'prova') - assert transport.isfile('prova') - transport.remove('prova') + transport.put(str(local_base_dir / 'a.txt'), str(remote_workdir / 'prova')) + assert transport.isfile(str(remote_workdir / 'prova')) + transport.remove(str(remote_workdir / 'prova')) + # put of folder into an existing folder + # NOTE: the command cp has a different behavior on Mac vs Ubuntu + # tests performed locally on a Mac may result in a failure. + transport.mkdir(str(remote_workdir / 'prova')) + transport.put(str(local_base_dir), str(remote_workdir / 'prova')) + assert set(['origin']) == set(transport.listdir(str(remote_workdir / 'prova'))) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set(transport.listdir(str(remote_workdir / 'prova' / 'origin'))) + transport.rmtree(str(remote_workdir / 'prova')) # exit - transport.chdir('..') - transport.rmtree(directory) + transport.rmtree(str(remote_workdir)) -def test_get(custom_transport): +def test_get(custom_transport, tmp_path_remote, tmp_path_local): """Test getting files.""" - # exactly the same tests of copy, just with the put function - # and therefore the local path must be absolute - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + local_workdir: Path = local_dir / directory + remote_workdir: Path = remote_dir / directory - transport.mkdir(directory) - transport.chdir(directory) + local_workdir.mkdir() - local_base_dir = os.path.join(local_dir, directory, 'local') - local_destination = os.path.join(local_dir, directory) - os.mkdir(local_base_dir) + remote_base_dir: Path = remote_workdir / 'origin' + remote_base_dir.mkdir(parents=True) - # first test put: I create three files in local - file_1 = os.path.join(local_base_dir, 'a.txt') - file_2 = os.path.join(local_base_dir, 'b.tmp') - file_3 = os.path.join(local_base_dir, 'c.txt') + # first test put: I create three files in remote + file_1 = remote_base_dir / 'a.txt' + file_2 = remote_base_dir / 'b.tmp' + file_3 = remote_base_dir / 'c.txt' text = 'Viva Verdi\n' for filename in [file_1, file_2, file_3]: with open(filename, 'w', encoding='utf8') as fhandle: fhandle.write(text) - # first test put. Copy of two files matching patterns, into a folder - transport.get(os.path.join('local', '*.txt'), local_destination) - assert set(['a.txt', 'c.txt', 'local']) == set(os.listdir(local_destination)) - os.remove(os.path.join(local_destination, 'a.txt')) - os.remove(os.path.join(local_destination, 'c.txt')) + # first test get. Get two files matching patterns, from mocked remote folder into a local folder + transport.get(str(remote_base_dir / '*.txt'), str(local_workdir)) + assert set(['a.txt', 'c.txt']) == set([p.name for p in (local_workdir).iterdir()]) + (local_workdir / 'a.txt').unlink() + (local_workdir / 'c.txt').unlink() + # second. Copy of folder into a non existing folder - transport.get('local', os.path.join(local_destination, 'prova')) - assert set(['prova', 'local']) == set(os.listdir(local_destination)) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(os.listdir(os.path.join(local_destination, 'prova'))) - shutil.rmtree(os.path.join(local_destination, 'prova')) + transport.get(str(remote_base_dir), str(local_workdir / 'prova')) + assert set(['prova']) == set([p.name for p in local_workdir.iterdir()]) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set([p.name for p in (local_workdir / 'prova').iterdir()]) + shutil.rmtree(local_workdir / 'prova') + # third. copy of folder into an existing folder - os.mkdir(os.path.join(local_destination, 'prova')) - transport.get('local', os.path.join(local_destination, 'prova')) - assert set(['prova', 'local']) == set(os.listdir(local_destination)) - assert set(['local']) == set(os.listdir(os.path.join(local_destination, 'prova'))) - assert set(['a.txt', 'b.tmp', 'c.txt']) == set(os.listdir(os.path.join(local_destination, 'prova', 'local'))) - shutil.rmtree(os.path.join(local_destination, 'prova')) - # third test copy. Can copy one file into a new file - transport.get(os.path.join('local', '*.tmp'), os.path.join(local_destination, 'prova')) - assert set(['prova', 'local']) == set(os.listdir(local_destination)) - os.remove(os.path.join(local_destination, 'prova')) + (local_workdir / 'prova').mkdir() + transport.get(str(remote_base_dir), str(local_workdir / 'prova')) + assert set(['prova']) == set([p.name for p in local_workdir.iterdir()]) + assert set(['origin']) == set([p.name for p in (local_workdir / 'prova').iterdir()]) + assert set(['a.txt', 'b.tmp', 'c.txt']) == set([p.name for p in (local_workdir / 'prova' / 'origin').iterdir()]) + shutil.rmtree(local_workdir / 'prova') + + # test get one file into a new file prova + transport.get(str(remote_base_dir / '*.tmp'), str(local_workdir / 'prova')) + assert set(['prova']) == set([p.name for p in local_workdir.iterdir()]) + assert (local_workdir / 'prova').is_file() + (local_workdir / 'prova').unlink() + # fourth test copy: can't copy more than one file on the same file, # i.e., the destination should be a folder with pytest.raises(OSError): - transport.get(os.path.join('local', '*.txt'), os.path.join(local_destination, 'prova')) + transport.get(str(remote_base_dir / '*.txt'), str(local_workdir / 'prova')) # copy of folder into file - with open(os.path.join(local_destination, 'existing.txt'), 'w', encoding='utf8') as fhandle: + with open(local_workdir / 'existing.txt', 'w', encoding='utf8') as fhandle: fhandle.write(text) with pytest.raises(OSError): - transport.get('local', os.path.join(local_destination, 'existing.txt')) - os.remove(os.path.join(local_destination, 'existing.txt')) + transport.get(str(remote_base_dir), str(local_workdir / 'existing.txt')) + (local_workdir / 'existing.txt').unlink() + # fifth test, copying one file into a folder - os.mkdir(os.path.join(local_destination, 'prova')) - transport.get(os.path.join('local', 'a.txt'), os.path.join(local_destination, 'prova')) - assert set(os.listdir(os.path.join(local_destination, 'prova'))) == set(['a.txt']) - shutil.rmtree(os.path.join(local_destination, 'prova')) - # sixth test, copying one file into a file - transport.get(os.path.join('local', 'a.txt'), os.path.join(local_destination, 'prova')) - assert os.path.isfile(os.path.join(local_destination, 'prova')) - os.remove(os.path.join(local_destination, 'prova')) + (local_workdir / 'prova').mkdir() + transport.get(str(remote_base_dir / 'a.txt'), str(local_workdir / 'prova')) + assert set(['a.txt']) == set([p.name for p in (local_workdir / 'prova').iterdir()]) + shutil.rmtree(local_workdir / 'prova') - # exit - transport.chdir('..') - transport.rmtree(directory) + # sixth test, copying one file into a file + transport.get(str(remote_base_dir / 'a.txt'), str(local_workdir / 'prova')) + assert (local_workdir / 'prova').is_file() + (local_workdir / 'prova').unlink() -def test_put_get_abs_path_tree(custom_transport): +def test_put_get_abs_path_tree(custom_transport, tmp_path_remote, tmp_path_local): """Test of exception for non existing files and abs path""" - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) + local_subfolder = str(local_dir / directory / 'tmp1') + remote_subfolder = str(remote_dir / 'tmp2') + retrieved_subfolder = str(local_dir / directory / 'tmp3') - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + (local_dir / directory / local_subfolder).mkdir(parents=True) - local_subfolder = os.path.join(local_dir, directory, 'tmp1') - remote_subfolder = 'tmp2' - retrieved_subfolder = os.path.join(local_dir, directory, 'tmp3') + local_file_name = Path(local_subfolder) / 'file.txt' - os.mkdir(os.path.join(local_dir, directory)) - os.mkdir(os.path.join(local_dir, directory, local_subfolder)) - - transport.chdir(directory) - local_file_name = os.path.join(local_subfolder, 'file.txt') - pathlib.Path(local_file_name).touch() + text = 'Viva Verdi\n' + with open(local_file_name, 'w', encoding='utf8') as fhandle: + fhandle.write(text) + # here use full path in src and dst # 'tmp1' is not an abs path with pytest.raises(ValueError): transport.put('tmp1', remote_subfolder) @@ -989,40 +892,24 @@ def test_put_get_abs_path_tree(custom_transport): with pytest.raises(ValueError): transport.gettree(remote_subfolder, 'delete_me_tree') - os.remove(os.path.join(local_subfolder, 'file.txt')) - os.rmdir(local_subfolder) - transport.rmtree(remote_subfolder) - transport.chdir('..') - transport.rmdir(directory) - - -def test_put_get_empty_string_tree(custom_transport): +def test_put_get_empty_string_tree(custom_transport, tmp_path_remote, tmp_path_local): """Test of exception put/get of empty strings""" - # TODO : verify the correctness of \n at the end of a file - local_dir = os.path.join('/', 'tmp') - remote_dir = local_dir + local_dir = tmp_path_local + remote_dir = tmp_path_remote directory = 'tmp_try' with custom_transport as transport: - transport.chdir(remote_dir) - - while os.path.exists(os.path.join(local_dir, directory)): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) + local_subfolder: Path = local_dir / directory / 'tmp1' + remote_subfolder: Path = remote_dir / 'tmp2' + retrieved_subfolder: Path = local_dir / directory / 'tmp3' - local_subfolder = os.path.join(local_dir, directory, 'tmp1') - remote_subfolder = 'tmp2' - retrieved_subfolder = os.path.join(local_dir, directory, 'tmp3') + local_subfolder.mkdir(parents=True) - os.mkdir(os.path.join(local_dir, directory)) - os.mkdir(os.path.join(local_dir, directory, local_subfolder)) - - transport.chdir(directory) - local_file_name = os.path.join(local_subfolder, 'file.txt') + local_file = local_subfolder / 'file.txt' text = 'Viva Verdi\n' - with open(local_file_name, 'w', encoding='utf8') as fhandle: + with open(local_file, 'w', encoding='utf8') as fhandle: fhandle.write(text) # localpath is an empty string @@ -1034,7 +921,7 @@ def test_put_get_empty_string_tree(custom_transport): with pytest.raises(OSError): transport.puttree(local_subfolder, '') - transport.puttree(local_subfolder, remote_subfolder) + transport.puttree(str(local_subfolder), str(remote_subfolder)) # remote path is an empty string with pytest.raises(OSError): @@ -1047,73 +934,50 @@ def test_put_get_empty_string_tree(custom_transport): # TODO : get doesn't retrieve empty files. # Is it what we want? - transport.gettree(remote_subfolder, retrieved_subfolder) - - os.remove(os.path.join(local_subfolder, 'file.txt')) - os.rmdir(local_subfolder) - transport.remove(os.path.join(remote_subfolder, 'file.txt')) - transport.rmdir(remote_subfolder) - # If it couldn't end the copy, it leaves what he did on local file - # here I am mixing local with remote - assert 'file.txt' in transport.listdir('tmp3') - os.remove(os.path.join(retrieved_subfolder, 'file.txt')) - os.rmdir(retrieved_subfolder) + transport.gettree(str(remote_subfolder), str(retrieved_subfolder)) - transport.chdir('..') - transport.rmdir(directory) + assert 'file.txt' in [p.name for p in retrieved_subfolder.iterdir()] -def test_gettree_nested_directory(custom_transport): +def test_gettree_nested_directory(custom_transport, tmp_path_remote, tmp_path_local): """Test `gettree` for a nested directory.""" - with tempfile.TemporaryDirectory() as dir_remote, tempfile.TemporaryDirectory() as dir_local: - content = b'dummy\ncontent' - filepath = os.path.join(dir_remote, 'sub', 'path', 'filename.txt') - os.makedirs(os.path.dirname(filepath)) + content = b'dummy\ncontent' + dir_path = tmp_path_remote / 'sub' / 'path' + dir_path.mkdir(parents=True) + + file_path = str(dir_path / 'filename.txt') + + with open(file_path, 'wb') as handle: + handle.write(content) - with open(filepath, 'wb') as handle: - handle.write(content) + with custom_transport as transport: + transport.gettree(str(tmp_path_remote), str(tmp_path_local)) - with custom_transport as transport: - transport.gettree(os.path.join(dir_remote, 'sub/path'), os.path.join(dir_local, 'sub/path')) + assert (tmp_path_local / 'sub' / 'path' / 'filename.txt').is_file -def test_exec_pwd(custom_transport): +def test_exec_pwd(custom_transport, tmp_path_remote): """I create a strange subfolder with a complicated name and - then see if I can run pwd. This also checks the correct + then see if I can run ``ls``. This also checks the correct escaping of funny characters, both in the directory creation (which should be done by paramiko) and in the command execution (done in this module, in the _exec_command_internal function). """ # Start value - delete_at_end = False - with custom_transport as transport: # To compare with: getcwd uses the normalized ('realpath') path - location = transport.normalize('/tmp') subfolder = """_'s f"#""" # A folder with characters to escape - subfolder_fullpath = os.path.join(location, subfolder) + subfolder_fullpath = str(tmp_path_remote / subfolder) - transport.chdir(location) - if not transport.isdir(subfolder): - # Since I created the folder, I will remember to - # delete it at the end of this test - delete_at_end = True - transport.mkdir(subfolder) + transport.mkdir(subfolder_fullpath) - assert transport.isdir(subfolder) - transport.chdir(subfolder) + assert transport.isdir(subfolder_fullpath) - assert subfolder_fullpath == transport.getcwd() - retcode, stdout, stderr = transport.exec_command_wait('pwd') + retcode, stdout, stderr = transport.exec_command_wait(f'ls {tmp_path_remote!s}') assert retcode == 0 - # I have to strip it because 'pwd' returns a trailing \n - assert stdout.strip() == subfolder_fullpath + assert stdout.strip() in subfolder_fullpath assert stderr == '' - if delete_at_end: - transport.chdir(location) - transport.rmdir(subfolder) - def test_exec_with_stdin_string(custom_transport): """Test command execution with a stdin string.""" @@ -1194,7 +1058,7 @@ def test_exec_with_wrong_stdin(custom_transport): transport.exec_command_wait('cat', stdin=1) -def test_transfer_big_stdout(custom_transport): +def test_transfer_big_stdout(custom_transport, tmp_path_remote): """Test the transfer of a large amount of data on stdout.""" # Create a "big" file of > 2MB (10MB here; in general, larger than the buffer size) min_file_size_bytes = 5 * 1024 * 1024 @@ -1209,25 +1073,18 @@ def test_transfer_big_stdout(custom_transport): line_repetitions = min_file_size_bytes // len(file_line_binary) + 1 fcontent = (file_line_binary * line_repetitions).decode('utf8') - with custom_transport as trans: + with custom_transport as transport: # We cannot use tempfile.mkdtemp because we're on a remote folder - location = trans.normalize(os.path.join('/', 'tmp')) - trans.chdir(location) - assert location == trans.getcwd() - - directory = 'temp_dir_test_transfer_big_stdout' - while trans.isdir(directory): - # I append a random letter/number until it is unique - directory += random.choice(string.ascii_uppercase + string.digits) - trans.mkdir(directory) - trans.chdir(directory) + directory_name = 'temp_dir_test_transfer_big_stdout' + directory_path = tmp_path_remote / directory_name + transport.mkdir(str(directory_path)) with tempfile.NamedTemporaryFile(mode='wb') as tmpf: tmpf.write(fcontent.encode('utf8')) tmpf.flush() # I put a file with specific content there at the right file name - trans.putfile(tmpf.name, fname) + transport.putfile(tmpf.name, str(directory_path / fname)) python_code = r"""import sys @@ -1251,16 +1108,20 @@ def test_transfer_big_stdout(custom_transport): tmpf.flush() # I put a file with specific content there at the right file name - trans.putfile(tmpf.name, script_fname) + transport.putfile(tmpf.name, str(directory_path / script_fname)) # I get its content via the stdout; emulate also network slowness (note I cat twice) - retcode, stdout, stderr = trans.exec_command_wait(f'cat {fname} ; sleep 1 ; cat {fname}') + retcode, stdout, stderr = transport.exec_command_wait( + f'cat {fname} ; sleep 1 ; cat {fname}', workdir=str(directory_path) + ) assert stderr == '' assert stdout == fcontent + fcontent assert retcode == 0 # I get its content via the stderr; emulate also network slowness (note I cat twice) - retcode, stdout, stderr = trans.exec_command_wait(f'cat {fname} >&2 ; sleep 1 ; cat {fname} >&2') + retcode, stdout, stderr = transport.exec_command_wait( + f'cat {fname} >&2 ; sleep 1 ; cat {fname} >&2', workdir=str(directory_path) + ) assert stderr == fcontent + fcontent assert stdout == '' assert retcode == 0 @@ -1273,20 +1134,14 @@ def test_transfer_big_stdout(custom_transport): # line_repetitions, file_line, file_line)) # However this is pretty slow (and using 'cat' of a file containing only one line is even slower) - retcode, stdout, stderr = trans.exec_command_wait(f'python3 {script_fname}') + retcode, stdout, stderr = transport.exec_command_wait(f'python3 {script_fname}', workdir=str(directory_path)) assert stderr == fcontent assert stdout == fcontent assert retcode == 0 - # Clean-up - trans.remove(fname) - trans.remove(script_fname) - trans.chdir('..') - trans.rmdir(directory) - -def test_asynchronous_execution(custom_transport): +def test_asynchronous_execution(custom_transport, tmp_path): """Test that the execution of a long(ish) command via the direct scheduler does not block. This is a regression test for #3094, where running a long job on the direct scheduler @@ -1294,6 +1149,8 @@ def test_asynchronous_execution(custom_transport): """ # Use a unique name, using a UUID, to avoid concurrent tests (or very rapid # tests that follow each other) to overwrite the same destination + import os + script_fname = f'sleep-submit-{uuid.uuid4().hex}-{custom_transport.__class__.__name__}.sh' scheduler = SchedulerFactory('core.direct')() @@ -1305,10 +1162,10 @@ def test_asynchronous_execution(custom_transport): tmpf.write(b'#!/bin/bash\nsleep 10\n') tmpf.flush() - transport.putfile(tmpf.name, os.path.join('/tmp', script_fname)) + transport.putfile(tmpf.name, str(tmp_path / script_fname)) timestamp_before = time.time() - job_id_string = scheduler.submit_job('/tmp', script_fname) + job_id_string = scheduler.submit_job(str(tmp_path), script_fname) elapsed_time = time.time() - timestamp_before # We want to get back control. If it takes < 5 seconds, it means that it is not blocking @@ -1343,10 +1200,3 @@ def test_asynchronous_execution(custom_transport): except ProcessLookupError: # If the process is already dead (or has never run), I just ignore the error pass - - # Also remove the script - try: - transport.remove(f'/tmp/{script_fname}') - except FileNotFoundError: - # If the file wasn't even created, I just ignore this error - pass From c915a97348ffb344ab48fe031711120238f6976a Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 30 Nov 2024 22:23:00 +0000 Subject: [PATCH 2/5] Bump ruff version (#6614) Most of the changes here are minor import sorting changes (I checked all of them and they all look more correct than before). The only other change were couple type-comparison (E721) fixes --- .github/system_tests/test_daemon.py | 20 +++++++++---------- .molecule/default/files/polish/cli.py | 1 + .pre-commit-config.yaml | 8 +------- .../scripts/performance_benchmark_base.py | 1 + .../source/internals/includes/snippets/api.py | 3 ++- .../serialize/run_workchain_serialize.py | 3 ++- .../snippets/expose_inputs/complex_parent.py | 3 ++- .../snippets/expose_inputs/run_complex.py | 3 ++- .../snippets/expose_inputs/run_simple.py | 3 ++- .../snippets/expose_inputs/simple_parent.py | 3 ++- pyproject.toml | 4 ++++ src/aiida/cmdline/groups/dynamic.py | 2 +- .../engine/processes/workchains/workchain.py | 2 +- .../main_0002_recompute_hash_calc_job_node.py | 6 ++++-- tests/benchmark/test_archive.py | 1 + tests/benchmark/test_engine.py | 1 + tests/benchmark/test_nodes.py | 1 + tests/brokers/test_rabbitmq.py | 5 +++-- tests/calculations/arithmetic/test_add.py | 1 + tests/calculations/test_templatereplacer.py | 1 + tests/calculations/test_transfer.py | 1 + tests/cmdline/commands/test_archive_create.py | 2 +- tests/cmdline/commands/test_archive_import.py | 2 +- tests/cmdline/commands/test_calcjob.py | 4 ++-- tests/cmdline/commands/test_code.py | 1 + tests/cmdline/commands/test_computer.py | 1 + tests/cmdline/commands/test_config.py | 1 + tests/cmdline/commands/test_daemon.py | 1 + tests/cmdline/commands/test_data.py | 2 +- tests/cmdline/commands/test_devel.py | 1 + tests/cmdline/commands/test_group.py | 1 + tests/cmdline/commands/test_group_ls.py | 3 ++- tests/cmdline/commands/test_node.py | 1 + tests/cmdline/commands/test_plugin.py | 1 + tests/cmdline/commands/test_presto.py | 1 + tests/cmdline/commands/test_process.py | 2 +- tests/cmdline/commands/test_profile.py | 3 ++- tests/cmdline/commands/test_rabbitmq.py | 3 ++- tests/cmdline/commands/test_run.py | 1 + tests/cmdline/commands/test_setup.py | 3 ++- tests/cmdline/commands/test_status.py | 1 + tests/cmdline/commands/test_storage.py | 1 + tests/cmdline/commands/test_user.py | 1 + tests/cmdline/commands/test_verdi.py | 1 + tests/cmdline/groups/test_dynamic.py | 3 ++- tests/cmdline/params/options/test_callable.py | 3 ++- .../params/options/test_conditional.py | 1 + tests/cmdline/params/options/test_config.py | 1 + .../params/options/test_interactive.py | 1 + .../cmdline/params/options/test_verbosity.py | 1 + .../cmdline/params/types/test_calculation.py | 1 + tests/cmdline/params/types/test_code.py | 1 + tests/cmdline/params/types/test_computer.py | 1 + tests/cmdline/params/types/test_data.py | 1 + tests/cmdline/params/types/test_group.py | 1 + tests/cmdline/params/types/test_identifier.py | 1 + tests/cmdline/params/types/test_node.py | 1 + tests/cmdline/params/types/test_path.py | 1 + tests/cmdline/params/types/test_plugin.py | 1 + tests/cmdline/utils/test_common.py | 1 + tests/cmdline/utils/test_decorators.py | 1 + tests/cmdline/utils/test_multiline.py | 1 + tests/cmdline/utils/test_repository.py | 1 + tests/common/test_escaping.py | 1 + tests/common/test_folders.py | 1 + tests/common/test_hashing.py | 1 + tests/common/test_links.py | 1 + tests/common/test_timezone.py | 1 + tests/conftest.py | 4 +++- tests/engine/daemon/test_client.py | 1 + tests/engine/daemon/test_execmanager.py | 1 + tests/engine/daemon/test_worker.py | 1 + .../processes/calcjobs/test_calc_job.py | 1 + .../processes/calcjobs/test_monitors.py | 1 + tests/engine/processes/test_builder.py | 3 ++- tests/engine/processes/test_control.py | 4 ++-- tests/engine/processes/test_exit_code.py | 1 + .../processes/workchains/test_restart.py | 1 + .../engine/processes/workchains/test_utils.py | 9 +++++---- tests/engine/test_calcfunctions.py | 1 + tests/engine/test_class_loader.py | 3 ++- tests/engine/test_daemon.py | 2 +- tests/engine/test_futures.py | 2 +- tests/engine/test_launch.py | 1 + tests/engine/test_manager.py | 1 + tests/engine/test_memory_leaks.py | 2 +- tests/engine/test_persistence.py | 2 +- tests/engine/test_ports.py | 1 + tests/engine/test_process.py | 4 ++-- tests/engine/test_process_function.py | 1 + tests/engine/test_process_spec.py | 1 + tests/engine/test_rmq.py | 2 +- tests/engine/test_run.py | 2 +- tests/engine/test_runners.py | 1 + tests/engine/test_transport.py | 1 + tests/engine/test_utils.py | 1 + tests/engine/test_work_chain.py | 1 + tests/engine/test_workfunctions.py | 1 + .../migrations/test_migrations.py | 1 + tests/manage/configuration/test_config.py | 1 + .../configuration/test_configuration.py | 3 ++- tests/manage/configuration/test_options.py | 1 + tests/manage/external/test_postgres.py | 1 + tests/manage/test_caching_config.py | 1 + tests/manage/test_manager.py | 1 + tests/manage/test_profile_access.py | 1 + tests/manage/tests/test_pytest_fixtures.py | 1 + tests/orm/data/code/test_abstract.py | 1 + tests/orm/data/code/test_containerized.py | 1 + tests/orm/data/code/test_installed.py | 1 + tests/orm/data/code/test_portable.py | 1 + tests/orm/data/test_code.py | 1 + tests/orm/data/test_enum.py | 1 + tests/orm/implementation/test_backend.py | 1 + tests/orm/implementation/test_comments.py | 1 + tests/orm/implementation/test_logs.py | 1 + tests/orm/implementation/test_nodes.py | 1 + tests/orm/implementation/test_utils.py | 1 + tests/orm/nodes/data/test_array.py | 1 + tests/orm/nodes/data/test_array_bands.py | 1 + tests/orm/nodes/data/test_base.py | 3 ++- tests/orm/nodes/data/test_cif.py | 1 + tests/orm/nodes/data/test_data.py | 2 +- tests/orm/nodes/data/test_dict.py | 1 + tests/orm/nodes/data/test_folder.py | 1 + tests/orm/nodes/data/test_jsonable.py | 3 ++- tests/orm/nodes/data/test_kpoints.py | 1 + tests/orm/nodes/data/test_list.py | 1 + tests/orm/nodes/data/test_orbital.py | 1 + tests/orm/nodes/data/test_remote.py | 1 + tests/orm/nodes/data/test_remote_stash.py | 1 + tests/orm/nodes/data/test_singlefile.py | 1 + tests/orm/nodes/data/test_structure.py | 1 + tests/orm/nodes/data/test_to_aiida_type.py | 1 + tests/orm/nodes/data/test_trajectory.py | 1 + tests/orm/nodes/data/test_upf.py | 4 ++-- tests/orm/nodes/data/test_xy.py | 1 + tests/orm/nodes/process/test_process.py | 1 + tests/orm/nodes/test_calcjob.py | 1 + tests/orm/nodes/test_node.py | 1 + tests/orm/nodes/test_repository.py | 1 + tests/orm/test_authinfos.py | 1 + tests/orm/test_autogroups.py | 1 + tests/orm/test_comments.py | 1 + tests/orm/test_computers.py | 1 + tests/orm/test_entities.py | 1 + tests/orm/test_fields.py | 3 ++- tests/orm/test_groups.py | 1 + tests/orm/test_logs.py | 1 + tests/orm/test_mixins.py | 1 + tests/orm/test_querybuilder.py | 1 + tests/orm/utils/test_calcjob.py | 1 + tests/orm/utils/test_loaders.py | 1 + tests/orm/utils/test_managers.py | 1 + tests/orm/utils/test_node.py | 1 + tests/orm/utils/test_serialize.py | 1 + tests/parsers/test_parser.py | 1 + tests/plugins/test_entry_point.py | 5 +++-- tests/plugins/test_factories.py | 1 + tests/plugins/test_utils.py | 1 + tests/repository/backend/test_abstract.py | 1 + .../backend/test_disk_object_store.py | 1 + tests/repository/backend/test_sandbox.py | 1 + tests/repository/test_common.py | 1 + tests/repository/test_repository.py | 1 + tests/restapi/conftest.py | 3 ++- tests/restapi/test_config.py | 1 + tests/restapi/test_identifiers.py | 1 + tests/restapi/test_routes.py | 3 ++- tests/schedulers/test_all.py | 1 + tests/schedulers/test_datastructures.py | 1 + tests/schedulers/test_direct.py | 4 +++- tests/schedulers/test_lsf.py | 1 + tests/schedulers/test_slurm.py | 1 + tests/storage/psql_dos/migrations/conftest.py | 5 +++-- .../django_branch/test_0026_0027_traj_data.py | 1 + ...st_0037_attributes_extras_settings_json.py | 3 ++- .../migrations/django_branch/test_legacy.py | 1 + .../sqlalchemy_branch/test_4_dblog_update.py | 3 ++- .../test_6_trajectory_data.py | 1 + .../psql_dos/migrations/test_all_schema.py | 1 + tests/storage/psql_dos/test_alembic_cli.py | 3 ++- tests/storage/psql_dos/test_backend.py | 1 + tests/storage/psql_dos/test_nodes.py | 1 + tests/storage/psql_dos/test_query.py | 1 + tests/storage/psql_dos/test_schema.py | 3 ++- tests/storage/psql_dos/test_session.py | 3 ++- tests/storage/sqlite/test_orm.py | 1 + .../storage/sqlite_dos/migrations/conftest.py | 3 ++- .../sqlite_dos/migrations/test_all_schema.py | 1 + tests/storage/sqlite_dos/test_backend.py | 2 +- tests/storage/sqlite_zip/test_backend.py | 3 ++- tests/test_calculation_node.py | 1 + tests/test_conftest.py | 3 ++- tests/test_dataclasses.py | 1 + tests/test_generic.py | 1 + tests/test_nodes.py | 1 + tests/tools/archive/conftest.py | 1 + tests/tools/archive/migration/conftest.py | 2 +- .../archive/migration/test_legacy_funcs.py | 2 +- .../migration/test_legacy_migrations.py | 2 +- .../archive/migration/test_legacy_to_main.py | 2 +- .../archive/migration/test_prov_redesign.py | 2 +- .../archive/migration/test_v05_to_v06.py | 1 - .../archive/migration/test_v06_to_v07.py | 1 + tests/tools/archive/orm/test_authinfo.py | 1 + tests/tools/archive/orm/test_calculations.py | 1 + tests/tools/archive/orm/test_codes.py | 1 - tests/tools/archive/orm/test_comments.py | 1 + tests/tools/archive/orm/test_computers.py | 2 +- tests/tools/archive/orm/test_extras.py | 1 + tests/tools/archive/orm/test_groups.py | 1 + tests/tools/archive/orm/test_links.py | 1 - tests/tools/archive/test_abstract.py | 1 + tests/tools/archive/test_backend.py | 2 +- tests/tools/archive/test_complex.py | 1 + tests/tools/archive/test_schema.py | 8 ++++---- tests/tools/archive/test_simple.py | 3 ++- tests/tools/archive/test_specific_import.py | 1 + tests/tools/archive/test_utils.py | 3 ++- tests/tools/data/orbital/test_orbitals.py | 1 + tests/tools/dbimporters/test_icsd.py | 1 + .../dbimporters/test_materialsproject.py | 1 + tests/tools/dumping/test_processes.py | 2 ++ tests/tools/graph/test_age.py | 1 + tests/tools/graph/test_graph_traversers.py | 1 + tests/tools/groups/test_paths.py | 1 + tests/tools/ipython/test_ipython_magics.py | 3 ++- tests/tools/visualization/test_graph.py | 1 + tests/transports/test_all_plugins.py | 1 + tests/transports/test_local.py | 1 + tests/transports/test_ssh.py | 1 + tests/utils/processes.py | 1 + .../workflows/arithmetic/test_add_multiply.py | 1 + utils/validate_consistency.py | 3 ++- 235 files changed, 301 insertions(+), 99 deletions(-) diff --git a/.github/system_tests/test_daemon.py b/.github/system_tests/test_daemon.py index 7f652f8d69..c49cb1f750 100644 --- a/.github/system_tests/test_daemon.py +++ b/.github/system_tests/test_daemon.py @@ -16,16 +16,6 @@ import tempfile import time -from aiida.common import StashMode, exceptions -from aiida.engine import run, submit -from aiida.engine.daemon.client import get_daemon_client -from aiida.engine.persistence import ObjectLoader -from aiida.engine.processes import CalcJob, Process -from aiida.manage.caching import enable_caching -from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node -from aiida.orm.nodes.caching import NodeCaching -from aiida.plugins import CalculationFactory, WorkflowFactory -from aiida.workflows.arithmetic.add_multiply import add, add_multiply from workchains import ( ArithmeticAddBaseWorkChain, CalcFunctionRunnerWorkChain, @@ -39,6 +29,16 @@ WorkFunctionRunnerWorkChain, ) +from aiida.common import StashMode, exceptions +from aiida.engine import run, submit +from aiida.engine.daemon.client import get_daemon_client +from aiida.engine.persistence import ObjectLoader +from aiida.engine.processes import CalcJob, Process +from aiida.manage.caching import enable_caching +from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node +from aiida.orm.nodes.caching import NodeCaching +from aiida.plugins import CalculationFactory, WorkflowFactory +from aiida.workflows.arithmetic.add_multiply import add, add_multiply from tests.utils.memory import get_instances CODENAME_ADD = 'add@localhost' diff --git a/.molecule/default/files/polish/cli.py b/.molecule/default/files/polish/cli.py index e162fc97d8..4ef0485b00 100755 --- a/.molecule/default/files/polish/cli.py +++ b/.molecule/default/files/polish/cli.py @@ -14,6 +14,7 @@ import time import click + from aiida.cmdline.params import options, types from aiida.cmdline.utils import decorators diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a88a2ab99..307403bdac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,16 +37,10 @@ repos: args: [--line-length=120, --fail-on-change] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.7.3 hooks: - id: ruff-format - exclude: &exclude_ruff > - (?x)^( - docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.py| - docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.py| - )$ - id: ruff - exclude: *exclude_ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks diff --git a/docs/source/howto/include/scripts/performance_benchmark_base.py b/docs/source/howto/include/scripts/performance_benchmark_base.py index 7fda09f54b..ad59f6b370 100755 --- a/docs/source/howto/include/scripts/performance_benchmark_base.py +++ b/docs/source/howto/include/scripts/performance_benchmark_base.py @@ -2,6 +2,7 @@ """Script to benchmark the performance of the AiiDA workflow engine on a given installation.""" import click + from aiida.cmdline.params import options from aiida.cmdline.utils import decorators, echo diff --git a/docs/source/internals/includes/snippets/api.py b/docs/source/internals/includes/snippets/api.py index 3a034508c4..5de6967751 100755 --- a/docs/source/internals/includes/snippets/api.py +++ b/docs/source/internals/includes/snippets/api.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import click +from flask_restful import Resource + from aiida import load_profile from aiida.restapi import common from aiida.restapi.api import AiidaApi, App from aiida.restapi.run_api import run_api -from flask_restful import Resource class NewResource(Resource): diff --git a/docs/source/topics/processes/include/snippets/serialize/run_workchain_serialize.py b/docs/source/topics/processes/include/snippets/serialize/run_workchain_serialize.py index e16f9e03fe..3ffb35770c 100644 --- a/docs/source/topics/processes/include/snippets/serialize/run_workchain_serialize.py +++ b/docs/source/topics/processes/include/snippets/serialize/run_workchain_serialize.py @@ -1,7 +1,8 @@ #!/usr/bin/env runaiida -from aiida.engine import run from serialize_workchain import SerializeWorkChain +from aiida.engine import run + if __name__ == '__main__': print(run(SerializeWorkChain, a=1, b=1.2, c=True)) # Result: {'a': 1, 'b': 1.2, 'c': True} diff --git a/docs/source/topics/workflows/include/snippets/expose_inputs/complex_parent.py b/docs/source/topics/workflows/include/snippets/expose_inputs/complex_parent.py index 887d2c702b..4c0e150f50 100644 --- a/docs/source/topics/workflows/include/snippets/expose_inputs/complex_parent.py +++ b/docs/source/topics/workflows/include/snippets/expose_inputs/complex_parent.py @@ -1,6 +1,7 @@ -from aiida.engine import ToContext, WorkChain from child import ChildWorkChain +from aiida.engine import ToContext, WorkChain + class ComplexParentWorkChain(WorkChain): @classmethod diff --git a/docs/source/topics/workflows/include/snippets/expose_inputs/run_complex.py b/docs/source/topics/workflows/include/snippets/expose_inputs/run_complex.py index 2e5fa33cba..ace2e52c61 100755 --- a/docs/source/topics/workflows/include/snippets/expose_inputs/run_complex.py +++ b/docs/source/topics/workflows/include/snippets/expose_inputs/run_complex.py @@ -1,8 +1,9 @@ #!/usr/bin/env runaiida +from complex_parent import ComplexParentWorkChain + from aiida.engine import run from aiida.orm import Bool, Float, Int -from complex_parent import ComplexParentWorkChain if __name__ == '__main__': result = run( diff --git a/docs/source/topics/workflows/include/snippets/expose_inputs/run_simple.py b/docs/source/topics/workflows/include/snippets/expose_inputs/run_simple.py index 638b2aa574..81b028a681 100755 --- a/docs/source/topics/workflows/include/snippets/expose_inputs/run_simple.py +++ b/docs/source/topics/workflows/include/snippets/expose_inputs/run_simple.py @@ -1,8 +1,9 @@ #!/usr/bin/env runaiida +from simple_parent import SimpleParentWorkChain + from aiida.engine import run from aiida.orm import Bool, Float, Int -from simple_parent import SimpleParentWorkChain if __name__ == '__main__': result = run(SimpleParentWorkChain, a=Int(1), b=Float(1.2), c=Bool(True)) diff --git a/docs/source/topics/workflows/include/snippets/expose_inputs/simple_parent.py b/docs/source/topics/workflows/include/snippets/expose_inputs/simple_parent.py index 7eadb8c406..70c054e522 100644 --- a/docs/source/topics/workflows/include/snippets/expose_inputs/simple_parent.py +++ b/docs/source/topics/workflows/include/snippets/expose_inputs/simple_parent.py @@ -1,6 +1,7 @@ -from aiida.engine import ToContext, WorkChain from child import ChildWorkChain +from aiida.engine import ToContext, WorkChain + class SimpleParentWorkChain(WorkChain): @classmethod diff --git a/pyproject.toml b/pyproject.toml index 7a884cc1f8..35f2f23b7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -388,6 +388,10 @@ testpaths = [ xfail_strict = true [tool.ruff] +exclude = [ + 'docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.py', + 'docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.py' +] line-length = 120 [tool.ruff.format] diff --git a/src/aiida/cmdline/groups/dynamic.py b/src/aiida/cmdline/groups/dynamic.py index f87787460b..4a1e0eb600 100644 --- a/src/aiida/cmdline/groups/dynamic.py +++ b/src/aiida/cmdline/groups/dynamic.py @@ -177,7 +177,7 @@ def list_options(self, entry_point: str) -> list: # ``typing.Union[str, None].__args__`` will return the tuple ``(str, NoneType)``. So to get the real type, # we simply remove all ``NoneType`` and the remaining type should be the type of the option. if hasattr(field_info.annotation, '__args__'): - args = list(filter(lambda e: e != type(None), field_info.annotation.__args__)) + args = list(filter(lambda e: e is not type(None), field_info.annotation.__args__)) # Click parameters only support specifying a single type, so we default to the first one even if the # pydantic model defines multiple. field_type = args[0] diff --git a/src/aiida/engine/processes/workchains/workchain.py b/src/aiida/engine/processes/workchains/workchain.py index 1107204ab0..6a82a703e0 100644 --- a/src/aiida/engine/processes/workchains/workchain.py +++ b/src/aiida/engine/processes/workchains/workchain.py @@ -208,7 +208,7 @@ def _resolve_nested_context(self, key: str) -> tuple[AttributeDict, str]: # (subclasses of AttributeDict) but after resolution of an Awaitable this will be the value itself # * assumption: a resolved value is never a plain AttributeDict, on the other hand if a resolved Awaitable # would be an AttributeDict we can append things to it since the order of tasks is maintained. - if type(ctx) != AttributeDict: + if type(ctx) is not AttributeDict: raise ValueError( f'Can not update the context for key `{key}`:' f' found instance of `{type(ctx)}` at `{".".join(ctx_path[:index+1])}`, expected AttributeDict' diff --git a/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py b/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py index ae70c45c4c..fae7cb056e 100644 --- a/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py +++ b/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py @@ -18,9 +18,10 @@ from __future__ import annotations -from aiida.common.log import AIIDA_LOGGER from alembic import op +from aiida.common.log import AIIDA_LOGGER + LOGGER = AIIDA_LOGGER.getChild(__file__) revision = 'main_0002' @@ -39,9 +40,10 @@ def drop_hashes(conn, hash_extra_key: str, entry_point_string: str | None = None value should be a complete entry point string, e.g., ``aiida.node:process.calculation.calcjob`` to drop the hash of all ``CalcJobNode`` rows. """ + from sqlalchemy.sql import text + from aiida.orm.utils.node import get_type_string_from_class from aiida.plugins import load_entry_point_from_string - from sqlalchemy.sql import text if entry_point_string is not None: entry_point = load_entry_point_from_string(entry_point_string) diff --git a/tests/benchmark/test_archive.py b/tests/benchmark/test_archive.py index 2a01818ac1..4b1a091183 100644 --- a/tests/benchmark/test_archive.py +++ b/tests/benchmark/test_archive.py @@ -15,6 +15,7 @@ from io import StringIO import pytest + from aiida.common.links import LinkType from aiida.engine import ProcessState from aiida.orm import CalcFunctionNode, Dict, load_node diff --git a/tests/benchmark/test_engine.py b/tests/benchmark/test_engine.py index dad34848a7..e0b7f1db4a 100644 --- a/tests/benchmark/test_engine.py +++ b/tests/benchmark/test_engine.py @@ -13,6 +13,7 @@ """ import pytest + from aiida.engine import WorkChain, run_get_node, while_ from aiida.orm import InstalledCode, Int from aiida.plugins.factories import CalculationFactory diff --git a/tests/benchmark/test_nodes.py b/tests/benchmark/test_nodes.py index 133ddae1f0..cb619b6f8e 100644 --- a/tests/benchmark/test_nodes.py +++ b/tests/benchmark/test_nodes.py @@ -15,6 +15,7 @@ from io import StringIO import pytest + from aiida.common import NotExistent from aiida.orm import Data, load_node diff --git a/tests/brokers/test_rabbitmq.py b/tests/brokers/test_rabbitmq.py index 00ee662338..2417d27748 100644 --- a/tests/brokers/test_rabbitmq.py +++ b/tests/brokers/test_rabbitmq.py @@ -13,11 +13,12 @@ import pytest import requests +from kiwipy.rmq import RmqThreadCommunicator +from packaging.version import parse + from aiida.brokers.rabbitmq import client, utils from aiida.engine.processes import ProcessState, control from aiida.orm import Int -from kiwipy.rmq import RmqThreadCommunicator -from packaging.version import parse pytestmark = pytest.mark.requires_rmq diff --git a/tests/calculations/arithmetic/test_add.py b/tests/calculations/arithmetic/test_add.py index 90dd8d6265..925467eb0e 100644 --- a/tests/calculations/arithmetic/test_add.py +++ b/tests/calculations/arithmetic/test_add.py @@ -9,6 +9,7 @@ """Tests for the `ArithmeticAddCalculation` plugin.""" import pytest + from aiida import orm from aiida.calculations.arithmetic.add import ArithmeticAddCalculation from aiida.common import datastructures diff --git a/tests/calculations/test_templatereplacer.py b/tests/calculations/test_templatereplacer.py index 15f5bb4318..cad048debc 100644 --- a/tests/calculations/test_templatereplacer.py +++ b/tests/calculations/test_templatereplacer.py @@ -11,6 +11,7 @@ import io import pytest + from aiida import orm from aiida.common import datastructures diff --git a/tests/calculations/test_transfer.py b/tests/calculations/test_transfer.py index 367eba5630..3fc43649b8 100644 --- a/tests/calculations/test_transfer.py +++ b/tests/calculations/test_transfer.py @@ -11,6 +11,7 @@ import os import pytest + from aiida import orm from aiida.common import datastructures diff --git a/tests/cmdline/commands/test_archive_create.py b/tests/cmdline/commands/test_archive_create.py index 4a9f4133d3..5fea646714 100644 --- a/tests/cmdline/commands/test_archive_create.py +++ b/tests/cmdline/commands/test_archive_create.py @@ -13,11 +13,11 @@ import zipfile import pytest + from aiida.cmdline.commands import cmd_archive from aiida.orm import Computer, Dict, Group, InstalledCode from aiida.storage.sqlite_zip.migrator import list_versions from aiida.tools.archive import ArchiveFormatSqlZip - from tests.utils.archives import get_archive_file pytest.mark.usefixtures('chdir_tmp_path') diff --git a/tests/cmdline/commands/test_archive_import.py b/tests/cmdline/commands/test_archive_import.py index 90c524781a..64ea7a7043 100644 --- a/tests/cmdline/commands/test_archive_import.py +++ b/tests/cmdline/commands/test_archive_import.py @@ -9,11 +9,11 @@ """Tests for `verdi archive import`.""" import pytest + from aiida.cmdline.commands import cmd_archive from aiida.orm import Group from aiida.storage.sqlite_zip.migrator import list_versions from aiida.tools.archive import ArchiveFormatSqlZip - from tests.utils.archives import get_archive_file ARCHIVE_PATH = 'export/migrate' diff --git a/tests/cmdline/commands/test_calcjob.py b/tests/cmdline/commands/test_calcjob.py index c3f45f195e..641d9546f1 100644 --- a/tests/cmdline/commands/test_calcjob.py +++ b/tests/cmdline/commands/test_calcjob.py @@ -11,6 +11,8 @@ import io import pytest +from click.testing import CliRunner + from aiida import orm from aiida.cmdline.commands import cmd_calcjob as command from aiida.common.datastructures import CalcJobState @@ -19,8 +21,6 @@ from aiida.orm.nodes.data.remote.base import RemoteData from aiida.plugins import CalculationFactory from aiida.plugins.entry_point import get_entry_point_string_from_class -from click.testing import CliRunner - from tests.utils.archives import import_test_archive diff --git a/tests/cmdline/commands/test_code.py b/tests/cmdline/commands/test_code.py index d2f09c0f24..e84fcd1e14 100644 --- a/tests/cmdline/commands/test_code.py +++ b/tests/cmdline/commands/test_code.py @@ -17,6 +17,7 @@ import click import pytest + from aiida.cmdline.commands import cmd_code from aiida.cmdline.params.options.commands.code import validate_label_uniqueness from aiida.common.exceptions import MultipleObjectsError, NotExistent diff --git a/tests/cmdline/commands/test_computer.py b/tests/cmdline/commands/test_computer.py index 422a155214..cf8fb01df9 100644 --- a/tests/cmdline/commands/test_computer.py +++ b/tests/cmdline/commands/test_computer.py @@ -16,6 +16,7 @@ import pytest import yaml + from aiida import orm from aiida.cmdline.commands.cmd_computer import ( computer_configure, diff --git a/tests/cmdline/commands/test_config.py b/tests/cmdline/commands/test_config.py index 744d14d4a6..ffa744beee 100644 --- a/tests/cmdline/commands/test_config.py +++ b/tests/cmdline/commands/test_config.py @@ -9,6 +9,7 @@ """Tests for ``verdi config``.""" import pytest + from aiida import get_profile from aiida.cmdline.commands import cmd_verdi diff --git a/tests/cmdline/commands/test_daemon.py b/tests/cmdline/commands/test_daemon.py index 8bf5a9cbff..f4013e84b1 100644 --- a/tests/cmdline/commands/test_daemon.py +++ b/tests/cmdline/commands/test_daemon.py @@ -12,6 +12,7 @@ from unittest.mock import patch import pytest + from aiida import get_profile from aiida.cmdline.commands import cmd_daemon from aiida.engine.daemon.client import DaemonClient diff --git a/tests/cmdline/commands/test_data.py b/tests/cmdline/commands/test_data.py index d2b00c9c7c..28f33c40a5 100644 --- a/tests/cmdline/commands/test_data.py +++ b/tests/cmdline/commands/test_data.py @@ -16,6 +16,7 @@ import numpy as np import pytest + from aiida import orm from aiida.cmdline.commands import cmd_group from aiida.cmdline.commands.cmd_data import ( @@ -33,7 +34,6 @@ from aiida.engine import calcfunction from aiida.orm import ArrayData, BandsData, CifData, Dict, Group, KpointsData, RemoteData, StructureData, TrajectoryData from aiida.orm.nodes.data.cif import has_pycifrw - from tests.static import STATIC_DIR diff --git a/tests/cmdline/commands/test_devel.py b/tests/cmdline/commands/test_devel.py index 856073f524..6c30f4cdc3 100644 --- a/tests/cmdline/commands/test_devel.py +++ b/tests/cmdline/commands/test_devel.py @@ -11,6 +11,7 @@ import re import pytest + from aiida.cmdline.commands import cmd_devel from aiida.orm import Node, ProcessNode, QueryBuilder, WorkChainNode diff --git a/tests/cmdline/commands/test_group.py b/tests/cmdline/commands/test_group.py index fa319276f1..49b8018ab5 100644 --- a/tests/cmdline/commands/test_group.py +++ b/tests/cmdline/commands/test_group.py @@ -9,6 +9,7 @@ """Tests for the `verdi group` command.""" import pytest + from aiida import orm from aiida.cmdline.commands import cmd_group from aiida.cmdline.utils.echo import ExitCode diff --git a/tests/cmdline/commands/test_group_ls.py b/tests/cmdline/commands/test_group_ls.py index 4caf2ea995..f16d348579 100644 --- a/tests/cmdline/commands/test_group_ls.py +++ b/tests/cmdline/commands/test_group_ls.py @@ -11,9 +11,10 @@ from textwrap import dedent import pytest +from click.testing import CliRunner + from aiida import orm from aiida.cmdline.commands.cmd_group import group_path_ls -from click.testing import CliRunner @pytest.fixture diff --git a/tests/cmdline/commands/test_node.py b/tests/cmdline/commands/test_node.py index 66ca83b686..0e52708051 100644 --- a/tests/cmdline/commands/test_node.py +++ b/tests/cmdline/commands/test_node.py @@ -16,6 +16,7 @@ import warnings import pytest + from aiida import orm from aiida.cmdline.commands import cmd_node from aiida.cmdline.utils.echo import ExitCode diff --git a/tests/cmdline/commands/test_plugin.py b/tests/cmdline/commands/test_plugin.py index 9960db7acd..b0b0d9453c 100644 --- a/tests/cmdline/commands/test_plugin.py +++ b/tests/cmdline/commands/test_plugin.py @@ -9,6 +9,7 @@ """Tests for the `verdi plugin list` command.""" import pytest + from aiida.cmdline.commands import cmd_plugin from aiida.parsers import Parser from aiida.plugins import BaseFactory diff --git a/tests/cmdline/commands/test_presto.py b/tests/cmdline/commands/test_presto.py index 80c61eaa4b..366babcb38 100644 --- a/tests/cmdline/commands/test_presto.py +++ b/tests/cmdline/commands/test_presto.py @@ -3,6 +3,7 @@ import textwrap import pytest + from aiida.cmdline.commands.cmd_presto import get_default_presto_profile_name, verdi_presto from aiida.manage.configuration import profile_context from aiida.manage.configuration.config import Config diff --git a/tests/cmdline/commands/test_process.py b/tests/cmdline/commands/test_process.py index b70991aa4d..15ff8911eb 100644 --- a/tests/cmdline/commands/test_process.py +++ b/tests/cmdline/commands/test_process.py @@ -15,6 +15,7 @@ import uuid import pytest + from aiida import get_profile from aiida.cmdline.commands import cmd_process from aiida.cmdline.utils.echo import ExitCode @@ -23,7 +24,6 @@ from aiida.engine import Process, ProcessState from aiida.engine.processes import control as process_control from aiida.orm import CalcJobNode, Group, WorkChainNode, WorkflowNode, WorkFunctionNode - from tests.utils.processes import WaitProcess diff --git a/tests/cmdline/commands/test_profile.py b/tests/cmdline/commands/test_profile.py index 07b45c2818..f04ffbd1e3 100644 --- a/tests/cmdline/commands/test_profile.py +++ b/tests/cmdline/commands/test_profile.py @@ -9,11 +9,12 @@ """Tests for ``verdi profile``.""" import pytest +from pgtest.pgtest import PGTest + from aiida.cmdline.commands import cmd_profile, cmd_verdi from aiida.manage import configuration from aiida.plugins import StorageFactory from aiida.tools.archive.create import create_archive -from pgtest.pgtest import PGTest # NOTE: Most of these tests would work with sqlite_dos, # but would require generalizing a bunch of fixtures ('profile_factory' et al) in tests/conftest.py diff --git a/tests/cmdline/commands/test_rabbitmq.py b/tests/cmdline/commands/test_rabbitmq.py index 990fb50767..8bb443cfe2 100644 --- a/tests/cmdline/commands/test_rabbitmq.py +++ b/tests/cmdline/commands/test_rabbitmq.py @@ -9,11 +9,12 @@ """Tests for ``verdi devel rabbitmq``.""" import pytest +from plumpy.process_comms import RemoteProcessThreadController + from aiida.cmdline.commands import cmd_rabbitmq from aiida.engine import ProcessState, submit from aiida.engine.processes import control from aiida.orm import Int -from plumpy.process_comms import RemoteProcessThreadController @pytest.mark.requires_rmq diff --git a/tests/cmdline/commands/test_run.py b/tests/cmdline/commands/test_run.py index 32084c9937..4ad0111d75 100644 --- a/tests/cmdline/commands/test_run.py +++ b/tests/cmdline/commands/test_run.py @@ -12,6 +12,7 @@ import textwrap import pytest + from aiida.cmdline.commands import cmd_run from aiida.common.log import override_log_level diff --git a/tests/cmdline/commands/test_setup.py b/tests/cmdline/commands/test_setup.py index 6c7a668542..9e7be9809b 100644 --- a/tests/cmdline/commands/test_setup.py +++ b/tests/cmdline/commands/test_setup.py @@ -13,11 +13,12 @@ import uuid import pytest +from pgtest.pgtest import PGTest + from aiida import orm from aiida.cmdline.commands import cmd_setup from aiida.manage import configuration from aiida.manage.external.postgres import Postgres -from pgtest.pgtest import PGTest pytestmark = pytest.mark.requires_psql diff --git a/tests/cmdline/commands/test_status.py b/tests/cmdline/commands/test_status.py index a4b81dbfc6..babbeb8446 100644 --- a/tests/cmdline/commands/test_status.py +++ b/tests/cmdline/commands/test_status.py @@ -9,6 +9,7 @@ """Tests for `verdi status`.""" import pytest + from aiida import __version__, get_profile from aiida.cmdline.commands import cmd_status from aiida.cmdline.utils.echo import ExitCode diff --git a/tests/cmdline/commands/test_storage.py b/tests/cmdline/commands/test_storage.py index 59698343e6..9bbf6dfee8 100644 --- a/tests/cmdline/commands/test_storage.py +++ b/tests/cmdline/commands/test_storage.py @@ -11,6 +11,7 @@ import json import pytest + from aiida import get_profile from aiida.cmdline.commands import cmd_storage from aiida.common import exceptions diff --git a/tests/cmdline/commands/test_user.py b/tests/cmdline/commands/test_user.py index 0e6fc90137..d5aecae911 100644 --- a/tests/cmdline/commands/test_user.py +++ b/tests/cmdline/commands/test_user.py @@ -12,6 +12,7 @@ import secrets import pytest + from aiida import orm from aiida.cmdline.commands import cmd_user diff --git a/tests/cmdline/commands/test_verdi.py b/tests/cmdline/commands/test_verdi.py index 559d975515..9ceaa6a5cb 100644 --- a/tests/cmdline/commands/test_verdi.py +++ b/tests/cmdline/commands/test_verdi.py @@ -10,6 +10,7 @@ import click import pytest + from aiida import get_version from aiida.cmdline.commands import cmd_verdi diff --git a/tests/cmdline/groups/test_dynamic.py b/tests/cmdline/groups/test_dynamic.py index 48c18f0941..f66c0ee4fa 100644 --- a/tests/cmdline/groups/test_dynamic.py +++ b/tests/cmdline/groups/test_dynamic.py @@ -2,10 +2,11 @@ import typing as t -from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup from pydantic import BaseModel, Field from pydantic_core import PydanticUndefined +from aiida.cmdline.groups.dynamic import DynamicEntryPointCommandGroup + class CustomClass: """Test plugin class.""" diff --git a/tests/cmdline/params/options/test_callable.py b/tests/cmdline/params/options/test_callable.py index 59382639a9..2e5e1c8c8b 100644 --- a/tests/cmdline/params/options/test_callable.py +++ b/tests/cmdline/params/options/test_callable.py @@ -11,9 +11,10 @@ import sys import pytest -from aiida.cmdline.commands.cmd_verdi import verdi from click.shell_completion import ShellComplete +from aiida.cmdline.commands.cmd_verdi import verdi + SLOW_IMPORTS = ['pydantic'] diff --git a/tests/cmdline/params/options/test_conditional.py b/tests/cmdline/params/options/test_conditional.py index abbf2bc7af..9aa08b3c4a 100644 --- a/tests/cmdline/params/options/test_conditional.py +++ b/tests/cmdline/params/options/test_conditional.py @@ -12,6 +12,7 @@ import click import pytest + from aiida.cmdline.params.options.conditional import ConditionalOption diff --git a/tests/cmdline/params/options/test_config.py b/tests/cmdline/params/options/test_config.py index 10e026716f..81260a650f 100644 --- a/tests/cmdline/params/options/test_config.py +++ b/tests/cmdline/params/options/test_config.py @@ -13,6 +13,7 @@ import click import pytest + from aiida.cmdline.params.options import CONFIG_FILE diff --git a/tests/cmdline/params/options/test_interactive.py b/tests/cmdline/params/options/test_interactive.py index 05df89dd4b..cb5d76ee2d 100644 --- a/tests/cmdline/params/options/test_interactive.py +++ b/tests/cmdline/params/options/test_interactive.py @@ -12,6 +12,7 @@ import click import pytest + from aiida.cmdline.params.options import NON_INTERACTIVE from aiida.cmdline.params.options.interactive import InteractiveOption from aiida.cmdline.params.types.plugin import PluginParamType diff --git a/tests/cmdline/params/options/test_verbosity.py b/tests/cmdline/params/options/test_verbosity.py index 3573edd54f..dc8a3fe0fd 100644 --- a/tests/cmdline/params/options/test_verbosity.py +++ b/tests/cmdline/params/options/test_verbosity.py @@ -12,6 +12,7 @@ import logging import pytest + from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.utils import echo from aiida.common import log diff --git a/tests/cmdline/params/types/test_calculation.py b/tests/cmdline/params/types/test_calculation.py index 5328b69bf7..974a9ff7ac 100644 --- a/tests/cmdline/params/types/test_calculation.py +++ b/tests/cmdline/params/types/test_calculation.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida.cmdline.params.types import CalculationParamType from aiida.orm import CalculationNode from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_code.py b/tests/cmdline/params/types/test_code.py index 6b86949e3a..cf1f457f8e 100644 --- a/tests/cmdline/params/types/test_code.py +++ b/tests/cmdline/params/types/test_code.py @@ -12,6 +12,7 @@ import click import pytest + from aiida.cmdline.params.types import CodeParamType from aiida.orm import InstalledCode from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_computer.py b/tests/cmdline/params/types/test_computer.py index c889d321c1..289ce12321 100644 --- a/tests/cmdline/params/types/test_computer.py +++ b/tests/cmdline/params/types/test_computer.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida import orm from aiida.cmdline.params.types import ComputerParamType from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_data.py b/tests/cmdline/params/types/test_data.py index d17f2857ce..2dff371325 100644 --- a/tests/cmdline/params/types/test_data.py +++ b/tests/cmdline/params/types/test_data.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida.cmdline.params.types import DataParamType from aiida.orm import Data from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_group.py b/tests/cmdline/params/types/test_group.py index eabd47e134..6913008894 100644 --- a/tests/cmdline/params/types/test_group.py +++ b/tests/cmdline/params/types/test_group.py @@ -12,6 +12,7 @@ import click import pytest + from aiida.cmdline.params.types import GroupParamType from aiida.orm import AutoGroup, Group, ImportGroup from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_identifier.py b/tests/cmdline/params/types/test_identifier.py index b2e71af071..f80809bcbe 100644 --- a/tests/cmdline/params/types/test_identifier.py +++ b/tests/cmdline/params/types/test_identifier.py @@ -10,6 +10,7 @@ import click import pytest + from aiida.cmdline.params.types import IdentifierParamType, NodeParamType from aiida.orm import Bool, Float, Int diff --git a/tests/cmdline/params/types/test_node.py b/tests/cmdline/params/types/test_node.py index be6cac18dc..376c942809 100644 --- a/tests/cmdline/params/types/test_node.py +++ b/tests/cmdline/params/types/test_node.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida.cmdline.params.types import NodeParamType from aiida.orm import Data from aiida.orm.utils.loaders import OrmEntityLoader diff --git a/tests/cmdline/params/types/test_path.py b/tests/cmdline/params/types/test_path.py index d9fbb1fb07..8130828998 100644 --- a/tests/cmdline/params/types/test_path.py +++ b/tests/cmdline/params/types/test_path.py @@ -10,6 +10,7 @@ import click import pytest + from aiida.cmdline.params.types.path import PathOrUrl, check_timeout_seconds diff --git a/tests/cmdline/params/types/test_plugin.py b/tests/cmdline/params/types/test_plugin.py index 8ceb5e3443..f05660325a 100644 --- a/tests/cmdline/params/types/test_plugin.py +++ b/tests/cmdline/params/types/test_plugin.py @@ -10,6 +10,7 @@ import click import pytest + from aiida.cmdline.params.types.plugin import PluginParamType from aiida.plugins.entry_point import get_entry_point_from_string diff --git a/tests/cmdline/utils/test_common.py b/tests/cmdline/utils/test_common.py index 7b012dcc0f..926660d47d 100644 --- a/tests/cmdline/utils/test_common.py +++ b/tests/cmdline/utils/test_common.py @@ -11,6 +11,7 @@ from pathlib import Path import pytest + from aiida.cmdline.utils import common from aiida.cmdline.utils.common import validate_output_filename from aiida.common import LinkType diff --git a/tests/cmdline/utils/test_decorators.py b/tests/cmdline/utils/test_decorators.py index b2798c6ba3..baf6536105 100644 --- a/tests/cmdline/utils/test_decorators.py +++ b/tests/cmdline/utils/test_decorators.py @@ -11,6 +11,7 @@ from unittest import mock import pytest + from aiida.cmdline.utils.decorators import load_backend_if_not_loaded from aiida.common.exceptions import InvalidOperation from aiida.manage import get_manager diff --git a/tests/cmdline/utils/test_multiline.py b/tests/cmdline/utils/test_multiline.py index c84cd0242b..0dabeedf61 100644 --- a/tests/cmdline/utils/test_multiline.py +++ b/tests/cmdline/utils/test_multiline.py @@ -10,6 +10,7 @@ import click import pytest + from aiida.cmdline.utils.multi_line_input import edit_comment, edit_multiline_template COMMAND = 'sleep 1 ; vim -c "g!/^#=/s/$/Test" -cwq' # Appends `Test` to every line NOT starting with `#=` diff --git a/tests/cmdline/utils/test_repository.py b/tests/cmdline/utils/test_repository.py index e9bba6e53c..c5fcce0905 100644 --- a/tests/cmdline/utils/test_repository.py +++ b/tests/cmdline/utils/test_repository.py @@ -11,6 +11,7 @@ import io import pytest + from aiida.cmdline.utils.repository import list_repository_contents from aiida.orm import FolderData diff --git a/tests/common/test_escaping.py b/tests/common/test_escaping.py index aa5d21f57f..591345cbce 100644 --- a/tests/common/test_escaping.py +++ b/tests/common/test_escaping.py @@ -9,6 +9,7 @@ """Tests for the :mod:`aiida.common.escaping`.""" import pytest + from aiida.common.escaping import escape_for_bash diff --git a/tests/common/test_folders.py b/tests/common/test_folders.py index 2fc8cb902c..0d95023b94 100644 --- a/tests/common/test_folders.py +++ b/tests/common/test_folders.py @@ -14,6 +14,7 @@ import tempfile import pytest + from aiida.common.folders import Folder, SandboxFolder diff --git a/tests/common/test_hashing.py b/tests/common/test_hashing.py index d51c543195..70d928140e 100644 --- a/tests/common/test_hashing.py +++ b/tests/common/test_hashing.py @@ -17,6 +17,7 @@ import numpy as np import pytest + from aiida.common.exceptions import HashingError from aiida.common.folders import SandboxFolder from aiida.common.hashing import chunked_file_hash, float_to_text, make_hash diff --git a/tests/common/test_links.py b/tests/common/test_links.py index 1e2836226e..3fe7a3cca3 100644 --- a/tests/common/test_links.py +++ b/tests/common/test_links.py @@ -9,6 +9,7 @@ """Tests for the links utilities.""" import pytest + from aiida.common.links import validate_link_label diff --git a/tests/common/test_timezone.py b/tests/common/test_timezone.py index 8115007de7..273a86dcf2 100644 --- a/tests/common/test_timezone.py +++ b/tests/common/test_timezone.py @@ -12,6 +12,7 @@ from time import time import pytest + from aiida.common.timezone import delta, localtime, make_aware, now, timezone_from_name diff --git a/tests/conftest.py b/tests/conftest.py index b50cf90843..89b0a1bad7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,7 @@ import click import pytest + from aiida import get_profile from aiida.common.folders import Folder from aiida.common.links import LinkType @@ -782,9 +783,10 @@ def run_cli_command_subprocess(command, parameters, user_input, profile_name, su def run_cli_command_runner(command, parameters, user_input, initialize_ctx_obj, kwargs): """Run CLI command through ``click.testing.CliRunner``.""" + from click.testing import CliRunner + from aiida.cmdline.commands.cmd_verdi import VerdiCommandGroup from aiida.cmdline.groups.verdi import LazyVerdiObjAttributeDict - from click.testing import CliRunner if initialize_ctx_obj: config = get_config() diff --git a/tests/engine/daemon/test_client.py b/tests/engine/daemon/test_client.py index 80a66a8632..f10d0b0c2e 100644 --- a/tests/engine/daemon/test_client.py +++ b/tests/engine/daemon/test_client.py @@ -12,6 +12,7 @@ import pytest import zmq + from aiida.engine.daemon.client import ( DaemonClient, DaemonNotRunningException, diff --git a/tests/engine/daemon/test_execmanager.py b/tests/engine/daemon/test_execmanager.py index 79692b689b..b0b4e4f1bd 100644 --- a/tests/engine/daemon/test_execmanager.py +++ b/tests/engine/daemon/test_execmanager.py @@ -12,6 +12,7 @@ import pathlib import pytest + from aiida.common.datastructures import CalcInfo, CodeInfo, FileCopyOperation from aiida.common.folders import SandboxFolder from aiida.engine.daemon import execmanager diff --git a/tests/engine/daemon/test_worker.py b/tests/engine/daemon/test_worker.py index f8807fccab..4debe28e3b 100644 --- a/tests/engine/daemon/test_worker.py +++ b/tests/engine/daemon/test_worker.py @@ -9,6 +9,7 @@ """Unit tests for the :mod:`aiida.engine.daemon.worker` module.""" import pytest + from aiida.engine.daemon.worker import shutdown_worker from aiida.orm import Log from aiida.workflows.arithmetic.multiply_add import MultiplyAddWorkChain diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index 4e679d5cdf..9a2078fec6 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -19,6 +19,7 @@ from unittest.mock import patch import pytest + from aiida import orm from aiida.common import CalcJobState, LinkType, StashMode, exceptions from aiida.common.datastructures import FileCopyOperation diff --git a/tests/engine/processes/calcjobs/test_monitors.py b/tests/engine/processes/calcjobs/test_monitors.py index cd1d4f51ce..7f6678917b 100644 --- a/tests/engine/processes/calcjobs/test_monitors.py +++ b/tests/engine/processes/calcjobs/test_monitors.py @@ -5,6 +5,7 @@ import time import pytest + from aiida.calculations.arithmetic.add import ArithmeticAddCalculation from aiida.calculations.monitors import base from aiida.common.exceptions import EntryPointError diff --git a/tests/engine/processes/test_builder.py b/tests/engine/processes/test_builder.py index 9a57362248..20f4d7c58c 100644 --- a/tests/engine/processes/test_builder.py +++ b/tests/engine/processes/test_builder.py @@ -12,12 +12,13 @@ from collections.abc import Mapping, MutableMapping import pytest +from IPython.lib.pretty import pretty + from aiida import orm from aiida.common import LinkType from aiida.engine import Process, WorkChain, run_get_node from aiida.engine.processes.builder import ProcessBuilderNamespace from aiida.plugins import CalculationFactory -from IPython.lib.pretty import pretty DEFAULT_INT = 256 diff --git a/tests/engine/processes/test_control.py b/tests/engine/processes/test_control.py index 51222c653c..5bb9b8b7a6 100644 --- a/tests/engine/processes/test_control.py +++ b/tests/engine/processes/test_control.py @@ -1,12 +1,12 @@ """Tests for the :mod:`aiida.engine.processes.control` module.""" import pytest +from plumpy.process_comms import RemoteProcessThreadController + from aiida.engine import ProcessState from aiida.engine.launch import submit from aiida.engine.processes import control from aiida.orm import Int -from plumpy.process_comms import RemoteProcessThreadController - from tests.utils.processes import WaitProcess diff --git a/tests/engine/processes/test_exit_code.py b/tests/engine/processes/test_exit_code.py index b44f29df2a..d8cce0a281 100644 --- a/tests/engine/processes/test_exit_code.py +++ b/tests/engine/processes/test_exit_code.py @@ -9,6 +9,7 @@ """Tests for `aiida.engine.processes.exit_code.ExitCode`.""" import pytest + from aiida.engine import ExitCode diff --git a/tests/engine/processes/workchains/test_restart.py b/tests/engine/processes/workchains/test_restart.py index d468b744c3..bf870292d7 100644 --- a/tests/engine/processes/workchains/test_restart.py +++ b/tests/engine/processes/workchains/test_restart.py @@ -11,6 +11,7 @@ import warnings import pytest + from aiida import engine, orm from aiida.engine.processes.workchains.awaitable import Awaitable diff --git a/tests/engine/processes/workchains/test_utils.py b/tests/engine/processes/workchains/test_utils.py index a4593e8f04..4397c50828 100644 --- a/tests/engine/processes/workchains/test_utils.py +++ b/tests/engine/processes/workchains/test_utils.py @@ -9,6 +9,7 @@ """Tests for `aiida.engine.processes.workchains.utils` module.""" import pytest + from aiida.engine import ExitCode, ProcessState from aiida.engine.processes.workchains.restart import BaseRestartWorkChain from aiida.engine.processes.workchains.utils import ProcessHandlerReport, process_handler @@ -31,7 +32,7 @@ class SomeWorkChain(BaseRestartWorkChain): def _(self, node): pass - class SomeWorkChain(BaseRestartWorkChain): # noqa: F811 + class SomeWorkChain(BaseRestartWorkChain): @process_handler(priority=400) def _(self, node): pass @@ -110,7 +111,7 @@ class SomeWorkChain(BaseRestartWorkChain): def _(self, node): pass - class SomeWorkChain(BaseRestartWorkChain): # noqa: F811 + class SomeWorkChain(BaseRestartWorkChain): @process_handler(exit_codes=ExitCode()) def _(self, node): pass @@ -132,7 +133,7 @@ class SomeWorkChain(BaseRestartWorkChain): def _(self, node): pass - class SomeWorkChain(BaseRestartWorkChain): # noqa: F811 + class SomeWorkChain(BaseRestartWorkChain): @process_handler(exit_codes=ExitCode(400, 'Some exit code')) def _(self, node): pass @@ -183,7 +184,7 @@ class SomeWorkChain(BaseRestartWorkChain): def _(self, node): pass - class SomeWorkChain(BaseRestartWorkChain): # noqa: F811 + class SomeWorkChain(BaseRestartWorkChain): @process_handler(enabled=False) def _(self, node): pass diff --git a/tests/engine/test_calcfunctions.py b/tests/engine/test_calcfunctions.py index e5ccfac01a..98c53aefa4 100644 --- a/tests/engine/test_calcfunctions.py +++ b/tests/engine/test_calcfunctions.py @@ -9,6 +9,7 @@ """Tests for the calcfunction decorator and CalcFunctionNode.""" import pytest + from aiida.common import exceptions from aiida.common.links import LinkType from aiida.engine import Process, calcfunction diff --git a/tests/engine/test_class_loader.py b/tests/engine/test_class_loader.py index 688e35cf08..f396d35a39 100644 --- a/tests/engine/test_class_loader.py +++ b/tests/engine/test_class_loader.py @@ -8,8 +8,9 @@ ########################################################################### """A module to test class loader factories.""" -import aiida import pytest + +import aiida from aiida.engine import Process from aiida.plugins import CalculationFactory diff --git a/tests/engine/test_daemon.py b/tests/engine/test_daemon.py index f556b55ade..c9a8570e32 100644 --- a/tests/engine/test_daemon.py +++ b/tests/engine/test_daemon.py @@ -11,9 +11,9 @@ import asyncio import pytest -from aiida.manage import get_manager from plumpy.process_states import ProcessState +from aiida.manage import get_manager from tests.utils import processes as test_processes diff --git a/tests/engine/test_futures.py b/tests/engine/test_futures.py index 9f9a752f62..b8ba78aa8f 100644 --- a/tests/engine/test_futures.py +++ b/tests/engine/test_futures.py @@ -11,9 +11,9 @@ import asyncio import pytest + from aiida.engine import processes, run from aiida.manage import get_manager - from tests.utils import processes as test_processes diff --git a/tests/engine/test_launch.py b/tests/engine/test_launch.py index 92fba4bd6b..6aba82cdcc 100644 --- a/tests/engine/test_launch.py +++ b/tests/engine/test_launch.py @@ -12,6 +12,7 @@ import shutil import pytest + from aiida import orm from aiida.common import exceptions from aiida.engine import CalcJob, Process, WorkChain, calcfunction, launch diff --git a/tests/engine/test_manager.py b/tests/engine/test_manager.py index 1df0348a64..97ac97c11a 100644 --- a/tests/engine/test_manager.py +++ b/tests/engine/test_manager.py @@ -12,6 +12,7 @@ import time import pytest + from aiida.engine.processes.calcjobs.manager import JobManager, JobsList from aiida.engine.transports import TransportQueue from aiida.orm import User diff --git a/tests/engine/test_memory_leaks.py b/tests/engine/test_memory_leaks.py index 4eece939f3..e59ff98366 100644 --- a/tests/engine/test_memory_leaks.py +++ b/tests/engine/test_memory_leaks.py @@ -11,10 +11,10 @@ import sys import pytest + from aiida import orm from aiida.engine import processes, run_get_node from aiida.plugins import CalculationFactory - from tests.utils import processes as test_processes from tests.utils.memory import get_instances diff --git a/tests/engine/test_persistence.py b/tests/engine/test_persistence.py index 870597557f..7ca9908376 100644 --- a/tests/engine/test_persistence.py +++ b/tests/engine/test_persistence.py @@ -10,9 +10,9 @@ import plumpy import pytest + from aiida.engine import Process, run from aiida.engine.persistence import AiiDAPersister - from tests.utils.processes import DummyProcess diff --git a/tests/engine/test_ports.py b/tests/engine/test_ports.py index 147717f8a4..5b3c6ae59b 100644 --- a/tests/engine/test_ports.py +++ b/tests/engine/test_ports.py @@ -9,6 +9,7 @@ """Tests for process spec ports.""" import pytest + from aiida.engine.processes.ports import InputPort, PortNamespace from aiida.orm import Dict, Int, to_aiida_type diff --git a/tests/engine/test_process.py b/tests/engine/test_process.py index 9b1f230041..1d7008478b 100644 --- a/tests/engine/test_process.py +++ b/tests/engine/test_process.py @@ -12,6 +12,8 @@ import plumpy import pytest +from plumpy.utils import AttributesFrozendict + from aiida import orm from aiida.common.lang import override from aiida.engine import ExitCode, ExitCodesNamespace, Process, run, run_get_node, run_get_pk @@ -20,8 +22,6 @@ from aiida.orm import to_aiida_type from aiida.orm.nodes.caching import NodeCaching from aiida.plugins import CalculationFactory -from plumpy.utils import AttributesFrozendict - from tests.utils import processes as test_processes diff --git a/tests/engine/test_process_function.py b/tests/engine/test_process_function.py index 9f1cfadbc8..33581868aa 100644 --- a/tests/engine/test_process_function.py +++ b/tests/engine/test_process_function.py @@ -24,6 +24,7 @@ import typing as t import pytest + from aiida import orm from aiida.engine import ExitCode, calcfunction, run, run_get_node, submit, workfunction from aiida.orm.nodes.data.bool import get_true_node diff --git a/tests/engine/test_process_spec.py b/tests/engine/test_process_spec.py index 56b6b8ca3f..22fbae4f25 100644 --- a/tests/engine/test_process_spec.py +++ b/tests/engine/test_process_spec.py @@ -9,6 +9,7 @@ """Tests for the `ProcessSpec` class.""" import pytest + from aiida.engine import Process from aiida.orm import Data, Node diff --git a/tests/engine/test_rmq.py b/tests/engine/test_rmq.py index 1fbf821aff..a2edc2fa41 100644 --- a/tests/engine/test_rmq.py +++ b/tests/engine/test_rmq.py @@ -12,10 +12,10 @@ import plumpy import pytest + from aiida.engine import ProcessState from aiida.manage import get_manager from aiida.orm import Int - from tests.utils import processes as test_processes diff --git a/tests/engine/test_run.py b/tests/engine/test_run.py index d15036cd5d..fd4fd5ddfa 100644 --- a/tests/engine/test_run.py +++ b/tests/engine/test_run.py @@ -9,9 +9,9 @@ """Tests for the `run` functions.""" import pytest + from aiida.engine import run, run_get_node from aiida.orm import Int, ProcessNode, Str - from tests.utils.processes import DummyProcess diff --git a/tests/engine/test_runners.py b/tests/engine/test_runners.py index 4f746e5d34..db6e0e7493 100644 --- a/tests/engine/test_runners.py +++ b/tests/engine/test_runners.py @@ -13,6 +13,7 @@ import plumpy import pytest + from aiida.calculations.arithmetic.add import ArithmeticAddCalculation from aiida.engine import Process, launch from aiida.manage import get_manager diff --git a/tests/engine/test_transport.py b/tests/engine/test_transport.py index f4749e9820..02d6cee928 100644 --- a/tests/engine/test_transport.py +++ b/tests/engine/test_transport.py @@ -11,6 +11,7 @@ import asyncio import pytest + from aiida import orm from aiida.engine.transports import TransportQueue diff --git a/tests/engine/test_utils.py b/tests/engine/test_utils.py index cca5866343..fe18020445 100644 --- a/tests/engine/test_utils.py +++ b/tests/engine/test_utils.py @@ -12,6 +12,7 @@ import contextlib import pytest + from aiida import orm from aiida.engine import calcfunction, workfunction from aiida.engine.utils import ( diff --git a/tests/engine/test_work_chain.py b/tests/engine/test_work_chain.py index 1fbb578b2a..4fb8c70667 100644 --- a/tests/engine/test_work_chain.py +++ b/tests/engine/test_work_chain.py @@ -14,6 +14,7 @@ import plumpy import pytest + from aiida import orm from aiida.common import exceptions from aiida.common.links import LinkType diff --git a/tests/engine/test_workfunctions.py b/tests/engine/test_workfunctions.py index c8595aadbc..b84438ac6a 100644 --- a/tests/engine/test_workfunctions.py +++ b/tests/engine/test_workfunctions.py @@ -9,6 +9,7 @@ """Tests for the workfunction decorator and WorkFunctionNode.""" import pytest + from aiida.common.links import LinkType from aiida.engine import Process, calcfunction, workfunction from aiida.manage.caching import enable_caching diff --git a/tests/manage/configuration/migrations/test_migrations.py b/tests/manage/configuration/migrations/test_migrations.py index 2d3612ed72..9c6f243347 100644 --- a/tests/manage/configuration/migrations/test_migrations.py +++ b/tests/manage/configuration/migrations/test_migrations.py @@ -14,6 +14,7 @@ import uuid import pytest + from aiida.common.exceptions import ConfigurationError from aiida.manage.configuration.migrations import check_and_migrate_config from aiida.manage.configuration.migrations.migrations import MIGRATIONS, Initial, downgrade_config, upgrade_config diff --git a/tests/manage/configuration/test_config.py b/tests/manage/configuration/test_config.py index 7fedb4bf2a..77adc4bf6d 100644 --- a/tests/manage/configuration/test_config.py +++ b/tests/manage/configuration/test_config.py @@ -14,6 +14,7 @@ import uuid import pytest + from aiida.common import exceptions from aiida.manage.configuration import Profile, settings from aiida.manage.configuration.config import Config diff --git a/tests/manage/configuration/test_configuration.py b/tests/manage/configuration/test_configuration.py index 4f6e38cd5f..1226fe2db3 100644 --- a/tests/manage/configuration/test_configuration.py +++ b/tests/manage/configuration/test_configuration.py @@ -1,7 +1,8 @@ """Tests for the :mod:`aiida.manage.configuration` module.""" -import aiida import pytest + +import aiida from aiida.manage.configuration import Profile, create_profile, get_profile, profile_context from aiida.manage.manager import get_manager diff --git a/tests/manage/configuration/test_options.py b/tests/manage/configuration/test_options.py index 7a76ceeec5..788281b72e 100644 --- a/tests/manage/configuration/test_options.py +++ b/tests/manage/configuration/test_options.py @@ -9,6 +9,7 @@ """Tests for the configuration options.""" import pytest + from aiida import get_profile from aiida.common.exceptions import ConfigurationError from aiida.manage.configuration import get_config, get_config_option diff --git a/tests/manage/external/test_postgres.py b/tests/manage/external/test_postgres.py index be829ae4c9..ce3c7ef348 100644 --- a/tests/manage/external/test_postgres.py +++ b/tests/manage/external/test_postgres.py @@ -11,6 +11,7 @@ from unittest import TestCase import pytest + from aiida.manage.external.postgres import Postgres diff --git a/tests/manage/test_caching_config.py b/tests/manage/test_caching_config.py index 932a32873c..1138cf393d 100644 --- a/tests/manage/test_caching_config.py +++ b/tests/manage/test_caching_config.py @@ -14,6 +14,7 @@ import pytest import yaml + from aiida.common import exceptions from aiida.manage.caching import _validate_identifier_pattern, disable_caching, enable_caching, get_use_cache diff --git a/tests/manage/test_manager.py b/tests/manage/test_manager.py index 4359a2ab48..3a8f4949cf 100644 --- a/tests/manage/test_manager.py +++ b/tests/manage/test_manager.py @@ -1,6 +1,7 @@ """Tests for :mod:`aiida.manage.manager`.""" import pytest + from aiida import engine, orm diff --git a/tests/manage/test_profile_access.py b/tests/manage/test_profile_access.py index 0700333838..e8f34908fb 100644 --- a/tests/manage/test_profile_access.py +++ b/tests/manage/test_profile_access.py @@ -14,6 +14,7 @@ import psutil import pytest + from aiida.common.exceptions import LockedProfileError, LockingProfileError from aiida.manage.profile_access import ProfileAccessManager diff --git a/tests/manage/tests/test_pytest_fixtures.py b/tests/manage/tests/test_pytest_fixtures.py index ab2e9d82b6..93ea7c1b80 100644 --- a/tests/manage/tests/test_pytest_fixtures.py +++ b/tests/manage/tests/test_pytest_fixtures.py @@ -3,6 +3,7 @@ import uuid import pytest + from aiida.manage.configuration import get_config from aiida.manage.configuration.config import Config from aiida.orm import Computer diff --git a/tests/orm/data/code/test_abstract.py b/tests/orm/data/code/test_abstract.py index 7b4dd745ee..0985f90f40 100644 --- a/tests/orm/data/code/test_abstract.py +++ b/tests/orm/data/code/test_abstract.py @@ -11,6 +11,7 @@ import pathlib import pytest + from aiida.orm.nodes.data.code.abstract import AbstractCode diff --git a/tests/orm/data/code/test_containerized.py b/tests/orm/data/code/test_containerized.py index 9eae096cde..68ae32b76d 100644 --- a/tests/orm/data/code/test_containerized.py +++ b/tests/orm/data/code/test_containerized.py @@ -11,6 +11,7 @@ import pathlib import pytest + from aiida.orm.nodes.data.code.containerized import ContainerizedCode diff --git a/tests/orm/data/code/test_installed.py b/tests/orm/data/code/test_installed.py index 15c297f43e..c2c60f6154 100644 --- a/tests/orm/data/code/test_installed.py +++ b/tests/orm/data/code/test_installed.py @@ -11,6 +11,7 @@ import pathlib import pytest + from aiida.common.exceptions import ModificationNotAllowed, ValidationError from aiida.common.warnings import AiidaDeprecationWarning from aiida.orm import Computer diff --git a/tests/orm/data/code/test_portable.py b/tests/orm/data/code/test_portable.py index 4bffa04515..a63bb2b111 100644 --- a/tests/orm/data/code/test_portable.py +++ b/tests/orm/data/code/test_portable.py @@ -12,6 +12,7 @@ import pathlib import pytest + from aiida.common.exceptions import ModificationNotAllowed, ValidationError from aiida.common.warnings import AiidaDeprecationWarning from aiida.orm.nodes.data.code.portable import PortableCode diff --git a/tests/orm/data/test_code.py b/tests/orm/data/test_code.py index 0041cd6957..a45bf8d4ec 100644 --- a/tests/orm/data/test_code.py +++ b/tests/orm/data/test_code.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida.common.exceptions import ValidationError from aiida.orm import Code, Computer diff --git a/tests/orm/data/test_enum.py b/tests/orm/data/test_enum.py index 99a6824386..fcfa3275a1 100644 --- a/tests/orm/data/test_enum.py +++ b/tests/orm/data/test_enum.py @@ -3,6 +3,7 @@ import enum import pytest + from aiida.common import links from aiida.orm import load_node from aiida.orm.nodes.data.enum import EnumData diff --git a/tests/orm/implementation/test_backend.py b/tests/orm/implementation/test_backend.py index 001564f057..a98d090254 100644 --- a/tests/orm/implementation/test_backend.py +++ b/tests/orm/implementation/test_backend.py @@ -15,6 +15,7 @@ import uuid import pytest + from aiida import orm from aiida.common import exceptions from aiida.common.links import LinkType diff --git a/tests/orm/implementation/test_comments.py b/tests/orm/implementation/test_comments.py index 4d3db03bcc..aaba91d0da 100644 --- a/tests/orm/implementation/test_comments.py +++ b/tests/orm/implementation/test_comments.py @@ -12,6 +12,7 @@ from uuid import UUID, uuid4 import pytest + from aiida import orm from aiida.common import exceptions, timezone diff --git a/tests/orm/implementation/test_logs.py b/tests/orm/implementation/test_logs.py index 1281493e58..5d1a61f74e 100644 --- a/tests/orm/implementation/test_logs.py +++ b/tests/orm/implementation/test_logs.py @@ -13,6 +13,7 @@ from uuid import UUID, uuid4 import pytest + from aiida import orm from aiida.common import exceptions, timezone from aiida.common.log import LOG_LEVEL_REPORT diff --git a/tests/orm/implementation/test_nodes.py b/tests/orm/implementation/test_nodes.py index 7437b4dc09..92b71dbe1f 100644 --- a/tests/orm/implementation/test_nodes.py +++ b/tests/orm/implementation/test_nodes.py @@ -13,6 +13,7 @@ from uuid import UUID, uuid4 import pytest + from aiida.common import exceptions, timezone from aiida.orm.implementation.nodes import BackendNode diff --git a/tests/orm/implementation/test_utils.py b/tests/orm/implementation/test_utils.py index 22841009f4..315637dbe0 100644 --- a/tests/orm/implementation/test_utils.py +++ b/tests/orm/implementation/test_utils.py @@ -11,6 +11,7 @@ import math import pytest + from aiida.common import exceptions from aiida.orm.implementation.utils import FIELD_SEPARATOR, clean_value, validate_attribute_extra_key diff --git a/tests/orm/nodes/data/test_array.py b/tests/orm/nodes/data/test_array.py index 4f934d5c1f..2294518425 100644 --- a/tests/orm/nodes/data/test_array.py +++ b/tests/orm/nodes/data/test_array.py @@ -10,6 +10,7 @@ import numpy import pytest + from aiida.orm import ArrayData, load_node diff --git a/tests/orm/nodes/data/test_array_bands.py b/tests/orm/nodes/data/test_array_bands.py index e609ffeaa8..3f8f6e343e 100644 --- a/tests/orm/nodes/data/test_array_bands.py +++ b/tests/orm/nodes/data/test_array_bands.py @@ -12,6 +12,7 @@ from argparse import Namespace import pytest + from aiida.common.exceptions import NotExistent from aiida.orm import BandsData, Group, User from aiida.orm.nodes.data.array.bands import get_bands_and_parents_structure diff --git a/tests/orm/nodes/data/test_base.py b/tests/orm/nodes/data/test_base.py index 29a6701d61..154847fabe 100644 --- a/tests/orm/nodes/data/test_base.py +++ b/tests/orm/nodes/data/test_base.py @@ -11,6 +11,7 @@ import operator import pytest + from aiida.orm import Bool, Float, Int, NumericType, Str, load_node @@ -214,7 +215,7 @@ def test_operator(opera): for node_x, node_y in [(node_a, node_b), (node_b, node_a)]: res = opera(node_x, node_y) c_val = opera(node_x.value, node_y.value) - assert res._type == type(c_val) + assert res._type is type(c_val) assert res == opera(node_x.value, node_y.value) diff --git a/tests/orm/nodes/data/test_cif.py b/tests/orm/nodes/data/test_cif.py index b7a34084f1..cfe01728c9 100644 --- a/tests/orm/nodes/data/test_cif.py +++ b/tests/orm/nodes/data/test_cif.py @@ -9,6 +9,7 @@ """Tests for cif related functions.""" import pytest + from aiida.orm.nodes.data.cif import parse_formula diff --git a/tests/orm/nodes/data/test_data.py b/tests/orm/nodes/data/test_data.py index 44101df4b1..313b3f38e4 100644 --- a/tests/orm/nodes/data/test_data.py +++ b/tests/orm/nodes/data/test_data.py @@ -12,8 +12,8 @@ import numpy import pytest -from aiida import orm, plugins +from aiida import orm, plugins from tests.static import STATIC_DIR diff --git a/tests/orm/nodes/data/test_dict.py b/tests/orm/nodes/data/test_dict.py index 05a60f6e2d..7c565daffe 100644 --- a/tests/orm/nodes/data/test_dict.py +++ b/tests/orm/nodes/data/test_dict.py @@ -9,6 +9,7 @@ """Tests for :class:`aiida.orm.nodes.data.dict.Dict` class.""" import pytest + from aiida.orm import Dict diff --git a/tests/orm/nodes/data/test_folder.py b/tests/orm/nodes/data/test_folder.py index 991a35d028..a8dd40d2ef 100644 --- a/tests/orm/nodes/data/test_folder.py +++ b/tests/orm/nodes/data/test_folder.py @@ -9,6 +9,7 @@ """Tests for the `FolderData` class.""" import pytest + from aiida.orm import FolderData diff --git a/tests/orm/nodes/data/test_jsonable.py b/tests/orm/nodes/data/test_jsonable.py index bacedac73a..0b113cce48 100644 --- a/tests/orm/nodes/data/test_jsonable.py +++ b/tests/orm/nodes/data/test_jsonable.py @@ -4,9 +4,10 @@ import math import pytest +from pymatgen.core.structure import Molecule + from aiida.orm import load_node from aiida.orm.nodes.data.jsonable import JsonableData -from pymatgen.core.structure import Molecule class JsonableClass: diff --git a/tests/orm/nodes/data/test_kpoints.py b/tests/orm/nodes/data/test_kpoints.py index 112a46859e..73d25f82ea 100644 --- a/tests/orm/nodes/data/test_kpoints.py +++ b/tests/orm/nodes/data/test_kpoints.py @@ -10,6 +10,7 @@ import numpy as np import pytest + from aiida.orm import KpointsData, StructureData, load_node diff --git a/tests/orm/nodes/data/test_list.py b/tests/orm/nodes/data/test_list.py index 657b77f127..d5dfa1c85a 100644 --- a/tests/orm/nodes/data/test_list.py +++ b/tests/orm/nodes/data/test_list.py @@ -9,6 +9,7 @@ """Tests for :class:`aiida.orm.nodes.data.list.List` class.""" import pytest + from aiida.common.exceptions import ModificationNotAllowed from aiida.orm import List, load_node diff --git a/tests/orm/nodes/data/test_orbital.py b/tests/orm/nodes/data/test_orbital.py index 62f20cc4bf..710a583a5e 100644 --- a/tests/orm/nodes/data/test_orbital.py +++ b/tests/orm/nodes/data/test_orbital.py @@ -11,6 +11,7 @@ import copy import pytest + from aiida.common import ValidationError from aiida.orm import OrbitalData from aiida.plugins import OrbitalFactory diff --git a/tests/orm/nodes/data/test_remote.py b/tests/orm/nodes/data/test_remote.py index 8f08fee37b..333e59c547 100644 --- a/tests/orm/nodes/data/test_remote.py +++ b/tests/orm/nodes/data/test_remote.py @@ -9,6 +9,7 @@ """Tests for the :mod:`aiida.orm.nodes.data.remote.base.RemoteData` module.""" import pytest + from aiida.orm import RemoteData diff --git a/tests/orm/nodes/data/test_remote_stash.py b/tests/orm/nodes/data/test_remote_stash.py index bc2c49e452..0cb555f0e4 100644 --- a/tests/orm/nodes/data/test_remote_stash.py +++ b/tests/orm/nodes/data/test_remote_stash.py @@ -9,6 +9,7 @@ """Tests for the :mod:`aiida.orm.nodes.data.remote.stash` module.""" import pytest + from aiida.common.datastructures import StashMode from aiida.common.exceptions import StoringNotAllowed from aiida.orm import RemoteStashData, RemoteStashFolderData diff --git a/tests/orm/nodes/data/test_singlefile.py b/tests/orm/nodes/data/test_singlefile.py index 373b91eabb..304050648a 100644 --- a/tests/orm/nodes/data/test_singlefile.py +++ b/tests/orm/nodes/data/test_singlefile.py @@ -14,6 +14,7 @@ import tempfile import pytest + from aiida.orm import SinglefileData, load_node diff --git a/tests/orm/nodes/data/test_structure.py b/tests/orm/nodes/data/test_structure.py index 45df92dff6..47cfbec4e9 100644 --- a/tests/orm/nodes/data/test_structure.py +++ b/tests/orm/nodes/data/test_structure.py @@ -9,6 +9,7 @@ """Tests for StructureData-related functions.""" import pytest + from aiida.orm.nodes.data.structure import StructureData, get_formula diff --git a/tests/orm/nodes/data/test_to_aiida_type.py b/tests/orm/nodes/data/test_to_aiida_type.py index f063a83943..7f1afd93ee 100644 --- a/tests/orm/nodes/data/test_to_aiida_type.py +++ b/tests/orm/nodes/data/test_to_aiida_type.py @@ -10,6 +10,7 @@ import numpy import pytest + from aiida import orm from aiida.common.links import LinkType diff --git a/tests/orm/nodes/data/test_trajectory.py b/tests/orm/nodes/data/test_trajectory.py index 0594677fa0..eb0a78384a 100644 --- a/tests/orm/nodes/data/test_trajectory.py +++ b/tests/orm/nodes/data/test_trajectory.py @@ -2,6 +2,7 @@ import numpy as np import pytest + from aiida.orm import StructureData, TrajectoryData, load_node diff --git a/tests/orm/nodes/data/test_upf.py b/tests/orm/nodes/data/test_upf.py index 94e550ddad..fb0ed01abd 100644 --- a/tests/orm/nodes/data/test_upf.py +++ b/tests/orm/nodes/data/test_upf.py @@ -14,11 +14,11 @@ import numpy import pytest +from numpy import array, isclose + from aiida import orm from aiida.common.exceptions import ParsingError from aiida.orm.nodes.data.upf import parse_upf -from numpy import array, isclose - from tests.static import STATIC_DIR diff --git a/tests/orm/nodes/data/test_xy.py b/tests/orm/nodes/data/test_xy.py index c2d1a12ed9..3842157b78 100644 --- a/tests/orm/nodes/data/test_xy.py +++ b/tests/orm/nodes/data/test_xy.py @@ -10,6 +10,7 @@ import numpy import pytest + from aiida.common.exceptions import NotExistent from aiida.orm import XyData, load_node diff --git a/tests/orm/nodes/process/test_process.py b/tests/orm/nodes/process/test_process.py index 7fe3f30703..ada60bbadc 100644 --- a/tests/orm/nodes/process/test_process.py +++ b/tests/orm/nodes/process/test_process.py @@ -1,6 +1,7 @@ """Tests for :mod:`aiida.orm.nodes.process.process`.""" import pytest + from aiida.engine import ExitCode, ProcessState, launch from aiida.orm import Int from aiida.orm.nodes.caching import NodeCaching diff --git a/tests/orm/nodes/test_calcjob.py b/tests/orm/nodes/test_calcjob.py index 6d5006996e..5321368d6d 100644 --- a/tests/orm/nodes/test_calcjob.py +++ b/tests/orm/nodes/test_calcjob.py @@ -11,6 +11,7 @@ import io import pytest + from aiida.common import CalcJobState, LinkType from aiida.orm import CalcJobNode, FolderData diff --git a/tests/orm/nodes/test_node.py b/tests/orm/nodes/test_node.py index ddcc586b84..6b52875c78 100644 --- a/tests/orm/nodes/test_node.py +++ b/tests/orm/nodes/test_node.py @@ -14,6 +14,7 @@ from io import BytesIO import pytest + from aiida.common import LinkType, exceptions, timezone from aiida.manage import get_manager from aiida.orm import CalculationNode, Computer, Data, Int, Log, Node, User, WorkflowNode, load_node diff --git a/tests/orm/nodes/test_repository.py b/tests/orm/nodes/test_repository.py index 08d1eb6268..062fa6ad9f 100644 --- a/tests/orm/nodes/test_repository.py +++ b/tests/orm/nodes/test_repository.py @@ -3,6 +3,7 @@ import pathlib import pytest + from aiida.common import exceptions from aiida.common.warnings import AiidaDeprecationWarning from aiida.engine import ProcessState diff --git a/tests/orm/test_authinfos.py b/tests/orm/test_authinfos.py index ef0bfaacc3..c64274c720 100644 --- a/tests/orm/test_authinfos.py +++ b/tests/orm/test_authinfos.py @@ -9,6 +9,7 @@ """Unit tests for the AuthInfo ORM class.""" import pytest + from aiida.common import exceptions from aiida.orm import authinfos, computers diff --git a/tests/orm/test_autogroups.py b/tests/orm/test_autogroups.py index 8d4b310f7c..484c49ab7e 100644 --- a/tests/orm/test_autogroups.py +++ b/tests/orm/test_autogroups.py @@ -9,6 +9,7 @@ """Tests for the Autogroup functionality.""" import pytest + from aiida.orm import AutoGroup, QueryBuilder from aiida.orm.autogroup import AutogroupManager diff --git a/tests/orm/test_comments.py b/tests/orm/test_comments.py index 23bad1423c..de9db9c30b 100644 --- a/tests/orm/test_comments.py +++ b/tests/orm/test_comments.py @@ -9,6 +9,7 @@ """Unit tests for the Comment ORM class.""" import pytest + from aiida import orm from aiida.common import exceptions from aiida.orm.comments import Comment diff --git a/tests/orm/test_computers.py b/tests/orm/test_computers.py index adb1cbaae8..cd26797f1c 100644 --- a/tests/orm/test_computers.py +++ b/tests/orm/test_computers.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida.common import exceptions from aiida.orm import AuthInfo, Computer, User from aiida.plugins import TransportFactory diff --git a/tests/orm/test_entities.py b/tests/orm/test_entities.py index aa6e5705cb..00fee47612 100644 --- a/tests/orm/test_entities.py +++ b/tests/orm/test_entities.py @@ -11,6 +11,7 @@ import pickle import pytest + from aiida import orm from aiida.common.exceptions import InvalidOperation diff --git a/tests/orm/test_fields.py b/tests/orm/test_fields.py index df74f58b07..0d5e934f8b 100644 --- a/tests/orm/test_fields.py +++ b/tests/orm/test_fields.py @@ -9,10 +9,11 @@ """Test for entity fields""" import pytest +from importlib_metadata import entry_points + from aiida import orm from aiida.orm.fields import add_field from aiida.plugins import load_entry_point -from importlib_metadata import entry_points EPS = entry_points() diff --git a/tests/orm/test_groups.py b/tests/orm/test_groups.py index b70758b08e..c62f903400 100644 --- a/tests/orm/test_groups.py +++ b/tests/orm/test_groups.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida import orm from aiida.common import exceptions from aiida.tools.graph.deletions import delete_nodes diff --git a/tests/orm/test_logs.py b/tests/orm/test_logs.py index 7767510639..cbb3863b04 100644 --- a/tests/orm/test_logs.py +++ b/tests/orm/test_logs.py @@ -12,6 +12,7 @@ import logging import pytest + from aiida import orm from aiida.common import exceptions from aiida.common.log import LOG_LEVEL_REPORT diff --git a/tests/orm/test_mixins.py b/tests/orm/test_mixins.py index b72b87cf23..7c6cce79c0 100644 --- a/tests/orm/test_mixins.py +++ b/tests/orm/test_mixins.py @@ -9,6 +9,7 @@ """Tests for the ORM mixin classes.""" import pytest + from aiida.common import exceptions from aiida.common.links import LinkType from aiida.orm import CalculationNode, Int diff --git a/tests/orm/test_querybuilder.py b/tests/orm/test_querybuilder.py index 8797fe4e03..c434c93411 100644 --- a/tests/orm/test_querybuilder.py +++ b/tests/orm/test_querybuilder.py @@ -17,6 +17,7 @@ from itertools import chain import pytest + from aiida import orm, plugins from aiida.common.links import LinkType from aiida.orm.querybuilder import _get_ormclass diff --git a/tests/orm/utils/test_calcjob.py b/tests/orm/utils/test_calcjob.py index 10f1a0a7bf..46c68bc9f2 100644 --- a/tests/orm/utils/test_calcjob.py +++ b/tests/orm/utils/test_calcjob.py @@ -9,6 +9,7 @@ """Tests for the `CalcJob` utils.""" import pytest + from aiida.common.links import LinkType from aiida.orm import Dict from aiida.orm.utils.calcjob import CalcJobResultManager diff --git a/tests/orm/utils/test_loaders.py b/tests/orm/utils/test_loaders.py index 7429e92034..63ca1afa4d 100644 --- a/tests/orm/utils/test_loaders.py +++ b/tests/orm/utils/test_loaders.py @@ -9,6 +9,7 @@ """Module to test orm utilities to load nodes, codes etc.""" import pytest + from aiida.common.exceptions import NotExistent from aiida.orm import Data, Group, Node from aiida.orm.utils import load_code, load_computer, load_entity, load_group, load_node diff --git a/tests/orm/utils/test_managers.py b/tests/orm/utils/test_managers.py index ea9806093b..74975aec02 100644 --- a/tests/orm/utils/test_managers.py +++ b/tests/orm/utils/test_managers.py @@ -11,6 +11,7 @@ import warnings import pytest + from aiida import orm from aiida.common import AttributeDict, LinkType from aiida.common.exceptions import NotExistent, NotExistentAttributeError, NotExistentKeyError diff --git a/tests/orm/utils/test_node.py b/tests/orm/utils/test_node.py index c2f0f608a2..780aa155f0 100644 --- a/tests/orm/utils/test_node.py +++ b/tests/orm/utils/test_node.py @@ -9,6 +9,7 @@ """Tests for the `Node` utils.""" import pytest + from aiida.orm import Data from aiida.orm.utils.node import load_node_class diff --git a/tests/orm/utils/test_serialize.py b/tests/orm/utils/test_serialize.py index 231a25d1bb..3375a9e3c6 100644 --- a/tests/orm/utils/test_serialize.py +++ b/tests/orm/utils/test_serialize.py @@ -14,6 +14,7 @@ import numpy as np import pytest + from aiida import orm from aiida.common.links import LinkType from aiida.orm.utils import serialize diff --git a/tests/parsers/test_parser.py b/tests/parsers/test_parser.py index f298acfe85..e4b6af5be9 100644 --- a/tests/parsers/test_parser.py +++ b/tests/parsers/test_parser.py @@ -11,6 +11,7 @@ import io import pytest + from aiida import orm from aiida.common import LinkType from aiida.engine import CalcJob diff --git a/tests/plugins/test_entry_point.py b/tests/plugins/test_entry_point.py index 2570194ff3..ebab1b138c 100644 --- a/tests/plugins/test_entry_point.py +++ b/tests/plugins/test_entry_point.py @@ -9,12 +9,13 @@ """Tests for the :mod:`~aiida.plugins.entry_point` module.""" import pytest +from importlib_metadata import EntryPoint as EP # noqa: N817 +from importlib_metadata import EntryPoints + from aiida.common.exceptions import MissingEntryPointError, MultipleEntryPointError from aiida.common.warnings import AiidaDeprecationWarning from aiida.plugins import entry_point from aiida.plugins.entry_point import get_entry_point, validate_registered_entry_points -from importlib_metadata import EntryPoint as EP # noqa: N817 -from importlib_metadata import EntryPoints def test_validate_registered_entry_points(): diff --git a/tests/plugins/test_factories.py b/tests/plugins/test_factories.py index 5ad4a83c13..49f3140fbe 100644 --- a/tests/plugins/test_factories.py +++ b/tests/plugins/test_factories.py @@ -9,6 +9,7 @@ """Tests for the :py:mod:`~aiida.plugins.factories` module.""" import pytest + from aiida.common.exceptions import InvalidEntryPointTypeError from aiida.engine import CalcJob, CalcJobImporter, WorkChain, calcfunction, workfunction from aiida.orm import CalcFunctionNode, Data, Node, WorkFunctionNode diff --git a/tests/plugins/test_utils.py b/tests/plugins/test_utils.py index 485291e8d6..5c1d248a26 100644 --- a/tests/plugins/test_utils.py +++ b/tests/plugins/test_utils.py @@ -9,6 +9,7 @@ """Tests for utilities dealing with plugins and entry points.""" import pytest + from aiida import __version__ as version_core from aiida.common.exceptions import EntryPointError from aiida.engine import WorkChain, calcfunction diff --git a/tests/repository/backend/test_abstract.py b/tests/repository/backend/test_abstract.py index a7ae0bff5d..aca48217cb 100644 --- a/tests/repository/backend/test_abstract.py +++ b/tests/repository/backend/test_abstract.py @@ -5,6 +5,7 @@ from typing import BinaryIO, Iterable, List, Optional import pytest + from aiida.repository.backend import AbstractRepositoryBackend diff --git a/tests/repository/backend/test_disk_object_store.py b/tests/repository/backend/test_disk_object_store.py index 61654fbd6f..071cbe747d 100644 --- a/tests/repository/backend/test_disk_object_store.py +++ b/tests/repository/backend/test_disk_object_store.py @@ -4,6 +4,7 @@ import pathlib import pytest + from aiida.repository.backend.disk_object_store import DiskObjectStoreRepositoryBackend diff --git a/tests/repository/backend/test_sandbox.py b/tests/repository/backend/test_sandbox.py index 3409cf4a77..360a6dc613 100644 --- a/tests/repository/backend/test_sandbox.py +++ b/tests/repository/backend/test_sandbox.py @@ -4,6 +4,7 @@ import pathlib import pytest + from aiida.repository.backend.sandbox import SandboxRepositoryBackend diff --git a/tests/repository/test_common.py b/tests/repository/test_common.py index 061c5d7406..cc97014f4a 100644 --- a/tests/repository/test_common.py +++ b/tests/repository/test_common.py @@ -1,6 +1,7 @@ """Tests for the :mod:`aiida.repository.common` module.""" import pytest + from aiida.repository import File, FileType diff --git a/tests/repository/test_repository.py b/tests/repository/test_repository.py index 2f829f42de..f86b2456a8 100644 --- a/tests/repository/test_repository.py +++ b/tests/repository/test_repository.py @@ -6,6 +6,7 @@ import typing as t import pytest + from aiida.repository import File, FileType, Repository from aiida.repository.backend import DiskObjectStoreRepositoryBackend, SandboxRepositoryBackend diff --git a/tests/restapi/conftest.py b/tests/restapi/conftest.py index 8c8b69fa3b..8abe8f9296 100644 --- a/tests/restapi/conftest.py +++ b/tests/restapi/conftest.py @@ -14,9 +14,10 @@ @pytest.fixture(scope='function') def restapi_server(): """Make REST API server""" + from werkzeug.serving import make_server + from aiida.restapi.common.config import CLI_DEFAULTS from aiida.restapi.run_api import configure_api - from werkzeug.serving import make_server def _restapi_server(restapi=None): if restapi is None: diff --git a/tests/restapi/test_config.py b/tests/restapi/test_config.py index a0a81c95dd..cf5ef4fac9 100644 --- a/tests/restapi/test_config.py +++ b/tests/restapi/test_config.py @@ -9,6 +9,7 @@ """Tests for the configuration options from `aiida.restapi.common.config` when running the REST API.""" import pytest + from aiida import orm from aiida.manage import get_manager diff --git a/tests/restapi/test_identifiers.py b/tests/restapi/test_identifiers.py index 21a65b8a00..8e3f245f6b 100644 --- a/tests/restapi/test_identifiers.py +++ b/tests/restapi/test_identifiers.py @@ -12,6 +12,7 @@ import pytest import requests + from aiida import orm from aiida.restapi.common.identifiers import FULL_TYPE_CONCATENATOR, LIKE_OPERATOR_CHARACTER, get_full_type_filters diff --git a/tests/restapi/test_routes.py b/tests/restapi/test_routes.py index 9b8649961f..50f4d2f2c8 100644 --- a/tests/restapi/test_routes.py +++ b/tests/restapi/test_routes.py @@ -14,12 +14,13 @@ import numpy as np import pytest +from flask_cors.core import ACL_ORIGIN + from aiida import orm from aiida.common.links import LinkType from aiida.manage import get_manager from aiida.orm.nodes.data.array.array import clean_array from aiida.restapi.run_api import configure_api -from flask_cors.core import ACL_ORIGIN class TestRestApi: diff --git a/tests/schedulers/test_all.py b/tests/schedulers/test_all.py index 2259b65725..702c212bf3 100644 --- a/tests/schedulers/test_all.py +++ b/tests/schedulers/test_all.py @@ -11,6 +11,7 @@ from __future__ import annotations import pytest + from aiida.common.datastructures import CodeRunMode from aiida.plugins import SchedulerFactory, entry_point from aiida.schedulers import Scheduler diff --git a/tests/schedulers/test_datastructures.py b/tests/schedulers/test_datastructures.py index afe0c23d3d..81d69c7190 100644 --- a/tests/schedulers/test_datastructures.py +++ b/tests/schedulers/test_datastructures.py @@ -9,6 +9,7 @@ """Tests for the :mod:`aiida.schedulers.test_datastructures` module.""" import pytest + from aiida.schedulers.datastructures import NodeNumberJobResource, ParEnvJobResource diff --git a/tests/schedulers/test_direct.py b/tests/schedulers/test_direct.py index 175f5ede48..2a01883f8a 100644 --- a/tests/schedulers/test_direct.py +++ b/tests/schedulers/test_direct.py @@ -9,6 +9,7 @@ """Tests for the ``DirectScheduler`` plugin.""" import pytest + from aiida.common.datastructures import CodeRunMode from aiida.schedulers import SchedulerError from aiida.schedulers.datastructures import JobTemplate, JobTemplateCodeInfo @@ -85,9 +86,10 @@ def test_kill_job(scheduler, tmpdir): import multiprocessing import time - from aiida.transports.plugins.local import LocalTransport from psutil import Process + from aiida.transports.plugins.local import LocalTransport + def run_sleep_100(): import subprocess diff --git a/tests/schedulers/test_lsf.py b/tests/schedulers/test_lsf.py index 013298f8ac..af7b8274bc 100644 --- a/tests/schedulers/test_lsf.py +++ b/tests/schedulers/test_lsf.py @@ -12,6 +12,7 @@ import uuid import pytest + from aiida.common.exceptions import ConfigurationError from aiida.schedulers.datastructures import JobState from aiida.schedulers.plugins.lsf import LsfScheduler diff --git a/tests/schedulers/test_slurm.py b/tests/schedulers/test_slurm.py index 1ba991a5d3..4da51c331c 100644 --- a/tests/schedulers/test_slurm.py +++ b/tests/schedulers/test_slurm.py @@ -15,6 +15,7 @@ import uuid import pytest + from aiida.engine import CalcJob from aiida.schedulers import JobState, SchedulerError from aiida.schedulers.plugins.slurm import SlurmJobResource, SlurmScheduler diff --git a/tests/storage/psql_dos/migrations/conftest.py b/tests/storage/psql_dos/migrations/conftest.py index 5ca772dc5f..9b3f38e287 100644 --- a/tests/storage/psql_dos/migrations/conftest.py +++ b/tests/storage/psql_dos/migrations/conftest.py @@ -11,11 +11,12 @@ from uuid import uuid4 import pytest +from pgtest.pgtest import PGTest +from sqlalchemy import text + from aiida.manage.configuration import Profile from aiida.storage.psql_dos.migrator import PsqlDosMigrator from aiida.storage.psql_dos.utils import create_sqlalchemy_engine -from pgtest.pgtest import PGTest -from sqlalchemy import text @pytest.fixture(scope='session') diff --git a/tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py b/tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py index 0f3cdf5e9f..dbe200d0be 100644 --- a/tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py +++ b/tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py @@ -10,6 +10,7 @@ import numpy import pytest + from aiida.common import timezone from aiida.common.utils import get_new_uuid from aiida.storage.psql_dos.backend import get_filepath_container diff --git a/tests/storage/psql_dos/migrations/django_branch/test_0037_attributes_extras_settings_json.py b/tests/storage/psql_dos/migrations/django_branch/test_0037_attributes_extras_settings_json.py index b81681a892..a8edbc8557 100644 --- a/tests/storage/psql_dos/migrations/django_branch/test_0037_attributes_extras_settings_json.py +++ b/tests/storage/psql_dos/migrations/django_branch/test_0037_attributes_extras_settings_json.py @@ -10,9 +10,10 @@ from datetime import datetime +from sqlalchemy import select + from aiida.common import timezone from aiida.storage.psql_dos.migrator import PsqlDosMigrator -from sqlalchemy import select def test_attr_extra_migration(perform_migrations: PsqlDosMigrator): diff --git a/tests/storage/psql_dos/migrations/django_branch/test_legacy.py b/tests/storage/psql_dos/migrations/django_branch/test_legacy.py index ce30a70f15..504f6360e9 100644 --- a/tests/storage/psql_dos/migrations/django_branch/test_legacy.py +++ b/tests/storage/psql_dos/migrations/django_branch/test_legacy.py @@ -19,6 +19,7 @@ """ import sqlalchemy as sa + from aiida.storage.psql_dos.migrator import PsqlDosMigrator diff --git a/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_4_dblog_update.py b/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_4_dblog_update.py index 91f6121ea0..0a2e6f5e16 100644 --- a/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_4_dblog_update.py +++ b/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_4_dblog_update.py @@ -12,9 +12,10 @@ import json import pytest +from sqlalchemy import column + from aiida.storage.psql_dos.migrations.utils import dblog_update from aiida.storage.psql_dos.migrator import PsqlDosMigrator -from sqlalchemy import column # The values that will be exported for the log records that will be deleted values_to_export = ('id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata') diff --git a/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_6_trajectory_data.py b/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_6_trajectory_data.py index 0f1ccd3288..d83987c185 100644 --- a/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_6_trajectory_data.py +++ b/tests/storage/psql_dos/migrations/sqlalchemy_branch/test_6_trajectory_data.py @@ -12,6 +12,7 @@ import numpy as np import pytest + from aiida.storage.psql_dos.backend import get_filepath_container from aiida.storage.psql_dos.migrations.utils import utils from aiida.storage.psql_dos.migrator import PsqlDosMigrator diff --git a/tests/storage/psql_dos/migrations/test_all_schema.py b/tests/storage/psql_dos/migrations/test_all_schema.py index 7af6174cdc..36f6096496 100644 --- a/tests/storage/psql_dos/migrations/test_all_schema.py +++ b/tests/storage/psql_dos/migrations/test_all_schema.py @@ -9,6 +9,7 @@ """Basic tests for all migrations""" import pytest + from aiida.storage.psql_dos.migrator import PsqlDosMigrator diff --git a/tests/storage/psql_dos/test_alembic_cli.py b/tests/storage/psql_dos/test_alembic_cli.py index 709c10bfbf..a31e90ca76 100644 --- a/tests/storage/psql_dos/test_alembic_cli.py +++ b/tests/storage/psql_dos/test_alembic_cli.py @@ -8,9 +8,10 @@ ########################################################################### """Basic tests for the alembic_cli module.""" -from aiida.storage.psql_dos.alembic_cli import alembic_cli from click.testing import CliRunner +from aiida.storage.psql_dos.alembic_cli import alembic_cli + def test_history(): """Test the 'history' command.""" diff --git a/tests/storage/psql_dos/test_backend.py b/tests/storage/psql_dos/test_backend.py index bbc77b1a14..fd55e38a14 100644 --- a/tests/storage/psql_dos/test_backend.py +++ b/tests/storage/psql_dos/test_backend.py @@ -9,6 +9,7 @@ """Testing the general methods of the psql_dos backend.""" import pytest + from aiida.manage import get_manager from aiida.orm import User diff --git a/tests/storage/psql_dos/test_nodes.py b/tests/storage/psql_dos/test_nodes.py index 1bbed60601..6a7090d971 100644 --- a/tests/storage/psql_dos/test_nodes.py +++ b/tests/storage/psql_dos/test_nodes.py @@ -9,6 +9,7 @@ """Tests for nodes, attributes and links.""" import pytest + from aiida import orm from aiida.orm import Data, load_node diff --git a/tests/storage/psql_dos/test_query.py b/tests/storage/psql_dos/test_query.py index 935b55636d..4a0390a9d0 100644 --- a/tests/storage/psql_dos/test_query.py +++ b/tests/storage/psql_dos/test_query.py @@ -9,6 +9,7 @@ """Tests for generic queries.""" import pytest + from aiida.orm import Computer, Data, Group, Node, ProcessNode, QueryBuilder, User diff --git a/tests/storage/psql_dos/test_schema.py b/tests/storage/psql_dos/test_schema.py index ef73855b27..9a2c23e365 100644 --- a/tests/storage/psql_dos/test_schema.py +++ b/tests/storage/psql_dos/test_schema.py @@ -10,13 +10,14 @@ import warnings +from sqlalchemy import exc as sa_exc + from aiida.common.links import LinkType from aiida.common.utils import get_new_uuid from aiida.manage import get_manager from aiida.orm import CalculationNode, Data from aiida.storage.psql_dos.models.node import DbNode from aiida.storage.psql_dos.models.user import DbUser -from sqlalchemy import exc as sa_exc class TestRelationshipsSQLA: diff --git a/tests/storage/psql_dos/test_session.py b/tests/storage/psql_dos/test_session.py index cd237a86ec..8092d18962 100644 --- a/tests/storage/psql_dos/test_session.py +++ b/tests/storage/psql_dos/test_session.py @@ -11,9 +11,10 @@ import uuid import pytest -from aiida.storage.psql_dos.utils import create_scoped_session_factory from sqlalchemy.orm import sessionmaker +from aiida.storage.psql_dos.utils import create_scoped_session_factory + class TestSessionSqla: """The following tests check that the session works as expected in some diff --git a/tests/storage/sqlite/test_orm.py b/tests/storage/sqlite/test_orm.py index 0d859d6bac..8a03ce034f 100644 --- a/tests/storage/sqlite/test_orm.py +++ b/tests/storage/sqlite/test_orm.py @@ -11,6 +11,7 @@ import json import pytest + from aiida.orm import Dict, QueryBuilder from aiida.storage.sqlite_temp import SqliteTempBackend diff --git a/tests/storage/sqlite_dos/migrations/conftest.py b/tests/storage/sqlite_dos/migrations/conftest.py index bba974705f..5cc74616e2 100644 --- a/tests/storage/sqlite_dos/migrations/conftest.py +++ b/tests/storage/sqlite_dos/migrations/conftest.py @@ -12,9 +12,10 @@ import pathlib import pytest +from sqlalchemy import text + from aiida.manage.configuration import Profile from aiida.storage.sqlite_zip.utils import create_sqla_engine -from sqlalchemy import text @pytest.fixture diff --git a/tests/storage/sqlite_dos/migrations/test_all_schema.py b/tests/storage/sqlite_dos/migrations/test_all_schema.py index 51351f918e..fab3221fbd 100644 --- a/tests/storage/sqlite_dos/migrations/test_all_schema.py +++ b/tests/storage/sqlite_dos/migrations/test_all_schema.py @@ -9,6 +9,7 @@ """Basic tests for all migrations""" import pytest + from aiida.storage.sqlite_dos.backend import SqliteDosMigrator diff --git a/tests/storage/sqlite_dos/test_backend.py b/tests/storage/sqlite_dos/test_backend.py index 61988db1e9..cabf692bf6 100644 --- a/tests/storage/sqlite_dos/test_backend.py +++ b/tests/storage/sqlite_dos/test_backend.py @@ -3,6 +3,7 @@ import pathlib import pytest + from aiida.storage.sqlite_dos.backend import FILENAME_CONTAINER, FILENAME_DATABASE, SqliteDosStorage @@ -18,7 +19,6 @@ def test_archive_import(aiida_config, aiida_profile_factory): """Test that archives can be imported.""" from aiida.orm import Node, QueryBuilder from aiida.tools.archive.imports import import_archive - from tests.utils.archives import get_archive_file with aiida_profile_factory(aiida_config, storage_backend='core.sqlite_dos'): diff --git a/tests/storage/sqlite_zip/test_backend.py b/tests/storage/sqlite_zip/test_backend.py index 7b42f8455c..64fca2844f 100644 --- a/tests/storage/sqlite_zip/test_backend.py +++ b/tests/storage/sqlite_zip/test_backend.py @@ -3,9 +3,10 @@ import pathlib import pytest +from pydantic_core import ValidationError + from aiida.storage.sqlite_zip.backend import SqliteZipBackend from aiida.storage.sqlite_zip.migrator import validate_storage -from pydantic_core import ValidationError def test_initialise(tmp_path, caplog): diff --git a/tests/test_calculation_node.py b/tests/test_calculation_node.py index d1120499f8..2bd1bd3e32 100644 --- a/tests/test_calculation_node.py +++ b/tests/test_calculation_node.py @@ -9,6 +9,7 @@ """Tests for the CalculationNode and CalcJobNode class.""" import pytest + from aiida.common.datastructures import CalcJobState from aiida.common.exceptions import ModificationNotAllowed from aiida.orm import CalcJobNode, CalculationNode diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 22d990d323..59d56a5708 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -1,9 +1,10 @@ """Tests for fixtures in the ``conftest.py``.""" import pytest +from importlib_metadata import EntryPoint + from aiida.common.exceptions import MissingEntryPointError from aiida.plugins.entry_point import get_entry_point, load_entry_point -from importlib_metadata import EntryPoint ENTRY_POINT_GROUP = 'aiida.calculations.importers' diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 799b6454d7..b4e9ef402c 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -14,6 +14,7 @@ import numpy as np import pytest + from aiida.common.exceptions import ModificationNotAllowed from aiida.common.utils import Capturing from aiida.orm import ArrayData, BandsData, CifData, Dict, KpointsData, StructureData, TrajectoryData, load_node diff --git a/tests/test_generic.py b/tests/test_generic.py index 8f434130bb..6ccac8770c 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -9,6 +9,7 @@ """Generic tests that need the use of the DB.""" import pytest + from aiida import orm diff --git a/tests/test_nodes.py b/tests/test_nodes.py index 6f64ab6d2d..ea85e6bb22 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -11,6 +11,7 @@ import tempfile import pytest + from aiida import get_profile, orm from aiida.common.exceptions import InvalidOperation, ModificationNotAllowed, StoringNotAllowed, ValidationError from aiida.common.links import LinkType diff --git a/tests/tools/archive/conftest.py b/tests/tools/archive/conftest.py index b65970fa23..108b768bc7 100644 --- a/tests/tools/archive/conftest.py +++ b/tests/tools/archive/conftest.py @@ -11,6 +11,7 @@ import logging import pytest + from aiida.tools.archive import EXPORT_LOGGER, IMPORT_LOGGER diff --git a/tests/tools/archive/migration/conftest.py b/tests/tools/archive/migration/conftest.py index a324ff512a..2e599d7bdd 100644 --- a/tests/tools/archive/migration/conftest.py +++ b/tests/tools/archive/migration/conftest.py @@ -9,8 +9,8 @@ """Module with tests for export archive migrations.""" import pytest -from aiida.storage.sqlite_zip.migrations.utils import verify_metadata_version +from aiida.storage.sqlite_zip.migrations.utils import verify_metadata_version from tests.utils.archives import get_archive_file, read_json_files diff --git a/tests/tools/archive/migration/test_legacy_funcs.py b/tests/tools/archive/migration/test_legacy_funcs.py index a3167a1ba8..d91e562fc5 100644 --- a/tests/tools/archive/migration/test_legacy_funcs.py +++ b/tests/tools/archive/migration/test_legacy_funcs.py @@ -9,10 +9,10 @@ """Test migrating all export archives included in `tests/static/export/migrate`.""" import pytest + from aiida import get_version from aiida.storage.sqlite_zip.migrations.legacy import LEGACY_MIGRATE_FUNCTIONS from aiida.storage.sqlite_zip.migrations.utils import verify_metadata_version - from tests.utils.archives import get_archive_file, read_json_files diff --git a/tests/tools/archive/migration/test_legacy_migrations.py b/tests/tools/archive/migration/test_legacy_migrations.py index fb4a95d235..70d686f001 100644 --- a/tests/tools/archive/migration/test_legacy_migrations.py +++ b/tests/tools/archive/migration/test_legacy_migrations.py @@ -9,10 +9,10 @@ """Test archive file migration from old export versions to the newest""" import pytest + from aiida import orm from aiida.common.exceptions import StorageMigrationError from aiida.tools.archive import ArchiveFormatSqlZip - from tests.utils.archives import get_archive_file # archives to test migration against diff --git a/tests/tools/archive/migration/test_legacy_to_main.py b/tests/tools/archive/migration/test_legacy_to_main.py index a0b51134a4..617abd1f78 100644 --- a/tests/tools/archive/migration/test_legacy_to_main.py +++ b/tests/tools/archive/migration/test_legacy_to_main.py @@ -9,9 +9,9 @@ """Test archive file migration from legacy format (JSON) to main format (SQLite).""" import pytest + from aiida.common.exceptions import StorageMigrationError from aiida.storage.sqlite_zip.migrator import migrate - from tests.utils.archives import get_archive_file diff --git a/tests/tools/archive/migration/test_prov_redesign.py b/tests/tools/archive/migration/test_prov_redesign.py index 6beb0de2e4..357a0b1b39 100644 --- a/tests/tools/archive/migration/test_prov_redesign.py +++ b/tests/tools/archive/migration/test_prov_redesign.py @@ -17,6 +17,7 @@ """ import pytest + from aiida import orm from aiida.tools.archive import create_archive, import_archive @@ -85,7 +86,6 @@ def test_base_data_type_change(tmp_path, aiida_profile): def test_node_process_type(aiida_profile, tmp_path): """Column `process_type` added to `Node` entity DB table""" from aiida.engine import run_get_node - from tests.utils.processes import AddProcess aiida_profile.reset_storage() diff --git a/tests/tools/archive/migration/test_v05_to_v06.py b/tests/tools/archive/migration/test_v05_to_v06.py index a7e72a62d6..11f7b5d351 100644 --- a/tests/tools/archive/migration/test_v05_to_v06.py +++ b/tests/tools/archive/migration/test_v05_to_v06.py @@ -11,7 +11,6 @@ from aiida.storage.psql_dos.migrations.utils.calc_state import STATE_MAPPING from aiida.storage.sqlite_zip.migrations.legacy import migrate_v5_to_v6 # type: ignore[attr-defined] from aiida.storage.sqlite_zip.migrations.utils import verify_metadata_version - from tests.utils.archives import get_archive_file, read_json_files diff --git a/tests/tools/archive/migration/test_v06_to_v07.py b/tests/tools/archive/migration/test_v06_to_v07.py index d617e2126e..687817137f 100644 --- a/tests/tools/archive/migration/test_v06_to_v07.py +++ b/tests/tools/archive/migration/test_v06_to_v07.py @@ -9,6 +9,7 @@ """Test archive file migration from export version 0.6 to 0.7""" import pytest + from aiida.common.exceptions import CorruptStorage from aiida.storage.sqlite_zip.migrations.legacy.v06_to_v07 import ( data_migration_legacy_process_attributes, diff --git a/tests/tools/archive/orm/test_authinfo.py b/tests/tools/archive/orm/test_authinfo.py index e20c51270a..650724a514 100644 --- a/tests/tools/archive/orm/test_authinfo.py +++ b/tests/tools/archive/orm/test_authinfo.py @@ -9,6 +9,7 @@ """orm.AuthInfo tests for the export and import routines""" import pytest + from aiida import orm from aiida.tools.archive import create_archive, import_archive from aiida.tools.archive.abstract import get_format diff --git a/tests/tools/archive/orm/test_calculations.py b/tests/tools/archive/orm/test_calculations.py index de7b9058fe..0f1c3bc63a 100644 --- a/tests/tools/archive/orm/test_calculations.py +++ b/tests/tools/archive/orm/test_calculations.py @@ -9,6 +9,7 @@ """orm.CalcNode tests for the export and import routines""" import pytest + from aiida import orm from aiida.common.exceptions import NotExistent from aiida.common.links import LinkType diff --git a/tests/tools/archive/orm/test_codes.py b/tests/tools/archive/orm/test_codes.py index 4ac55584d2..27ae5f7882 100644 --- a/tests/tools/archive/orm/test_codes.py +++ b/tests/tools/archive/orm/test_codes.py @@ -13,7 +13,6 @@ from aiida import orm from aiida.common.links import LinkType from aiida.tools.archive import create_archive, import_archive - from tests.tools.archive.utils import get_all_node_links diff --git a/tests/tools/archive/orm/test_comments.py b/tests/tools/archive/orm/test_comments.py index 61935827d2..eb55b21f6e 100644 --- a/tests/tools/archive/orm/test_comments.py +++ b/tests/tools/archive/orm/test_comments.py @@ -9,6 +9,7 @@ """orm.Comment tests for the export and import routines""" import pytest + from aiida import orm from aiida.tools.archive import create_archive, import_archive diff --git a/tests/tools/archive/orm/test_computers.py b/tests/tools/archive/orm/test_computers.py index e3807c9a48..928275d173 100644 --- a/tests/tools/archive/orm/test_computers.py +++ b/tests/tools/archive/orm/test_computers.py @@ -9,10 +9,10 @@ """orm.Computer tests for the export and import routines""" import pytest + from aiida import orm from aiida.tools.archive import create_archive, import_archive from aiida.tools.archive.imports import DUPLICATE_LABEL_TEMPLATE - from tests.utils.archives import import_test_archive diff --git a/tests/tools/archive/orm/test_extras.py b/tests/tools/archive/orm/test_extras.py index 8435e44e32..e36e49fde6 100644 --- a/tests/tools/archive/orm/test_extras.py +++ b/tests/tools/archive/orm/test_extras.py @@ -9,6 +9,7 @@ """Extras tests for the export and import routines""" import pytest + from aiida import orm from aiida.tools.archive import create_archive, import_archive diff --git a/tests/tools/archive/orm/test_groups.py b/tests/tools/archive/orm/test_groups.py index f4651248b6..e40e1ba4be 100644 --- a/tests/tools/archive/orm/test_groups.py +++ b/tests/tools/archive/orm/test_groups.py @@ -11,6 +11,7 @@ import uuid import pytest + from aiida import orm from aiida.common.links import LinkType from aiida.orm import load_group diff --git a/tests/tools/archive/orm/test_links.py b/tests/tools/archive/orm/test_links.py index 83203def2a..75e7ada226 100644 --- a/tests/tools/archive/orm/test_links.py +++ b/tests/tools/archive/orm/test_links.py @@ -12,7 +12,6 @@ from aiida.common.links import LinkType from aiida.orm.entities import EntityTypes from aiida.tools.archive import ArchiveFormatSqlZip, create_archive, import_archive - from tests.tools.archive.utils import get_all_node_links diff --git a/tests/tools/archive/test_abstract.py b/tests/tools/archive/test_abstract.py index e913e65141..943b55ce0c 100644 --- a/tests/tools/archive/test_abstract.py +++ b/tests/tools/archive/test_abstract.py @@ -14,6 +14,7 @@ from io import BytesIO import pytest + from aiida import orm from aiida.common.exceptions import IntegrityError from aiida.orm.entities import EntityTypes diff --git a/tests/tools/archive/test_backend.py b/tests/tools/archive/test_backend.py index 88bddb8ccd..1cf39d1fcf 100644 --- a/tests/tools/archive/test_backend.py +++ b/tests/tools/archive/test_backend.py @@ -9,11 +9,11 @@ """Test using the archive backend directly.""" import pytest + from aiida import orm from aiida.common.exceptions import NotExistent from aiida.orm.implementation import StorageBackend from aiida.tools.archive import ArchiveFormatSqlZip, ArchiveReaderAbstract - from tests.utils.archives import get_archive_file diff --git a/tests/tools/archive/test_complex.py b/tests/tools/archive/test_complex.py index 3bb132c308..c6206953b9 100644 --- a/tests/tools/archive/test_complex.py +++ b/tests/tools/archive/test_complex.py @@ -13,6 +13,7 @@ from datetime import datetime import numpy as np + from aiida import orm from aiida.common.exceptions import NotExistent from aiida.common.hashing import make_hash diff --git a/tests/tools/archive/test_schema.py b/tests/tools/archive/test_schema.py index b117a9553a..bd667a72a6 100644 --- a/tests/tools/archive/test_schema.py +++ b/tests/tools/archive/test_schema.py @@ -12,15 +12,15 @@ import pytest import yaml -from aiida import get_profile -from aiida.storage.psql_dos.utils import create_sqlalchemy_engine -from aiida.storage.sqlite_zip import models, utils -from aiida.storage.sqlite_zip.migrator import get_schema_version_head, migrate from archive_path import extract_file_in_zip from sqlalchemy import String, inspect from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.engine import Inspector +from aiida import get_profile +from aiida.storage.psql_dos.utils import create_sqlalchemy_engine +from aiida.storage.sqlite_zip import models, utils +from aiida.storage.sqlite_zip.migrator import get_schema_version_head, migrate from tests.utils.archives import get_archive_file diff --git a/tests/tools/archive/test_simple.py b/tests/tools/archive/test_simple.py index 0e97ad9409..2f7377eb22 100644 --- a/tests/tools/archive/test_simple.py +++ b/tests/tools/archive/test_simple.py @@ -12,11 +12,12 @@ import json import pytest +from archive_path import ZipPath + from aiida import orm from aiida.common.exceptions import IncompatibleStorageSchema, LicensingException from aiida.common.links import LinkType from aiida.tools.archive import create_archive, import_archive -from archive_path import ZipPath @pytest.mark.parametrize('entities', ['all', 'specific']) diff --git a/tests/tools/archive/test_specific_import.py b/tests/tools/archive/test_specific_import.py index c7fed34a8e..61abef951b 100644 --- a/tests/tools/archive/test_specific_import.py +++ b/tests/tools/archive/test_specific_import.py @@ -9,6 +9,7 @@ """Tests for the export and import routines""" import numpy as np + from aiida import orm from aiida.tools.archive import create_archive, import_archive diff --git a/tests/tools/archive/test_utils.py b/tests/tools/archive/test_utils.py index ee494c74fa..8a931b0a1a 100644 --- a/tests/tools/archive/test_utils.py +++ b/tests/tools/archive/test_utils.py @@ -8,9 +8,10 @@ ########################################################################### """Test utility functions.""" -from aiida.storage.sqlite_zip.migrations.utils import copy_tar_to_zip, copy_zip_to_zip from archive_path import TarPath, ZipPath +from aiida.storage.sqlite_zip.migrations.utils import copy_tar_to_zip, copy_zip_to_zip + def test_copy_zip_to_zip(tmp_path): """Test copying a zipfile to a new zipfile""" diff --git a/tests/tools/data/orbital/test_orbitals.py b/tests/tools/data/orbital/test_orbitals.py index 02e992c4c6..8dc239e5d0 100644 --- a/tests/tools/data/orbital/test_orbitals.py +++ b/tests/tools/data/orbital/test_orbitals.py @@ -10,6 +10,7 @@ """Test for the `Orbital` class and subclasses.""" import pytest + from aiida.common.exceptions import ValidationError from aiida.plugins import OrbitalFactory from aiida.tools.data.orbital import Orbital diff --git a/tests/tools/dbimporters/test_icsd.py b/tests/tools/dbimporters/test_icsd.py index ec6245f418..6c7f08fbae 100644 --- a/tests/tools/dbimporters/test_icsd.py +++ b/tests/tools/dbimporters/test_icsd.py @@ -11,6 +11,7 @@ import urllib.request import pytest + from aiida import get_profile from aiida.tools.dbimporters.plugins import icsd diff --git a/tests/tools/dbimporters/test_materialsproject.py b/tests/tools/dbimporters/test_materialsproject.py index cfef6c599b..dee9a1f1cc 100644 --- a/tests/tools/dbimporters/test_materialsproject.py +++ b/tests/tools/dbimporters/test_materialsproject.py @@ -9,6 +9,7 @@ """Module that contains the class definitions necessary to offer support for queries to Materials Project.""" import pytest + from aiida.plugins import DbImporterFactory diff --git a/tests/tools/dumping/test_processes.py b/tests/tools/dumping/test_processes.py index be05d50441..accfbd17d2 100644 --- a/tests/tools/dumping/test_processes.py +++ b/tests/tools/dumping/test_processes.py @@ -15,6 +15,7 @@ from pathlib import Path import pytest + from aiida.common.links import LinkType from aiida.tools.dumping.processes import ProcessDumper @@ -42,6 +43,7 @@ def generate_calculation_node_io(generate_calculation_node, tmp_path): def _generate_calculation_node_io(entry_point: str | None = None, attach_outputs: bool = True): import numpy as np + from aiida.orm import ArrayData, FolderData, SinglefileData singlefiledata_input = SinglefileData.from_string(content=filecontent, filename=filename) diff --git a/tests/tools/graph/test_age.py b/tests/tools/graph/test_age.py index 460c72c582..a025d16107 100644 --- a/tests/tools/graph/test_age.py +++ b/tests/tools/graph/test_age.py @@ -10,6 +10,7 @@ import numpy as np import pytest + from aiida import orm from aiida.common.links import LinkType from aiida.tools.graph.age_entities import AiidaEntitySet, Basket, DirectedEdgeSet, GroupNodeEdge diff --git a/tests/tools/graph/test_graph_traversers.py b/tests/tools/graph/test_graph_traversers.py index dd87ac1f94..615fe023f6 100644 --- a/tests/tools/graph/test_graph_traversers.py +++ b/tests/tools/graph/test_graph_traversers.py @@ -9,6 +9,7 @@ """Tests for aiida.tools.graph.graph_traversers""" import pytest + from aiida.common.links import LinkType from aiida.tools.graph.graph_traversers import get_nodes_delete, traverse_graph diff --git a/tests/tools/groups/test_paths.py b/tests/tools/groups/test_paths.py index 9879fb2910..1cad363d46 100644 --- a/tests/tools/groups/test_paths.py +++ b/tests/tools/groups/test_paths.py @@ -9,6 +9,7 @@ """Tests for GroupPath""" import pytest + from aiida import orm from aiida.tools.groups.paths import GroupAttr, GroupNotFoundError, GroupPath, InvalidPath, NoGroupsInPathError diff --git a/tests/tools/ipython/test_ipython_magics.py b/tests/tools/ipython/test_ipython_magics.py index 022e739fff..56232679cb 100644 --- a/tests/tools/ipython/test_ipython_magics.py +++ b/tests/tools/ipython/test_ipython_magics.py @@ -2,9 +2,10 @@ import textwrap -from aiida.tools.ipython.ipython_magics import register_ipython_extension from IPython.testing.globalipapp import get_ipython +from aiida.tools.ipython.ipython_magics import register_ipython_extension + def test_ipython_magics(): """Test that the ``%aiida`` magic can be loaded and imports the ``QueryBuilder`` and ``Node`` classes.""" diff --git a/tests/tools/visualization/test_graph.py b/tests/tools/visualization/test_graph.py index ba47b335b2..db082017a9 100644 --- a/tests/tools/visualization/test_graph.py +++ b/tests/tools/visualization/test_graph.py @@ -11,6 +11,7 @@ import re import pytest + from aiida import orm from aiida.common import AttributeDict from aiida.common.links import LinkType diff --git a/tests/transports/test_all_plugins.py b/tests/transports/test_all_plugins.py index 8aea2529cb..7b4fa6954f 100644 --- a/tests/transports/test_all_plugins.py +++ b/tests/transports/test_all_plugins.py @@ -21,6 +21,7 @@ import psutil import pytest + from aiida.plugins import SchedulerFactory, TransportFactory, entry_point from aiida.transports import Transport diff --git a/tests/transports/test_local.py b/tests/transports/test_local.py index 97e821e749..0765302dfb 100644 --- a/tests/transports/test_local.py +++ b/tests/transports/test_local.py @@ -11,6 +11,7 @@ import getpass import pytest + from aiida.transports.plugins.local import LocalTransport from aiida.transports.transport import TransportInternalError diff --git a/tests/transports/test_ssh.py b/tests/transports/test_ssh.py index 27698dfa54..68205900d3 100644 --- a/tests/transports/test_ssh.py +++ b/tests/transports/test_ssh.py @@ -12,6 +12,7 @@ import paramiko import pytest + from aiida.transports.plugins.ssh import SshTransport from aiida.transports.transport import TransportInternalError diff --git a/tests/utils/processes.py b/tests/utils/processes.py index 43582eea45..7471560d5d 100644 --- a/tests/utils/processes.py +++ b/tests/utils/processes.py @@ -9,6 +9,7 @@ """Utilities for testing components from the workflow engine""" import plumpy + from aiida.engine import Process from aiida.orm import Bool, CalcJobNode, Data, WorkflowNode diff --git a/tests/workflows/arithmetic/test_add_multiply.py b/tests/workflows/arithmetic/test_add_multiply.py index 4000617856..1de06ff369 100644 --- a/tests/workflows/arithmetic/test_add_multiply.py +++ b/tests/workflows/arithmetic/test_add_multiply.py @@ -9,6 +9,7 @@ """Tests for the `aiida.workflows.arithmetic.add_multiply` work function.""" import pytest + from aiida.orm import Int from aiida.plugins import WorkflowFactory from aiida.workflows.arithmetic.add_multiply import add_multiply diff --git a/utils/validate_consistency.py b/utils/validate_consistency.py index 297dfccab8..71fd64a98f 100644 --- a/utils/validate_consistency.py +++ b/utils/validate_consistency.py @@ -98,9 +98,10 @@ def cli(): @cli.command('verdi-autodocs') def validate_verdi_documentation(): """Auto-generate the documentation for `verdi` through `click`.""" - from aiida.cmdline.commands.cmd_verdi import verdi from click import Context + from aiida.cmdline.commands.cmd_verdi import verdi + width = 90 # The maximum width of the formatted help strings in characters # Set the `verdi data` command to isolated mode such that external plugin commands are not discovered From 090dc1c7300ab9d25ee75498d040ecf1cd3bb1d7 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Sun, 1 Dec 2024 02:33:33 +0100 Subject: [PATCH 3/5] DevOp: Using xdist to run pytest in parallel (#6620) --- .github/workflows/ci-code.yml | 4 ++-- .github/workflows/test-install.yml | 2 +- pyproject.toml | 1 + requirements/requirements-py-3.10.txt | 1 + requirements/requirements-py-3.11.txt | 1 + requirements/requirements-py-3.12.txt | 1 + requirements/requirements-py-3.9.txt | 1 + tests/orm/test_querybuilder.py | 4 ++++ tests/restapi/conftest.py | 16 ++++++++++++++-- tests/restapi/test_identifiers.py | 8 ++++++-- tests/restapi/test_statistics.py | 6 ++++-- tests/restapi/test_threaded_restapi.py | 6 ++++-- tests/tools/archive/orm/test_authinfo.py | 4 ++++ tests/tools/archive/orm/test_groups.py | 16 ++++++++-------- tests/tools/archive/orm/test_logs.py | 14 +++++++------- tests/tools/archive/test_simple.py | 4 ++-- tests/tools/groups/test_paths.py | 1 + tests/tools/visualization/test_graph.py | 3 ++- 18 files changed, 64 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index a8d0cb9a08..57e85bb004 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -104,7 +104,7 @@ jobs: AIIDA_WARN_v3: 1 # Python 3.12 has a performance regression when running with code coverage # so run code coverage only for python 3.9. - run: pytest --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }} + run: pytest -n auto --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }} - name: Upload coverage report if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core' @@ -139,7 +139,7 @@ jobs: - name: Run test suite env: AIIDA_WARN_v3: 0 - run: pytest -m 'presto' tests/ + run: pytest -n auto -m 'presto' tests/ verdi: diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index d315a50691..3abeb4a2ba 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -229,7 +229,7 @@ jobs: env: AIIDA_TEST_PROFILE: test_aiida AIIDA_WARN_v3: 1 - run: pytest --db-backend psql tests -m 'not nightly' tests/ + run: pytest -n auto --db-backend psql tests -m 'not nightly' tests/ - name: Freeze test environment run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt diff --git a/pyproject.toml b/pyproject.toml index 35f2f23b7f..7641b7ed72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -241,6 +241,7 @@ tests = [ 'pytest-rerunfailures~=12.0', 'pytest-benchmark~=4.0', 'pytest-regressions~=2.2', + 'pytest-xdist~=3.6', 'pympler~=1.0', 'coverage~=7.0', 'sphinx~=7.2.0', diff --git a/requirements/requirements-py-3.10.txt b/requirements/requirements-py-3.10.txt index b2408e8087..23e3b24c45 100644 --- a/requirements/requirements-py-3.10.txt +++ b/requirements/requirements-py-3.10.txt @@ -149,6 +149,7 @@ pytest-datadir==1.4.1 pytest-regressions==2.4.2 pytest-rerunfailures==12.0 pytest-timeout==2.2.0 +pytest-xdist==3.6.1 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 diff --git a/requirements/requirements-py-3.11.txt b/requirements/requirements-py-3.11.txt index 24acc25a6b..a0f1aa7a75 100644 --- a/requirements/requirements-py-3.11.txt +++ b/requirements/requirements-py-3.11.txt @@ -148,6 +148,7 @@ pytest-datadir==1.4.1 pytest-regressions==2.4.2 pytest-rerunfailures==12.0 pytest-timeout==2.2.0 +pytest-xdist==3.6.1 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 diff --git a/requirements/requirements-py-3.12.txt b/requirements/requirements-py-3.12.txt index 3f5d72ebb6..8047763fea 100644 --- a/requirements/requirements-py-3.12.txt +++ b/requirements/requirements-py-3.12.txt @@ -148,6 +148,7 @@ pytest-datadir==1.5.0 pytest-regressions==2.5.0 pytest-rerunfailures==12.0 pytest-timeout==2.2.0 +pytest-xdist==3.6.1 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 diff --git a/requirements/requirements-py-3.9.txt b/requirements/requirements-py-3.9.txt index 3087e62844..7f6c5bbc6f 100644 --- a/requirements/requirements-py-3.9.txt +++ b/requirements/requirements-py-3.9.txt @@ -151,6 +151,7 @@ pytest-datadir==1.4.1 pytest-regressions==2.4.2 pytest-rerunfailures==12.0 pytest-timeout==2.2.0 +pytest-xdist==3.6.1 python-dateutil==2.8.2 python-json-logger==2.0.7 python-memcached==1.59 diff --git a/tests/orm/test_querybuilder.py b/tests/orm/test_querybuilder.py index c434c93411..6d5d7cefc4 100644 --- a/tests/orm/test_querybuilder.py +++ b/tests/orm/test_querybuilder.py @@ -372,6 +372,7 @@ def test_dict_multiple_projections(self): assert dictionary['*'].pk == node.pk assert dictionary['id'] == node.pk + @pytest.mark.usefixtures('aiida_profile_clean') def test_operators_eq_lt_gt(self): nodes = [orm.Data() for _ in range(8)] @@ -394,6 +395,7 @@ def test_operators_eq_lt_gt(self): assert orm.QueryBuilder().append(orm.Node, filters={'attributes.fa': {'>': 1.02}}).count() == 4 assert orm.QueryBuilder().append(orm.Node, filters={'attributes.fa': {'>=': 1.02}}).count() == 5 + @pytest.mark.usefixtures('aiida_profile_clean') def test_subclassing(self): s = orm.StructureData() s.base.attributes.set('cat', 'miau') @@ -514,6 +516,7 @@ def test_append_validation(self): # So this should work now: qb.append(orm.StructureData, tag='s').limit(2).dict() + @pytest.mark.usefixtures('aiida_profile_clean') def test_tuples(self): """Test appending ``cls`` tuples.""" orm.Group(label='helloworld').store() @@ -696,6 +699,7 @@ def test_query_links(self): class TestMultipleProjections: """Unit tests for the QueryBuilder ORM class.""" + @pytest.mark.usefixtures('aiida_profile_clean') def test_first_multiple_projections(self): """Test `first()` returns correct types and numbers for multiple projections.""" orm.Data().store() diff --git a/tests/restapi/conftest.py b/tests/restapi/conftest.py index 8abe8f9296..442814a808 100644 --- a/tests/restapi/conftest.py +++ b/tests/restapi/conftest.py @@ -8,18 +8,27 @@ ########################################################################### """pytest fixtures for use with the aiida.restapi tests""" +from typing import Optional + import pytest @pytest.fixture(scope='function') def restapi_server(): """Make REST API server""" + import socket + from werkzeug.serving import make_server from aiida.restapi.common.config import CLI_DEFAULTS from aiida.restapi.run_api import configure_api def _restapi_server(restapi=None): + # Dynamically find a free port + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('', 0)) # Bind to a free port provided by the OS + _, port = sock.getsockname() # Get the dynamically assigned port + if restapi is None: flask_restapi = configure_api() else: @@ -27,7 +36,7 @@ def _restapi_server(restapi=None): return make_server( host=CLI_DEFAULTS['HOST_NAME'], - port=int(CLI_DEFAULTS['PORT']), + port=port, app=flask_restapi.app, threaded=True, processes=1, @@ -44,7 +53,10 @@ def _restapi_server(restapi=None): def server_url(): from aiida.restapi.common.config import API_CONFIG, CLI_DEFAULTS - return f"http://{CLI_DEFAULTS['HOST_NAME']}:{CLI_DEFAULTS['PORT']}{API_CONFIG['PREFIX']}" + def _server_url(hostname: Optional[str] = None, port: Optional[int] = None): + return f"http://{hostname or CLI_DEFAULTS['HOST_NAME']}:{port or CLI_DEFAULTS['PORT']}{API_CONFIG['PREFIX']}" + + return _server_url @pytest.fixture diff --git a/tests/restapi/test_identifiers.py b/tests/restapi/test_identifiers.py index 8e3f245f6b..0c593e433e 100644 --- a/tests/restapi/test_identifiers.py +++ b/tests/restapi/test_identifiers.py @@ -105,9 +105,11 @@ def test_full_type_unregistered(process_class, restapi_server, server_url): server = restapi_server() server_thread = Thread(target=server.serve_forever) + _server_url = server_url(port=server.server_port) + try: server_thread.start() - type_count_response = requests.get(f'{server_url}/nodes/full_types', timeout=10) + type_count_response = requests.get(f'{_server_url}/nodes/full_types', timeout=10) finally: server.shutdown() @@ -189,9 +191,11 @@ def test_full_type_backwards_compatibility(node_class, restapi_server, server_ur server = restapi_server() server_thread = Thread(target=server.serve_forever) + _server_url = server_url(port=server.server_port) + try: server_thread.start() - type_count_response = requests.get(f'{server_url}/nodes/full_types', timeout=10) + type_count_response = requests.get(f'{_server_url}/nodes/full_types', timeout=10) finally: server.shutdown() diff --git a/tests/restapi/test_statistics.py b/tests/restapi/test_statistics.py index e059de70ac..f125ccff1d 100644 --- a/tests/restapi/test_statistics.py +++ b/tests/restapi/test_statistics.py @@ -42,10 +42,12 @@ def test_count_consistency(restapi_server, server_url): server = restapi_server() server_thread = Thread(target=server.serve_forever) + _server_url = server_url(port=server.server_port) + try: server_thread.start() - type_count_response = requests.get(f'{server_url}/nodes/full_types_count', timeout=10) - statistics_response = requests.get(f'{server_url}/nodes/statistics', timeout=10) + type_count_response = requests.get(f'{_server_url}/nodes/full_types_count', timeout=10) + statistics_response = requests.get(f'{_server_url}/nodes/statistics', timeout=10) finally: server.shutdown() diff --git a/tests/restapi/test_threaded_restapi.py b/tests/restapi/test_threaded_restapi.py index bad1a8a76f..40a994776b 100644 --- a/tests/restapi/test_threaded_restapi.py +++ b/tests/restapi/test_threaded_restapi.py @@ -36,17 +36,19 @@ def test_run_threaded_server(restapi_server, server_url, aiida_localhost): This test will fail, if database connections are not being properly closed by the end-point calls. """ server = restapi_server() - computer_id = aiida_localhost.uuid # Create a thread that will contain the running server, # since we do not wish to block the main thread server_thread = Thread(target=server.serve_forever) + _server_url = server_url(port=server.server_port) + + computer_id = aiida_localhost.uuid try: server_thread.start() for _ in range(NO_OF_REQUESTS): - response = requests.get(f'{server_url}/computers/{computer_id}', timeout=10) + response = requests.get(f'{_server_url}/computers/{computer_id}', timeout=10) assert response.status_code == 200 diff --git a/tests/tools/archive/orm/test_authinfo.py b/tests/tools/archive/orm/test_authinfo.py index 650724a514..e2bb653cbb 100644 --- a/tests/tools/archive/orm/test_authinfo.py +++ b/tests/tools/archive/orm/test_authinfo.py @@ -16,6 +16,7 @@ @pytest.mark.usefixtures('aiida_localhost') +@pytest.mark.usefixtures('aiida_profile_clean') def test_create_all_no_authinfo(tmp_path): """Test archive creation that does not include authinfo.""" filename1 = tmp_path / 'export1.aiida' @@ -25,6 +26,7 @@ def test_create_all_no_authinfo(tmp_path): @pytest.mark.usefixtures('aiida_localhost') +@pytest.mark.usefixtures('aiida_profile_clean') def test_create_all_with_authinfo(tmp_path): """Test archive creation that does include authinfo.""" filename1 = tmp_path / 'export1.aiida' @@ -33,6 +35,7 @@ def test_create_all_with_authinfo(tmp_path): assert archive.querybuilder().append(orm.AuthInfo).count() == 1 +@pytest.mark.usefixtures('aiida_profile_clean') def test_create_comp_with_authinfo(tmp_path, aiida_localhost): """Test archive creation that does include authinfo.""" filename1 = tmp_path / 'export1.aiida' @@ -41,6 +44,7 @@ def test_create_comp_with_authinfo(tmp_path, aiida_localhost): assert archive.querybuilder().append(orm.AuthInfo).count() == 1 +@pytest.mark.usefixtures('aiida_profile_clean') def test_import_authinfo(aiida_profile, tmp_path, aiida_localhost): """Test archive import, including authinfo""" filename1 = tmp_path / 'export1.aiida' diff --git a/tests/tools/archive/orm/test_groups.py b/tests/tools/archive/orm/test_groups.py index e40e1ba4be..556d1617b3 100644 --- a/tests/tools/archive/orm/test_groups.py +++ b/tests/tools/archive/orm/test_groups.py @@ -18,7 +18,7 @@ from aiida.tools.archive import create_archive, import_archive -def test_nodes_in_group(aiida_profile, tmp_path, aiida_localhost): +def test_nodes_in_group(aiida_profile_clean, tmp_path, aiida_localhost): """This test checks that nodes that belong to a specific group are correctly imported and exported. """ @@ -52,7 +52,7 @@ def test_nodes_in_group(aiida_profile, tmp_path, aiida_localhost): filename1 = tmp_path / 'export1.aiida' create_archive([sd1, jc1, gr1], filename=filename1) n_uuids = [sd1.uuid, jc1.uuid] - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(filename1) # Check that the imported nodes are correctly imported and that @@ -66,7 +66,7 @@ def test_nodes_in_group(aiida_profile, tmp_path, aiida_localhost): assert builder.count() == 1, 'The group was not found.' -def test_group_export(tmp_path, aiida_profile): +def test_group_export(tmp_path, aiida_profile_clean): """Exporting a group includes its extras and nodes.""" # Create a new user new_email = uuid.uuid4().hex @@ -90,7 +90,7 @@ def test_group_export(tmp_path, aiida_profile): filename = tmp_path / 'export.aiida' create_archive([group], filename=filename) n_uuids = [sd1.uuid] - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(filename) # Check that the imported nodes are correctly imported and that @@ -106,7 +106,7 @@ def test_group_export(tmp_path, aiida_profile): assert imported_group.base.extras.get('test') == 1, 'Extra missing on imported group' -def test_group_import_existing(tmp_path, aiida_profile): +def test_group_import_existing(tmp_path, aiida_profile_clean): """Testing what happens when I try to import a group that already exists in the database. This should raise an appropriate exception """ @@ -131,7 +131,7 @@ def test_group_import_existing(tmp_path, aiida_profile): # At this point we export the generated data filename = tmp_path / 'export1.aiida' create_archive([group], filename=filename) - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() # Creating a group of the same name group = orm.Group(label='node_group_existing') @@ -155,7 +155,7 @@ def test_group_import_existing(tmp_path, aiida_profile): assert builder.count() == 2 -def test_import_to_group(tmp_path, aiida_profile): +def test_import_to_group(tmp_path, aiida_profile_clean): """Test `group` parameter Make sure an unstored Group is stored by the import function, forwarding the Group object. Make sure the Group is correctly handled and used for imported nodes. @@ -168,7 +168,7 @@ def test_import_to_group(tmp_path, aiida_profile): # Export Nodes filename = tmp_path / 'export.aiida' create_archive([data1, data2], filename=filename) - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() # Create Group, do not store group_label = 'import_madness' diff --git a/tests/tools/archive/orm/test_logs.py b/tests/tools/archive/orm/test_logs.py index a6c80edb6b..b355ca8ddb 100644 --- a/tests/tools/archive/orm/test_logs.py +++ b/tests/tools/archive/orm/test_logs.py @@ -12,7 +12,7 @@ from aiida.tools.archive import create_archive, import_archive -def test_critical_log_msg_and_metadata(tmp_path, aiida_profile): +def test_critical_log_msg_and_metadata(tmp_path, aiida_profile_clean): """Testing logging of critical message""" message = 'Testing logging of critical failure' calc = orm.CalculationNode() @@ -33,7 +33,7 @@ def test_critical_log_msg_and_metadata(tmp_path, aiida_profile): export_file = tmp_path.joinpath('export.aiida') create_archive([calc], filename=export_file) - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(export_file) @@ -45,7 +45,7 @@ def test_critical_log_msg_and_metadata(tmp_path, aiida_profile): assert logs[0].metadata == log_metadata -def test_exclude_logs_flag(tmp_path, aiida_profile): +def test_exclude_logs_flag(tmp_path, aiida_profile_clean): """Test that the `include_logs` argument for `export` works.""" log_msg = 'Testing logging of critical failure' @@ -65,7 +65,7 @@ def test_exclude_logs_flag(tmp_path, aiida_profile): create_archive([calc], filename=export_file, include_logs=False) # Clean database and reimport exported data - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(export_file) # Finding all the log messages @@ -80,7 +80,7 @@ def test_exclude_logs_flag(tmp_path, aiida_profile): assert str(import_calcs[0][0]) == calc_uuid -def test_export_of_imported_logs(tmp_path, aiida_profile): +def test_export_of_imported_logs(tmp_path, aiida_profile_clean): """Test export of imported Log""" log_msg = 'Testing export of imported log' @@ -102,7 +102,7 @@ def test_export_of_imported_logs(tmp_path, aiida_profile): create_archive([calc], filename=export_file) # Clean database and reimport exported data - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(export_file) # Finding all the log messages @@ -123,7 +123,7 @@ def test_export_of_imported_logs(tmp_path, aiida_profile): create_archive([calc], filename=re_export_file) # Clean database and reimport exported data - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() import_archive(re_export_file) # Finding all the log messages diff --git a/tests/tools/archive/test_simple.py b/tests/tools/archive/test_simple.py index 2f7377eb22..ac6209ab95 100644 --- a/tests/tools/archive/test_simple.py +++ b/tests/tools/archive/test_simple.py @@ -21,7 +21,7 @@ @pytest.mark.parametrize('entities', ['all', 'specific']) -def test_base_data_nodes(aiida_profile, tmp_path, entities): +def test_base_data_nodes(aiida_profile_clean, tmp_path, entities): """Test ex-/import of Base Data nodes""" # producing values for each base type values = ('Hello', 6, -1.2399834e12, False) @@ -46,7 +46,7 @@ def test_base_data_nodes(aiida_profile, tmp_path, entities): # actually export now create(filename=filename) # cleaning: - aiida_profile.reset_storage() + aiida_profile_clean.reset_storage() # Importing back the data: import_archive(filename) # Checking whether values are preserved: diff --git a/tests/tools/groups/test_paths.py b/tests/tools/groups/test_paths.py index 1cad363d46..6ff2459650 100644 --- a/tests/tools/groups/test_paths.py +++ b/tests/tools/groups/test_paths.py @@ -125,6 +125,7 @@ def test_walk_with_invalid_path(): assert [c.path for c in sorted(group_path.walk())] == expected +@pytest.mark.usefixtures('aiida_profile_clean') def test_walk_nodes(): """Test the ``GroupPath.walk_nodes()`` function.""" group, _ = orm.Group.collection.get_or_create('a') diff --git a/tests/tools/visualization/test_graph.py b/tests/tools/visualization/test_graph.py index db082017a9..276c000044 100644 --- a/tests/tools/visualization/test_graph.py +++ b/tests/tools/visualization/test_graph.py @@ -354,6 +354,7 @@ def test_graph_node_identifiers(self, node_id_type, monkeypatch, file_regression # The order of certain output lines can be randomly ordered so we split the file in lines, sort, and then join # them into a single string again. The node identifiers generated by the engine are of the form ``N{pk}`` and # they also clearly vary, so they are replaced with the ``NODE`` placeholder. - string = '\n'.join(sorted(graph.graphviz.source.strip().split('\n'))) + string = graph.graphviz.source string = re.sub(r'N\d+', 'NODE', string) + string = '\n'.join(sorted(string.strip().split('\n'))) file_regression.check(string) From 835d13b735e068883ed414755717b0fc366642b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 00:58:20 +0100 Subject: [PATCH 4/5] Bump codecov/codecov-action from 4 to 5 in the gha-dependencies group (#6648) Bumps the gha-dependencies group with 1 update: [codecov/codecov-action](https://github.com/codecov/codecov-action). Updates `codecov/codecov-action` from 4 to 5 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: gha-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jusong Yu --- .github/workflows/ci-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index 57e85bb004..917582b0c7 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -108,7 +108,7 @@ jobs: - name: Upload coverage report if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} name: aiida-pytests-py3.9 From dbdc36c635ae3596905ab54f0905e97026b85f49 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 2 Dec 2024 11:34:00 +0100 Subject: [PATCH 5/5] Amend type call error after using AiiDAConfigDir (#6646) --- src/aiida/manage/tests/pytest_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiida/manage/tests/pytest_fixtures.py b/src/aiida/manage/tests/pytest_fixtures.py index 6c5d04d3bc..8cc44dc608 100644 --- a/src/aiida/manage/tests/pytest_fixtures.py +++ b/src/aiida/manage/tests/pytest_fixtures.py @@ -173,7 +173,7 @@ def aiida_instance( if configuration.CONFIG is not None: reset = True current_config = configuration.CONFIG - current_config_path = current_config.dirpath + current_config_path = pathlib.Path(current_config.dirpath) current_profile = configuration.get_profile() current_path_variable = os.environ.get(settings.DEFAULT_AIIDA_PATH_VARIABLE, None)