From 898b48a4d7f2086ae06f521d32dedad7bd8d6550 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 6 Dec 2023 17:42:02 +0100 Subject: [PATCH] b0 - reworked Package and Mod --- glia/_cli.py | 19 ++++------------ glia/_fs.py | 6 +---- glia/_glia.py | 45 +++++++++++++++---------------------- glia/_hash.py | 4 ++-- glia/assets.py | 35 ++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/test_glia_runnable.py | 9 +++++--- tests/test_resolve.py | 5 +++-- 8 files changed, 64 insertions(+), 61 deletions(-) diff --git a/glia/_cli.py b/glia/_cli.py index ca896d1..e8be137 100644 --- a/glia/_cli.py +++ b/glia/_cli.py @@ -81,22 +81,12 @@ def compile(args): _manager.compile() if _mpi.main_node: print("Compilation complete!") - assets, _, _ = _manager._collect_asset_state() + assets, _ = _manager._precompile_cache() if _mpi.main_node: print( "Compiled assets:", ", ".join( - list( - set( - map( - lambda a: a[0].name - + "." - + a[1].asset_name - + "({})".format(a[1].variant), - assets, - ) - ) - ) + set(f"{mod.pkg.name}.{mod.asset_name}({mod.variant})" for mod in assets) ), ) print("Testing assets ...") @@ -194,12 +184,11 @@ def _show_pkg(pkg_name): pstr = "Package: " + _colors.OKGREEN + candidate.name + _colors.ENDC print(pstr) print("=" * len(pstr)) - print("Path: " + candidate.path) - print("Module path: " + candidate.mod_path) + print("Location: " + candidate.root) print() print("Available modules:") for mod in candidate.mods: - print(" *", mod.mod_name) + print(" *", mod.mod_name, "=", mod.path) print() diff --git a/glia/_fs.py b/glia/_fs.py index cfb2e0d..2d8d374 100644 --- a/glia/_fs.py +++ b/glia/_fs.py @@ -30,10 +30,6 @@ def get_data_path(*subfolders): return os.path.join(_install_dirs.user_data_dir, *subfolders) -def get_mod_path(pkg): - return os.path.abspath(os.path.join(pkg.path, "mod")) - - def get_neuron_mod_path(*paths): return get_cache_path(*paths) @@ -43,7 +39,7 @@ def _read_shared_storage(*path): try: with open(_path, "r") as f: return json.load(f) - except IOError: + except (IOError, json.JSONDecodeError): return {} diff --git a/glia/_glia.py b/glia/_glia.py index 1974309..d535b78 100644 --- a/glia/_glia.py +++ b/glia/_glia.py @@ -14,12 +14,10 @@ create_preferences, get_cache_path, get_data_path, - get_mod_path, get_neuron_mod_path, read_cache, update_cache, ) -from ._hash import get_directory_hash from .assets import Catalogue, Mod, Package from .exceptions import ( CatalogueError, @@ -87,23 +85,22 @@ def resolver(self): def packages(self) -> typing.List[Package]: packages = [] for pkg_ptr in entry_points().get("glia.package", []): - advert = pkg_ptr.load() - self.entry_points.append(advert) - packages.append(Package.from_remote(self, advert)) + self.entry_points.append(pkg_ptr) + packages.append(pkg_ptr.load()) return packages @property @lru_cache(maxsize=1) def catalogues(self) -> typing.Mapping[str, Catalogue]: - catalogues = {} + catalogues: typing.Mapping[str, Catalogue] = {} for pkg_ptr in entry_points().get("glia.catalogue", []): advert = pkg_ptr.load() - self.entry_points.append(advert) + self.entry_points.append(pkg_ptr) if advert.name in catalogues: raise RuntimeError( f"Duplicate installations of `{advert.name}` catalogue:" - + f"\n{catalogues[advert.name].path}" - + f"\n{advert.path}" + + f"\n{catalogues[advert.name].FIXMEpath}" + + f"\n{advert.FIXMEpath}" ) catalogues[advert.name] = advert return catalogues @@ -159,16 +156,16 @@ def compile(self, check_cache=False): @_requires_install def _compile(self): - assets, mod_files, cache_data = self._collect_asset_state() + assets, cache_data = self._precompile_cache() if _should_skip_compile(): return update_cache(cache_data) - if len(mod_files) == 0: + if len(assets) == 0: return neuron_mod_path = get_neuron_mod_path() _remove_tree(neuron_mod_path) # Copy over fresh mods - for file in mod_files: - copy_file(file, neuron_mod_path) + for asset in assets: + copy_file(asset.path, neuron_mod_path) # Platform specific compile if sys.platform == "win32": self._compile_nrn_windows(neuron_mod_path) @@ -215,22 +212,17 @@ def _compile_nrn_linux(self, neuron_mod_path): if process.returncode != 0: raise CompileError(stderr.decode("UTF-8")) - def _collect_asset_state(self): + def _precompile_cache(self): cache_data = read_cache() - mod_files = [] assets = [] # Iterate over all discovered packages to collect the mod files. for pkg in self.packages: if pkg.builtin: continue - mod_path = get_mod_path(pkg) - for mod in pkg.mods: - assets.append((pkg, mod)) - mod_file = mod.mod_path - mod_files.append(mod_file) - # Hash mod directories and their contents to update the cache data. - cache_data["mod_hashes"][pkg.path] = get_directory_hash(mod_path) - return assets, mod_files, cache_data + assets.extend(pkg.mods) + # Update the package's hash to the current modfile contents + cache_data["mod_hashes"][pkg.hash] = pkg.mod_hash + return assets, cache_data def _resolve_mod(self, asset, variant=None, pkg=None): if isinstance(asset, str) and asset.startswith("glia__"): @@ -397,12 +389,11 @@ def get_libraries(self): def is_cache_fresh(self) -> bool: try: cache_data = read_cache() - hashes = cache_data["mod_hashes"] + mod_hashes = cache_data["mod_hashes"] for pkg in self.packages: - if pkg.path not in hashes: + if pkg.hash not in mod_hashes: return False - hash = get_directory_hash(os.path.join(pkg.path, "mod")) - if hash != hashes[pkg.path]: + if pkg.mod_hash != mod_hashes[pkg.hash]: return False return True except FileNotFoundError as _: diff --git a/glia/_hash.py b/glia/_hash.py index 2b4d45d..4ef5be2 100644 --- a/glia/_hash.py +++ b/glia/_hash.py @@ -46,11 +46,11 @@ def get_package_mods_hash(package: "Package"): raise FileNotFoundError( f"Modfile {package.name}.{mod.mech_id} not found at '{mod.path}'" ) - hash_update_from_file(mod.mod_path, h) + hash_update_from_file(mod.path, h) return h.hexdigest() def get_package_hash(package: "Package"): h = hashlib.md5() - h.update(package.root) + h.update(bytes(package.root)) return h.hexdigest() diff --git a/glia/assets.py b/glia/assets.py index 5a1ad6c..0584b0d 100644 --- a/glia/assets.py +++ b/glia/assets.py @@ -14,26 +14,49 @@ import arbor +class _ModList(list): + def __init__(self, package: "Package", *args, **kwargs): + super().__init__(*args, **kwargs) + self.pkg = package + for item in self: + if not isinstance(item, Mod): + raise PackageFileError(f"Package mod '{item}' is not a valid `glia.Mod`.") + item.set_package(package) + + def __setitem__(self, key, value): + if not isinstance(value, Mod): + raise PackageFileError(f"Package mod '{value}' is not a valid `glia.Mod`.") + super().__setitem__(key, value) + value.set_package(self.pkg) + + def append(self, item): + item.set_package(self.pkg) + super().append(item) + + def extend(self, itr): + super().extend(i.set_package(self.pkg) or i for i in itr) + + class Package: def __init__(self, name: str, root: Path, *, mods: list["Mod"] = None, builtin=False): self._name = name self._root = Path(root) - self.path = None - self.mods: list["Mod"] = [] if mods is None else mods + self.mods: list["Mod"] = _ModList(self, [] if mods is None else mods) # Exceptional flag for the NEURON builtins. # They need a definition to be `insert`ed, # but have no mod files to be compiled. self.builtin = builtin - def __hash__(self): - return get_package_hash(self) - @property def name(self): return self._name @property def hash(self): + return get_package_hash(self) + + @property + def mod_hash(self): return get_package_mods_hash(self) @property @@ -83,7 +106,7 @@ def mod_name(self): return f"glia__{self.asset_name}__{self.variant}" @property - def mod_path(self): + def path(self): return self.pkg.root / self._relpath diff --git a/pyproject.toml b/pyproject.toml index b09075d..b90af5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ parallel = [ "mpi4py~=3.0", ] neuron = [ - "nrn-patch>=4.0.0b0" + "nrn-patch>=4.0.0b1" ] arbor = [ "arbor>=0.6" diff --git a/tests/test_glia_runnable.py b/tests/test_glia_runnable.py index 80019be..f4f4116 100644 --- a/tests/test_glia_runnable.py +++ b/tests/test_glia_runnable.py @@ -5,11 +5,13 @@ import glia._glia from glia._fs import read_cache - -@unittest.skipIf( +skipWithoutMods = unittest.skipIf( not importlib.util.find_spec("glia_test_mods"), - "Package discovery should be tested with the `glia_test_mods` package installed.", + "Test requires `glia_test_mods` package to be installed.", ) + + +@skipWithoutMods class TestPackageDiscovery(unittest.TestCase): """ Check if packages can be discovered. @@ -42,6 +44,7 @@ def test_compilation(self): with self.subTest(path=path): self.assertTrue(os.path.exists(path), "Missing library file") + @skipWithoutMods def test_insert(self): from patch import p diff --git a/tests/test_resolve.py b/tests/test_resolve.py index 939ee5e..954b6d7 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -15,8 +15,9 @@ class TestResolver(unittest.TestCase): """ def test_resolve(self): - pkg = Package("test", __file__, False) - m = Mod(pkg, "hello", "test_v") + pkg = Package( + "test", __file__, mods=[m := Mod("./doesntexist", "hello", "test_v")] + ) mname = m.mod_name pkg.mods = [m] resolver = Resolver(ManagerMock([pkg]))