Fix CI

.github/workflows/black.yml

@@ -10,6 +10,6 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Black Check
-      uses: jpetrucciani/black-check@20.8b1
+      uses: jpetrucciani/black-check@master
       with:
         path: 'glia'

.github/workflows/main.yml

@@ -8,8 +8,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        py: [3.9, 3.10, 3.11]
-        nrn: [8.0.0, 8.1.0, 8.2.0]
+        py: ["3.9", "3.10"]
+        nrn: ["8.1.0", "8.2.3"]
+        include:
+          - py: "3.9"
+            nrn: "8.0.2"
+          - py: "3.11"
+            nrn: "8.2.3"
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.py }}

glia/__init__.py

@@ -2,7 +2,7 @@
 NMODL Asset Manager.
 """
 
-__version__ = "4.0.0b0"
+__version__ = "4.0.0b1"
 
 from ._glia import Glia
 from .exceptions import *

glia/_cli.py

@@ -1,19 +1,9 @@
 import argparse
 import sys
 
-from . import _manager
+from . import _manager, _mpi
 from .exceptions import *
 
-try:
-    from patch import p
-
-    p.nrnmpi_init()
-    import mpi4py.MPI
-
-    main_node = not mpi4py.MPI.COMM_WORLD.Get_rank()
-except ImportError:
-    main_node = True
-
 
 def glia_cli():
     parser = argparse.ArgumentParser()
@@ -81,13 +71,15 @@ def glia_cli():
 
 
 def compile(args):
-    if main_node:
+    if _mpi.main_node:
         print("Glia is compiling...")
-    _manager._compile()
-    if main_node:
+    print("huh")
+    _manager.compile()
+    print("huh2")
+    if _mpi.main_node:
         print("Compilation complete!")
     assets, _, _ = _manager._collect_asset_state()
-    if main_node:
+    if _mpi.main_node:
         print(
             "Compiled assets:",
             ", ".join(
@@ -130,12 +122,12 @@ def test(*args, verbose=False):
         except LookupError as e:
             mstr = "[X]"
             estr = str(e)
-        if main_node:
+        if _mpi.main_node:
             print(mstr, mechanism)
         if verbose and estr != "":
-            if main_node:
+            if _mpi.main_node:
                 print("  -- " + estr)
-    if main_node:
+    if _mpi.main_node:
         print("Tests finished:", successes, "out of", tests, "passed")
 
 

glia/_fs.py

@@ -26,16 +26,19 @@ def get_cache_path(*subfolders, prefix=""):
         *subfolders,
     )
 
+
 def get_data_path(*subfolders):
     return os.path.join(_install_dirs.user_data_dir, *subfolders)
 
 
 def get_mod_path(pkg):
     return os.path.abspath(os.path.join(pkg.path, "mod"))
 
+
 def get_neuron_mod_path(*paths):
     return get_cache_path(*paths)
 
+
 def _read_shared_storage(*path):
     _path = get_data_path(*path)
     try:
@@ -44,48 +47,58 @@ def _read_shared_storage(*path):
     except IOError:
         return {}
 
+
 def _write_shared_storage(data, *path):
     _path = get_data_path(*path)
     with open(_path, "w") as f:
         f.write(json.dumps(data))
 
+
 def read_storage(*path):
     data = _read_shared_storage(*path)
     glia_path = get_glia_path()
     if glia_path not in data:
         return {}
     return data[glia_path]
 
+
 def write_storage(data, *path):
     _path = get_data_path(*path)
     glia_path = get_glia_path()
     shared_data = _read_shared_storage(*path)
     shared_data[glia_path] = data
     _write_shared_storage(shared_data, *path)
 
+
 def read_cache():
     cache = read_storage("cache.json")
     if "mod_hashes" not in cache:
         cache["mod_hashes"] = {}
     return cache
 
+
 def write_cache(cache_data):
     write_storage(cache_data, "cache.json")
 
+
 def update_cache(cache_data):
     cache = read_cache()
     cache.update(cache_data)
     write_cache(cache)
 
+
 def create_cache():
     empty_cache = {"mod_hashes": {}, "cat_hashes": {}}
     write_cache(empty_cache)
 
+
 def read_preferences():
     return read_storage("preferences.json")
 
+
 def write_preferences(preferences):
     write_storage(preferences, "preferences.json")
 
+
 def create_preferences():
-    write_storage({}, "preferences.json")
+    write_storage({}, "preferences.json")

glia/_glia.py

@@ -1,13 +1,14 @@
 import os
 import subprocess
 import sys
+import warnings
 import weakref
 from functools import wraps
-from glob import glob
 from importlib.metadata import entry_points
 from shutil import copy2 as copy_file
 from shutil import rmtree as rmdir
 
+from . import _mpi
 from ._fs import (
     create_cache,
     create_preferences,
@@ -92,8 +93,11 @@ def catalogues(self):
 
     def discover_packages(self):
         self._packages = []
+        print("AND?")
         for pkg_ptr in entry_points().get("glia.package", []):
+            print("AND2?", pkg_ptr)
             advert = pkg_ptr.load()
+            print("stuck??")
             self.entry_points.append(advert)
             self._packages.append(Package.from_remote(self, advert))
 
@@ -126,7 +130,7 @@ def catalogue(self, name):
     def build_catalogue(self, name, debug=False, verbose=False, gpu=None):
         return self._get_catalogue(name).build(verbose=verbose, debug=debug, gpu=gpu)
 
-    def start(self, load_dll=True):
+    def start(self):
         self.compile(check_cache=True)
 
     @_requires_install
@@ -155,7 +159,9 @@ def compile(self, check_cache=False):
         """
         self._compiled = True
         if not check_cache or not self.is_cache_fresh():
-            self._compile()
+            if _mpi.main_node:
+                self._compile()
+            _mpi.barrier()
 
     @_requires_install
     def _compile(self):
@@ -164,78 +170,54 @@ def _compile(self):
             return update_cache(cache_data)
         if len(mod_files) == 0:
             return
-        for i in self._distribute_n(len(mod_files)):
-            self._compile_nrn_mod(assets[i], mod_files[i])
-        # Update the cache with the new mod directory hashes.
-        update_cache(cache_data)
-
-    def _compile_nrn_mod(self, asset, file):
-        mod_path = get_neuron_mod_path(asset[1].mod_name)
-        os.makedirs(mod_path, exist_ok=True)
-        # Clean out previous files inside of the mod path
-        _remove_tree(mod_path)
-        # Copy over the mod file
-        copy_file(file, mod_path)
+        neuron_mod_path = get_neuron_mod_path()
+        _remove_tree(neuron_mod_path)
+        # Copy over fresh mods
+        for file in mod_files:
+            copy_file(file, neuron_mod_path)
         # Platform specific compile
         if sys.platform == "win32":
-            self._compile_nrn_windows(mod_path)
+            self._compile_nrn_windows(neuron_mod_path)
         elif sys.platform in ("linux", "darwin"):
-            self._compile_nrn_linux(mod_path)
+            self._compile_nrn_linux(neuron_mod_path)
         else:
             raise NotImplementedError(
-                "Only linux, darwin and win32 are supported. You are using "
-                + sys.platform
+                "Only linux and win32 are supported. You are using " + sys.platform
             )
-
-    def _distribute_n(self, n):
-        try:
-            from mpi4py.MPI import COMM_WORLD
-        except ImportError:
-            return range(0, n)
-        else:
-            r = COMM_WORLD.Get_rank()
-            s = COMM_WORLD.Get_size()
-            return range(r, n, s)
+        # Update the cache with the new mod directory hashes.
+        update_cache(cache_data)
 
     def _compile_nrn_windows(self, neuron_mod_path):
         # Compile the glia cache for Linux.
-        # Swap the python process's current working directory to the glia mod directory
-        # and run mknrndll.sh in mingw. This approach works even when the PATH isn't set
-        # properly by the installer.
+        # Runs %NEURONHOME%/nrnivmodl.bat
         from patch import p
 
-        nrn_path = p.neuronhome()
-        current_dir = os.getcwd()
-        os.chdir(neuron_mod_path)
         process = subprocess.Popen(
-            [os.path.join(nrn_path, "bin", "nrnivmodl.bat")],
+            [os.path.join(p.neuronhome(), "bin", "nrnivmodl.bat")],
+            cwd=neuron_mod_path,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
 
         stdout, stderr = process.communicate(input=b"\n")
         self._compilation_failed = process.returncode != 0
-        os.chdir(current_dir)
         if process.returncode != 0:
             raise CompileError(stderr.decode("UTF-8"))
 
     def _compile_nrn_linux(self, neuron_mod_path):
         # Compile the glia cache for Linux.
-        # Swap the python process's current working directory to the glia mod directory
-        # and run nrnivmodl.
-        current_dir = os.getcwd()
-        os.chdir(neuron_mod_path)
+        # Runs nrnivmodl.
         process = subprocess.Popen(
             ["nrnivmodl"],
+            cwd=neuron_mod_path,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
 
         stdout, stderr = process.communicate()
         self._compiled = process.returncode == 0
-        os.chdir(current_dir)
         if process.returncode != 0:
             raise CompileError(stderr.decode("UTF-8"))
 
@@ -295,11 +277,9 @@ def insert(self, section, asset, variant=None, pkg=None, /, attributes=None, x=N
         :type x: float
         :raises: LibraryError if the asset isn't found or was incorrectly marked as a point process.
         """
-        # Transform the given section into a NEURON section.
-        nrn_section = _transform(section)
         if attributes is None:
             attributes = {}
-        if asset.startswith("glia"):
+        if asset.startswith("glia__"):
             mod_name = asset
         else:
             mod_name = self.resolver.resolve(asset, pkg=pkg, variant=variant)
@@ -387,15 +367,14 @@ def _load_all_libraries(self):
 
     def get_libraries(self):
         """
-        Return the locations of the library paths (dll/so) to be loaded into NEURON. Or
-        perhaps for use in dark neuroscientific rituals, who knows.
+        Return the locations of the library paths (dll/so) to be loaded into NEURON.
         """
         if sys.platform == "win32":
             path = ["nrnmech.dll"]
         else:
             path = ["x86_64", ".libs", "libnrnmech.so"]
 
-        return [os.path.join(folder, *path) for folder in glob(get_cache_path("*/"))]
+        return [get_neuron_mod_path(*path)]
 
     def is_cache_fresh(self):
         try:
@@ -433,6 +412,7 @@ def _add_neuron_pkg(self):
         except ImportError:
             pass
         else:
+            import patch
             from patch import is_density_mechanism, is_point_process
 
             nrn_pkg = Package("NEURON", neuron.__path__[0], builtin=True)
@@ -477,7 +457,6 @@ def _transform(obj):
 
 
 def _remove_tree(path):
-    # Clear compiled mods
     for root, dirs, files in os.walk(path):
         for dir in dirs:
             try:
@@ -488,7 +467,7 @@ def _remove_tree(path):
             try:
                 os.remove(os.path.join(path, file))
             except PermissionError as _:
-                print("Couldn't remove", file)
+                warnings.warn(f"Couldn't remove {file}")
 
 
 class MechAccessor:

glia/_mpi.py

@@ -0,0 +1,30 @@
+try:
+    import mpi4py.MPI
+except ImportError as e:
+    _comm = None
+    has_mpi = False
+    main_node = True
+    parallel_run = False
+else:
+    _comm = mpi4py.MPI.COMM_WORLD
+    has_mpi = True
+    main_node = not _comm.Get_rank()
+    parallel_run = _comm.Get_size() > 1
+
+
+def set_comm(comm):
+    global _comm
+
+    _comm = comm
+
+
+def barrier():
+    if _comm:
+        _comm.barrier()
+
+
+def bcast(data, root=0):
+    if not _comm:
+        return data
+    else:
+        return _comm.bcast(data, root=root)