Merge branch 'topic/bbannier/testing-on-macos'

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
   - id: check-yaml
   - id: check-added-large-files
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.7
+  rev: v0.5.2
   hooks:
     - id: ruff
       args: [--fix]

CHANGES

@@ -1,3 +1,22 @@
+3.0.1-39 | 2024-07-18 18:48:44 +0200
+
+  * Use documented Git package exception API (Benjamin Bannier, Corelight)
+
+  * Bump pre-commit hooks (Benjamin Bannier, Corelight)
+
+  * Use f-strings in more places (Benjamin Bannier, Corelight)
+
+  * Add installation workaround for tar files with macos metadata (Benjamin Bannier, Corelight)
+
+  * Fix test use of `sed` for macos (Benjamin Bannier, Corelight)
+
+    This fixes the invocation of `sed` in the test to be compatible with
+    macos' `sed` version. Previously this test would fail with
+
+    ```
+    sed: 1: "./install-dir/bin/zkg": invalid command code
+    ```
+
 3.0.1-33 | 2024-05-13 13:52:59 -0700
 
   * Pull in `btest` when installing package manager (Benjamin Bannier, Corelight)

VERSION

@@ -1 +1 @@
-3.0.1-33
+3.0.1-39

doc/ext/sphinxarg/ext.py

@@ -42,7 +42,7 @@ def map_nested_definitions(nested_content):
                 "@before",
                 "@after",
             ):
-                raise Exception("Unknown classifier: %s" % classifier)
+                raise Exception(f"Unknown classifier: {classifier}")
             idx = subitem.first_child_matching_class(nodes.term)
             if idx is not None:
                 ch = subitem[idx]
@@ -70,8 +70,9 @@ def print_arg_list(data, nested_content):
                 my_def.append(
                     nodes.paragraph(
                         text=(
-                            "Possible choices: %s"
-                            % ", ".join([str(c) for c in arg["choices"]])
+                            "Possible choices: {}".format(
+                                ", ".join([str(c) for c in arg["choices"]]),
+                            )
                         ),
                     ),
                 )
@@ -115,8 +116,9 @@ def print_opt_list(data, nested_content):
                 my_def.append(
                     nodes.paragraph(
                         text=(
-                            "Possible choices: %s"
-                            % ", ".join([str(c) for c in opt["choices"]])
+                            "Possible choices: {}".format(
+                                ", ".join([str(c) for c in opt["choices"]]),
+                            )
                         ),
                     ),
                 )
@@ -169,7 +171,7 @@ def apply_definition(definitions, my_def, name):
             return my_def + definition.children
         if classifier == "@before":
             return definition.children + my_def
-        raise Exception("Unknown classifier: %s" % classifier)
+        raise Exception(f"Unknown classifier: {classifier}")
     return my_def
 
 

doc/ext/sphinxarg/parser.py

@@ -16,8 +16,9 @@ def parser_navigate(parser_result, path, current_path=None):
         return parser_result
     if "children" not in parser_result:
         raise NavigationError(
-            "Current parser have no children elements.  (path: %s)"
-            % " ".join(current_path),
+            "Current parser have no children elements.  (path: {})".format(
+                " ".join(current_path),
+            ),
         )
     next_hop = path.pop(0)
     for child in parser_result["children"]:

testing/tests/bundled-zkg

@@ -9,7 +9,7 @@ set -eu
 mkdir -p install-dir/{bin,lib/python}
 cp -R $TEST_BASE/../zeekpkg ./install-dir/lib/python
 cp  $TEST_BASE/../zkg ./install-dir/bin
-sed -i "s,@PY_MOD_INSTALL_DIR@,$(pwd)/install-dir/lib/python," ./install-dir/bin/zkg
+sed -i -e "s,@PY_MOD_INSTALL_DIR@,$(pwd)/install-dir/lib/python," ./install-dir/bin/zkg
 
 export PATH=$(pwd)/install-dir/bin:$PATH
 # Ensure we're using the right zkg

zeekpkg/__init__.py

@@ -9,7 +9,7 @@
 
 import logging
 
-__version__ = "3.0.1-33"
+__version__ = "3.0.1-39"
 __all__ = ["manager", "package", "source", "template", "uservar"]  # noqa: F405
 
 LOG = logging.getLogger(__name__)

zeekpkg/_util.py

@@ -132,7 +132,7 @@ def git_clone(git_url, dst_path, shallow=False):
                 recursive=True,
                 depth=1,
             )
-        except git.exc.GitCommandError:
+        except git.GitCommandError:
             if not git_url.startswith(".") and not git_url.startswith("/"):
                 # Not a local repo
                 raise
@@ -171,7 +171,7 @@ def git_checkout(clone, version):
         version (str): the branch, tag, or commit to checkout
 
     Raises:
-        git.exc.GitCommandError: if the git repo is invalid
+        git.GitCommandError: if the git repo is invalid
     """
     clone.git.checkout(version)
     clone.git.submodule("sync", "--recursive")
@@ -255,7 +255,7 @@ def git_pull(repo):
         clone (git.Repo): the git clone on which to operate
 
     Raises:
-        git.exc.GitCommandError: in case of git trouble
+        git.GitCommandError: in case of git trouble
     """
     repo.git.pull()
     repo.git.submodule("sync", "--recursive")

zeekpkg/manager.py

@@ -580,7 +580,7 @@ def add_source(self, name, git_url):
                 git_url=git_url,
                 version=version,
             )
-        except git.exc.GitCommandError as error:
+        except git.GitCommandError as error:
             LOG.warning("failed to clone git repo: %s", error)
             return "failed to clone git repo"
         else:
@@ -1068,7 +1068,7 @@ def _refresh_source(self, name, aggregate=False, push=False):
         try:
             source.clone.git.fetch("--recurse-submodules=yes")
             git_pull(source.clone)
-        except git.exc.GitCommandError as error:
+        except git.GitCommandError as error:
             LOG.error("failed to pull source %s: %s", name, error)
             return self.SourceAggregationResults(
                 f"failed to pull from remote source: {error}",
@@ -1125,7 +1125,7 @@ def _refresh_source(self, name, aggregate=False, push=False):
 
                     try:
                         clone = git_clone(url, clonepath, shallow=True)
-                    except git.exc.GitCommandError as error:
+                    except git.GitCommandError as error:
                         LOG.warn(
                             "failed to clone %s, skipping aggregation: %s",
                             url,
@@ -1143,7 +1143,7 @@ def _refresh_source(self, name, aggregate=False, push=False):
 
                     try:
                         git_checkout(clone, version)
-                    except git.exc.GitCommandError as error:
+                    except git.GitCommandError as error:
                         LOG.warn(
                             'failed to checkout branch/version "%s" of %s, '
                             "skipping aggregation: %s",
@@ -1264,7 +1264,7 @@ def refresh_installed_packages(self):
 
             try:
                 clone.git.fetch("--recurse-submodules=yes")
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 LOG.warn(
                     "failed to fetch package %s: %s",
                     ipkg.package.qualified_name(),
@@ -1859,7 +1859,7 @@ def info(self, pkg_path, version="", prefer_installed=True):
 
             try:
                 return self._info(package, status, version)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 LOG.info(
                     'getting info on "%s": invalid git repo path: %s',
                     pkg_path,
@@ -1892,7 +1892,7 @@ def info(self, pkg_path, version="", prefer_installed=True):
 
         try:
             return self._info(package, status, version)
-        except git.exc.GitCommandError as error:
+        except git.GitCommandError as error:
             LOG.info('getting info on "%s": invalid git repo path: %s', pkg_path, error)
             reason = "git repository is either invalid or unreachable"
             return PackageInfo(package=package, invalid_reason=reason, status=status)
@@ -1904,7 +1904,7 @@ def _info(self, package, status, version):
             A :class:`.package.PackageInfo` object.
 
         Raises:
-            git.exc.GitCommandError: when failing to clone the package repo
+            git.GitCommandError: when failing to clone the package repo
         """
         clonepath = os.path.join(self.scratch_dir, package.name)
         clone = _clone_package(package, clonepath, version)
@@ -1918,7 +1918,7 @@ def _info(self, package, status, version):
 
         try:
             git_checkout(clone, version)
-        except git.exc.GitCommandError:
+        except git.GitCommandError:
             reason = f'no such commit, branch, or version tag: "{version}"'
             return PackageInfo(package=package, status=status, invalid_reason=reason)
 
@@ -2421,7 +2421,7 @@ def match_package_url_and_version(git_url, version):
 
             try:
                 git_clone(git_url, clonepath, shallow=(not is_sha1(version)))
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 return f"failed to clone {git_url}: {error}"
 
         # Record the built-in packages expected by this bundle (or simply
@@ -2607,7 +2607,7 @@ def test(self, pkg_path, version="", test_dependencies=False):
 
             try:
                 clone = _clone_package(info.package, clonepath, version)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 LOG.warning("failed to clone git repo: %s", error)
                 return (
                     f"failed to clone {info.package.git_url}",
@@ -2617,7 +2617,7 @@ def test(self, pkg_path, version="", test_dependencies=False):
 
             try:
                 git_checkout(clone, version)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 LOG.warning("failed to checkout git repo version: %s", error)
                 return (
                     f"failed to checkout {version} of {info.package.git_url}",
@@ -2941,7 +2941,7 @@ def install(self, pkg_path, version=""):
             try:
                 package = Package(git_url=pkg_path)
                 return self._install(package, version)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 LOG.info('installing "%s": invalid git repo path: %s', pkg_path, error)
 
             LOG.info('installing "%s": matched no source package', pkg_path)
@@ -2962,7 +2962,7 @@ def install(self, pkg_path, version=""):
 
         try:
             return self._install(matches[0], version)
-        except git.exc.GitCommandError as error:
+        except git.GitCommandError as error:
             LOG.warning('installing "%s": source package git repo is invalid', pkg_path)
             return f'failed to clone package "{pkg_path}": {error}'
 
@@ -3022,7 +3022,7 @@ def _install(self, package, version, use_existing_clone=False):
             string explaining why it failed.
 
         Raises:
-            git.exc.GitCommandError: if the git repo is invalid
+            git.GitCommandError: if the git repo is invalid
             IOError: if the package manifest file can't be written
         """
         clonepath = os.path.join(self.package_clonedir, package.name)
@@ -3240,7 +3240,15 @@ def _copy_package_dir(package, dirname, src, dst, scratch_dir):
         ld = os.listdir(tmp_dir)
 
         if len(ld) != 1:
-            return f"failed to copy package {dirname}: invalid tarfile"
+            # Apple `tar` might store HFS+ extended metadata in tar files.
+            # These metadata files have the names `._FOO` for each entry `FOO`.
+            # Since we expect a single top-level directory for the extracted
+            # plugin, ignore the metadata file if we see it.
+            ld.sort()
+            if len(ld) == 2 and ld[0] == f"._{ld[1]}":
+                ld = ld[1:]
+            else:
+                return f"failed to copy package {dirname}: invalid tarfile"
 
         src = os.path.join(tmp_dir, ld[0])
 
@@ -3289,7 +3297,7 @@ def _clone_package(package, clonepath, version):
         git.Repo: the cloned package
 
     Raises:
-        git.exc.GitCommandError: if the git repo is invalid
+        git.GitCommandError: if the git repo is invalid
     """
     delete_path(clonepath)
     shallow = not is_sha1(version)

zeekpkg/source.py

@@ -42,7 +42,7 @@ def __init__(self, name, clone_path, git_url, version=None):
         """Create a package source.
 
         Raises:
-            git.exc.GitCommandError: if the git repo is invalid
+            git.GitCommandError: if the git repo is invalid
             OSError: if the git repo is invalid and can't be re-initialized
         """
         git_url = os.path.expanduser(git_url)
@@ -52,10 +52,10 @@ def __init__(self, name, clone_path, git_url, version=None):
 
         try:
             self.clone = git.Repo(clone_path)
-        except git.exc.NoSuchPathError:
+        except git.NoSuchPathError:
             LOG.debug('creating source clone of "%s" at %s', name, clone_path)
             self.clone = git_clone(git_url, clone_path, shallow=True)
-        except git.exc.InvalidGitRepositoryError:
+        except git.InvalidGitRepositoryError:
             LOG.debug('deleting invalid source clone of "%s" at %s', name, clone_path)
             shutil.rmtree(clone_path)
             self.clone = git_clone(git_url, clone_path, shallow=True)

zeekpkg/template.py

@@ -105,7 +105,7 @@ def load(config, template, version=None):
                 repo = git.Repo(template)
                 if not repo.is_dirty():
                     version = repo.head.ref.commit.hexsha[:8]
-            except git.exc.InvalidGitRepositoryError:
+            except git.InvalidGitRepositoryError:
                 pass
             templatedir = template
         else:
@@ -139,7 +139,7 @@ def load(config, template, version=None):
                         repo = None
                 if repo is None:
                     repo = git_clone(template, templatedir)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 msg = f'failed to update template "{template}": {error}'
                 LOG.error(msg)
                 raise GitError(msg) from error
@@ -154,7 +154,7 @@ def load(config, template, version=None):
 
             try:
                 git_checkout(repo, version)
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 msg = f'failed to checkout branch/version "{version}" of template {template}: {error}'
                 LOG.warn(msg)
                 raise GitError(msg) from error
@@ -168,7 +168,7 @@ def load(config, template, version=None):
                 git_pull(repo)
             except TypeError:
                 pass  # Not on a branch, do nothing
-            except git.exc.GitCommandError as error:
+            except git.GitCommandError as error:
                 msg = f'failed to update branch "{version}" of template {template}: {error}'
                 LOG.warning(msg)
                 raise GitError(msg) from error

zkg

@@ -309,7 +309,7 @@ def create_config(args, configfile):
 def active_git_branch(path):
     try:
         repo = git.Repo(path)
-    except git.exc.NoSuchPathError:
+    except git.NoSuchPathError:
         return None
 
     if not repo.working_tree_dir:
@@ -341,7 +341,7 @@ def is_local_git_repo(git_url):
         # accessible, (2) it's not the root directory of a git repo.
         git.Repo(git_url)
         return True
-    except (git.exc.InvalidGitRepositoryError, git.exc.NoSuchPathError):
+    except (git.InvalidGitRepositoryError, git.NoSuchPathError):
         return False
 
 
@@ -350,7 +350,7 @@ def is_local_git_repo_dirty(git_url):
         return False
     try:
         repo = git.Repo(git_url)
-    except git.exc.NoSuchPathError:
+    except git.NoSuchPathError:
         return False
 
     return repo.is_dirty(untracked_files=True)