Update --sources to allow restricting to specific GitRepository (#284)

Allows restricting kustomizations to a specific set that should be
included in the cluster.

Issue #283

.github/workflows/flux-local-diff.yaml

@@ -34,7 +34,7 @@ jobs:
         path: ${{ matrix.cluster_path }}
         resource: ${{ matrix.resource }}
         debug: true
-        sources: cluster=tests/testdata/cluster3
+        sources: cluster=tests/testdata/cluster3,flux-system,home-ops-kubernetes
         api-versions: batch/v1/CronJob
     - name: PR Comments
       uses: mshick/add-pr-comment@v2

.github/workflows/flux-local-test.yaml

@@ -27,6 +27,6 @@ jobs:
       with:
         enable-helm: true
         enable-kyverno: false
-        sources: cluster=tests/testdata/cluster3
+        sources: cluster=tests/testdata/cluster3,flux-system,home-ops-kubernetes
         path: ${{ matrix.cluster_path }}
         api-versions: batch/v1/CronJob

action/diff/action.yml

@@ -41,7 +41,7 @@ inputs:
     description: Additional flags to pass to kustomize build
     default: ''
   sources:
-    description: OCIRepository source mappings like `cluster=./kubernetes/`
+    description: GitRepository or OCIRepository to include with optional source mappings like `flux-system` or `cluster=./kubernetes/`
     default: ''
 outputs:
   diff:

action/test/action.yml

@@ -30,7 +30,7 @@ inputs:
     description: Additional flags to pass to kustomize build
     default: ''
   sources:
-    description: OCIRepository source mappings like `cluster=./kubernetes/`
+    description: GitRepository or OCIRepository to include with optional source mappings like `flux-system` or `cluster=./kubernetes/`
     default: ''
 runs:
   using: "composite"

flux_local/git_repo.py

@@ -96,10 +96,10 @@ class Source:
     name: str
     """The name of the repository source."""
 
-    root: Path
+    root: Path | None
     """The path name within the repo root."""
 
-    namespace: str
+    namespace: str | None
     """The namespace of the repository source."""
 
     @property
@@ -110,13 +110,16 @@ def source_name(self) -> str:
     @classmethod
     def from_str(self, value: str) -> "Source":
         """Parse a Source from key=value string."""
-        if "=" not in value:
-            raise ValueError("Expected source name=root")
-        namespace = "flux-system"
-        name, root = value.split("=")
+        root: Path | None = None
+        if "=" in value:
+            name, root_str = value.split("=")
+            root = Path(root_str)
+        else:
+            name = value
+        namespace: str | None = None
         if "/" in name:
             namespace, name = name.split("/")
-        return Source(name=name, root=Path(root), namespace=namespace)
+        return Source(name=name, root=root, namespace=namespace)
 
 
 @cache
@@ -304,7 +307,10 @@ class ResourceSelector:
 
 
 async def get_fluxtomizations(
-    root: Path, relative_path: Path, build: bool
+    root: Path,
+    relative_path: Path,
+    build: bool,
+    sources: list[Source],
 ) -> list[Kustomization]:
     """Find all flux Kustomizations in the specified path.
 
@@ -324,13 +330,27 @@ async def get_fluxtomizations(
             f"kind={CLUSTER_KUSTOMIZE_KIND}", root / relative_path
         ).grep(GREP_SOURCE_REF_KIND)
     docs = await cmd.objects()
-    return [
-        Kustomization.parse_doc(doc) for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs)
-    ]
+    results = []
+    for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs):
+        ks = Kustomization.parse_doc(doc)
+        if not is_allowed_source(ks, sources):
+            continue
+        results.append(ks)
+    return results
+
+
+def is_allowed_source(doc: Kustomization, sources: list[Source]) -> bool:
+    """Return true if this Kustomization is from an allowed source."""
+    if not sources:
+        return True
+    for source in sources:
+        if source.name == doc.source_name and (
+            source.namespace is None or source.namespace == doc.source_namespace
+        ):
+            return True
+    return False
 
 
-# default_path=root_path_selector.relative_path
-# sources=path-selector.sources or ()
 def adjust_ks_path(
     doc: Kustomization, default_path: Path, sources: list[Source]
 ) -> Path | None:
@@ -347,6 +367,9 @@ def adjust_ks_path(
                 _LOGGER.debug(
                     "Updated Source for OCIRepository %s: %s", doc.name, doc.path
                 )
+                if not source.root:
+                    _LOGGER.info("OCIRepository source has no root specified")
+                    continue
                 return source.root / doc.path
 
         _LOGGER.info(
@@ -372,7 +395,12 @@ async def kustomization_traversal(
         path = path_queue.get()
         _LOGGER.debug("Visiting path (%s) %s", root_path_selector.path, path)
         try:
-            docs = await get_fluxtomizations(root_path_selector.root, path, build=build)
+            docs = await get_fluxtomizations(
+                root_path_selector.root,
+                path,
+                build=build,
+                sources=path_selector.sources or [],
+            )
         except FluxException as err:
             detail = ERROR_DETAIL_BAD_KS if visited_paths else ERROR_DETAIL_BAD_PATH
             raise FluxException(
@@ -430,7 +458,10 @@ async def get_clusters(
     """Load Cluster objects from the specified path."""
     try:
         roots = await get_fluxtomizations(
-            path_selector.root, path_selector.relative_path, build=False
+            path_selector.root,
+            path_selector.relative_path,
+            build=False,
+            sources=path_selector.sources or [],
         )
     except FluxException as err:
         raise FluxException(

flux_local/manifest.py

@@ -10,10 +10,11 @@
 
 import aiofiles
 import yaml
+
 try:
     from pydantic.v1 import BaseModel, Field
 except ImportError:
-    from pydantic import BaseModel, Field
+    from pydantic import BaseModel, Field  # type: ignore
 
 from .exceptions import InputException
 
@@ -283,6 +284,9 @@ class Kustomization(BaseManifest):
     source_name: str | None = None
     """The name of the sourceRef that provides this Kustomization."""
 
+    source_namespace: str | None = None
+    """The namespace of the sourceRef that provides this Kustomization."""
+
     @classmethod
     def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization":
         """Parse a partial Kustomization from a kubernetes resource."""
@@ -305,6 +309,7 @@ def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization":
             source_path=source_path,
             source_kind=source_ref.get("kind"),
             source_name=source_ref.get("name"),
+            source_namespace=source_ref.get("namespace", namespace),
         )
 
     @property
@@ -326,6 +331,7 @@ def namespaced_name(self, sep: str = "/") -> str:
         },
         "source_path": True,
         "source_name": True,
+        "source_namespace": True,
         "source_kind": True,
     }
 

flux_local/tool/selector.py

@@ -32,12 +32,15 @@ def __call__(
         values = values.split(",")
         if not values[0]:
             return
-        try:
-            source = git_repo.Source.from_str(values[0])
-        except ValueError:
-            raise ArgumentError(self, f"Expected key=value format from '{values[0]}'")
         result = getattr(namespace, self.dest) or []
-        result.append(source)
+        for value in values:
+            try:
+                source = git_repo.Source.from_str(value)
+            except ValueError:
+                raise ArgumentError(
+                    self, f"Expected key or key=value format from '{value}'"
+                )
+            result.append(source)
         setattr(namespace, self.dest, result)
 
 
@@ -52,7 +55,9 @@ def add_selector_flags(args: ArgumentParser) -> None:
     )
     args.add_argument(
         "--sources",
-        help="Optional map of repository source to relative path e.g. cluster=./k8s/",
+        help="Optional GitRepository or OCIRepository sources to restrict "
+        "to e.g. `flux-system`. Can include optional map of repository "
+        "source to relative path e.g. `cluster=./k8s/`",
         action=SourceAppendAction,
     )
     args.add_argument(

tests/test_git_repo.py

@@ -13,6 +13,7 @@
     kustomization_traversal,
     Source,
     PathSelector,
+    is_allowed_source,
 )
 from flux_local.kustomize import Kustomize
 from flux_local.manifest import Kustomization
@@ -273,7 +274,6 @@ async def test_kustomization_traversal(path: str) -> None:
                 name="flux-system",
                 namespace="flux-system",
                 path="./kubernetes/cluster",
-                source_path="apps.yaml",
             ),
         ],
         # Second traversal
@@ -282,13 +282,11 @@ async def test_kustomization_traversal(path: str) -> None:
                 name="cluster-apps",
                 namespace="flux-system",
                 path="./kubernetes/apps",
-                source_path="apps.yaml",
             ),
             Kustomization(
                 name="cluster",
                 namespace="flux-system",
                 path="./kubernetes/flux",
-                source_path="config/cluster.yaml",
             ),
         ],
         # Third traversal
@@ -297,13 +295,11 @@ async def test_kustomization_traversal(path: str) -> None:
                 name="cluster-apps-rook-ceph",
                 namespace="flux-system",
                 path="./kubernetes/apps/rook-ceph/rook-ceph/app",
-                source_path="rook-ceph/rook-ceph/ks.yaml",
             ),
             Kustomization(
                 name="cluster-apps-volsync",
                 namespace="flux-system",
                 path="./kubernetes/apps/volsync/volsync/app",
-                source_path="volsync/volsync/ks.yaml",
             ),
         ],
         [],
@@ -313,7 +309,9 @@ async def test_kustomization_traversal(path: str) -> None:
     ]
     paths = []
 
-    async def fetch(root: Path, p: Path, build: bool) -> list[Kustomization]:
+    async def fetch(
+        root: Path, p: Path, build: bool, sources: list[Source]
+    ) -> list[Kustomization]:
         nonlocal paths, results
         paths.append((str(root), str(p)))
         return results.pop(0)
@@ -340,7 +338,7 @@ def test_source() -> None:
     """Test parsing a source from a string."""
     source = Source.from_str("cluster=./k8s")
     assert source.name == "cluster"
-    assert source.namespace == "flux-system"
+    assert source.namespace is None
     assert str(source.root) == "k8s"
 
 
@@ -350,3 +348,47 @@ def test_source_with_namespace() -> None:
     assert source.name == "cluster"
     assert source.namespace == "flux-system2"
     assert str(source.root) == "k8s"
+
+
+def test_source_without_path() -> None:
+    """Test parsing a source without a path."""
+    source = Source.from_str("cluster")
+    assert source.name == "cluster"
+    assert source.namespace is None
+    assert source.root is None
+
+
+def test_is_allowed_source() -> None:
+    """Test GitRepository sources allowed."""
+    ks = Kustomization(
+        name="ks",
+        namespace="flux-system",
+        path="./kubernetes/apps/volsync/volsync/app",
+        source_name="flux-system",
+    )
+    assert is_allowed_source(ks, [Source.from_str("flux-system")])
+
+
+def test_is_not_allowed_source() -> None:
+    """Test GitRepository sources allowed."""
+    ks = Kustomization(
+        name="ks",
+        namespace="flux-system",
+        path="./kubernetes/apps/volsync/volsync/app",
+        source_name="flux-system",
+    )
+    assert not is_allowed_source(ks, [Source.from_str("flux-system-other")])
+
+
+def test_is_allowed_source_namespace_optional() -> None:
+    """Test GitRepository sources allowed."""
+    ks = Kustomization(
+        name="ks",
+        namespace="flux-system",
+        path="./kubernetes/apps/volsync/volsync/app",
+        source_name="flux-system",
+        source_namespace="flux-system2",
+    )
+    assert is_allowed_source(ks, [Source.from_str("flux-system")])
+    assert is_allowed_source(ks, [Source.from_str("flux-system2/flux-system")])
+    assert not is_allowed_source(ks, [Source.from_str("flux-system3/flux-system")])

tests/tool/testdata/get_cluster3.yaml

@@ -6,5 +6,5 @@ args:
 - --sources
 - cluster=tests/testdata/cluster3
 stdout: |
-  NAME           PATH                                           KUSTOMIZATIONS    
-  flux-system    ./tests/testdata/cluster3/clusters/cluster3    2                 
+  NAME       PATH                       KUSTOMIZATIONS    
+  cluster    tests/testdata/cluster3    2