diff --git a/.github/workflows/flux-local-diff.yaml b/.github/workflows/flux-local-diff.yaml index fddae090..4f1f3534 100644 --- a/.github/workflows/flux-local-diff.yaml +++ b/.github/workflows/flux-local-diff.yaml @@ -34,7 +34,7 @@ jobs: path: ${{ matrix.cluster_path }} resource: ${{ matrix.resource }} debug: true - sources: cluster=tests/testdata/cluster3 + sources: cluster=tests/testdata/cluster3,flux-system,home-ops-kubernetes api-versions: batch/v1/CronJob - name: PR Comments uses: mshick/add-pr-comment@v2 diff --git a/.github/workflows/flux-local-test.yaml b/.github/workflows/flux-local-test.yaml index 9c4c79e2..5cd8b0ad 100644 --- a/.github/workflows/flux-local-test.yaml +++ b/.github/workflows/flux-local-test.yaml @@ -27,6 +27,6 @@ jobs: with: enable-helm: true enable-kyverno: false - sources: cluster=tests/testdata/cluster3 + sources: cluster=tests/testdata/cluster3,flux-system,home-ops-kubernetes path: ${{ matrix.cluster_path }} api-versions: batch/v1/CronJob diff --git a/action/diff/action.yml b/action/diff/action.yml index 61486b60..74c2b347 100644 --- a/action/diff/action.yml +++ b/action/diff/action.yml @@ -41,7 +41,7 @@ inputs: description: Additional flags to pass to kustomize build default: '' sources: - description: OCIRepository source mappings like `cluster=./kubernetes/` + description: GitRepository or OCIRepository to include with optional source mappings like `flux-system` or `cluster=./kubernetes/` default: '' outputs: diff: diff --git a/action/test/action.yml b/action/test/action.yml index 03979492..cafe304f 100644 --- a/action/test/action.yml +++ b/action/test/action.yml @@ -30,7 +30,7 @@ inputs: description: Additional flags to pass to kustomize build default: '' sources: - description: OCIRepository source mappings like `cluster=./kubernetes/` + description: GitRepository or OCIRepository to include with optional source mappings like `flux-system` or `cluster=./kubernetes/` default: '' runs: using: "composite" diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index e9bbce9f..8c7bf5bf 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -96,10 +96,10 @@ class Source: name: str """The name of the repository source.""" - root: Path + root: Path | None """The path name within the repo root.""" - namespace: str + namespace: str | None """The namespace of the repository source.""" @property @@ -110,13 +110,16 @@ def source_name(self) -> str: @classmethod def from_str(self, value: str) -> "Source": """Parse a Source from key=value string.""" - if "=" not in value: - raise ValueError("Expected source name=root") - namespace = "flux-system" - name, root = value.split("=") + root: Path | None = None + if "=" in value: + name, root_str = value.split("=") + root = Path(root_str) + else: + name = value + namespace: str | None = None if "/" in name: namespace, name = name.split("/") - return Source(name=name, root=Path(root), namespace=namespace) + return Source(name=name, root=root, namespace=namespace) @cache @@ -304,7 +307,10 @@ class ResourceSelector: async def get_fluxtomizations( - root: Path, relative_path: Path, build: bool + root: Path, + relative_path: Path, + build: bool, + sources: list[Source], ) -> list[Kustomization]: """Find all flux Kustomizations in the specified path. @@ -324,13 +330,27 @@ async def get_fluxtomizations( f"kind={CLUSTER_KUSTOMIZE_KIND}", root / relative_path ).grep(GREP_SOURCE_REF_KIND) docs = await cmd.objects() - return [ - Kustomization.parse_doc(doc) for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs) - ] + results = [] + for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs): + ks = Kustomization.parse_doc(doc) + if not is_allowed_source(ks, sources): + continue + results.append(ks) + return results + + +def is_allowed_source(doc: Kustomization, sources: list[Source]) -> bool: + """Return true if this Kustomization is from an allowed source.""" + if not sources: + return True + for source in sources: + if source.name == doc.source_name and ( + source.namespace is None or source.namespace == doc.source_namespace + ): + return True + return False -# default_path=root_path_selector.relative_path -# sources=path-selector.sources or () def adjust_ks_path( doc: Kustomization, default_path: Path, sources: list[Source] ) -> Path | None: @@ -347,6 +367,9 @@ def adjust_ks_path( _LOGGER.debug( "Updated Source for OCIRepository %s: %s", doc.name, doc.path ) + if not source.root: + _LOGGER.info("OCIRepository source has no root specified") + continue return source.root / doc.path _LOGGER.info( @@ -372,7 +395,12 @@ async def kustomization_traversal( path = path_queue.get() _LOGGER.debug("Visiting path (%s) %s", root_path_selector.path, path) try: - docs = await get_fluxtomizations(root_path_selector.root, path, build=build) + docs = await get_fluxtomizations( + root_path_selector.root, + path, + build=build, + sources=path_selector.sources or [], + ) except FluxException as err: detail = ERROR_DETAIL_BAD_KS if visited_paths else ERROR_DETAIL_BAD_PATH raise FluxException( @@ -430,7 +458,10 @@ async def get_clusters( """Load Cluster objects from the specified path.""" try: roots = await get_fluxtomizations( - path_selector.root, path_selector.relative_path, build=False + path_selector.root, + path_selector.relative_path, + build=False, + sources=path_selector.sources or [], ) except FluxException as err: raise FluxException( diff --git a/flux_local/manifest.py b/flux_local/manifest.py index 96b18e9a..67ca1691 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -10,10 +10,11 @@ import aiofiles import yaml + try: from pydantic.v1 import BaseModel, Field except ImportError: - from pydantic import BaseModel, Field + from pydantic import BaseModel, Field # type: ignore from .exceptions import InputException @@ -283,6 +284,9 @@ class Kustomization(BaseManifest): source_name: str | None = None """The name of the sourceRef that provides this Kustomization.""" + source_namespace: str | None = None + """The namespace of the sourceRef that provides this Kustomization.""" + @classmethod def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization": """Parse a partial Kustomization from a kubernetes resource.""" @@ -305,6 +309,7 @@ def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization": source_path=source_path, source_kind=source_ref.get("kind"), source_name=source_ref.get("name"), + source_namespace=source_ref.get("namespace", namespace), ) @property @@ -326,6 +331,7 @@ def namespaced_name(self, sep: str = "/") -> str: }, "source_path": True, "source_name": True, + "source_namespace": True, "source_kind": True, } diff --git a/flux_local/tool/selector.py b/flux_local/tool/selector.py index bc170a08..690ca571 100644 --- a/flux_local/tool/selector.py +++ b/flux_local/tool/selector.py @@ -32,12 +32,15 @@ def __call__( values = values.split(",") if not values[0]: return - try: - source = git_repo.Source.from_str(values[0]) - except ValueError: - raise ArgumentError(self, f"Expected key=value format from '{values[0]}'") result = getattr(namespace, self.dest) or [] - result.append(source) + for value in values: + try: + source = git_repo.Source.from_str(value) + except ValueError: + raise ArgumentError( + self, f"Expected key or key=value format from '{value}'" + ) + result.append(source) setattr(namespace, self.dest, result) @@ -52,7 +55,9 @@ def add_selector_flags(args: ArgumentParser) -> None: ) args.add_argument( "--sources", - help="Optional map of repository source to relative path e.g. cluster=./k8s/", + help="Optional GitRepository or OCIRepository sources to restrict " + "to e.g. `flux-system`. Can include optional map of repository " + "source to relative path e.g. `cluster=./k8s/`", action=SourceAppendAction, ) args.add_argument( diff --git a/tests/test_git_repo.py b/tests/test_git_repo.py index 01a19a9b..7f401fc1 100644 --- a/tests/test_git_repo.py +++ b/tests/test_git_repo.py @@ -13,6 +13,7 @@ kustomization_traversal, Source, PathSelector, + is_allowed_source, ) from flux_local.kustomize import Kustomize from flux_local.manifest import Kustomization @@ -273,7 +274,6 @@ async def test_kustomization_traversal(path: str) -> None: name="flux-system", namespace="flux-system", path="./kubernetes/cluster", - source_path="apps.yaml", ), ], # Second traversal @@ -282,13 +282,11 @@ async def test_kustomization_traversal(path: str) -> None: name="cluster-apps", namespace="flux-system", path="./kubernetes/apps", - source_path="apps.yaml", ), Kustomization( name="cluster", namespace="flux-system", path="./kubernetes/flux", - source_path="config/cluster.yaml", ), ], # Third traversal @@ -297,13 +295,11 @@ async def test_kustomization_traversal(path: str) -> None: name="cluster-apps-rook-ceph", namespace="flux-system", path="./kubernetes/apps/rook-ceph/rook-ceph/app", - source_path="rook-ceph/rook-ceph/ks.yaml", ), Kustomization( name="cluster-apps-volsync", namespace="flux-system", path="./kubernetes/apps/volsync/volsync/app", - source_path="volsync/volsync/ks.yaml", ), ], [], @@ -313,7 +309,9 @@ async def test_kustomization_traversal(path: str) -> None: ] paths = [] - async def fetch(root: Path, p: Path, build: bool) -> list[Kustomization]: + async def fetch( + root: Path, p: Path, build: bool, sources: list[Source] + ) -> list[Kustomization]: nonlocal paths, results paths.append((str(root), str(p))) return results.pop(0) @@ -340,7 +338,7 @@ def test_source() -> None: """Test parsing a source from a string.""" source = Source.from_str("cluster=./k8s") assert source.name == "cluster" - assert source.namespace == "flux-system" + assert source.namespace is None assert str(source.root) == "k8s" @@ -350,3 +348,47 @@ def test_source_with_namespace() -> None: assert source.name == "cluster" assert source.namespace == "flux-system2" assert str(source.root) == "k8s" + + +def test_source_without_path() -> None: + """Test parsing a source without a path.""" + source = Source.from_str("cluster") + assert source.name == "cluster" + assert source.namespace is None + assert source.root is None + + +def test_is_allowed_source() -> None: + """Test GitRepository sources allowed.""" + ks = Kustomization( + name="ks", + namespace="flux-system", + path="./kubernetes/apps/volsync/volsync/app", + source_name="flux-system", + ) + assert is_allowed_source(ks, [Source.from_str("flux-system")]) + + +def test_is_not_allowed_source() -> None: + """Test GitRepository sources allowed.""" + ks = Kustomization( + name="ks", + namespace="flux-system", + path="./kubernetes/apps/volsync/volsync/app", + source_name="flux-system", + ) + assert not is_allowed_source(ks, [Source.from_str("flux-system-other")]) + + +def test_is_allowed_source_namespace_optional() -> None: + """Test GitRepository sources allowed.""" + ks = Kustomization( + name="ks", + namespace="flux-system", + path="./kubernetes/apps/volsync/volsync/app", + source_name="flux-system", + source_namespace="flux-system2", + ) + assert is_allowed_source(ks, [Source.from_str("flux-system")]) + assert is_allowed_source(ks, [Source.from_str("flux-system2/flux-system")]) + assert not is_allowed_source(ks, [Source.from_str("flux-system3/flux-system")]) diff --git a/tests/tool/testdata/get_cluster3.yaml b/tests/tool/testdata/get_cluster3.yaml index 25187b88..7424ffc8 100644 --- a/tests/tool/testdata/get_cluster3.yaml +++ b/tests/tool/testdata/get_cluster3.yaml @@ -6,5 +6,5 @@ args: - --sources - cluster=tests/testdata/cluster3 stdout: | - NAME PATH KUSTOMIZATIONS - flux-system ./tests/testdata/cluster3/clusters/cluster3 2 + NAME PATH KUSTOMIZATIONS + cluster tests/testdata/cluster3 2