Collect resource types also for non-preferred API version (#67)

* #61 add a test for the bug

* #61 collect non-preferred API versions, too

* #61 simplify code

* #61 also test the API version

* #61 use same debug log message as before

* update dependencies incl. pykube-ng

* fix build: ekalinin/nodeenv#253

Author: hjacobs

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
   # formatters
 
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.9.0
+    rev: v2.1.0
     hooks:
       - id: reorder-python-imports
 
@@ -27,7 +27,7 @@ repos:
       - id: black
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v1.26.2
+    rev: v2.1.0
     hooks:
       - id: pyupgrade
         stages: [push]
@@ -69,7 +69,7 @@ repos:
         stages: [push]
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.20.0
+    rev: v1.21.0
     hooks:
       - id: yamllint
         args: ["--strict", "-d", "{rules: {line-length: {max: 180}}}"]
@@ -82,16 +82,10 @@ repos:
         #
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.761
+    rev: v0.770
     hooks:
       - id: mypy
 
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: v0.1.0
-    hooks:
-      - id: dockerfilelint
-        stages: [commit]  # required
-
   # miscellaneous
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -125,6 +119,6 @@ repos:
   # http://jorisroovers.com/gitlint/#using-gitlint-through-pre-commit
 
   - repo: https://github.com/jorisroovers/gitlint
-    rev:  v0.12.0
+    rev:  v0.13.1
     hooks:
       - id: gitlint

Makefile

@@ -15,7 +15,6 @@ lint:
 	poetry run pre-commit run --all-files
 
 test: install lint
-	poetry run mypy --ignore-missing-imports kube_janitor
 	poetry run coverage run --source=kube_janitor -m py.test -v
 	poetry run coverage report
 

kube_janitor/resources.py

@@ -14,6 +14,13 @@ def namespaced_object_factory(kind: str, name: str, api_version: str):
     )
 
 
+def discover_api_group(api, group_version: str):
+    logger.debug(f"Collecting resources in API group {group_version}..")
+    response = api.get(version=group_version)
+    response.raise_for_status()
+    return response.json()["resources"]
+
+
 def discover_namespaced_api_resources(api):
     core_version = "v1"
     r = api.get(version=core_version)
@@ -22,30 +29,49 @@ def discover_namespaced_api_resources(api):
         # ignore subresources like pods/proxy
         if (
             resource["namespaced"]
-            and "delete" in resource["verbs"]
             and "/" not in resource["name"]
+            and "delete" in resource["verbs"]
         ):
             yield core_version, resource
 
     r = api.get(version="/apis")
     r.raise_for_status()
+    group_versions = set()
     for group in r.json()["groups"]:
+        pref_version = group["preferredVersion"]["groupVersion"]
+        group_versions.add((pref_version, pref_version))
+        for version in group.get("versions", []):
+            group_version = version["groupVersion"]
+            group_versions.add((group_version, pref_version))
+
+    yielded = set()
+    non_preferred = []
+    for group_version, pref_version in sorted(group_versions):
         try:
-            pref_version = group["preferredVersion"]["groupVersion"]
-            logger.debug(f"Collecting resources in API group {pref_version}..")
-            r2 = api.get(version=pref_version)
-            r2.raise_for_status()
-            for resource in r2.json()["resources"]:
-                if (
-                    resource["namespaced"]
-                    and "delete" in resource["verbs"]
-                    and "/" not in resource["name"]
-                ):
-                    yield pref_version, resource
+            resources = discover_api_group(api, group_version)
         except Exception as e:
-            logger.error(
-                f"Could not collect resources in API group {pref_version}: {e}"
+            # do not crash if one API group is not available
+            # see https://codeberg.org/hjacobs/kube-web-view/issues/64
+            logger.warning(
+                f"Could not collect resources in API group {group_version}: {e}"
             )
+            continue
+
+        for resource in resources:
+            if (
+                resource["namespaced"]
+                and "/" not in resource["name"]
+                and "delete" in resource["verbs"]
+            ):
+                if group_version == pref_version:
+                    yield group_version, resource
+                    yielded.add((group_version, resource["name"]))
+                else:
+                    non_preferred.append((group_version, resource))
+
+    for group_version, resource in non_preferred:
+        if (group_version, resource["name"]) not in yielded:
+            yield group_version, resource
 
 
 def get_namespaced_resource_types(api):

poetry.lock

@@ -557,7 +557,7 @@ def get(**kwargs):
     api_mock.delete.assert_called_once_with(
         data='{"propagationPolicy": "Background"}',
         namespace="ns-1",
-        url="customfoos/foo-1",
+        url="/customfoos/foo-1",
         version="srcco.de/v1",
     )
 
@@ -640,7 +640,7 @@ def get(**kwargs):
     api_mock.delete.assert_called_once_with(
         data='{"propagationPolicy": "Background"}',
         namespace="ns-1",
-        url="customfoos/foo-1",
+        url="/customfoos/foo-1",
         version="srcco.de/v1",
     )
 
@@ -733,6 +733,6 @@ def get(**kwargs):
     api_mock.delete.assert_called_once_with(
         data='{"propagationPolicy": "Background"}',
         namespace="ns-1",
-        url="customfoos/foo-1",
+        url="/customfoos/foo-1",
         version="srcco.de/v1",
     )