Make service list APIs handle empty items case (#752)

**Pull Request Checklist** - [x] Fixes #720 - [ ] Tests added - [ ] Documentation/examples added - [ ] [Good commit messages](https://cbea.ms/git-commit/) and/or PR title See inline documentation for explanation Signed-off-by: Flaviu Vadan <[email protected]>
argoproj-labs · Sep 2, 2023 · 046b49f · 046b49f
1 parent d42c182
commit 046b49f
Show file tree

Hide file tree

Showing 46 changed files with 167 additions and 199 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,6 @@
-OPENAPI_SPEC_URL="https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json"
+ARGO_WORKFLOWS_VERSION="3.4.4"
+OPENAPI_SPEC_URL="https://raw.githubusercontent.com/argoproj/argo-workflows/v$(ARGO_WORKFLOWS_VERSION)/api/openapi-spec/swagger.json"
+SPEC_PATH="$(shell pwd)/argo-workflows-$(ARGO_WORKFLOWS_VERSION).json"
 
 .PHONY: help
 help: ## Showcase the help instructions for all the available `make` commands
@@ -46,8 +48,10 @@ test:  ## Run tests for Hera
 
 .PHONY: workflows-models
 workflows-models: ## Generate the Workflows models portion of Argo Workflows
+	@touch $(SPEC_PATH)
+	@poetry run python scripts/spec.py $(OPENAPI_SPEC_URL) $(SPEC_PATH)
 	@poetry run datamodel-codegen \
-		--url $(OPENAPI_SPEC_URL) \
+		--input $(SPEC_PATH) \
 		--snake-case-field \
 		--target-python-version 3.8 \
 		--output src/hera/workflows/models \
@@ -58,12 +62,15 @@ workflows-models: ## Generate the Workflows models portion of Argo Workflows
 		--use-default-kwarg
 	@poetry run python scripts/models.py $(OPENAPI_SPEC_URL) workflows
 	@poetry run stubgen -o src -p hera.workflows.models && find src/hera/workflows/models -name '__init__.pyi' -delete
+	@rm $(SPEC_PATH)
 	@$(MAKE) format
 
 .PHONY: events-models
 events-models: ## Generate the Events models portion of Argo Workflows
+	@touch $(SPEC_PATH)
+	@poetry run python scripts/spec.py $(OPENAPI_SPEC_URL) $(SPEC_PATH)
 	@poetry run datamodel-codegen \
-		--url $(OPENAPI_SPEC_URL) \
+		--input $(SPEC_PATH) \
 		--snake-case-field \
 		--target-python-version 3.8 \
 		--output src/hera/events/models \
@@ -74,6 +81,7 @@ events-models: ## Generate the Events models portion of Argo Workflows
 		--use-default-kwarg
 	@poetry run python scripts/models.py $(OPENAPI_SPEC_URL) events
 	@poetry run stubgen -o src -p hera.events.models && find src/hera/events/models -name '__init__.pyi' -delete
+	@rm $(SPEC_PATH)
 	@$(MAKE) format
 
 .PHONY: models

diff --git a/scripts/service.py b/scripts/service.py
@@ -169,36 +169,6 @@ def __str__(self) -> str:
             ret_val = "str(resp.content)"
         elif "Response" in self.response.ref:
             ret_val = f"{self.response}()"
-        elif "CronWorkflow" in self.response.ref:
-            # when users schedule cron workflows that have not executed the moment they are scheduled, the response
-            # does contain `CronWorkflowStatus` but its fields are empty. However, the `CronWorkflowStatus` object,
-            # while optional on `CronWorkflow`, has *required* fields. Here, we overwrite the response with a special
-            # case that handles setting the `CronWorkflowStatus` to `None` if the response is empty.
-            return f"""
-    {signature}
-        assert valid_host_scheme(self.host), "The host scheme is required for service usage"
-        resp = requests.{self.method}(
-            url={req_url},
-            params={params},
-            headers={headers},
-            data={body},
-            verify=self.verify_ssl
-        )
-
-        if resp.ok:
-            resp_json = resp.json()
-            if "status" in resp_json or \
-                resp_json["status"]['active'] is None or \
-                resp_json["status"]['lastScheduledTime'] is None or \
-                resp_json["status"]['conditions'] is None:
-                # this is a necessary special case as the status fields cannot be empty on the `CronWorkflowStatus`
-                # object. So, we overwrite the response with a value that allows the response to pass through safely.
-                # See `hera.scripts.service.ServiceEndpoint.__str__` for more details.
-                resp_json['status'] = None
-            return {self.response}(**resp_json)
-        
-        raise exception_from_server_response(resp)
-            """
         else:
             ret_val = f"{self.response}(**resp.json())"
 

diff --git a/scripts/spec.py b/scripts/spec.py
@@ -0,0 +1,59 @@
+"""This small module downloads and adjusts the OpenAPI spec of a given Argo Workflows version."""
+
+import json
+import logging
+import sys
+from typing import Dict, List, Set
+
+import requests
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+# get the OpenAPI spec URI from the command line, along with the output file
+open_api_spec_url = sys.argv[1]
+assert open_api_spec_url is not None, "Expected the OpenAPI spec URL to be passed as the first argument"
+
+output_file = sys.argv[2]
+assert output_file is not None, "Expected the output file to be passed as the second argument"
+
+# download the spec
+response = requests.get(open_api_spec_url)
+
+# get the spec into a dictionary
+spec = response.json()
+
+# these are specifications of objects with fields that are marked as required. However, it is possible for the Argo
+# Server to not return anything for those fields. In those cases, Pydantic fails type validation for those objects.
+# Here, we maintain a map of objects specifications whose fields must be marked as optional i.e. removed from the
+# `required` list in the OpenAPI specification.
+DEFINITION_TO_OPTIONAL_FIELDS: Dict[str, List[str]] = {
+    "io.argoproj.workflow.v1alpha1.CronWorkflowStatus": ["active", "lastScheduledTime", "conditions"],
+    "io.argoproj.workflow.v1alpha1.CronWorkflowList": ["items"],
+    "io.argoproj.workflow.v1alpha1.ClusterWorkflowTemplateList": ["items"],
+    "io.argoproj.workflow.v1alpha1.WorkflowList": ["items"],
+    "io.argoproj.workflow.v1alpha1.WorkflowTemplateList": ["items"],
+    "io.argoproj.workflow.v1alpha1.WorkflowEventBindingList": ["items"],
+    "io.argoproj.workflow.v1alpha1.Metrics": ["prometheus"],
+}
+for definition, optional_fields in DEFINITION_TO_OPTIONAL_FIELDS.items():
+    try:
+        curr_required: Set[str] = set(spec["definitions"][definition]["required"])
+    except KeyError as e:
+        raise KeyError(
+            f"Could not find definition {definition} in Argo specification for OpenAPI URI {open_api_spec_url}, "
+            f"caught error: {e}"
+        )
+    for optional_field in optional_fields:
+        if optional_field in curr_required:
+            curr_required.remove(optional_field)
+        else:
+            logger.warning(
+                f"Expected to find and change field {optional_fields} of {definition} from required to optional, "
+                f"but it was not found"
+            )
+    spec["definitions"][definition]["required"] = list(curr_required)
+
+# finally, we write the spec to the output file that is passed to use assuming the client wants to perform
+# something with this file
+with open(output_file, "w+") as f:
+    json.dump(spec, f, indent=2)
diff --git a/src/hera/events/models/eventsource.py b/src/hera/events/models/eventsource.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/google/__init__.py b/src/hera/events/models/google/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/google/protobuf.py b/src/hera/events/models/google/protobuf.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/grpc/gateway/__init__.py b/src/hera/events/models/grpc/gateway/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/grpc/gateway/runtime.py b/src/hera/events/models/grpc/gateway/runtime.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/argoproj/events/__init__.py b/src/hera/events/models/io/argoproj/events/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/argoproj/events/v1alpha1.py b/src/hera/events/models/io/argoproj/events/v1alpha1.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/argoproj/workflow/__init__.py b/src/hera/events/models/io/argoproj/workflow/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/argoproj/workflow/v1alpha1.py b/src/hera/events/models/io/argoproj/workflow/v1alpha1.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 
@@ -674,14 +674,16 @@ class ContainerSetRetryStrategy(BaseModel):
 
 
 class CronWorkflowStatus(BaseModel):
-    active: List[v1.ObjectReference] = Field(
-        ..., description="Active is a list of active workflows stemming from this CronWorkflow"
+    active: Optional[List[v1.ObjectReference]] = Field(
+        default=None, description="Active is a list of active workflows stemming from this CronWorkflow"
     )
-    conditions: List[Condition] = Field(
-        ..., description="Conditions is a list of conditions the CronWorkflow may have"
+    conditions: Optional[List[Condition]] = Field(
+        default=None, description="Conditions is a list of conditions the CronWorkflow may have"
     )
-    last_scheduled_time: v1_1.Time = Field(
-        ..., alias="lastScheduledTime", description="LastScheduleTime is the last time the CronWorkflow was scheduled"
+    last_scheduled_time: Optional[v1_1.Time] = Field(
+        default=None,
+        alias="lastScheduledTime",
+        description="LastScheduleTime is the last time the CronWorkflow was scheduled",
     )
 
 
@@ -898,7 +900,9 @@ class Memoize(BaseModel):
 
 
 class Metrics(BaseModel):
-    prometheus: List[Prometheus] = Field(..., description="Prometheus is a list of prometheus metrics to be emitted")
+    prometheus: Optional[List[Prometheus]] = Field(
+        default=None, description="Prometheus is a list of prometheus metrics to be emitted"
+    )
 
 
 class OAuth2Auth(BaseModel):
@@ -2469,7 +2473,7 @@ class WorkflowEventBindingList(BaseModel):
             " https://git.io.k8s.community/contributors/devel/sig-architecture/api-conventions.md#resources"
         ),
     )
-    items: List[WorkflowEventBinding]
+    items: Optional[List[WorkflowEventBinding]] = None
     kind: Optional[str] = Field(
         default=None,
         description=(
@@ -2523,7 +2527,7 @@ class ClusterWorkflowTemplateList(BaseModel):
             " https://git.io.k8s.community/contributors/devel/sig-architecture/api-conventions.md#resources"
         ),
     )
-    items: List[ClusterWorkflowTemplate]
+    items: Optional[List[ClusterWorkflowTemplate]] = None
     kind: Optional[str] = Field(
         default=None,
         description=(
@@ -2579,7 +2583,7 @@ class CronWorkflowList(BaseModel):
             " https://git.io.k8s.community/contributors/devel/sig-architecture/api-conventions.md#resources"
         ),
     )
-    items: List[CronWorkflow]
+    items: Optional[List[CronWorkflow]] = None
     kind: Optional[str] = Field(
         default=None,
         description=(
@@ -2955,7 +2959,7 @@ class WorkflowList(BaseModel):
             " https://git.io.k8s.community/contributors/devel/sig-architecture/api-conventions.md#resources"
         ),
     )
-    items: List[Workflow]
+    items: Optional[List[Workflow]] = None
     kind: Optional[str] = Field(
         default=None,
         description=(
@@ -3405,7 +3409,7 @@ class WorkflowTemplateList(BaseModel):
             " https://git.io.k8s.community/contributors/devel/sig-architecture/api-conventions.md#resources"
         ),
     )
-    items: List[WorkflowTemplate]
+    items: Optional[List[WorkflowTemplate]] = None
     kind: Optional[str] = Field(
         default=None,
         description=(

diff --git a/src/hera/events/models/io/argoproj/workflow/v1alpha1.pyi b/src/hera/events/models/io/argoproj/workflow/v1alpha1.pyi
@@ -339,9 +339,9 @@ class ContainerSetRetryStrategy(BaseModel):
     retries: intstr.IntOrString
 
 class CronWorkflowStatus(BaseModel):
-    active: List[v1.ObjectReference]
-    conditions: List[Condition]
-    last_scheduled_time: v1_1.Time
+    active: Optional[List[v1.ObjectReference]]
+    conditions: Optional[List[Condition]]
+    last_scheduled_time: Optional[v1_1.Time]
 
 class GCSArtifact(BaseModel):
     bucket: Optional[str]
@@ -405,7 +405,7 @@ class Memoize(BaseModel):
     max_age: str
 
 class Metrics(BaseModel):
-    prometheus: List[Prometheus]
+    prometheus: Optional[List[Prometheus]]
 
 class OAuth2Auth(BaseModel):
     client_id_secret: Optional[v1.SecretKeySelector]
@@ -814,7 +814,7 @@ class WorkflowEventBinding(BaseModel):
 
 class WorkflowEventBindingList(BaseModel):
     api_version: Optional[str]
-    items: List[WorkflowEventBinding]
+    items: Optional[List[WorkflowEventBinding]]
     kind: Optional[str]
     metadata: v1_1.ListMeta
 
@@ -834,7 +834,7 @@ class ClusterWorkflowTemplateLintRequest(BaseModel):
 
 class ClusterWorkflowTemplateList(BaseModel):
     api_version: Optional[str]
-    items: List[ClusterWorkflowTemplate]
+    items: Optional[List[ClusterWorkflowTemplate]]
     kind: Optional[str]
     metadata: v1_1.ListMeta
 
@@ -856,7 +856,7 @@ class CronWorkflow(BaseModel):
 
 class CronWorkflowList(BaseModel):
     api_version: Optional[str]
-    items: List[CronWorkflow]
+    items: Optional[List[CronWorkflow]]
     kind: Optional[str]
     metadata: v1_1.ListMeta
 
@@ -965,7 +965,7 @@ class WorkflowLintRequest(BaseModel):
 
 class WorkflowList(BaseModel):
     api_version: Optional[str]
-    items: List[Workflow]
+    items: Optional[List[Workflow]]
     kind: Optional[str]
     metadata: v1_1.ListMeta
 
@@ -1068,7 +1068,7 @@ class WorkflowTemplateLintRequest(BaseModel):
 
 class WorkflowTemplateList(BaseModel):
     api_version: Optional[str]
-    items: List[WorkflowTemplate]
+    items: Optional[List[WorkflowTemplate]]
     kind: Optional[str]
     metadata: v1_1.ListMeta
 

diff --git a/src/hera/events/models/io/k8s/api/core/__init__.py b/src/hera/events/models/io/k8s/api/core/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/k8s/api/core/v1.py b/src/hera/events/models/io/k8s/api/core/v1.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/k8s/api/policy/__init__.py b/src/hera/events/models/io/k8s/api/policy/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/k8s/api/policy/v1beta1.py b/src/hera/events/models/io/k8s/api/policy/v1beta1.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/api/__init__.py b/src/hera/events/models/io/k8s/apimachinery/pkg/api/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/api/resource.py b/src/hera/events/models/io/k8s/apimachinery/pkg/api/resource.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/apis/meta/__init__.py b/src/hera/events/models/io/k8s/apimachinery/pkg/apis/meta/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/apis/meta/v1.py b/src/hera/events/models/io/k8s/apimachinery/pkg/apis/meta/v1.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations
 

diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/util/__init__.py b/src/hera/events/models/io/k8s/apimachinery/pkg/util/__init__.py
@@ -1,2 +1,2 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
diff --git a/src/hera/events/models/io/k8s/apimachinery/pkg/util/intstr.py b/src/hera/events/models/io/k8s/apimachinery/pkg/util/intstr.py
@@ -1,5 +1,5 @@
 # generated by datamodel-codegen:
-#   filename:  https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.4/api/openapi-spec/swagger.json
+#   filename:  argo-workflows-3.4.4.json
 
 from __future__ import annotations