improvement(argus-client): ship argus client within sct

In order to ease update argus client, it was removed from docker image
and included in SCT repository. This way we may update argus client
without rebuilding hydra image and easily backport it to other sct
branches.

Added simple script to streamline this process.

fixes: scylladb/qa-tasks#1766

.pre-commit-config.yaml

@@ -8,10 +8,11 @@ repos:
         exclude: >
             (?x)^(
                 \.git/.*|
-                data_dir/latte_stress_output\.log
+                data_dir/latte_stress_output\.log|
+                argus/.*
             )$
     -   id: end-of-file-fixer
-        exclude: ^\.git/.*$
+        exclude: ^(\.git/.*|argus/.*)$
     -   id: check-yaml
         args:
         - --allow-multiple-documents
@@ -41,13 +42,14 @@ repos:
         entry: autopep8 -i -j 2 --max-line-length=120 --ignore=E226,E24,W50,W690,E402,E731
         language: system
         types: [python]
-        exclude: '\.sh$'
+        exclude: '(\.sh$|argus/.*)'
 
     -   id: ruff
         name: ruff
         entry: ruff check --fix --preview
         language: system
         types: [python]
+        exclude: ^argus/.*$
 
 - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
   rev: v5.0.0

argus/client/__init__.py

@@ -0,0 +1 @@
+from argus.client.base import ArgusClient

argus/client/base.py

@@ -0,0 +1,225 @@
+import re
+import logging
+from dataclasses import asdict
+from typing import Any, Type
+from uuid import UUID
+
+import requests
+
+from argus.common.enums import TestStatus
+from argus.client.generic_result import GenericResultTable
+from argus.client.sct.types import LogLink
+
+JSON = dict[str, Any] | list[Any] | int | str | float | bool | Type[None]
+LOGGER = logging.getLogger(__name__)
+
+
+class ArgusClientError(Exception):
+    pass
+
+
+class ArgusClient:
+    schema_version: str | None = None
+
+    class Routes():
+        # pylint: disable=too-few-public-methods
+        SUBMIT = "/testrun/$type/submit"
+        GET = "/testrun/$type/$id/get"
+        HEARTBEAT = "/testrun/$type/$id/heartbeat"
+        GET_STATUS = "/testrun/$type/$id/get_status"
+        SET_STATUS = "/testrun/$type/$id/set_status"
+        SET_PRODUCT_VERSION = "/testrun/$type/$id/update_product_version"
+        SUBMIT_LOGS = "/testrun/$type/$id/logs/submit"
+        SUBMIT_RESULTS = "/testrun/$type/$id/submit_results"
+        FETCH_RESULTS = "/testrun/$type/$id/fetch_results"
+        FINALIZE = "/testrun/$type/$id/finalize"
+
+    def __init__(self, auth_token: str, base_url: str, api_version="v1") -> None:
+        self._auth_token = auth_token
+        self._base_url = base_url
+        self._api_ver = api_version
+
+    @property
+    def auth_token(self) -> str:
+        return self._auth_token
+
+    def verify_location_params(self, endpoint: str, location_params: dict[str, str]) -> bool:
+        required_params: list[str] = re.findall(r"\$[\w_]+", endpoint)
+        for param in required_params:
+            if param.lstrip("$") not in location_params.keys():
+                raise ArgusClientError(f"Missing required location argument for endpoint {endpoint}: {param}")
+
+        return True
+
+    @staticmethod
+    def check_response(response: requests.Response, expected_code: int = 200):
+        if response.status_code != expected_code:
+            raise ArgusClientError(
+                f"Unexpected HTTP Response encountered - expected: {expected_code}, got: {response.status_code}",
+                expected_code,
+                response.status_code,
+                response.request,
+            )
+
+        response_data: JSON = response.json()
+        LOGGER.debug("API Response: %s", response_data)
+        if response_data.get("status") != "ok":
+            exc_args = response_data["response"]["arguments"]
+            raise ArgusClientError(
+                f"API Error encountered using endpoint: {response.request.method} {response.request.path_url}",
+                exc_args[0] if len(exc_args) > 0 else response_data.get("response", {}).get("exception", "#NoMessage"),
+            )
+
+    def get_url_for_endpoint(self, endpoint: str, location_params: dict[str, str] | None) -> str:
+        if self.verify_location_params(endpoint, location_params):
+            for param, value in location_params.items():
+                endpoint = endpoint.replace(f"${param}", str(value))
+        return f"{self._base_url}/api/{self._api_ver}/client{endpoint}"
+
+    @property
+    def generic_body(self) -> dict:
+        return {
+            "schema_version": self.schema_version
+        }
+
+    @property
+    def request_headers(self):
+        return {
+            "Authorization": f"token {self.auth_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+
+    def get(self, endpoint: str, location_params: dict[str, str] = None, params: dict = None) -> requests.Response:
+        response = requests.get(
+            url=self.get_url_for_endpoint(
+                endpoint=endpoint,
+                location_params=location_params
+            ),
+            params=params,
+            headers=self.request_headers
+        )
+
+        return response
+
+    def post(
+        self,
+        endpoint: str,
+        location_params: dict = None,
+        params: dict = None,
+        body: dict = None,
+    ) -> requests.Response:
+        response = requests.post(
+            url=self.get_url_for_endpoint(
+                endpoint=endpoint,
+                location_params=location_params
+            ),
+            params=params,
+            json=body,
+            headers=self.request_headers,
+        )
+
+        return response
+
+    def submit_run(self, run_type: str, run_body: dict) -> requests.Response:
+        return self.post(endpoint=self.Routes.SUBMIT, location_params={"type": run_type}, body={
+            **self.generic_body,
+            **run_body
+        })
+
+    def get_run(self, run_type: str = None, run_id: UUID | str = None) -> requests.Response:
+
+        if not run_type and hasattr(self, "test_type"):
+            run_type = self.test_type
+
+        if not run_id and hasattr(self, "run_id"):
+            run_id = self.run_id
+
+        if not (run_type and run_id):
+            raise ValueError("run_type and run_id must be set in func params or object attributes")
+
+        response = self.get(endpoint=self.Routes.GET, location_params={"type": run_type, "id": run_id })
+        self.check_response(response)
+
+        return response.json()["response"]
+
+    def get_status(self, run_type: str = None, run_id: UUID = None) -> TestStatus:
+        if not run_type and hasattr(self, "test_type"):
+            run_type = self.test_type
+
+        if not run_id and hasattr(self, "run_id"):
+            run_id = self.run_id
+
+        if not (run_type and run_id):
+            raise ValueError("run_type and run_id must be set in func params or object attributes")
+
+        response = self.get(
+            endpoint=self.Routes.GET_STATUS,
+            location_params={"type": run_type, "id": str(run_id)},
+        )
+        self.check_response(response)
+        return TestStatus(response.json()["response"])
+
+    def set_status(self, run_type: str, run_id: UUID, new_status: TestStatus) -> requests.Response:
+        return self.post(
+            endpoint=self.Routes.SET_STATUS,
+            location_params={"type": run_type, "id": str(run_id)},
+            body={
+                **self.generic_body,
+                "new_status": new_status.value
+            }
+        )
+
+    def update_product_version(self, run_type: str, run_id: UUID, product_version: str) -> requests.Response:
+        return self.post(
+            endpoint=self.Routes.SET_PRODUCT_VERSION,
+            location_params={"type": run_type, "id": str(run_id)},
+            body={
+                **self.generic_body,
+                "product_version": product_version
+            }
+        )
+
+    def submit_logs(self, run_type: str, run_id: UUID, logs: list[LogLink]) -> requests.Response:
+        return self.post(
+            endpoint=self.Routes.SUBMIT_LOGS,
+            location_params={"type": run_type, "id": str(run_id)},
+            body={
+                **self.generic_body,
+                "logs": [asdict(l) for l in logs]
+            }
+        )
+
+    def finalize_run(self, run_type: str, run_id: UUID, body: dict = None) -> requests.Response:
+        body = body if body else {}
+        return self.post(
+            endpoint=self.Routes.FINALIZE,
+            location_params={"type": run_type, "id": str(run_id)},
+            body={
+                **self.generic_body,
+                **body,
+            }
+        )
+
+    def heartbeat(self, run_type: str, run_id: UUID) -> None:
+        response = self.post(
+            endpoint=self.Routes.HEARTBEAT,
+            location_params={"type": run_type, "id": str(run_id)},
+            body={
+                **self.generic_body,
+            }
+        )
+        self.check_response(response)
+
+    def submit_results(self, result: GenericResultTable) -> None:
+        response = self.post(
+            endpoint=self.Routes.SUBMIT_RESULTS,
+            location_params={"type": self.test_type, "id": str(self.run_id)},
+            body={
+                **self.generic_body,
+                "run_id": str(self.run_id),
+                ** result.as_dict(),
+            }
+        )
+        self.check_response(response)
+

argus/client/driver_matrix_tests/cli.py

@@ -0,0 +1,110 @@
+import base64
+import json
+from pathlib import Path
+import click
+import logging
+from argus.common.enums import TestStatus
+
+from argus.client.driver_matrix_tests.client import ArgusDriverMatrixClient
+
+LOGGER = logging.getLogger(__name__)
+
+
+@click.group
+def cli():
+    pass
+
+
+def _submit_driver_result_internal(api_key: str, base_url: str, run_id: str, metadata_path: str):
+    metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
+    LOGGER.info("Submitting results for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
+    raw_xml = (Path(metadata_path).parent / metadata["junit_result"]).read_bytes()
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client.submit_driver_result(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], raw_junit_data=base64.encodebytes(raw_xml))
+    LOGGER.info("Done.")
+
+def _submit_driver_failure_internal(api_key: str, base_url: str, run_id: str, metadata_path: str):
+    metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
+    LOGGER.info("Submitting failure for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client.submit_driver_failure(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], failure_reason=metadata["failure_reason"])
+    LOGGER.info("Done.")
+
+
+@click.command("submit-run")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--build-id", required=True, help="Unique job identifier in the build system, e.g. scylla-master/group/job for jenkins (The full path)")
+@click.option("--build-url", required=True, help="Job URL in the build system")
+def submit_driver_matrix_run(api_key: str, base_url: str, run_id: str, build_id: str, build_url: str):
+    LOGGER.info("Submitting %s (%s) to Argus...", build_id, run_id)
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client.submit_driver_matrix_run(job_name=build_id, job_url=build_url)
+    LOGGER.info("Done.")
+
+
+@click.command("submit-driver")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
+def submit_driver_result(api_key: str, base_url: str, run_id: str, metadata_path: str):
+    _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+
+
+@click.command("fail-driver")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
+def submit_driver_failure(api_key: str, base_url: str, run_id: str, metadata_path: str):
+    _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+
+
+@click.command("submit-or-fail-driver")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
+def submit_or_fail_driver(api_key: str, base_url: str, run_id: str, metadata_path: str):
+    metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
+    if metadata.get("failure_reason"):
+        _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+    else:
+        _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path)
+
+
+@click.command("submit-env")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--env-path", required=True, help="Path to the Build-00.txt file that contains environment information about Scylla")
+def submit_driver_env(api_key: str, base_url: str, run_id: str, env_path: str):
+    LOGGER.info("Submitting environment for run %s to Argus...", run_id)
+    raw_env = Path(env_path).read_text()
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client.submit_env(raw_env)
+    LOGGER.info("Done.")
+
+
+@click.command("finish-run")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--status", required=True, help="Resulting job status")
+def finish_driver_matrix_run(api_key: str, base_url: str, run_id: str, status: str):
+    client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url)
+    client.finalize_run(run_type=ArgusDriverMatrixClient.test_type, run_id=run_id, body={"status": TestStatus(status)})
+
+
+cli.add_command(submit_driver_matrix_run)
+cli.add_command(submit_driver_result)
+cli.add_command(submit_or_fail_driver)
+cli.add_command(submit_driver_failure)
+cli.add_command(submit_driver_env)
+cli.add_command(finish_driver_matrix_run)
+
+
+if __name__ == "__main__":
+    cli()