From 9c2eab01c50d7b0d7c6e8e74cd3e2efa9f25908c Mon Sep 17 00:00:00 2001 From: mimir-d Date: Sun, 13 Oct 2024 15:47:06 +0100 Subject: [PATCH] add spec validation for the examples - fix extension example since it was outputting a string extension which is invalid, per the spec (also a test) - fix the typing to disallow primitive types for extension content - added scripts/check.sh for easier validation before github actions get to run on any PRs - fixes #5, #12 Signed-off-by: mimir-d --- .github/workflows/tests.yaml | 32 ++++++++++++++++++++++++++++++++ DEVELOPER_NOTES.md | 6 ++++-- examples/extension.py | 5 ----- scripts/check.sh | 13 +++++++++++++ {.scripts => scripts}/gendoc.sh | 0 src/ocptv/output/objects.py | 7 ++++--- tests/output/test_run.py | 32 ++++++++++---------------------- 7 files changed, 63 insertions(+), 32 deletions(-) create mode 100755 scripts/check.sh rename {.scripts => scripts}/gendoc.sh (100%) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 890a300..db6731a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,3 +30,35 @@ jobs: - name: Test with pytest run: | pytest -v + + spec: + runs-on: ubuntu-latest + name: examples / spec validation + steps: + - uses: actions/checkout@v4 + - name: Install python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: pull validator + run: git clone https://github.com/opencomputeproject/ocp-diag-core.git --depth=1 + - name: Install go + uses: actions/setup-go@v2 + with: + go-version: "1.17.6" + - name: run validator against examples + run: | + ROOT="$(pwd)" + cd ocp-diag-core/validators/spec_validator + PYTHONPATH="$ROOT" python -m examples list | + grep -v demo_python_logging_io | + xargs -I{} bash -c " + echo validating output of example {}... + PYTHONPATH=\"$ROOT\" python -m examples {} | + tee /dev/stderr | + go run . -schema ../../json_spec/output/root.json - + " diff --git a/DEVELOPER_NOTES.md b/DEVELOPER_NOTES.md index 3e12e80..0e72503 100644 --- a/DEVELOPER_NOTES.md +++ b/DEVELOPER_NOTES.md @@ -32,7 +32,7 @@ In order to do so: ```bash $ pytest -v ``` -4. *[optiona]* deactivate/exit the env +4. *[optional]* deactivate/exit the env ```bash $ deactivate ``` @@ -61,6 +61,8 @@ Steps: $ black . # will reformat all the source files $ mypy ocptv tests examples --check-untyped-defs # check the type annotations ``` + Alternatively, use `scripts/check.sh` to run all the CI checks (aside from spec validation). + 4. if the tests above pass and everything is ready, push and make a PR. This can be done either from the Github website or using [gh cli](https://cli.github.com/manual/gh_pr_create). 5. the PR will now be reviewed. If everything is ok, a maintainer will merge it to the `dev` branch. @@ -81,5 +83,5 @@ Run it just by simply using the `act -j pytest` command in the repository root d When necessary, likely due to code/doc changes, regenerate the api reference by running: ```bash -$ .scripts/gendoc.sh +$ scripts/gendoc.sh ``` \ No newline at end of file diff --git a/examples/extension.py b/examples/extension.py index cb518b0..2db8286 100644 --- a/examples/extension.py +++ b/examples/extension.py @@ -13,11 +13,6 @@ def demo_step_extension(): with run.scope(dut=tv.Dut(id="dut0")): step = run.add_step("step0") with step.scope(): - step.add_extension( - name="simple", - content="extension_identifier", - ) - step.add_extension( name="complex", content={ diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100755 index 0000000..941cca6 --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eo pipefail + +# run all the CI checks, for local evaluation + +# linter +black . --check --diff + +# typings +mypy . --check-untyped-defs + +# tests +pytest -v --cov-fail-under=100 diff --git a/.scripts/gendoc.sh b/scripts/gendoc.sh similarity index 100% rename from .scripts/gendoc.sh rename to scripts/gendoc.sh diff --git a/src/ocptv/output/objects.py b/src/ocptv/output/objects.py index 1137f29..cc65ada 100644 --- a/src/ocptv/output/objects.py +++ b/src/ocptv/output/objects.py @@ -767,11 +767,12 @@ def __post_init__(self): # note: these must specify some bounds for the extension content in python, despite # the spec saying that it can be anything. This is necessary for the json serializer # to actually know how to output the data. - ExtensionContentType = ty.Union[ # pragma: no cover - ty.Dict[str, "ExtensionContentType"], - ty.List["ExtensionContentType"], + ExtensionContentTypeInner = ty.Union[ # pragma: no cover + ty.Dict[str, "ExtensionContentTypeInner"], + ty.List["ExtensionContentTypeInner"], ty.Union[float, int, bool, str, None], ] + ExtensionContentType = ty.Dict[str, ExtensionContentTypeInner] # pragma: no cover else: # the runtime checker cannot deal with recursive types, and this is meant to be any ExtensionContentType = ty.Any diff --git a/tests/output/test_run.py b/tests/output/test_run.py index cbd4068..11687d8 100644 --- a/tests/output/test_run.py +++ b/tests/output/test_run.py @@ -3,7 +3,14 @@ import ocptv.output as tv from ocptv.formatter import format_timestamp -from ocptv.output import DiagnosisType, LogSeverity, SoftwareType, TestResult, TestStatus, ValidatorType +from ocptv.output import ( + DiagnosisType, + LogSeverity, + SoftwareType, + TestResult, + TestStatus, + ValidatorType, +) from ocptv.output.emit import JSON from .checks import IgnoreAssert, LambdaAssert, RangeAssert, assert_json @@ -678,11 +685,6 @@ def test_step_produces_extensions(writer: MockWriter): with run.scope(dut=tv.Dut(id="dut0")): step = run.add_step("step0") with step.scope(): - step.add_extension( - name="simple", - content="extension_identifier", - ) - step.add_extension( name="complex", content={ @@ -692,23 +694,9 @@ def test_step_produces_extensions(writer: MockWriter): }, ) - assert len(writer.lines) == 7 + assert len(writer.lines) == 6 assert_json( writer.lines[3], - { - "testStepArtifact": { - "extension": { - "name": "simple", - "content": "extension_identifier", - }, - "testStepId": "0", - }, - "sequenceNumber": 3, - "timestamp": IgnoreAssert(), - }, - ) - assert_json( - writer.lines[4], { "testStepArtifact": { "extension": { @@ -721,7 +709,7 @@ def test_step_produces_extensions(writer: MockWriter): }, "testStepId": "0", }, - "sequenceNumber": 4, + "sequenceNumber": 3, "timestamp": IgnoreAssert(), }, )