Remove undefined values from evaluations. (#393)

Signed-off-by: dblock <[email protected]>

tools/src/tester/ChapterEvaluator.ts

@@ -80,11 +80,20 @@ export default class ChapterEvaluator {
   #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation {
     const expected_status = chapter.response?.status ?? 200
     if (response.status === expected_status) return { result: Result.PASSED }
-    return {
+
+    var result: Evaluation = {
       result: Result.ERROR,
-      message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}. ${response.message}`,
-      error: response.error as Error
+      message: _.join(_.compact([
+        `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`,
+        response.message
+      ]), ' ')
     }
+
+    if (response.error !== undefined) {
+      result.error = response.error as Error
+    }
+
+    return result
   }
 
   #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {

tools/src/tester/SchemaValidator.ts

@@ -45,9 +45,15 @@ export default class SchemaValidator {
       this.logger.info(`* ${to_json(data)}`)
       this.logger.info(`& ${to_json(validate.errors)}`)
     }
-    return {
+
+    var result: Evaluation = {
       result: valid ? Result.PASSED : Result.FAILED,
-      message: valid ? undefined : this.ajv.errorsText(validate.errors)
     }
+
+    if (!valid) {
+      result.message = this.ajv.errorsText(validate.errors)
+    }
+
+    return result
   }
 }

tools/tests/tester/fixtures/evals/error/chapter_error.yaml

@@ -12,8 +12,8 @@ prologues:
 chapters:
   - title: This chapter should fail.
     overall:
-      result: FAILED
       message: Operation "GET /{index}/settings" not found in the spec.
+      result: FAILED
   - title: This chapter show throw an error.
     overall:
       result: ERROR

tools/tests/tester/fixtures/evals/failed/invalid_data.yaml

@@ -1,7 +1,7 @@
 display_path: failed/invalid_data.yaml
 full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
 
-result: FAILED
+result: ERROR
 description: This story should failed due invalid data.
 
 prologues: []
@@ -59,6 +59,23 @@ chapters:
       payload_schema:
         result: FAILED
         message: data must NOT have additional properties
+  - title: This chapter should fail because the response status does not match.
+    overall:
+      result: ERROR
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: ERROR
+        message: 'Expected status 404, but received 200: application/json.'
+      payload_body:
+        result: SKIPPED
+      payload_schema:
+        result: SKIPPED
 
 epilogues:
   - title: DELETE /books

tools/tests/tester/fixtures/stories/failed/invalid_data.yaml

@@ -32,3 +32,10 @@ chapters:
       payload:
         acknowledged: false
         shards_acknowledged: true
+  - synopsis: This chapter should fail because the response status does not match.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: books
+    response:
+      status: 404

tools/tests/tester/helpers.ts

@@ -72,34 +72,51 @@ export function print_yaml (obj: any): void {
 }
 
 export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation {
-  const flatten = <T extends Evaluation | undefined>(e: T): T => (e !== undefined
-    ? {
-      ...e,
-      error: typeof e.error === 'object' ? e.error.message : e.error
+  const flatten = <T extends Evaluation | undefined>(e: T): T => {
+
+    var result = e
+
+    if (e !== undefined && result !== undefined) {
+      if (typeof e.error === 'object' && e.error.message !== undefined) {
+        result.error = e.error.message
+      } else if (e.error !== undefined) {
+        result.error = e.error
+      }
     }
-    : undefined as T)
+
+    return result
+  }
 
   const flatten_chapters = <T extends ChapterEvaluation[] | undefined> (chapters: T): T => {
     if (chapters === undefined) return undefined as T
-    return chapters.map((c: ChapterEvaluation): ChapterEvaluation => ({
-      ...c,
-      overall: flatten(c.overall),
-      request: c.request !== undefined
-        ? {
-          parameters: c.request.parameters !== undefined
-            ? Object.fromEntries(Object.entries(c.request.parameters).map(([k, v]) => [k, flatten(v)]))
-            : undefined,
+    return chapters.map((c: ChapterEvaluation): ChapterEvaluation => {
+      var result = {
+        ...c,
+        overall: flatten(c.overall),
+      }
+
+      if (c.request !== undefined) {
+        result.request = {
           request_body: flatten(c.request.request_body)
         }
-        : undefined,
-      response: c.response !== undefined
-        ? {
+
+        if (c.request.parameters !== undefined) {
+          result.request.parameters = Object.fromEntries(
+            Object.entries(c.request.parameters).map(([k, v]) => [k, flatten(v)])
+          )
+        }
+      }
+
+      if (c.response !== undefined) {
+        result.response = {
           status: flatten(c.response.status),
           payload_body: flatten(c.response.payload_body),
           payload_schema: flatten(c.response.payload_schema)
         }
-        : undefined
-    })) as T
+      }
+
+      return result;
+    }) as T
   }
 
   return {