From e99831e6d55e6b627e1f50a5fce1b40fdc11e884 Mon Sep 17 00:00:00 2001 From: Carl Gay Date: Sun, 1 Oct 2023 04:42:16 +0000 Subject: [PATCH] Add expect-* macros to match the assert-* macros These macros don't terminate the test, so in that respect they're similar to the `check-*` macros, but their signatures are like the `assert-*` macros, in which the assertion description is optional and if not supplied is auto-generated from the arguments. See discussion in https://github.com/dylan-lang/testworks/issues/86 --- assertions.dylan | 192 ++++++++++++++++++++++++++----- library.dylan | 21 +++- tests/testworks-test-suite.dylan | 166 +++++++++++++++++++++++--- 3 files changed, 332 insertions(+), 47 deletions(-) diff --git a/assertions.dylan b/assertions.dylan index 992baae..78ae25a 100644 --- a/assertions.dylan +++ b/assertions.dylan @@ -13,8 +13,14 @@ define class () end; /// Assertion macros -// The check-* macros require the caller to provide a name. -// The assert-* macros auto-generate a name by default. +// The check-* macros are non-terminating, require the caller to provide a +// name, and are DEPRECATED. Use expect-* instead in new code. +// +// The expect-* macros are non-terminating and auto-generate a description if +// none is provided. +// +// The assert-* macros are terminating (they cause the remainder of a test to +// be skipped when they fail) and they auto-generate a description by default. // Note that these macros wrap up the real macro arguments inside // methods to delay their evaluation until they are within the scope @@ -24,16 +30,13 @@ define function eval-check-description (thunk :: ) => (description :: ) let (description, #rest args) = thunk(); if (empty?(args)) - if (instance?(description, )) - description - else - format-to-string("%s", description) - end + format-to-string("%s", description) else apply(format-to-string, description, args) end end function; +// Deprecated; use expect. define macro check { check (?check-name:expression, ?check-function:expression, ?check-args:*) @@ -42,6 +45,15 @@ define macro check } end macro check; +define macro expect + { expect (?expr:expression) } + => { expect(?expr, ?"expr") } + + { expect (?expr:expression, ?description:*) } + => { expect-true(?expr, ?description) } +end macro; + +// Deprecated; use expect-equal. define macro check-equal { check-equal (?name:expression, ?want:expression, ?got:expression) } => { @@ -70,15 +82,26 @@ define macro assert-equal } end macro assert-equal; +define macro expect-equal + { expect-equal(?want:expression, ?got:expression) } + => { expect-equal(?want, ?got, ?"want" " = " ?"got") } + + { expect-equal(?want:expression, ?got:expression, ?description:*) } + => { do-check-equal(method () values(?description) end, + method () values(?want, ?got, ?"want", ?"got") end, + "expect-equal", + terminate?: #f) } +end macro; + define function do-check-equal - (description-thunk :: , arguments-thunk :: , - caller :: , #key terminate? :: ) + (description-thunk :: , arguments-thunk :: , caller :: , + #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (want, got, want-expr, got-expr) = arguments-thunk(); phase := format-to-string("while comparing %s and %s for equality", want-expr, got-expr); @@ -105,6 +128,17 @@ define function do-check-equal end block end function; +define macro expect-not-equal + { expect-not-equal(?expr1:expression, ?expr2:expression) } + => { expect-not-equal(?expr1, ?expr2, ?"expr1" " ~= " ?"expr2") } + + { expect-not-equal(?expr1:expression, ?expr2:expression, ?description:*) } + => { do-check-not-equal(method () values(?description) end, + method () values(?expr1, ?expr2, ?"expr1", ?"expr2") end, + "expect-not-equal", + terminate?: #f) } +end macro; + define macro assert-not-equal { assert-not-equal (?expr1:expression, ?expr2:expression) } => { @@ -116,26 +150,27 @@ define macro assert-not-equal method () values(?expr1, ?expr2, ?"expr1", ?"expr2") end, + "assert-not-equal", terminate?: #t) } end macro assert-not-equal; define function do-check-not-equal - (description-thunk :: , arguments-thunk :: , + (description-thunk :: , arguments-thunk :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (val1, val2, expr1, expr2) = arguments-thunk(); phase := format-to-string("while comparing %s and %s for inequality", expr1, expr2); if (val1 ~= val2) record-check(description, $passed, #f); else - phase := "getting assert-not-equal failure detail"; + phase := format-to-string("getting %s failure detail", caller); record-check(description, $failed, format-to-string("%= and %= are =.", val1, val2)); terminate? & signal(make()); @@ -205,6 +240,7 @@ define method check-equal-failure-detail join(choose(identity, vector(detail1, detail2, detail3)), "; ") end method; +// Deprecated; use expect-instance?. define macro check-instance? { check-instance? (?check-name:expression, ?type:expression, ?value:expression) } => { @@ -212,11 +248,24 @@ define macro check-instance? method () values(?type, ?value, ?"value") end, + "check-instance?", negate?: #f, terminate?: #f) } end macro check-instance?; +define macro expect-instance? + { expect-instance?(?type:expression, ?value:expression) } + => { expect-instance?(?type, ?value, ?"value" " is an instance of " ?"type") } + + { expect-instance?(?type:expression, ?value:expression, ?description:*) } + => { do-check-instance?(method () values(?description) end, + method () values(?type, ?value, ?"value") end, + "expect-instance?", + terminate?: #f) + } +end macro; + define macro assert-instance? { assert-instance? (?type:expression, ?value:expression) } => { @@ -228,11 +277,25 @@ define macro assert-instance? method () values(?type, ?value, ?"value") end, + "assert-instance?", negate?: #f, terminate?: #t) } end macro assert-instance?; +define macro expect-not-instance? + { expect-not-instance?(?type:expression, ?value:expression) } + => { expect-not-instance?(?type, ?value, ?"value" " is not an instance of " ?"type") } + + { expect-not-instance?(?type:expression, ?value:expression, ?description:*) } + => { do-check-instance?(method () values(?description) end, + method () values(?type, ?value, ?"value") end, + "expect-not-instance?", + negate?: #t, + terminate?: #f) + } +end macro; + define macro assert-not-instance? { assert-not-instance? (?type:expression, ?value:expression) } => { @@ -244,21 +307,22 @@ define macro assert-not-instance? method () values(?type, ?value, ?"value") end, + "assert-not-instance?", negate?: #t, terminate?: #t) } end macro assert-not-instance?; define function do-check-instance? - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key negate? :: , terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (type :: , value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= is %=an instance of %s", value-expr, if (negate?) "not " else "" end, type); @@ -278,6 +342,7 @@ define function do-check-instance? end block end function do-check-instance?; +// Deprecated; use expect-true. define macro check-true { check-true (?check-name:expression, ?expr:expression) } => { @@ -285,10 +350,23 @@ define macro check-true method () values(?expr, ?"expr") end, + "check-true", terminate?: #f) } end macro check-true; +define macro expect-true + { expect-true(?expr:expression) } + => { expect-true(?expr, ?"expr" " is true") } + + { expect-true(?expr:expression, ?description:*) } + => { do-check-true(method () values(?description) end, + method () values(?expr, ?"expr") end, + "expect-true", + terminate?: #f) + } +end macro; + define macro assert-true { assert-true (?expr:expression) } => { @@ -299,19 +377,20 @@ define macro assert-true } => { do-check-true(method () values(?description) end, method () values(?expr, ?"expr") end, + "assert-true", terminate?: #t) } end macro assert-true; define function do-check-true - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= evaluates to true", value-expr); if (value) @@ -329,6 +408,7 @@ define function do-check-true end block end function do-check-true; +// Deprecated; use expect-false. define macro check-false { check-false (?check-name:expression, ?expr:expression) } => { @@ -336,10 +416,23 @@ define macro check-false method () values(?expr, ?"expr") end, + "check-false", terminate?: #f) } end macro check-false; +define macro expect-false + { expect-false(?expr:expression) } + => { expect-false(?expr, ?"expr" " evaluates to #f") } + + { expect-false (?expr:expression, ?description:*) } + => { do-check-false(method () values(?description) end, + method () values(?expr, ?"expr") end, + "expect-false", + terminate?: #f) + } +end macro; + define macro assert-false { assert-false (?expr:expression) } => { @@ -352,19 +445,20 @@ define macro assert-false method () values(?expr, ?"expr") end, + "assert-false", terminate?: #t) } end macro assert-false; define function do-check-false - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= evaluates to #f", value-expr); if (~value) @@ -383,6 +477,7 @@ define function do-check-false end block end function do-check-false; +// Deprecated; use expect-condition. define macro check-condition { check-condition(?check-name:expression, ?condition:expression, ?expr:expression) @@ -391,10 +486,23 @@ define macro check-condition method () values(?condition, method () ?expr end, ?"expr") end, + "check-condition", terminate?: #f) } end macro check-condition; +define macro expect-condition + { expect-condition(?condition:expression, ?expr:expression) } + => { expect-condition(?condition, ?expr, ?"expr" " signals condition " ?"condition") } + + { expect-condition(?condition:expression, ?expr:expression, ?description:*) } + => { do-check-condition(method () values(?description) end, + method () values(?condition, method () ?expr end, ?"expr") end, + "expect-condition", + terminate?: #f) } +end macro; + +// Deprecated; use assert-condition. define macro assert-signals { assert-signals(?condition:expression, ?expr:expression) } => { @@ -407,19 +515,31 @@ define macro assert-signals method () values(?condition, method () ?expr end, ?"expr") end, + "assert-signals", terminate?: #t) } end macro assert-signals; +define macro assert-condition + { assert-condition(?condition:expression, ?expr:expression) } + => { assert-condition(?condition, ?expr, ?"expr" " signals condition " ?"condition") } + + { assert-condition(?condition:expression, ?expr:expression, ?description:*) } + => { do-check-condition(method () values(?description) end, + method () values(?condition, method () ?expr end, ?"expr") end, + "assert-condition", + terminate?: #t) } +end macro; + define function do-check-condition - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (condition-class, thunk :: , expr :: ) = get-arguments(); phase := format-to-string("checking if %= signals a condition of class %s", expr, condition-class); @@ -448,6 +568,7 @@ define function do-check-condition end function do-check-condition; +// Deprecated; use expect-no-errors. // Same as check-no-errors, for symmetry with check-condition... define macro check-no-condition { check-no-condition(?check-name:expression, ?check-body:expression) @@ -456,6 +577,7 @@ define macro check-no-condition } end macro check-no-condition; +// Deprecated; use expect-no-errors. define macro check-no-errors { check-no-errors(?check-name:expression, ?check-body:expression) } => { @@ -463,6 +585,15 @@ define macro check-no-errors } end macro check-no-errors; +define macro expect-no-condition + { expect-no-condition(?expr:expression) } + => { expect-true(begin ?expr; #t end) } + + { expect-no-condition(?expr:expression, ?description:*) } + => { expect-true(begin ?expr; #t end, ?description) } +end macro; + +// Deprecated; use assert-no-condition. define macro assert-no-errors { assert-no-errors(?expr:expression) } => { @@ -475,6 +606,13 @@ define macro assert-no-errors } end macro assert-no-errors; +define macro assert-no-condition + { assert-no-condition(?expr:expression) } + => { assert-no-condition(?expr, ?"expr" " doesn't signal an error ") } + + { assert-no-condition(?expr:expression, ?description:*) } + => { assert-true(begin ?expr; #t end, ?description) } +end macro; /// Check recording diff --git a/library.dylan b/library.dylan index f08bee7..34be7da 100644 --- a/library.dylan +++ b/library.dylan @@ -41,13 +41,12 @@ define module testworks runner-skip, runner-tags; - // Checks (deprecated, use assertions) + // Checks (deprecated, use assert-* or expect-*) create check, check-condition, check-no-condition, check-equal, - check-equal-failure-detail, check-false, check-no-errors, check-instance?, @@ -57,12 +56,26 @@ define module testworks create assert-equal, assert-not-equal, - assert-signals, + assert-condition, + assert-signals, // Deprecated; use assert-condition. assert-no-errors, assert-instance?, assert-not-instance?, assert-true, - assert-false; + assert-false, + expect, + expect-equal, + expect-not-equal, + expect-true, + expect-false, + expect-instance?, + expect-not-instance?, + expect-condition, + expect-no-condition; + + create + // Implement this to give detail to expect-equal failures for your own types. + check-equal-failure-detail; // Components create diff --git a/tests/testworks-test-suite.dylan b/tests/testworks-test-suite.dylan index 19ff055..53cda89 100644 --- a/tests/testworks-test-suite.dylan +++ b/tests/testworks-test-suite.dylan @@ -75,6 +75,12 @@ define test testworks-check-test () check("check(\\=, 3, 3)", \=, 3, 3); end test testworks-check-test; +define test testworks-expect-test () + expect(always(#t)); + expect(identity(#t)); + expect(3 = 3); +end test; + define test testworks-check-true-test () assert-equal($passed, with-result-status () @@ -93,6 +99,18 @@ define test testworks-check-true-test () "check-true of error crashes"); end test; +define test test-expect-true () + assert-equal($passed, + with-result-status () expect-true(#t) end, + "expect-true(#t) passes"); + assert-equal($failed, + with-result-status () expect-true(#f) end, + "expect-true(#f) fails"); + assert-equal($crashed, + with-result-status () expect-true(test-error()) end, + "expect-true of error crashes"); +end test; + define test testworks-assert-true-test () assert-true(#t); assert-true(#t, "#t is true with description"); @@ -119,6 +137,18 @@ define test testworks-check-false-test () "check-false of error crashes"); end test; +define test test-expect-false () + assert-equal($failed, + with-result-status () expect-false(#t) end, + "expect-false(#t) fails"); + assert-equal($passed, + with-result-status () expect-false(#f) end, + "expect-false(#f) passes"); + assert-equal($crashed, + with-result-status () expect-false(test-error()) end, + "expect-false of error crashes"); +end test; + define test testworks-assert-false-test () assert-false(#f); assert-false(#f, "#f is false with description"); @@ -150,6 +180,21 @@ define test testworks-check-equal-test () "check-equal of error crashes"); end test; +define test test-expect-equal () + assert-equal($passed, + with-result-status () expect-equal(1, 1) end, + "expect-equal(1, 1) passes"); + assert-equal($passed, + with-result-status () expect-equal("1", "1") end, + """expect-equal("1", "1") passes"""); + assert-equal($failed, + with-result-status () expect-equal(1, 2) end, + "expect-equal(1, 2) fails"); + assert-equal($crashed, + with-result-status () expect-equal(1, test-error()) end, + "expect-equal of error crashes"); +end test; + define test testworks-assert-equal-failure-detail () let result = with-result () assert-equal(#[1, 2, 3], #[1, 3, 2]) @@ -177,6 +222,15 @@ define test testworks-assert-not-equal-test () assert-equal($crashed, with-result-status () assert-not-equal(1, test-error()) end); end test; +define test test-expect-not-equal () + expect-not-equal(8, 9); + expect-not-equal(8, 9, "8 ~= 9 with description"); + assert-equal($passed, with-result-status () expect-not-equal(1, 2) end); + assert-equal($passed, with-result-status () expect-not-equal("1", "2") end); + assert-equal($failed, with-result-status () expect-not-equal(1, 1) end); + assert-equal($crashed, with-result-status () expect-not-equal(1, test-error()) end); +end test; + define test testworks-check-instance?-test () assert-equal($passed, with-result-status () @@ -195,6 +249,20 @@ define test testworks-check-instance?-test () "check-instance? of error crashes"); end test; +define test test-expect-instance? () + assert-equal($passed, + with-result-status () expect-instance?(, 1) end, + "expect-instance?(, 1) passes"); + assert-equal($failed, + with-result-status () expect-instance?(, 1) end, + "expect-instance?(, 1) fails"); + assert-equal($crashed, + with-result-status () + expect-instance?(, test-error()) + end, + "expect-instance? of error crashes"); +end test; + define test testworks-assert-instance?-test () assert-equal($passed, with-result-status () @@ -231,6 +299,24 @@ define test testworks-assert-not-instance?-test () "assert-not-instance? of error crashes"); end test; +define test test-assert-not-instance? () + assert-equal($passed, + with-result-status () + expect-not-instance?(, 1) + end, + "expect-not-instance?(, 1) passes"); + assert-equal($failed, + with-result-status () + expect-not-instance?(, 1) + end, + "expect-not-instance?(, 1) fails"); + assert-equal($crashed, + with-result-status () + expect-not-instance?(, test-error()) + end, + "expect-not-instance? of error crashes"); +end test; + define test testworks-check-condition-test () begin let success? = #f; @@ -261,30 +347,59 @@ define test testworks-check-condition-test () "check-condition doesn't catch wrong condition"); end test; -define test testworks-assert-signals-test () - assert-signals(, error("foo")); - assert-signals(, error("foo"), "error signals error w/ description"); +define test test-expect-condition () + begin + let success? = #f; + assert-equal($passed, + with-result-status () + expect-condition(, + begin + // default-handler for returns #f + test-warning(); + success? := #t; + test-error() + end) + end, + "expect-condition catches "); + assert-true(success?, + "expect-condition for doesn't catch "); + end; + assert-equal($failed, + with-result-status () + expect-condition(, #f) + end, + "expect-condition fails if no condition"); + assert-equal($failed, + with-result-status () + expect-condition(, test-warning()) + end, + "expect-condition doesn't catch wrong condition"); +end test; + +define test testworks-assert-condition-test () + assert-condition(, error("foo")); + assert-condition(, error("foo"), "error signals error w/ description"); begin let success? = #f; assert-equal($passed, with-result-status () - assert-signals(, - begin - // default-handler for returns #f - test-warning(); - success? := #t; - test-error() - end) + assert-condition(, + begin + // default-handler for returns #f + test-warning(); + success? := #t; + test-error() + end) end); assert-true(success?); end; assert-equal($failed, with-result-status () - assert-signals(, #f) + assert-condition(, #f) end); assert-equal($failed, with-result-status () - assert-signals(, test-warning()) + assert-condition(, test-warning()) end); end test; @@ -312,6 +427,13 @@ define test testworks-assert-no-errors-test () assert-equal($crashed, with-result-status () assert-no-errors(test-error()) end); end test; +define test test-expect-no-condition () + assert-equal($passed, with-result-status () expect-no-condition(#t) end); + assert-equal($passed, with-result-status () expect-no-condition(#f) end); + assert-equal($crashed, with-result-status () expect-no-condition(test-error()) end); +end test; + + define suite testworks-assertion-macros-suite () test testworks-check-test; test testworks-check-true-test; @@ -321,7 +443,6 @@ define suite testworks-assertion-macros-suite () test testworks-check-condition-test; test testworks-check-no-errors-test; - // Assert macros (newer). test testworks-assert-true-test; test testworks-assert-false-test; test testworks-assert-equal-test; @@ -329,11 +450,10 @@ define suite testworks-assertion-macros-suite () test testworks-assert-not-equal-test; test testworks-assert-instance?-test; test testworks-assert-not-instance?-test; - test testworks-assert-signals-test; + test testworks-assert-condition-test; test testworks-assert-no-errors-test; end suite testworks-assertion-macros-suite; - define benchmark basic-benchmark () "just exercise basic benchmark functionality" end; @@ -342,7 +462,6 @@ define suite testworks-benchmarks-suite () benchmark basic-benchmark; end; - /// Verify the result objects define test test-run-tests/test () @@ -479,6 +598,21 @@ define test test-check-failure-continues () end; end test; +// Make sure that if one `expect-*` expression fails the remaining assertions +// are still executed. +define test test-expect-failure-continues () + let x = #f; + block () + without-recording () + expect-true(#f); // fail + expect-true(error("blah")); // fail + end; + x := #t; + cleanup + assert-true(x); + end; +end test; + // Make sure that if one `assert-*` expression fails the rest of the test is // NOT executed. define test test-assertion-failure-terminates ()