From 79c9b77d1c081f08f2606e32154f0b56adac4bf8 Mon Sep 17 00:00:00 2001 From: Carl Gay Date: Sun, 1 Oct 2023 04:42:16 +0000 Subject: [PATCH] Add expect-* macros to match the assert-* macros These macros don't terminate the test, so in that respect they're similar to the `check-*` macros, but their signatures are like the `assert-*` macros, in which the assertion description is optional and if not supplied is auto-generated from the arguments. See discussion in https://github.com/dylan-lang/testworks/issues/86 --- assertions.dylan | 268 +++++++++++++++++++---------- documentation/source/reference.rst | 216 ++++++++++++++++++----- documentation/source/usage.rst | 87 +++++----- library.dylan | 20 ++- tests/testworks-test-suite.dylan | 175 ++++++++++++++++--- 5 files changed, 564 insertions(+), 202 deletions(-) diff --git a/assertions.dylan b/assertions.dylan index 36c8402..ad45cab 100644 --- a/assertions.dylan +++ b/assertions.dylan @@ -13,8 +13,14 @@ define class () end; /// Assertion macros -// The check-* macros require the caller to provide a name. -// The assert-* macros auto-generate a name by default. +// The check-* macros are non-terminating, require the caller to provide a +// name, and are DEPRECATED. Use expect-* instead in new code. +// +// The expect-* macros are non-terminating and auto-generate a description if +// none is provided. +// +// The assert-* macros are terminating (they cause the remainder of a test to +// be skipped when they fail) and they auto-generate a description by default. // Note that these macros wrap up the real macro arguments inside // methods to delay their evaluation until they are within the scope @@ -24,35 +30,35 @@ define function eval-check-description (thunk :: ) => (description :: ) let (description, #rest args) = thunk(); if (empty?(args)) - if (instance?(description, )) - description - else - format-to-string("%s", description) - end + format-to-string("%s", description) else apply(format-to-string, description, args) end end function; +// Deprecated; use expect. define macro check - { check (?check-name:expression, - ?check-function:expression, ?check-args:*) - } => { - check-true(?check-name, apply(?check-function, vector(?check-args))) + { check(?description:expression, ?fun:expression, ?args:*) } + => { expect(?fun(?args), ?description) } +end macro; + +define macro expect + { expect(?expr:expression) } + => { expect(?expr, ?"expr" " is true") } + + { expect(?expr:expression, ?description:*) } + => { do-check-true(method () values(?description) end, + method () values(?expr, ?"expr") end, + "expect", + terminate?: #f) } -end macro check; +end macro; +// Deprecated; use expect-equal. define macro check-equal - { check-equal (?name:expression, ?want:expression, ?got:expression) - } => { - do-check-equal(method () ?name end, - method () - values(?want, ?got, ?"want", ?"got") - end, - "check-equal", - terminate?: #f) - } -end macro check-equal; + { check-equal(?description:expression, ?want:expression, ?got:expression) } + => { expect-equal(?want, ?got, ?description) } +end macro; define macro assert-equal { assert-equal (?want:expression, ?got:expression) @@ -70,15 +76,26 @@ define macro assert-equal } end macro assert-equal; +define macro expect-equal + { expect-equal(?want:expression, ?got:expression) } + => { expect-equal(?want, ?got, ?"want" " = " ?"got") } + + { expect-equal(?want:expression, ?got:expression, ?description:*) } + => { do-check-equal(method () values(?description) end, + method () values(?want, ?got, ?"want", ?"got") end, + "expect-equal", + terminate?: #f) } +end macro; + define function do-check-equal - (description-thunk :: , arguments-thunk :: , - caller :: , #key terminate? :: ) + (description-thunk :: , arguments-thunk :: , caller :: , + #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (want, got, want-expr, got-expr) = arguments-thunk(); phase := format-to-string("while comparing %s and %s for equality", want-expr, got-expr); @@ -106,6 +123,17 @@ define function do-check-equal end block end function; +define macro expect-not-equal + { expect-not-equal(?expr1:expression, ?expr2:expression) } + => { expect-not-equal(?expr1, ?expr2, ?"expr1" " ~= " ?"expr2") } + + { expect-not-equal(?expr1:expression, ?expr2:expression, ?description:*) } + => { do-check-not-equal(method () values(?description) end, + method () values(?expr1, ?expr2, ?"expr1", ?"expr2") end, + "expect-not-equal", + terminate?: #f) } +end macro; + define macro assert-not-equal { assert-not-equal (?expr1:expression, ?expr2:expression) } => { @@ -117,26 +145,27 @@ define macro assert-not-equal method () values(?expr1, ?expr2, ?"expr1", ?"expr2") end, + "assert-not-equal", terminate?: #t) } end macro assert-not-equal; define function do-check-not-equal - (description-thunk :: , arguments-thunk :: , + (description-thunk :: , arguments-thunk :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (val1, val2, expr1, expr2) = arguments-thunk(); phase := format-to-string("while comparing %s and %s for inequality", expr1, expr2); if (val1 ~= val2) record-check(description, $passed, #f); else - phase := "getting assert-not-equal failure detail"; + phase := format-to-string("getting %s failure detail", caller); record-check(description, $failed, format-to-string("%= and %= are =.", val1, val2)); terminate? & signal(make()); @@ -206,17 +235,23 @@ define method check-equal-failure-detail join(choose(identity, vector(detail1, detail2, detail3)), "; ") end method; +// Deprecated; use expect-instance?. define macro check-instance? - { check-instance? (?check-name:expression, ?type:expression, ?value:expression) - } => { - do-check-instance?(method () ?check-name end, - method () - values(?type, ?value, ?"value") - end, - negate?: #f, - terminate?: #f) - } -end macro check-instance?; + { check-instance?(?description:expression, ?type:expression, ?value:expression) } + => { expect-instance?(?type, ?value, ?description) } +end macro; + +define macro expect-instance? + { expect-instance?(?type:expression, ?value:expression) } + => { expect-instance?(?type, ?value, ?"value" " is an instance of " ?"type") } + + { expect-instance?(?type:expression, ?value:expression, ?description:*) } + => { do-check-instance?(method () values(?description) end, + method () values(?type, ?value, ?"value") end, + "expect-instance?", + terminate?: #f) + } +end macro; define macro assert-instance? { assert-instance? (?type:expression, ?value:expression) @@ -229,11 +264,25 @@ define macro assert-instance? method () values(?type, ?value, ?"value") end, + "assert-instance?", negate?: #f, terminate?: #t) } end macro assert-instance?; +define macro expect-not-instance? + { expect-not-instance?(?type:expression, ?value:expression) } + => { expect-not-instance?(?type, ?value, ?"value" " is not an instance of " ?"type") } + + { expect-not-instance?(?type:expression, ?value:expression, ?description:*) } + => { do-check-instance?(method () values(?description) end, + method () values(?type, ?value, ?"value") end, + "expect-not-instance?", + negate?: #t, + terminate?: #f) + } +end macro; + define macro assert-not-instance? { assert-not-instance? (?type:expression, ?value:expression) } => { @@ -245,21 +294,22 @@ define macro assert-not-instance? method () values(?type, ?value, ?"value") end, + "assert-not-instance?", negate?: #t, terminate?: #t) } end macro assert-not-instance?; define function do-check-instance? - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key negate? :: , terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expressions"; + phase := format-to-string("evaluating %s expressions", caller); let (type :: , value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= is %=an instance of %s", value-expr, if (negate?) "not " else "" end, type); @@ -279,16 +329,11 @@ define function do-check-instance? end block end function do-check-instance?; +// Deprecated; use expect. define macro check-true - { check-true (?check-name:expression, ?expr:expression) - } => { - do-check-true(method () ?check-name end, - method () - values(?expr, ?"expr") - end, - terminate?: #f) - } -end macro check-true; + { check-true(?description:expression, ?expr:expression) } + => { expect(?expr, ?description) } +end macro; define macro assert-true { assert-true (?expr:expression) @@ -300,19 +345,20 @@ define macro assert-true } => { do-check-true(method () values(?description) end, method () values(?expr, ?"expr") end, + "assert-true", terminate?: #t) } end macro assert-true; define function do-check-true - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= evaluates to true", value-expr); if (value) @@ -330,16 +376,23 @@ define function do-check-true end block end function do-check-true; +// Deprecated; use expect-false. define macro check-false - { check-false (?check-name:expression, ?expr:expression) - } => { - do-check-false(method () ?check-name end, - method () - values(?expr, ?"expr") - end, - terminate?: #f) - } -end macro check-false; + { check-false(?description:expression, ?expr:expression) } + => { expect-false(?expr, ?description) } +end macro; + +define macro expect-false + { expect-false(?expr:expression) } + => { expect-false(?expr, ?"expr" " evaluates to #f") } + + { expect-false (?expr:expression, ?description:*) } + => { do-check-false(method () values(?description) end, + method () values(?expr, ?"expr") end, + "expect-false", + terminate?: #f) + } +end macro; define macro assert-false { assert-false (?expr:expression) @@ -353,19 +406,20 @@ define macro assert-false method () values(?expr, ?"expr") end, + "assert-false", terminate?: #t) } end macro assert-false; define function do-check-false - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (value, value-expr :: ) = get-arguments(); phase := format-to-string("checking if %= evaluates to #f", value-expr); if (~value) @@ -384,18 +438,24 @@ define function do-check-false end block end function do-check-false; +// Deprecated; use expect-condition. define macro check-condition - { check-condition(?check-name:expression, ?condition:expression, - ?expr:expression) - } => { - do-check-condition(method () ?check-name end, - method () - values(?condition, method () ?expr end, ?"expr") - end, - terminate?: #f) - } -end macro check-condition; - + { check-condition(?description:expression, ?condition:expression, ?expr:expression) } + => { expect-condition(?condition, ?expr, ?description) } +end macro; + +define macro expect-condition + { expect-condition(?condition:expression, ?expr:expression) } + => { expect-condition(?condition, ?expr, ?"expr" " signals condition " ?"condition") } + + { expect-condition(?condition:expression, ?expr:expression, ?description:*) } + => { do-check-condition(method () values(?description) end, + method () values(?condition, method () ?expr end, ?"expr") end, + "expect-condition", + terminate?: #f) } +end macro; + +// Deprecated; use assert-condition. define macro assert-signals { assert-signals(?condition:expression, ?expr:expression) } => { @@ -408,19 +468,31 @@ define macro assert-signals method () values(?condition, method () ?expr end, ?"expr") end, + "assert-signals", terminate?: #t) } end macro assert-signals; +define macro assert-condition + { assert-condition(?condition:expression, ?expr:expression) } + => { assert-condition(?condition, ?expr, ?"expr" " signals condition " ?"condition") } + + { assert-condition(?condition:expression, ?expr:expression, ?description:*) } + => { do-check-condition(method () values(?description) end, + method () values(?condition, method () ?expr end, ?"expr") end, + "assert-condition", + terminate?: #t) } +end macro; + define function do-check-condition - (description-thunk :: , get-arguments :: , + (description-thunk :: , get-arguments :: , caller :: , #key terminate? :: ) => () - let phase = "evaluating assertion description"; + let phase = format-to-string("evaluating %s description", caller); let description :: false-or() = #f; block () description := eval-check-description(description-thunk); - phase := "evaluating assertion expression"; + phase := format-to-string("evaluating %s expression", caller); let (condition-class, thunk :: , expr :: ) = get-arguments(); phase := format-to-string("checking if %= signals a condition of class %s", expr, condition-class); @@ -449,21 +521,28 @@ define function do-check-condition end function do-check-condition; +// Deprecated; use expect-no-condition. // Same as check-no-errors, for symmetry with check-condition... define macro check-no-condition - { check-no-condition(?check-name:expression, ?check-body:expression) - } => { - check-true(?check-name, begin ?check-body; #t end) - } -end macro check-no-condition; + { check-no-condition(?description:expression, ?expr:expression) } + => { expect-no-condition(?expr, ?description) } +end macro; +// Deprecated; use expect-no-condition. define macro check-no-errors - { check-no-errors(?check-name:expression, ?check-body:expression) - } => { - check-true(?check-name, begin ?check-body; #t end) - } -end macro check-no-errors; + { check-no-errors(?description:expression, ?expr:expression) } + => { expect-no-condition(?expr, ?description) } +end macro; + +define macro expect-no-condition + { expect-no-condition(?expr:expression) } + => { expect(begin ?expr; #t end) } + { expect-no-condition(?expr:expression, ?description:*) } + => { expect(begin ?expr; #t end, ?description) } +end macro; + +// Deprecated; use assert-no-condition. define macro assert-no-errors { assert-no-errors(?expr:expression) } => { @@ -476,6 +555,13 @@ define macro assert-no-errors } end macro assert-no-errors; +define macro assert-no-condition + { assert-no-condition(?expr:expression) } + => { assert-no-condition(?expr, ?"expr" " doesn't signal an error ") } + + { assert-no-condition(?expr:expression, ?description:*) } + => { assert-true(begin ?expr; #t end, ?description) } +end macro; /// Check recording diff --git a/documentation/source/reference.rst b/documentation/source/reference.rst index e475fe9..6ea6d22 100644 --- a/documentation/source/reference.rst +++ b/documentation/source/reference.rst @@ -22,7 +22,7 @@ Suites, Tests, and Benchmarks :signature: define test *test-name* (#key *expected-to-fail-reason, expected-to-fail-test, tags*) *body* end :parameter test-name: Name of the test; a Dylan variable name. - :parameter #key expected-to-fail-reason: A :drm:`` or ``#f``. + :parameter #key expected-to-fail-reason: A :drm:`` or :drm:`#f`. The reason this test is expected to fail. :parameter #key expected-to-fail-test: An instance of :drm:``. A function to decide whether the test is expected to fail. @@ -63,7 +63,7 @@ Suites, Tests, and Benchmarks :signature: define benchmark *benchmark-name* (#key *expected-to-fail-reason, expected-to-fail-test, tags*) *body* end :parameter benchmark-name: Name of the benchmark; a Dylan variable name. - :parameter #key expected-to-fail-reason: A :drm:`` or ``#f``. + :parameter #key expected-to-fail-reason: A :drm:`` or :drm:`#f`. The reason this benchmark is expected to fail. :parameter #key expected-to-fail-test: An instance of :drm:``. A function to decide whether the benchmark is expected to fail. @@ -280,15 +280,21 @@ These are the available assertion macros: * :macro:`assert-no-errors` * :macro:`assert-instance?` * :macro:`assert-not-instance?` + * :macro:`expect` + * :macro:`expect-false` + * :macro:`expect-equal` + * :macro:`expect-not-equal` + * :macro:`expect-condition` + * :macro:`expect-no-condition` + * :macro:`expect-instance?` + * :macro:`expect-not-instance?` .. macro:: assert-true - Assert that an expression evaluates to a true value. Importantly, - this does not mean the expression is exactly ``#t``, but rather - that it is *not* ``#f``. If you want to explicitly test for - equality to ``#t`` use ``assert-equal(#t, ...)`` . + Assert that an expression evaluates to a true value. Skip the remainder of + the test on failure. - :signature: assert-true *expression* [ *description* ] + :signature: assert-true *expression* [ *description* ... ] :parameter expression: any expression :parameter description: An optional description of what the assertion tests. @@ -297,6 +303,16 @@ These are the available assertion macros: than three". If no description is supplied one is automatically generated based on the text of the expression. + :description: + + Importantly, this does not test that the expression is exactly :drm:`#t`, + but rather that it is *not* :drm:`#f`. If you want to explicitly test for + equality to :drm:`#t` use ``assert-equal(#t, ...)`` . + + .. note:: It is also possible to use :macro:`assert` from the + ``common-dylan`` library, which will signal an error upon + failure and skip the remainder of the test. + :example: .. code-block:: dylan @@ -304,11 +320,22 @@ These are the available assertion macros: assert-true(has-fleas?(my-dog)) assert-true(has-fleas?(my-dog), "my dog has fleas") +.. macro:: expect + + Assert that an expression evaluates to a true value. **Do not skip** the + remainder of the test on failure. + + :description: + + The same as :macro:`assert-true` but does not terminate the test. Use + this only if subsequent assertions are independent of this one. + .. macro:: assert-false - Assert that an expression evaluates to ``#f``. + Assert that an expression evaluates to :drm:`#f`. Skip the remainder of the + test on failure. - :signature: assert-false *expression* [ *description* ] + :signature: assert-false *expression* [ *description* ... ] :parameter expression: any expression :parameter description: An optional description of what the assertion tests. @@ -324,6 +351,16 @@ These are the available assertion macros: assert-false(3 < 2) assert-false(6 = 7, "six equals seven") +.. macro:: expect-false + + Assert that an expression evaluates to :drm:`#f`. **Do not skip** the + remainder of the test on failure. + + :description: + + The same as :macro:`assert-false` but does not terminate the test. Use + this only if subsequent assertions are independent of this one. + .. macro:: assert-equal Assert that two values are equal using :drm:`=` as the comparison function. @@ -350,14 +387,22 @@ These are the available assertion macros: assert-equal(2, my-complicated-method()) assert-equal(want, f(), "f() returned %=", want) +.. macro:: expect-equal + + Assert that two values are equal using :drm:`=` as the comparison function. + **Do not skip** the remainder of the test on failure. + + :description: + + The same as :macro:`assert-equal` but does not terminate the test. Use + this only if subsequent assertions are independent of this one. + .. macro:: assert-not-equal Assert that two values are not equal using ``~=`` as the comparison - function. Using this macro is preferable to using ``assert-true(a - ~= b)`` or ``assert-false(a = b)`` because the generated failure - messages can be better. + function. Skip the remainder of the test on failure. - :signature: assert-not-equal *expression1* *expression2* [ *description* ] + :signature: assert-not-equal *expression1* *expression2* [ *description* ... ] :parameter expression1: any expression :parameter expression2: any expression @@ -367,6 +412,12 @@ These are the available assertion macros: than three". If no description is supplied one is automatically generated based on the text of the expression. + :description: + + Using this macro is preferable to using ``assert-true(a ~= b)`` or + ``assert-false(a = b)`` because the generated failure messages can be + better. + :example: .. code-block:: dylan @@ -374,11 +425,22 @@ These are the available assertion macros: assert-not-equal(2, my-complicated-method()) assert-not-equal(want, got, "want does not equal got") +.. macro:: expect-not-equal + + Assert that two values are not equal using :drm:`~=` as the comparison + function. **Do not skip** the remainder of the test on failure. + + :description: + + The same as :macro:`assert-not-equal` but does not terminate the + test. Use this only if subsequent assertions are independent of this one. + .. macro:: assert-signals - Assert that an expression signals a given condition class. + Assert that an expression signals a given condition class. Skip the + remainder of the test on failure. - :signature: assert-signals *condition*, *expression* [ *description* ] + :signature: assert-signals *condition*, *expression* [ *description* ... ] :parameter condition: an expression that yields a condition class :parameter expression: any expression @@ -399,29 +461,40 @@ These are the available assertion macros: assert-signals(, 3 / 0, "my super special description") +.. macro:: expect-condition + + Assert that an expression signals a given condition class. **Do not skip** + the remainder of the test on failure. + + :description: + + The same as :macro:`assert-signals` but does not terminate the test. Use + this only if subsequent assertions are independent of this one. + .. macro:: assert-no-errors - Assert that an expression does not signal any errors. + Assert that an expression does not signal any errors. Skip the remainder of + the test on failure. - :signature: assert-no-errors *expression* [ *description* ] + :signature: assert-no-errors *expression* [ *description* ... ] - :parameter expression: any expression + :parameter expression: any expression :parameter description: An optional description of what the assertion tests. This may be a single value of any type or a format string and format arguments. It should be stated in positive form, such as "f(3) does not - signal ". If no description is supplied one is automatically + signal ". If no description is supplied one is automatically generated based on the text of the expression. - The assertion succeeds if no error is signaled by the evaluation of - *expression*. + :description: + + The assertion succeeds if no condition is signaled by the evaluation of + *expression*. - Use of this macro is preferable to simply executing *expression* as - part of the test body for two reasons. First, it can clarify the - purpose of the test, by telling the reader "here's an expression - that is explicitly being tested, and not just part of the test - setup." Second, if the assertion signals an error the test will - record that fact and continue, as opposed to taking a non-local - exit. Third, it will show up in generated reports. + Use of this macro is preferable to simply executing *expression* as part + of the test body because it can clarify the purpose of the test by + telling the reader "here's an expression that is explicitly being tested, + and not just part of the test setup." Also, the assertion failure will + show up in generated reports. :example: @@ -431,12 +504,22 @@ These are the available assertion macros: assert-no-errors(my-hairy-logic(), "hairy logic completes without error") +.. macro:: expect-no-condition + + Assert that an expression does not signal a condition. **Do not skip** the + remainder of the test on failure. + + :description: + + The same as :macro:`assert-no-errors` but does not terminate the test. + Use this only if subsequent assertions are independent of this one. .. macro:: assert-instance? Assert that the result of an expression is an instance of a given type. + Skip the remainder of the test on failure. - :signature: assert-instance? *type* *expression* [ *description* ] + :signature: assert-instance? *type* *expression* [ *description* ... ] :parameter type: The expected type. :parameter expression: An expression. @@ -459,16 +542,24 @@ These are the available assertion macros: .. code-block:: dylan assert-instance?(, subclass()); + assert-instance?(, subclass(), "subclass returns a type"); + +.. macro:: expect-instance? + + Assert that the result of an expression is an instance of a given type. + **Do not skip** the remainder of the test on failure. - assert-instance?(, subclass(), - "subclass returns type"); + :description: + The same as :macro:`assert-instance?` but does not terminate the test. + Use this only if subsequent assertions are independent of this one. .. macro:: assert-not-instance? - Assert that the result of an expression is **not** an instance of a given class. + Assert that the result of an expression is **not** an instance of a given + class. Skip the remainder of the test on failure. - :signature: assert-not-instance? *type* *expression* [ *description* ] + :signature: assert-not-instance? *type* *expression* [ *description* ... ] :parameter type: The type. :parameter expression: An expression. @@ -491,14 +582,27 @@ These are the available assertion macros: .. code-block:: dylan assert-not-instance?(limited(, min: 0), -1); - assert-not-instance?(limited(, min: 0), -1, "values below lower bound are not instances"); +.. macro:: expect-not-instance? + + Assert that the result of an expression is **not** an instance of a given + class. **Do not skip** the remainder of the test on failure. + + :description: + + The same as :macro:`assert-not-instance?` but does not terminate the + test. Use this only if subsequent assertions are independent of this + one. + Checks ====== +.. deprecated:: 3.0 + Use the ``expect-*`` macros documented above. + Checks are like `Assertions`_ but they do not cause the test to terminate when they fail or crash. Only use them if later checks or assertions do not depend on them passing and they won't cause too many cascading failures (for example @@ -522,9 +626,12 @@ These are the available checks: Perform a check within a test. + .. deprecated:: 3.0 + Use :macro:`expect` instead. + :signature: check *name* *function* #rest *arguments* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter function: The function to check. :parameter #rest arguments: The arguments for ``function``. @@ -539,9 +646,12 @@ These are the available checks: Check that a given condition is signalled. + .. deprecated:: 3.0 + Use :macro:`expect-condition` instead. + :signature: check-condition *name* *expected* *expression* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter expected: The expected condition class. :parameter expression: An expression. @@ -557,9 +667,12 @@ These are the available checks: Check that 2 expressions are equal. + .. deprecated:: 3.0 + Use :macro:`expect-equal` instead. + :signature: check-equal *name* *expected* *expression* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter expected: The expected value of ``expression``. :parameter expression: An expression. @@ -574,11 +687,14 @@ These are the available checks: .. macro:: check-false - Check that an expression has a result of ``#f``. + Check that an expression has a result of :drm:`#f`. + + .. deprecated:: 3.0 + Use :macro:`expect-false` instead. :signature: check-false *name* *expression* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter expression: An expression. :example: @@ -592,9 +708,12 @@ These are the available checks: Check that the result of an expression is an instance of a given type. + .. deprecated:: 3.0 + Use :macro:`expect-instance?` instead. + :signature: check-instance? *name* *type* *expression* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter type: The expected type. :parameter expression: An expression. @@ -608,17 +727,20 @@ These are the available checks: .. macro:: check-true - Check that the result of an expression is not ``#f``. + Check that the result of an expression is not :drm:`#f`. + + .. deprecated:: 3.0 + Use :macro:`expect` instead. :signature: check-true *name* *expression* - :parameter name: An instance of ````. + :parameter name: An instance of :drm:``. :parameter expression: An expression. :description: Note that if you want to explicitly check if an expression - evaluates to ``#t``, you should use :func:`check-equal`. + evaluates to :drm:`#t`, you should use :func:`check-equal`. :example: @@ -672,9 +794,9 @@ Test Execution Retrieve a unique temporary directory for the current test to use. :signature: test-temp-directory => *directory* - :value directory: An instance of type ````. + :value directory: An instance of type :class:``. - Returns a directory (a ````) that may be used for + Returns a directory (a :class:``) that may be used for temporary files created by the test or benchmark. The directory is created the first time this function is called for each test or benchmark and is not deleted after the test run is complete in case it's useful for post-mortem @@ -690,13 +812,13 @@ Test Execution Writes a file in the current test's temp directory. :signature: write-test-file *filename* #key *contents* => *locator* - :parameter filename: An instance of ```` (i.e., a string or a + :parameter filename: An instance of :class:`` (i.e., a string or a locator). The name may be a relative path and if it contains the path separator character, subdirectories will be created. :parameter #key contents: An instance of :drm:`` to be written to the file. Defaults to the empty string. - :value locator: An instance of ```` which is the full, + :value locator: An instance of :class:`` which is the full, absolute pathname of the created file. When your test requires files to be present this is a handy utility to diff --git a/documentation/source/usage.rst b/documentation/source/usage.rst index 4124fe2..d693c00 100644 --- a/documentation/source/usage.rst +++ b/documentation/source/usage.rst @@ -14,22 +14,31 @@ See also: :doc:`reference` Quick Start =========== -For the impatient, this section summarizes most of what you need to know to use -Testworks. +For the impatient (i.e., most of us), this section summarizes most of what you +need to know to use Testworks. Add ``use testworks;`` to both your test library and test module. -Tests contain arbitrary code and at least one assertion: +Tests contain arbitrary code and at least one assertion or expectation. + +**Terminology:** An assertion failure terminates the test and skips any further +assertions or expectations. (This is normally preferred since it prevents +cascading failures.) An expectation failure does not terminate the +test. Assertions are Testworks macros that begin with ``assert-`` and +expectations begin with ``expect-``. When this documentation refers to +"assertions" it should be understood to include both kinds of macros. + +Example test: .. code-block:: dylan - define test test-fn1 () + define test test-my-function () let v = do-something(); - assert-equal("expected-value", fn1(v)); - assert-signals(, fn1(v, key: 7), "regression test for bug 12345"); - end; + assert-equal("expected-value", my-function(v)); + assert-condition(, my-function(v, key: 7), "regression test for bug 12345"); + end test; -If there are no assertions in a test it is considered "not implemented", which +If there are no assertions in a test it is considered "NOT IMPLEMENTED", which is displayed in the output (as a reminder to implement it) but is not considered a failure. @@ -40,7 +49,6 @@ Benchmarks do not require any assertions and are automatically given the .. code-block:: dylan - // Benchmark fn1 define benchmark fn1-benchmark () fn1() end; @@ -49,7 +57,8 @@ See also, :macro:`benchmark-repeat`. If you have a large or complex test library, "suites" may be used to organize tests into groups (for example one suite per module) and may be nested -arbitrarily. +arbitrarily. This does impose a small maintenance burden of adding each test to +a suite, and is not required. See `Suites`_ for details. .. code-block:: dylan @@ -84,49 +93,49 @@ accomplish this: In both cases :func:`run-test-application` parses the command line so the options are the same. Use ``--help`` to see all options. -See `Suites`_ for a way to organize large test suites. - Defining Tests ============== Assertions ---------- -An assertion accepts an expression to evaluate and report back on, -saying if the expression passed, failed, or crashed (i.e., signaled an -error). As an example, in +Assertions come in two forms: ``assert-*`` macros and ``expect-*`` macros. +When an ``assert-*`` macro fails, it causes the test to fail and the remainder +of the test to be skipped. This kind of assertion is generally preferred +because it prevents "cascading failures". In contrast, when an ``expect-*`` +macro fails, it causes a test failure but the test continues running, so there +may be multiple assertion failures recorded for the test. + +.. note:: You may also find ``check-*`` macros in existing test suites. These + are a deprecated form of assertion, replaced by ``expect-*``. + +The benefit of the ``expect-*`` macros is that when you're initially debugging +the test it may require fewer iterations if you can see multiple failures in +one pass. + +An assertion accepts an expression to evaluate and report back on, saying if +the expression passed, failed, or crashed (i.e., signaled an error). As an +example, in .. code-block:: dylan assert-true(foo > bar) + expect(foo > bar) the expression ``foo > bar`` is compared to ``#f``, and the result is recorded -by the test harness. Failing (or crashing) cause the test to terminate and -skip the remaining assertions in that test. - -.. note:: You may also find ``check-*`` macros in existing test suites. These - are deprecated assertions that do not cause the test to terminate and - require a description of the assertion as the first argument. - -See the :doc:`reference` for detailed documentation on the available -assertion macros: +by the test harness. - * :macro:`assert-true` - * :macro:`assert-false` - * :macro:`assert-equal` - * :macro:`assert-not-equal` - * :macro:`assert-signals` - * :macro:`assert-no-errors` - * :macro:`assert-instance?` - * :macro:`assert-not-instance?` +See the `reference docs `_ for detailed documentation on the +available assertion macros. -Each of these takes an optional description, after the required -arguments, which will be displayed if the assertion fails. If the -description isn't provided, Testworks makes one from the expressions -passed to the assertion macro. For example, ``assert-true(2 > 3)`` -produces this failure message:: +Each assertion macro accepts an optional description (a format string and +optional format arguments), after the required arguments, which is displayed if +the assertion fails. If the description isn't provided, Testworks makes one +from the expressions passed to the assertion macro. For example, +``assert-true(2 > 3)`` produces this failure message:: - (2 > 3) is true failed [expression "(2 > 3)" evaluates to #f] + FAILED: (2 > 3) + expression "(2 > 3)" evaluates to #f In general, Testworks should be pretty good at reporting the actual values that caused the failure so it shouldn't be necessary to include them in the @@ -141,7 +150,7 @@ arguments. These are all valid: .. code-block:: dylan assert-equal(want, got); // auto-generated description - assert-equal(want, got, foo); // a used as description + assert-equal(want, got, foo); // foo used as description assert-equal(want, got, "does %= = %=?", a, b); // formatted description Tests diff --git a/library.dylan b/library.dylan index de7c3d6..c587f55 100644 --- a/library.dylan +++ b/library.dylan @@ -42,13 +42,12 @@ define module testworks runner-skip, runner-tags; - // Checks (deprecated, use assertions) + // Checks (deprecated, use assert-* or expect-*) create check, check-condition, check-no-condition, check-equal, - check-equal-failure-detail, check-false, check-no-errors, check-instance?, @@ -58,12 +57,25 @@ define module testworks create assert-equal, assert-not-equal, - assert-signals, + assert-condition, + assert-signals, // Deprecated; use assert-condition. assert-no-errors, assert-instance?, assert-not-instance?, assert-true, - assert-false; + assert-false, + expect, + expect-equal, + expect-not-equal, + expect-false, + expect-instance?, + expect-not-instance?, + expect-condition, + expect-no-condition; + + create + // Implement this to give detail to expect-equal failures for your own types. + check-equal-failure-detail; // Components create diff --git a/tests/testworks-test-suite.dylan b/tests/testworks-test-suite.dylan index a0c1c12..9b69d9b 100644 --- a/tests/testworks-test-suite.dylan +++ b/tests/testworks-test-suite.dylan @@ -94,6 +94,21 @@ define test testworks-check-true-test () "check-true of error crashes"); end test; +define test test-expect () + expect(always(#t)); + expect(identity(#t)); + expect(3 = 3); + assert-equal($passed, + with-result-status () expect(#t) end, + "expect(#t) passes"); + assert-equal($failed, + with-result-status () expect(#f) end, + "expect(#f) fails"); + assert-equal($crashed, + with-result-status () expect(test-error()) end, + "expect of error crashes"); +end test; + define test testworks-assert-true-test () assert-true(#t); assert-true(#t, "#t is true with description"); @@ -120,6 +135,18 @@ define test testworks-check-false-test () "check-false of error crashes"); end test; +define test test-expect-false () + assert-equal($failed, + with-result-status () expect-false(#t) end, + "expect-false(#t) fails"); + assert-equal($passed, + with-result-status () expect-false(#f) end, + "expect-false(#f) passes"); + assert-equal($crashed, + with-result-status () expect-false(test-error()) end, + "expect-false of error crashes"); +end test; + define test testworks-assert-false-test () assert-false(#f); assert-false(#f, "#f is false with description"); @@ -151,6 +178,21 @@ define test testworks-check-equal-test () "check-equal of error crashes"); end test; +define test test-expect-equal () + assert-equal($passed, + with-result-status () expect-equal(1, 1) end, + "expect-equal(1, 1) passes"); + assert-equal($passed, + with-result-status () expect-equal("1", "1") end, + """expect-equal("1", "1") passes"""); + assert-equal($failed, + with-result-status () expect-equal(1, 2) end, + "expect-equal(1, 2) fails"); + assert-equal($crashed, + with-result-status () expect-equal(1, test-error()) end, + "expect-equal of error crashes"); +end test; + define test testworks-assert-equal-failure-detail () let result = with-result () assert-equal(#[1, 2, 3], #[1, 3, 2]) @@ -178,6 +220,15 @@ define test testworks-assert-not-equal-test () assert-equal($crashed, with-result-status () assert-not-equal(1, test-error()) end); end test; +define test test-expect-not-equal () + expect-not-equal(8, 9); + expect-not-equal(8, 9, "8 ~= 9 with description"); + assert-equal($passed, with-result-status () expect-not-equal(1, 2) end); + assert-equal($passed, with-result-status () expect-not-equal("1", "2") end); + assert-equal($failed, with-result-status () expect-not-equal(1, 1) end); + assert-equal($crashed, with-result-status () expect-not-equal(1, test-error()) end); +end test; + define test testworks-check-instance?-test () assert-equal($passed, with-result-status () @@ -196,6 +247,20 @@ define test testworks-check-instance?-test () "check-instance? of error crashes"); end test; +define test test-expect-instance? () + assert-equal($passed, + with-result-status () expect-instance?(, 1) end, + "expect-instance?(, 1) passes"); + assert-equal($failed, + with-result-status () expect-instance?(, 1) end, + "expect-instance?(, 1) fails"); + assert-equal($crashed, + with-result-status () + expect-instance?(, test-error()) + end, + "expect-instance? of error crashes"); +end test; + define test testworks-assert-instance?-test () assert-equal($passed, with-result-status () @@ -232,6 +297,24 @@ define test testworks-assert-not-instance?-test () "assert-not-instance? of error crashes"); end test; +define test test-assert-not-instance? () + assert-equal($passed, + with-result-status () + expect-not-instance?(, 1) + end, + "expect-not-instance?(, 1) passes"); + assert-equal($failed, + with-result-status () + expect-not-instance?(, 1) + end, + "expect-not-instance?(, 1) fails"); + assert-equal($crashed, + with-result-status () + expect-not-instance?(, test-error()) + end, + "expect-not-instance? of error crashes"); +end test; + define test testworks-check-condition-test () begin let success? = #f; @@ -262,30 +345,59 @@ define test testworks-check-condition-test () "check-condition doesn't catch wrong condition"); end test; -define test testworks-assert-signals-test () - assert-signals(, error("foo")); - assert-signals(, error("foo"), "error signals error w/ description"); +define test test-expect-condition () + begin + let success? = #f; + assert-equal($passed, + with-result-status () + expect-condition(, + begin + // default-handler for returns #f + test-warning(); + success? := #t; + test-error() + end) + end, + "expect-condition catches "); + assert-true(success?, + "expect-condition for doesn't catch "); + end; + assert-equal($failed, + with-result-status () + expect-condition(, #f) + end, + "expect-condition fails if no condition"); + assert-equal($failed, + with-result-status () + expect-condition(, test-warning()) + end, + "expect-condition doesn't catch wrong condition"); +end test; + +define test testworks-assert-condition-test () + assert-condition(, error("foo")); + assert-condition(, error("foo"), "error signals error w/ description"); begin let success? = #f; assert-equal($passed, with-result-status () - assert-signals(, - begin - // default-handler for returns #f - test-warning(); - success? := #t; - test-error() - end) + assert-condition(, + begin + // default-handler for returns #f + test-warning(); + success? := #t; + test-error() + end) end); assert-true(success?); end; assert-equal($failed, with-result-status () - assert-signals(, #f) + assert-condition(, #f) end); assert-equal($failed, with-result-status () - assert-signals(, test-warning()) + assert-condition(, test-warning()) end); end test; @@ -313,6 +425,13 @@ define test testworks-assert-no-errors-test () assert-equal($crashed, with-result-status () assert-no-errors(test-error()) end); end test; +define test test-expect-no-condition () + assert-equal($passed, with-result-status () expect-no-condition(#t) end); + assert-equal($passed, with-result-status () expect-no-condition(#f) end); + assert-equal($crashed, with-result-status () expect-no-condition(test-error()) end); +end test; + + define suite testworks-assertion-macros-suite () test testworks-check-test; test testworks-check-true-test; @@ -322,7 +441,6 @@ define suite testworks-assertion-macros-suite () test testworks-check-condition-test; test testworks-check-no-errors-test; - // Assert macros (newer). test testworks-assert-true-test; test testworks-assert-false-test; test testworks-assert-equal-test; @@ -330,11 +448,10 @@ define suite testworks-assertion-macros-suite () test testworks-assert-not-equal-test; test testworks-assert-instance?-test; test testworks-assert-not-instance?-test; - test testworks-assert-signals-test; + test testworks-assert-condition-test; test testworks-assert-no-errors-test; end suite testworks-assertion-macros-suite; - define benchmark basic-benchmark () "just exercise basic benchmark functionality" end; @@ -343,7 +460,6 @@ define suite testworks-benchmarks-suite () benchmark basic-benchmark; end; - /// Verify the result objects define test test-run-tests/test () @@ -464,6 +580,21 @@ define test test-check-failure-continues () end; end test; +// Make sure that if one `expect-*` expression fails the remaining assertions +// are still executed. +define test test-expect-failure-continues () + let x = #f; + block () + without-recording () + expect(#f); // fail + expect(error("blah")); // fail + end; + x := #t; + cleanup + assert-true(x); + end; +end test; + // Make sure that if one `assert-*` expression fails the rest of the test is // NOT executed. define test test-assertion-failure-terminates () @@ -776,11 +907,13 @@ end test; work to be done in this area so I expect to use these more. define test test-assert-equal-output () - check-instance?("b", , 1); - check-true("d", 3 < 2); - check-false("e", 3 == 3); - check-condition("f", , "no error"); - check-no-condition("g", error("foo")); + expect(2 > 3); + expect-instance?(, 1); + expect-not-instance?(, "b"); + expect-true(3 < 2); + expect-false(3 == 3); + expect-condition(, "no error"); + expect-no-condition(error("foo")); check-equal("list, same size, different elements", #("a", "b", "c", "d"), #("a", "b", "x", "d"));