From 95f38dd80190d1d8e80978dad37cf1fb7ea09ad1 Mon Sep 17 00:00:00 2001 From: Alan Acevedo Date: Mon, 24 Aug 2026 17:30:10 +0100 Subject: [PATCH] fix: accept flow functions in test case function call assertions Test case validation built its allowed function set from resources with the `fn` prefix, so an assertion naming a flow function under `flows//functions/` (prefix `ft`) was rejected as unknown. Agent Studio accepts either: such an assertion can be created in the UI and is read back by `poly pull`, so the ADK rejected files it had itself just written, and validation failed for the whole project until the test case was removed. Both prefixes now count. Unknown names still raise, so typos are still caught. DEVP-618 Co-Authored-By: Claude Opus 5 (1M context) --- docs/docs/reference/resources/tests.md | 8 ++-- src/poly/docs/tests.md | 2 +- src/poly/resources/test_suite.py | 8 ++-- src/poly/tests/resources_test.py | 66 ++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/docs/reference/resources/tests.md b/docs/docs/reference/resources/tests.md index 1b53aebd..151754ef 100644 --- a/docs/docs/reference/resources/tests.md +++ b/docs/docs/reference/resources/tests.md @@ -162,11 +162,11 @@ prompt_assertions: ## Function call assertions -Each function call assertion checks that a global function was called and, optionally, with specific argument values. +Each function call assertion checks that a function was called and, optionally, with specific argument values. | Field | Description | |---|---| -| `name` | Global function name. Must match a function in `functions/`. | +| `name` | Function name. Must match a global function in `functions/` or a flow function in `flows//functions/`. | | `arguments` | List of argument assertions. May be empty to check only that the function was called. | Argument assertion fields: @@ -235,7 +235,7 @@ prompt_assertions: - `simulated_at`, if set, must be a valid ISO 8601 datetime - `caller_number`, if set, must be text — an unquoted number is rejected rather than converted - `integration_attributes` values must be text, numbers, `true`/`false`, `null`, lists, or nested maps; an unquoted date is rejected with the quoted form to use instead; keys must be text -- each `function_call_assertions[*].name` must match a global function under `functions/` +- each `function_call_assertions[*].name` must match a global function under `functions/` or a flow function under `flows//functions/` - each argument's `value_type` must be one of `string`, `integer`, `number`, `boolean` - the filename must match the normalized `name` @@ -292,7 +292,7 @@ Good coverage of a project usually includes: --- - Reference for the global functions named in function call assertions. + Reference for the functions named in function call assertions. [Open functions](./functions.md) - **Variants** diff --git a/src/poly/docs/tests.md b/src/poly/docs/tests.md index 7e66aa1a..74c4ae45 100644 --- a/src/poly/docs/tests.md +++ b/src/poly/docs/tests.md @@ -164,7 +164,7 @@ On `push`, each test case is validated: - **scenario** is required (cannot be empty). - **language** is required and must match a configured project language (default or additional). - **variant**, if specified, must match an existing variant in the project. -- **function_call_assertions**: each function name must match a global function in the project, and each argument's `value_type` must be one of `string`, `integer`, `number`, or `boolean`. +- **function_call_assertions**: each function name must match a global function (`functions/`) or a flow function (`flows//functions/`) in the project, and each argument's `value_type` must be one of `string`, `integer`, `number`, or `boolean`. - **integration_attributes**: values must be text, numbers, `true`/`false`, `null`, lists or nested maps. An unquoted date is rejected with the quoted form to use instead, and keys must be text. ## Best practices diff --git a/src/poly/resources/test_suite.py b/src/poly/resources/test_suite.py index f699464f..e8ffab42 100644 --- a/src/poly/resources/test_suite.py +++ b/src/poly/resources/test_suite.py @@ -703,16 +703,16 @@ def validate(self, resource_mappings: list[ResourceMapping] = None, **kwargs): # Integration attributes carry JSON types through to the agent _validate_attribute_value(self.integration_attributes.attributes, "integration_attributes") - # Function name is valid - known_global_functions = { + # `fn` is a global function, `ft` a flow function. Both are assertable. + known_functions = { resource.resource_name for resource in resource_mappings or [] - if resource.resource_prefix == "fn" + if resource.resource_prefix in ("fn", "ft") } for function_call in self.assertions.function_calls: if not function_call.name: raise ValueError("Function call assertion must have a name") - if known_global_functions and function_call.name not in known_global_functions: + if known_functions and function_call.name not in known_functions: raise ValueError(f"Unknown function in assertion: {function_call.name}") for argument in function_call.arguments: if argument.value_type not in ALLOWED_TYPES: diff --git a/src/poly/tests/resources_test.py b/src/poly/tests/resources_test.py index 1bb94462..7d9a1aaa 100644 --- a/src/poly/tests/resources_test.py +++ b/src/poly/tests/resources_test.py @@ -7357,6 +7357,72 @@ def test_validate(self): ] ) + def _test_case_asserting_function(self, function_name: str) -> TestCase: + resource_id = "TEST-function_assertion" + return TestCase( + resource_id=resource_id, + name="Function assertion", + scenario="Give a phone number.", + channel="chat.polyai", + language="en-GB", + assertions=TestCaseAssertion( + resource_id=resource_id, + name="assertions", + prompts=[], + function_calls=[ + FunctionCallAssertion( + name=function_name, + arguments=[ + FunctionCallArgumentAssertion( + parameter_name="phone_number", + expected_value="123123", + value_type="string", + ) + ], + ) + ], + ), + tags=TestCaseTags(resource_id=resource_id, name="tags", tags=[]), + ) + + def _function_mappings(self) -> list[ResourceMapping]: + return [ + ResourceMapping( + resource_id="fn-transfer", + resource_name="transfer_call", + resource_type=Function, + resource_prefix="fn", + file_path="functions/transfer_call.py", + flow_name=None, + ), + ResourceMapping( + resource_id="ft-register", + resource_name="register_phone_number", + resource_type=Function, + resource_prefix="ft", + file_path="flows/idnv/functions/register_phone_number.py", + flow_name="idnv", + ), + ] + + def test_validate_accepts_global_function_assertion(self): + self._test_case_asserting_function("transfer_call").validate( + resource_mappings=self._function_mappings() + ) + + def test_validate_accepts_flow_function_assertion(self): + """A flow function is assertable: the platform accepts it, so the ADK must too.""" + self._test_case_asserting_function("register_phone_number").validate( + resource_mappings=self._function_mappings() + ) + + def test_validate_rejects_unknown_function_assertion(self): + with self.assertRaises(ValueError) as cm: + self._test_case_asserting_function("register_phone_numbr").validate( + resource_mappings=self._function_mappings() + ) + self.assertIn("Unknown function in assertion: register_phone_numbr", str(cm.exception)) + def test_get_new_updated_deleted_subresources(self): test_case = self._sample_test_case() new, updated, deleted = test_case.get_new_updated_deleted_subresources()