Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/docs/reference/resources/tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ prompt_assertions:

## Function call assertions

Each function call assertion checks that a global function was called and, optionally, with specific argument values.
Each function call assertion checks that a function was called and, optionally, with specific argument values.

| Field | Description |
|---|---|
| `name` | Global function name. Must match a function in `functions/`. |
| `name` | Function name. Must match a global function in `functions/` or a flow function in `flows/<flow>/functions/`. |
| `arguments` | List of argument assertions. May be empty to check only that the function was called. |

Argument assertion fields:
Expand Down Expand Up @@ -235,7 +235,7 @@ prompt_assertions:
- `simulated_at`, if set, must be a valid ISO 8601 datetime
- `caller_number`, if set, must be text — an unquoted number is rejected rather than converted
- `integration_attributes` values must be text, numbers, `true`/`false`, `null`, lists, or nested maps; an unquoted date is rejected with the quoted form to use instead; keys must be text
- each `function_call_assertions[*].name` must match a global function under `functions/`
- each `function_call_assertions[*].name` must match a global function under `functions/` or a flow function under `flows/<flow>/functions/`
- each argument's `value_type` must be one of `string`, `integer`, `number`, `boolean`
- the filename must match the normalized `name`

Expand Down Expand Up @@ -292,7 +292,7 @@ Good coverage of a project usually includes:

---

Reference for the global functions named in function call assertions.
Reference for the functions named in function call assertions.
[Open functions](./functions.md)

- **Variants**
Expand Down
2 changes: 1 addition & 1 deletion src/poly/docs/tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ On `push`, each test case is validated:
- **scenario** is required (cannot be empty).
- **language** is required and must match a configured project language (default or additional).
- **variant**, if specified, must match an existing variant in the project.
- **function_call_assertions**: each function name must match a global function in the project, and each argument's `value_type` must be one of `string`, `integer`, `number`, or `boolean`.
- **function_call_assertions**: each function name must match a global function (`functions/`) or a flow function (`flows/<flow>/functions/`) in the project, and each argument's `value_type` must be one of `string`, `integer`, `number`, or `boolean`.
- **integration_attributes**: values must be text, numbers, `true`/`false`, `null`, lists or nested maps. An unquoted date is rejected with the quoted form to use instead, and keys must be text.

## Best practices
Expand Down
8 changes: 4 additions & 4 deletions src/poly/resources/test_suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -703,16 +703,16 @@ def validate(self, resource_mappings: list[ResourceMapping] = None, **kwargs):
# Integration attributes carry JSON types through to the agent
_validate_attribute_value(self.integration_attributes.attributes, "integration_attributes")

# Function name is valid
known_global_functions = {
# `fn` is a global function, `ft` a flow function. Both are assertable.
known_functions = {
resource.resource_name
for resource in resource_mappings or []
if resource.resource_prefix == "fn"
if resource.resource_prefix in ("fn", "ft")
}
for function_call in self.assertions.function_calls:
if not function_call.name:
raise ValueError("Function call assertion must have a name")
if known_global_functions and function_call.name not in known_global_functions:
if known_functions and function_call.name not in known_functions:
raise ValueError(f"Unknown function in assertion: {function_call.name}")
for argument in function_call.arguments:
if argument.value_type not in ALLOWED_TYPES:
Expand Down
66 changes: 66 additions & 0 deletions src/poly/tests/resources_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -7357,6 +7357,72 @@ def test_validate(self):
]
)

def _test_case_asserting_function(self, function_name: str) -> TestCase:
resource_id = "TEST-function_assertion"
return TestCase(
resource_id=resource_id,
name="Function assertion",
scenario="Give a phone number.",
channel="chat.polyai",
language="en-GB",
assertions=TestCaseAssertion(
resource_id=resource_id,
name="assertions",
prompts=[],
function_calls=[
FunctionCallAssertion(
name=function_name,
arguments=[
FunctionCallArgumentAssertion(
parameter_name="phone_number",
expected_value="123123",
value_type="string",
)
],
)
],
),
tags=TestCaseTags(resource_id=resource_id, name="tags", tags=[]),
)

def _function_mappings(self) -> list[ResourceMapping]:
return [
ResourceMapping(
resource_id="fn-transfer",
resource_name="transfer_call",
resource_type=Function,
resource_prefix="fn",
file_path="functions/transfer_call.py",
flow_name=None,
),
ResourceMapping(
resource_id="ft-register",
resource_name="register_phone_number",
resource_type=Function,
resource_prefix="ft",
file_path="flows/idnv/functions/register_phone_number.py",
flow_name="idnv",
),
]

def test_validate_accepts_global_function_assertion(self):
self._test_case_asserting_function("transfer_call").validate(
resource_mappings=self._function_mappings()
)

def test_validate_accepts_flow_function_assertion(self):
"""A flow function is assertable: the platform accepts it, so the ADK must too."""
self._test_case_asserting_function("register_phone_number").validate(
resource_mappings=self._function_mappings()
)

def test_validate_rejects_unknown_function_assertion(self):
with self.assertRaises(ValueError) as cm:
self._test_case_asserting_function("register_phone_numbr").validate(
resource_mappings=self._function_mappings()
)
self.assertIn("Unknown function in assertion: register_phone_numbr", str(cm.exception))

def test_get_new_updated_deleted_subresources(self):
test_case = self._sample_test_case()
new, updated, deleted = test_case.get_new_updated_deleted_subresources()
Expand Down
Loading