From 18a1ce9581bc24cdbcf4e2a352fc5a1b711b7514 Mon Sep 17 00:00:00 2001 From: teocns <59549574+teocns@users.noreply.github.com> Date: Thu, 6 Feb 2025 04:10:26 +0200 Subject: [PATCH] tests: replace `assert len(mock_req.request_history) == N` checks with specific request filtering (#670) * test(record_tool): replace `assert len(mock_req.request_history) == N` checks with specific request filtering 1. Replaced `assert len(mock_req.request_history) == N` checks with specific request filtering 2. Added helper to find relevant requests using URL patterns 3. Made assertions more focused on the actual tool recording functionality 4. Maintained all existing functionality checks while removing dependency on request count * test(record_action): replace `assert len(mock_req.request_history) == N` checks with specific request filtering Replaced request count assertions with specific request filtering Added helper to find relevant requests using URL patterns Made assertions more focused on the actual action recording functionality Maintained all existing functionality checks while removing dependency on request count Signed-off-by: Teo * test(session): replace `assert len(mock_req.request_history) == N` checks with specific request filtering Signed-off-by: Teo * test(pre_init): replace `assert len(mock_req.request_history) == N` checks with specific request filtering Signed-off-by: Teo * test(canary): replace `assert len(mock_req.request_history) == N` checks with specific request filtering Signed-off-by: Teo --------- Signed-off-by: Teo --- tests/unit/test_canary.py | 10 +-- tests/unit/test_pre_init.py | 21 ++++-- tests/unit/test_record_action.py | 79 ++++++++++++--------- tests/unit/test_record_tool.py | 114 +++++++++++++++++-------------- tests/unit/test_session.py | 107 ++++++++++++++++------------- 5 files changed, 185 insertions(+), 146 deletions(-) diff --git a/tests/unit/test_canary.py b/tests/unit/test_canary.py index 2c79d3ee..90fcc65d 100644 --- a/tests/unit/test_canary.py +++ b/tests/unit/test_canary.py @@ -22,11 +22,13 @@ def test_agent_ops_record(self, mock_req): agentops.record(ActionEvent(event_type)) time.sleep(2) - # 3 requests: check_for_updates, create_session, create_events - assert len(mock_req.request_history) == 3 + # Find event requests + event_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(event_requests) > 0 + last_event_request = event_requests[-1] - request_json = mock_req.last_request.json() - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key + assert last_event_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_event_request.json() assert request_json["events"][0]["event_type"] == event_type agentops.end_session("Success") diff --git a/tests/unit/test_pre_init.py b/tests/unit/test_pre_init.py index c18fe659..5e8ce068 100644 --- a/tests/unit/test_pre_init.py +++ b/tests/unit/test_pre_init.py @@ -29,16 +29,23 @@ def test_track_agent(self, mock_req): agentops.init(api_key=self.api_key) time.sleep(1) - # Assert - # start session and create agent - agentops.end_session(end_state="Success") + # Find agent creation request + agent_requests = [r for r in mock_req.request_history if "/v2/create_agent" in r.url] + assert len(agent_requests) > 0 + last_agent_request = agent_requests[-1] + + # Assert agent creation + assert last_agent_request.headers["X-Agentops-Api-Key"] == self.api_key - # Wait for flush + # End session and wait for flush + agentops.end_session(end_state="Success") time.sleep(1.5) - # 4 requests: check_for_updates, create_session, create_agent, update_session - assert len(mock_req.request_history) == 4 + # Find session end request + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + assert len(end_session_requests) > 0 + last_end_request = end_session_requests[-1] - assert mock_req.request_history[-2].headers["X-Agentops-Api-Key"] == self.api_key + assert last_end_request.headers["X-Agentops-Api-Key"] == self.api_key mock_req.reset() diff --git a/tests/unit/test_record_action.py b/tests/unit/test_record_action.py index 63169142..0e781a57 100644 --- a/tests/unit/test_record_action.py +++ b/tests/unit/test_record_action.py @@ -25,10 +25,13 @@ def add_two(x, y): add_two(3, 4) time.sleep(0.1) - # 3 requests: check_for_updates, start_session, record_action - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find the record_action request + action_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(action_requests) > 0 + last_action_request = action_requests[-1] + + assert last_action_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_action_request.json() assert request_json["events"][0]["action_type"] == self.event_type assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -46,10 +49,13 @@ def add_two(x, y): add_two(3, 4) time.sleep(0.1) - # 3 requests: check_for_updates, start_session, record_action - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find the record_action request + action_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(action_requests) > 0 + last_action_request = action_requests[-1] + + assert last_action_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_action_request.json() assert request_json["events"][0]["action_type"] == "add_two" assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -70,10 +76,13 @@ def add_three(x, y, z=3): time.sleep(1.5) - # 3 requests: check_for_updates, start_session, record_action - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find the record_action request + action_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(action_requests) > 0 + last_action_request = action_requests[-1] + + assert last_action_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_action_request.json() assert request_json["events"][1]["action_type"] == self.event_type assert request_json["events"][1]["params"] == {"x": 1, "y": 2, "z": 4} @@ -100,10 +109,14 @@ async def async_add(x, y): # Assert assert result == 7 - # Assert - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + + # Find the record_action request + action_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(action_requests) > 0 + last_action_request = action_requests[-1] + + assert last_action_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_action_request.json() assert request_json["events"][0]["action_type"] == self.event_type assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -132,31 +145,29 @@ def add_three(x, y, z=3): add_three(1, 2, 3, session=session_2) time.sleep(0.1) - assert len(mock_req.request_history) == 5 + # Find action requests + action_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(action_requests) >= 2 # Should have at least 2 action requests - request_json = mock_req.last_request.json() - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - assert ( - mock_req.last_request.headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" - ) + # Verify session_2's request (last request) + last_request = action_requests[-1] + assert last_request.headers["X-Agentops-Api-Key"] == self.api_key + assert last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" + request_json = last_request.json() assert request_json["events"][0]["action_type"] == self.event_type assert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3} assert request_json["events"][0]["returns"] == 6 - second_last_request_json = mock_req.request_history[-2].json() - assert mock_req.request_history[-2].headers["X-Agentops-Api-Key"] == self.api_key + # Verify session_1's request (second to last request) + second_last_request = action_requests[-2] + assert second_last_request.headers["X-Agentops-Api-Key"] == self.api_key assert ( - mock_req.request_history[-2].headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" + second_last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" ) - assert second_last_request_json["events"][0]["action_type"] == self.event_type - assert second_last_request_json["events"][0]["params"] == { - "x": 1, - "y": 2, - "z": 3, - } - assert second_last_request_json["events"][0]["returns"] == 6 + request_json = second_last_request.json() + assert request_json["events"][0]["action_type"] == self.event_type + assert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3} + assert request_json["events"][0]["returns"] == 6 session_1.end_session(end_state="Success") session_2.end_session(end_state="Success") diff --git a/tests/unit/test_record_tool.py b/tests/unit/test_record_tool.py index aa5bb257..f2fca902 100644 --- a/tests/unit/test_record_tool.py +++ b/tests/unit/test_record_tool.py @@ -29,10 +29,13 @@ def add_two(x, y): add_two(3, 4) time.sleep(0.1) - # 3 requests: check_for_updates, start_session, record_tool - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find the record_tool request + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) > 0 + last_tool_request = tool_requests[-1] + + assert last_tool_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_tool_request.json() assert request_json["events"][0]["name"] == self.tool_name assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -50,10 +53,13 @@ def add_two(x, y): add_two(3, 4) time.sleep(0.1) - # Assert - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find the record_tool request + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) > 0 + last_tool_request = tool_requests[-1] + + assert last_tool_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_tool_request.json() assert request_json["events"][0]["name"] == "add_two" assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -74,10 +80,13 @@ def add_three(x, y, z=3): add_three(1, 2) time.sleep(0.1) - # 4 requests: check_for_updates, start_session, record_tool, record_tool - assert len(mock_req.request_history) == 4 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find all tool requests + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) > 0 + last_tool_request = tool_requests[-1] + + assert last_tool_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_tool_request.json() assert request_json["events"][0]["name"] == self.tool_name assert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3} assert request_json["events"][0]["returns"] == 6 @@ -99,10 +108,14 @@ async def async_add(x, y): # Assert assert result == 7 - # Assert - assert len(mock_req.request_history) == 3 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + + # Find the record_tool request + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) > 0 + last_tool_request = tool_requests[-1] + + assert last_tool_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_tool_request.json() assert request_json["events"][0]["name"] == self.tool_name assert request_json["events"][0]["params"] == {"x": 3, "y": 4} assert request_json["events"][0]["returns"] == 7 @@ -131,32 +144,29 @@ def add_three(x, y, z=3): add_three(1, 2, session=session_2) time.sleep(0.1) - # 6 requests: check_for_updates, start_session, record_tool, start_session, record_tool, end_session - assert len(mock_req.request_history) == 5 + # Find tool requests + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) >= 2 # Should have at least 2 tool requests - request_json = mock_req.last_request.json() - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - assert ( - mock_req.last_request.headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" - ) + # Verify session_2's request (last request) + last_request = tool_requests[-1] + assert last_request.headers["X-Agentops-Api-Key"] == self.api_key + assert last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" + request_json = last_request.json() assert request_json["events"][0]["name"] == self.tool_name assert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3} assert request_json["events"][0]["returns"] == 6 - second_last_request_json = mock_req.request_history[-2].json() - assert mock_req.request_history[-2].headers["X-Agentops-Api-Key"] == self.api_key + # Verify session_1's request (second to last request) + second_last_request = tool_requests[-2] + assert second_last_request.headers["X-Agentops-Api-Key"] == self.api_key assert ( - mock_req.request_history[-2].headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" + second_last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" ) - assert second_last_request_json["events"][0]["name"] == self.tool_name - assert second_last_request_json["events"][0]["params"] == { - "x": 1, - "y": 2, - "z": 3, - } - assert second_last_request_json["events"][0]["returns"] == 6 + request_json = second_last_request.json() + assert request_json["events"][0]["name"] == self.tool_name + assert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3} + assert request_json["events"][0]["returns"] == 6 session_1.end_session(end_state="Success") session_2.end_session(end_state="Success") @@ -180,31 +190,29 @@ async def async_add(x, y): await async_add(1, 2, session=session_2) time.sleep(0.1) - # Assert - assert len(mock_req.request_history) == 5 + # Find tool requests + tool_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(tool_requests) >= 2 # Should have at least 2 tool requests - request_json = mock_req.last_request.json() - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - assert ( - mock_req.last_request.headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" - ) + # Verify session_2's request (last request) + last_request = tool_requests[-1] + assert last_request.headers["X-Agentops-Api-Key"] == self.api_key + assert last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" + request_json = last_request.json() assert request_json["events"][0]["name"] == self.tool_name assert request_json["events"][0]["params"] == {"x": 1, "y": 2} assert request_json["events"][0]["returns"] == 3 - second_last_request_json = mock_req.request_history[-2].json() - assert mock_req.request_history[-2].headers["X-Agentops-Api-Key"] == self.api_key + # Verify session_1's request (second to last request) + second_last_request = tool_requests[-2] + assert second_last_request.headers["X-Agentops-Api-Key"] == self.api_key assert ( - mock_req.request_history[-2].headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" + second_last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" ) - assert second_last_request_json["events"][0]["name"] == self.tool_name - assert second_last_request_json["events"][0]["params"] == { - "x": 1, - "y": 2, - } - assert second_last_request_json["events"][0]["returns"] == 3 + request_json = second_last_request.json() + assert request_json["events"][0]["name"] == self.tool_name + assert request_json["events"][0]["params"] == {"x": 1, "y": 2} + assert request_json["events"][0]["returns"] == 3 session_1.end_session(end_state="Success") session_2.end_session(end_state="Success") diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 3d5d38dd..c8a1fc90 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -43,26 +43,27 @@ def test_session(self, mock_req): agentops.record(ActionEvent(self.event_type)) time.sleep(0.1) - # 3 Requests: check_for_updates, start_session, create_events (2 in 1) - assert len(mock_req.request_history) == 3 - time.sleep(0.15) - assert ( - mock_req.last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session.session_id)]}" - ) - request_json = mock_req.last_request.json() + # Find event requests + event_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(event_requests) > 0 + last_event_request = event_requests[-1] + + assert last_event_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session.session_id)]}" + request_json = last_event_request.json() assert request_json["events"][0]["event_type"] == self.event_type end_state = "Success" agentops.end_session(end_state) time.sleep(0.15) - # We should have 4 requests (additional end session) - assert len(mock_req.request_history) == 4 - assert ( - mock_req.last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session.session_id)]}" - ) - request_json = mock_req.last_request.json() + # Find session end request + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + assert len(end_session_requests) > 0 + last_end_request = end_session_requests[-1] + + assert last_end_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session.session_id)]}" + request_json = last_end_request.json() assert request_json["session"]["end_state"] == end_state assert len(request_json["session"]["tags"]) == 0 @@ -80,9 +81,13 @@ def test_add_tags(self, mock_req): agentops.end_session(end_state) time.sleep(0.15) - # Assert 3 requests, 1 for session init, 1 for event, 1 for end session - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find session end request + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + assert len(end_session_requests) > 0 + last_end_request = end_session_requests[-1] + + assert last_end_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_end_request.json() assert request_json["session"]["end_state"] == end_state assert request_json["session"]["tags"] == ["GPT-4", "test-tag", "dupe-tag"] @@ -101,10 +106,13 @@ def test_tags(self, mock_req): agentops.end_session(end_state) time.sleep(0.15) - # 4 requests: check_for_updates, start_session, record_event, end_session - assert len(mock_req.request_history) == 4 - assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key - request_json = mock_req.last_request.json() + # Find session end request + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + assert len(end_session_requests) > 0 + last_end_request = end_session_requests[-1] + + assert last_end_request.headers["X-Agentops-Api-Key"] == self.api_key + request_json = last_end_request.json() assert request_json["session"]["end_state"] == end_state assert request_json["session"]["tags"] == tags @@ -115,9 +123,14 @@ def test_inherit_session_id(self, mock_req): inherited_id = "4f72e834-ff26-4802-ba2d-62e7613446f1" agentops.start_session(tags=["test"], inherited_session_id=inherited_id) + # Find session start request + start_session_requests = [r for r in mock_req.request_history if "/v2/create_session" in r.url] + assert len(start_session_requests) > 0 + last_start_request = start_session_requests[-1] + # Act # session_id correct - request_json = mock_req.last_request.json() + request_json = last_start_request.json() assert request_json["session"]["session_id"] == inherited_id # Act @@ -247,54 +260,52 @@ def test_two_sessions(self, mock_req): ] time.sleep(0.1) - # Requests: check_for_updates, 2 start_session - assert len(mock_req.request_history) == 3 - session_1.record(ActionEvent(self.event_type)) session_2.record(ActionEvent(self.event_type)) time.sleep(1.5) - # 5 requests: check_for_updates, 2 start_session, 2 record_event - assert len(mock_req.request_history) == 5 - - # Check the last two requests instead of just the last one - last_request = mock_req.request_history[-1] - second_last_request = mock_req.request_history[-2] + # Find event requests + event_requests = [r for r in mock_req.request_history if "/v2/create_events" in r.url] + assert len(event_requests) >= 2 # Verify session_1's request + session_1_request = event_requests[-2] assert ( - second_last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" + session_1_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" ) - assert second_last_request.json()["events"][0]["event_type"] == self.event_type + assert session_1_request.json()["events"][0]["event_type"] == self.event_type # Verify session_2's request - assert last_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" - assert last_request.json()["events"][0]["event_type"] == self.event_type + session_2_request = event_requests[-1] + assert ( + session_2_request.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" + ) + assert session_2_request.json()["events"][0]["event_type"] == self.event_type end_state = "Success" session_1.end_session(end_state) time.sleep(1.5) - # Additional end session request - assert len(mock_req.request_history) == 6 - assert ( - mock_req.last_request.headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" - ) - request_json = mock_req.last_request.json() + # Find session end requests + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + assert len(end_session_requests) > 0 + session_1_end = end_session_requests[-1] + + assert session_1_end.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_1.session_id)]}" + request_json = session_1_end.json() assert request_json["session"]["end_state"] == end_state assert len(request_json["session"]["tags"]) == 0 session_2.end_session(end_state) - # Additional end session request - assert len(mock_req.request_history) == 7 - assert ( - mock_req.last_request.headers["Authorization"] - == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" - ) - request_json = mock_req.last_request.json() + time.sleep(0.1) + + # Verify session 2 end request + end_session_requests = [r for r in mock_req.request_history if "/v2/update_session" in r.url] + session_2_end = end_session_requests[-1] + assert session_2_end.headers["Authorization"] == f"Bearer {mock_req.session_jwts[str(session_2.session_id)]}" + request_json = session_2_end.json() assert request_json["session"]["end_state"] == end_state assert len(request_json["session"]["tags"]) == 0