cases.jsonl

{"id": "tri-01", "task": "json_schema", "split": "dev", "turns": [{"role": "user", "content": "A customer at dana.olsen@acme.io writes that their production dashboard has been down for two hours. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P1", "category": "outage", "customer_email": "dana.olsen@acme.io"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-02", "task": "json_schema", "split": "dev", "turns": [{"role": "user", "content": "omar.haddad@northwind.co.uk says he was charged twice for his March subscription. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P2", "category": "billing", "customer_email": "omar.haddad@northwind.co.uk"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-03", "task": "json_schema", "split": "test", "turns": [{"role": "user", "content": "priya.raman@lumen.dev asks whether you could add dark mode to the mobile app one day. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P3", "category": "feature_request", "customer_email": "priya.raman@lumen.dev"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-04", "task": "json_schema", "split": "test", "turns": [{"role": "user", "content": "sam.iyer@ferrous.systems cannot log in; the reset email never arrives. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P2", "category": "account", "customer_email": "sam.iyer@ferrous.systems"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-05", "task": "json_schema", "split": "dev", "turns": [{"role": "user", "content": "The export button throws a 500 for lea.moreau@bastion.fr, but only on Safari. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P2", "category": "bug", "customer_email": "lea.moreau@bastion.fr"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-06", "task": "json_schema", "split": "dev", "turns": [{"role": "user", "content": "All API requests from tomas.novak@svelt.cz are returning 503 since the deploy. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P1", "category": "outage", "customer_email": "tomas.novak@svelt.cz"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-07", "task": "json_schema", "split": "test", "turns": [{"role": "user", "content": "yuki.tanaka@orbit.jp was billed in USD instead of JPY and wants the difference back. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P2", "category": "billing", "customer_email": "yuki.tanaka@orbit.jp"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "tri-08", "task": "json_schema", "split": "test", "turns": [{"role": "user", "content": "A typo in the settings page tooltip was reported by nina.kovac@delta.hr. Triage this ticket. Reply with JSON only."}], "schema_ref": "triage", "expected": {"fields": {"priority": "P3", "category": "bug", "customer_email": "nina.kovac@delta.hr"}}, "rules": {"priority": {"match": "equiv", "equiv": [["P1", "p1", "urgent", "critical", "highest"], ["P2", "p2", "high", "normal"], ["P3", "p3", "low", "minor"]]}, "category": {"match": "exact"}, "customer_email": {"match": "exact"}}}
{"id": "esc-01", "task": "extraction", "split": "dev", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'Ticket SUP-3311 goes to Ada Okonkwo, due tomorrow at 14:00.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-3311", "assignee": "Ada Okonkwo", "due": "2026-03-21T14:00"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-02", "task": "extraction", "split": "dev", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'SUP-2870 -> Bjorn Halvorsen, needs an answer by today 09:30.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-2870", "assignee": "Bjorn Halvorsen", "due": "2026-03-20T09:30"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-03", "task": "extraction", "split": "test", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'Please put SUP-9902 on Grace Adeyemi; deadline tomorrow 17:45.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-9902", "assignee": "Grace Adeyemi", "due": "2026-03-21T17:45"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-04", "task": "extraction", "split": "test", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'SUP-1044 assigned to Mateo Ruiz, due today at 23:59.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-1044", "assignee": "Mateo Ruiz", "due": "2026-03-20T23:59"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-05", "task": "extraction", "split": "dev", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'Escalating SUP-7756 to Ingrid Larsen. Due tomorrow, 08:00 sharp.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-7756", "assignee": "Ingrid Larsen", "due": "2026-03-21T08:00"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-06", "task": "extraction", "split": "dev", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'SUP-6120, owner Kwame Mensah, must land today 16:15.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-6120", "assignee": "Kwame Mensah", "due": "2026-03-20T16:15"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-07", "task": "extraction", "split": "test", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'Hand SUP-4483 to Sofia Bianchi; due tomorrow 11:30.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-4483", "assignee": "Sofia Bianchi", "due": "2026-03-21T11:30"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "esc-08", "task": "extraction", "split": "test", "turns": [{"role": "user", "content": "Extract the escalation from this note: 'SUP-5309 for Hannah Weiss, due today 12:00.' Reply with JSON only."}], "schema_ref": "escalation", "expected": {"fields": {"ticket_id": "SUP-5309", "assignee": "Hannah Weiss", "due": "2026-03-20T12:00"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}, "due": {"match": "iso_minute"}}}
{"id": "act-01", "task": "function_call", "split": "dev", "turns": [{"role": "user", "content": "Put ticket SUP-3311 on Ada Okonkwo."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "assign_ticket", "args": {"ticket_id": "SUP-3311", "assignee": "Ada Okonkwo"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}}}
{"id": "act-02", "task": "function_call", "split": "dev", "turns": [{"role": "user", "content": "Refund omar.haddad@northwind.co.uk 49.00 for the duplicate charge."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "refund_customer", "args": {"customer_email": "omar.haddad@northwind.co.uk", "amount": 49.0}}, "rules": {"customer_email": {"match": "exact"}, "amount": {"match": "number", "tol": 0.01}}}
{"id": "act-03", "task": "function_call", "split": "test", "turns": [{"role": "user", "content": "Give yuki.tanaka@orbit.jp back the 128.40 she was overcharged."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "refund_customer", "args": {"customer_email": "yuki.tanaka@orbit.jp", "amount": 128.4}}, "rules": {"customer_email": {"match": "exact"}, "amount": {"match": "number", "tol": 0.01}}}
{"id": "act-04", "task": "function_call", "split": "test", "turns": [{"role": "user", "content": "Assign SUP-9902 to Grace Adeyemi, please."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "assign_ticket", "args": {"ticket_id": "SUP-9902", "assignee": "Grace Adeyemi"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}}}
{"id": "act-05", "task": "function_call", "split": "dev", "turns": [{"role": "user", "content": "SUP-6120 should go to Kwame Mensah."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "assign_ticket", "args": {"ticket_id": "SUP-6120", "assignee": "Kwame Mensah"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}}}
{"id": "act-06", "task": "function_call", "split": "dev", "turns": [{"role": "user", "content": "Send lea.moreau@bastion.fr a refund of 12.99."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "refund_customer", "args": {"customer_email": "lea.moreau@bastion.fr", "amount": 12.99}}, "rules": {"customer_email": {"match": "exact"}, "amount": {"match": "number", "tol": 0.01}}}
{"id": "act-07", "task": "function_call", "split": "test", "turns": [{"role": "user", "content": "Hand SUP-4483 over to Sofia Bianchi."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "assign_ticket", "args": {"ticket_id": "SUP-4483", "assignee": "Sofia Bianchi"}}, "rules": {"ticket_id": {"match": "exact"}, "assignee": {"match": "text"}}}
{"id": "act-08", "task": "function_call", "split": "test", "turns": [{"role": "user", "content": "Refund 300 to tomas.novak@svelt.cz for the outage credit."}], "tools": ["assign_ticket", "refund_customer"], "expected": {"tool": "refund_customer", "args": {"customer_email": "tomas.novak@svelt.cz", "amount": 300.0}}, "rules": {"customer_email": {"match": "exact"}, "amount": {"match": "number", "tol": 0.01}}}