Unified single-call date extraction (#466)

* Try unified approach

* Fix prompt

* Update dev eval results for unified date extraction

* Cap pages, fix date tests and docstring for unified extraction

* Fix date prompt

* Parallelize date eval with xdist and add per-jurisdiction results + JSON breakdown

* Simplify date extraction system prompt

* Trim eval commentary and extract PerJurisdictionResults to utilities

* Drop regression gate and CSV breakdown; JSON is the record

Author: rajeee

.gitignore

@@ -21,6 +21,9 @@ venv.bak/
 # Per-case eval run logs (debug artifacts, not committed)
 evals/results/*/logs/
 
+# Per-jurisdiction eval result shards (aggregated into the breakdown CSV)
+evals/results/*/per_jurisdiction/
+
 # Documentation
 docs/_build/
 docs/source/_autosummary/

compass/extraction/date.py

@@ -1,41 +1,34 @@
 """Ordinance date extraction logic"""
 
 import logging
-import asyncio
 from datetime import datetime
-from collections import Counter
 
 from compass.utilities.enums import LLMUsageCategory
 from compass.utilities.parsing import raw_pages_from_doc
 
 
 logger = logging.getLogger(__name__)
 
-# These domains contain the collection date in URL, not enactment date
-_BANNED_DATE_DOMAINS = ["https://energyzoning.org"]
+# Cap the text sent in a single date-extraction call. Enactment dates
+# live in the preamble (front) and signature/adoption block (back), so
+# keep the first and last pages and drop the middle for long documents.
+_MAX_HEAD_PAGES = 20
+_MAX_TAIL_PAGES = 10
 
 
 class DateExtractor:
     """Helper class to extract date info from document"""
 
     SYSTEM_MESSAGE = (
         "You are a legal scholar that reads ordinance text and extracts "
-        "structured date information. "
-        "Return your answer as a dictionary in JSON format (not markdown). "
-        "Your JSON file must include exactly four keys. The first "
-        "key is 'explanation', which contains a short summary of the most "
-        "relevant date information you found in the text. The second key is "
-        "'year', which should contain an integer value that represents the "
-        "latest year this ordinance was enacted/updated, or null if that "
-        "information cannot be found in the text. The third key is 'month', "
-        "which should contain an integer value that represents the latest "
-        "month of the year this ordinance was enacted/updated, or null if "
-        "that information cannot be found in the text. The fourth key is "
-        "'day', which should contain an integer value that represents the "
-        "latest day of the month this ordinance was enacted/updated, or null "
-        "if that information cannot be found in the text. Only provide values "
-        "if you are confident that they represent the latest date this "
-        "ordinance was enacted/updated"
+        "a single date. The date you report is the latest year the "
+        "ordinance was enacted or ammended or became effective. If no such "
+        "date is available return null."
+        "Return your answer in JSON format like this: "
+        '{"explanation": TEXT, "year": YY, "month": MM, "day": DD}'
+        "Where explanation contains a short summary/explanation of "
+        "the date information you found, including the exact text the "
+        "date is based on."
     )
     """System message for date extraction LLM calls"""
 
@@ -58,6 +51,10 @@ def __init__(self, json_llm_caller, text_splitter=None):
     async def parse(self, doc):
         """Extract date (year, month, day) from doc
 
+        The full document text is read in a single LLM call. The
+        document's ``source`` URL, if any, is passed along as a hint,
+        but the document text is the source of truth.
+
         Parameters
         ----------
         doc : BaseDocument
@@ -66,88 +63,72 @@ async def parse(self, doc):
         Returns
         -------
         tuple
-            3-tuple containing year, month, day, or ``None`` if any of
-            those are not found.
+            3-tuple of (year, month, day). Any element that cannot be
+            determined is ``None``.
         """
+        raw_pages = [
+            page
+            for page in raw_pages_from_doc(doc, self.text_splitter)
+            if page
+        ]
+        raw_pages = _trim_pages(raw_pages)
+        text = "\n\n".join(raw_pages)
+        if not text:
+            return None, None, None
+
+        content = "Please extract the enactment date for this ordinance."
         url = doc.attrs.get("source")
-        can_check_url_for_date = url and not any(
-            sub_str in url for sub_str in _BANNED_DATE_DOMAINS
+        if url:
+            content += f"\nThe document was downloaded from this URL: {url}"
+        content += f"\n\nOrdinance text:\n{text}"
+
+        response = await self.jlc.call(
+            sys_msg=self.SYSTEM_MESSAGE,
+            content=content,
+            usage_sub_label=LLMUsageCategory.DATE_EXTRACTION,
         )
-        if can_check_url_for_date:
-            logger.debug("Checking URL for date: %s", url)
-            response = await self.jlc.call(
-                sys_msg=self.SYSTEM_MESSAGE,
-                content=(
-                    "Please extract the date from the URL for this "
-                    f"ordinance, if possible:\n{url}"
-                ),
-                usage_sub_label=LLMUsageCategory.DATE_EXTRACTION,
+        if response:
+            logger.debug(
+                "Date extraction explanation: %s",
+                response.get("explanation"),
             )
-            if response:
-                date = _parse_date([response])
-                logger.debug("Parsed date from URL: %s", date)
-                return date
+        date = _parse_date(response)
+        logger.debug("Parsed date: %s", date)
+        return date
 
-        raw_pages = raw_pages_from_doc(doc, self.text_splitter)
-        if not raw_pages:
-            return None, None, None
-
-        outer_task_name = asyncio.current_task().get_name()
-        date_extractions = [
-            asyncio.create_task(
-                self.jlc.call(
-                    sys_msg=self.SYSTEM_MESSAGE,
-                    content=(
-                        f"Please extract the date for this ordinance:\n{text}"
-                    ),
-                    usage_sub_label=LLMUsageCategory.DATE_EXTRACTION,
-                ),
-                name=outer_task_name,
-            )
-            for text in raw_pages
-            if text
-        ]
-        all_years = await asyncio.gather(*date_extractions)
-        return _parse_date([y for y in all_years if y])
 
+def _trim_pages(pages):
+    """Keep the head and tail pages, dropping the middle if too long"""
+    if len(pages) <= _MAX_HEAD_PAGES + _MAX_TAIL_PAGES:
+        return pages
+    return pages[:_MAX_HEAD_PAGES] + pages[-_MAX_TAIL_PAGES:]
 
-def _parse_date(json_list):
-    """Parse all date elements
 
-    True date is determined to be the most frequent date. In the case of
-    a tie, the latest date is chosen.
-    """
-    if not json_list:
+def _parse_date(date_info):
+    """Validate and return the (year, month, day) from a response"""
+    if not date_info:
         return None, None, None
 
-    years = _parse_date_element(
-        json_list,
-        key="year",
-        max_len=4,
-        min_val=2000,
-        max_val=datetime.now().year,
-    )
-    months = _parse_date_element(
-        json_list, key="month", max_len=2, min_val=1, max_val=12
-    )
-    days = _parse_date_element(
-        json_list, key="day", max_len=2, min_val=1, max_val=31
+    year = _validated_element(
+        date_info, key="year", min_val=1950, max_val=datetime.now().year + 1
     )
+    month = _validated_element(date_info, key="month", min_val=1, max_val=12)
+    day = _validated_element(date_info, key="day", min_val=1, max_val=31)
+    return year, month, day
 
-    date_elements = Counter(zip(years, months, days, strict=False))
-    date = max(date_elements, key=lambda date: (date_elements[date], date))
-    return tuple(None if d < 0 else d for d in date)
-
-
-def _parse_date_element(json_list, key, max_len, min_val, max_val):
-    """Parse out a single date element"""
-    date_elements = [info.get(key) for info in json_list]
-    logger.debug("key=%r, date_elements=%r", key, date_elements)
-    return [
-        int(y)
-        if y is not None
-        and len(str(y)) <= max_len
-        and (min_val <= int(y) <= max_val)
-        else -1 * float("inf")
-        for y in date_elements
-    ]
+
+def _validated_element(date_info, key, min_val, max_val):
+    """Return a single date element if it falls within the valid range
+
+    Acts as a cheap safety net against an out-of-range or malformed
+    value in the model response.
+    """
+    value = date_info.get(key)
+    logger.debug("key=%r, value=%r", key, value)
+    if value is None:
+        return None
+    try:
+        value = int(float(value))
+    except (TypeError, ValueError):
+        return None
+    return value if min_val <= value <= max_val else None

evals/README.md

@@ -2,9 +2,10 @@
 
 Accuracy/quality evaluations of COMPASS extraction against real ordinance
 documents. Each suite is a `test_run_<name>_evals.py` file that drives a
-specific extractor end-to-end and writes a breakdown + metrics file to
-`results/`. A regression gate (run by the test module's autouse fixture)
-fails the run if the committed baseline gets worse.
+specific extractor end-to-end and writes a metrics file (and, for dev, a
+per-case breakdown) to `results/`. Cases run in parallel under
+pytest-xdist. There is no regression gate — accuracy changes show up as
+diffs in the committed `results/` files.
 
 ## Run
 
@@ -30,8 +31,7 @@ Each dataset is split into a frequently-run **dev** set and a sacred
 | --- | --- | --- |
 | Purpose | iterate, tune prompts/logic, debug failures | unbiased estimate of true performance |
 | Cadence | run frequently during development | run before a release |
-| Regression gate | yes — fails on aggregate or per-row regression | no — unbiased read, just prints + writes JSON |
-| Per-case breakdown | written + logged | hidden (no breakdown CSV, no per-case logs) |
+| Per-case breakdown | written + logged | hidden (metrics only, no per-case detail) |
 
 The held-out set only gives an **honest** read if we *don't* tune against it:
 
@@ -43,23 +43,24 @@ The held-out set only gives an **honest** read if we *don't* tune against it:
   before a release), not a development loop.
 
 The harness helps enforce this: a `--held-out` run writes **only summary
-metrics** (no per-case breakdown), per-case predictions are not logged,
-and there is no regression gate — so there is nothing to eyeball or tune
-against.
+metrics** (no per-case breakdown, no explanations, no per-case logs) — so
+there is nothing to eyeball or tune against.
 
 ## Layout
 
 ```
 test_run_<name>_evals.py   # one eval suite per extractor (e.g. test_run_date_extraction_evals.py)
-conftest.py                # registers the --held-out pytest flag
+conftest.py                # --held-out flag; session hooks that clear + aggregate results
 utilities/                 # shared, eval-agnostic plumbing
   base.py                  #   Result schema, SUCCESS/FAILURE, classify, load_doc
   metrics.py               #   compute_metrics, wilson_ci (pure math, no I/O)
-  reports.py               #   report_evals + load_baseline_failing + regressed_rows (I/O + formatting)
+  reports.py               #   report_evals + PerJurisdictionResults (I/O + formatting)
 results/
-  dev/<name>_evals.json              # committed baseline metrics (gate reads these)
-  dev/<name>_evals_breakdown.csv     # committed per-case dev breakdown
-  held_out/<name>_evals.json         # committed baseline held-out metrics (no per-case detail)
+  dev/<name>_evals.json              # committed metrics
+  dev/<name>_evals_breakdown.json    # committed per-case dev breakdown (+ explanations)
+  dev/per_jurisdiction/              # one Result JSON per case (xdist shards; gitignored)
+  dev/logs/                          # per-jurisdiction run logs (gitignored)
+  held_out/<name>_evals.json         # committed held-out metrics (no per-case detail)
 data/
   dev/<tech>/
     manifest.json5         # [{state, county, subdivision, jurisdiction_type, file, source, expected: {year, ...}}, ...]
@@ -78,27 +79,30 @@ exists" — the extractor should return no year for that document.
 
 ## How a suite is wired
 
-A `test_run_<name>_evals.py` file owns three pieces:
+Cases run in parallel across xdist worker processes, so results can't
+live in a module-level list (each worker is its own process). Instead:
 
 1. **`pytest_generate_tests(metafunc)`** reads `--held-out`, loads the
    right `manifest.json5`, and parametrizes the test's `case` argument.
    It also stamps each case with `case["fp"]` (the resolved document
    path), so the test body never has to know which dataset it came from.
 2. **`@pytest.mark.evals` test function** runs the extractor on one
-   case and appends a `Result` to the module-level `RESULTS` list.
-3. **Module-scoped autouse teardown fixture** calls
-   `report_evals(request, EVAL_NAME, RESULTS, results_dir,
-   write_breakdown=not held_out)` to compute metrics, write the
-   artifacts, and snapshot baselines. The returned dict's
-   `baseline_failing` / `fails_now` / `regressed_rows` fields drive
-   each suite's own gate -- the `reports` module does **not** decide
-   what counts as a regression. (Held-out runs skip the gate entirely.)
+   case and writes its `Result` to its own
+   `results/<set>/per_jurisdiction/<jurisdiction>.json` file (one file
+   per case, so concurrent workers never collide) via
+   `utilities.PerJurisdictionResults`.
+3. **`conftest.py` session hooks** (controller only): `sessionstart`
+   clears stale per-jurisdiction files and logs; `sessionfinish` reads
+   every per-jurisdiction file back, then calls the suite's `report(...)`
+   to write the metrics JSON and (dev only) the explanation-rich
+   breakdown JSON.
 
 ## Adding an eval suite
 
 1. Drop ground-truth docs + a `manifest.json5` under `data/{dev,held-out}/<tech>/`.
 2. Copy `test_run_date_extraction_evals.py` as a starting point. Swap in
    your extractor function and the `expected.<feature>` key you compare
    against.
-3. First run sets the baseline; commit the resulting
-   `results/{dev,held_out}/<name>_evals.json` (and the dev breakdown `.csv`).
+3. Commit the resulting `results/{dev,held_out}/<name>_evals.json` and
+   the dev `<name>_evals_breakdown.json`; accuracy changes then show up
+   as diffs on those files.

evals/conftest.py

@@ -10,3 +10,55 @@ def pytest_addoption(parser):
         default=False,
         help=("Run the eval against the held-out dataset"),
     )
+
+
+def _is_controller(config):
+    # xdist sets ``workerinput`` only on workers; its absence is the
+    # controller, which outlives all workers and aggregates their results.
+    return not hasattr(config, "workerinput")
+
+
+def _date_eval():
+    """Import the date eval module, or ``None`` if it isn't this run
+
+    The session hooks need a few eval-specific entry points
+    (``per_jurisdiction_results``, ``clear_logs``, ``report``); the
+    generic results I/O lives in ``utilities.PerJurisdictionResults``.
+    """
+    try:
+        import test_run_date_extraction_evals as module  # noqa: PLC0415
+    except ImportError:
+        return None
+    return module
+
+
+def pytest_sessionstart(session):
+    """Clear stale results and logs before a run (controller only)"""
+    config = session.config
+    if not _is_controller(config):
+        return
+    module = _date_eval()
+    if module is None:
+        return
+    held_out = config.getoption("--held-out")
+    module.per_jurisdiction_results(held_out).clear()
+    module.clear_logs(held_out)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Aggregate per-jurisdiction results and write reports (controller only)
+
+    Under xdist the per-case results are scattered across workers, so
+    reading the per-jurisdiction files here gives the full set.
+    """
+    config = session.config
+    if not _is_controller(config):
+        return
+    module = _date_eval()
+    if module is None:
+        return
+
+    held_out = config.getoption("--held-out")
+    results = module.per_jurisdiction_results(held_out).load()
+    if results:
+        module.report(session, results, held_out)

evals/data/dev/solar/manifest.json5

@@ -368,7 +368,7 @@
         "source": "https://www.townofglenville.org/home/files/gecc-solar-attachments",
         "document_satus": "Final",
         "expected": {
-            "year": 2023
+            "year": 2021
         }
     },
     {