Skip to content
This repository was archived by the owner on Jul 20, 2026. It is now read-only.
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion ngm/ngscrape/spiders/district_case_enrichment.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,17 @@ def start_requests(self):
or_(
CourtCase.status == "pending",
CourtCase.status.is_(None),
# Revisit already-enriched rows whose case_type never
# got populated: the daily cause-list only carries the
# (often blank) "मुद्दा विषय" subject column, so these
# need the detail page's "मुद्दाको किसिम" to backfill.
and_(
CourtCase.status == "enriched",
or_(
CourtCase.case_type.is_(None),
CourtCase.case_type == "",
),
),
),
)
)
Expand Down Expand Up @@ -258,9 +269,27 @@ def parse_case_detail(self, response):
self.logger.warning(f"Case {case_number} not found in database")
return

if case.status == "enriched":
already_enriched = case.status == "enriched"
has_case_type = bool(case.case_type)

if already_enriched:
if has_case_type:
self.logger.info(f"Case {case_number} already enriched, skipping")
return
# Already enriched but missing case_type: backfill just that field
# from the detail page without re-touching entities/hearings (whose
# rebuild deletes rows that may have downstream linkages).
case_type = self._extract_enrichment_data(soup).get("case_type")
if case_type:
self._backfill_case_type(case_number, code_name, case_type)
self.logger.info(
f"Backfilled case_type for {case_number} ({code_name}): {case_type}"
)
else:
self.logger.info(
f"Case {case_number} ({code_name}) has no case_type on detail page"
)
return

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Combining the status check and the backfill logic inside the existing transaction block avoids querying the database twice for the same CourtCase row and opening/closing multiple transactions.

Additionally, if a case is already enriched but has no case_type on the detail page, leaving it empty will cause it to be repeatedly queried and scraped on every subsequent run of the spider (due to the query in start_requests). Setting a placeholder like "-" prevents this infinite re-scraping loop.

            if case.status == "enriched":
                if case.case_type:
                    self.logger.info(f"Case {case_number} already enriched, skipping")
                    return

                # Already enriched but missing case_type: backfill just that field
                # from the detail page without re-touching entities/hearings (whose
                # rebuild deletes rows that may have downstream linkages).
                case_type = self._extract_enrichment_data(soup).get("case_type")
                if case_type:
                    case.case_type = case_type[:200]
                    case.updated_at = datetime.now(KATHMANDU_TZ).replace(tzinfo=None)
                    self.logger.info(
                        f"Backfilled case_type for {case_number} ({code_name}): {case_type}"
                    )
                else:
                    # Set a placeholder to prevent infinite re-scraping in start_requests
                    case.case_type = "-"
                    case.updated_at = datetime.now(KATHMANDU_TZ).replace(tzinfo=None)
                    self.logger.info(
                        f"Case {case_number} ({code_name}) has no case_type on detail page; set to '-'"
                    )
                return


# Extract enrichment data
enrichment_data = self._extract_enrichment_data(soup)
Expand Down Expand Up @@ -296,6 +325,11 @@ def _extract_enrichment_data(self, soup: BeautifulSoup) -> Dict:
# Map Nepali labels to database fields
if label == "रजिष्ट्रेशन नं" and value:
data["registration_number"] = value[:100]
elif label == "मुद्दाको किसिम" and value:
# True case type, only available on the detail page; the
# cause-list listing has no such column (it carries the
# subject), which is why district case_type is often blank.
data["case_type"] = value[:200]
elif label == "मुद्दाको बिषय" and value:
data["case_subject"] = value
elif label == "मुद्दाको स्थिति" and value:
Expand Down Expand Up @@ -395,6 +429,26 @@ def _extract_hearings_timeline(self, soup: BeautifulSoup) -> Dict[str, List[Dict

return data

def _backfill_case_type(self, case_number: str, code_name: str, case_type: str):
"""Set case_type on an already-enriched case, leaving everything else."""
now = datetime.now(KATHMANDU_TZ).replace(tzinfo=None)

with self.session.begin():
case = (
self.session.query(CourtCase)
.filter(
and_(
CourtCase.case_number == case_number,
CourtCase.court_identifier == code_name,
)
)
.first()
)

if case and not case.case_type:
case.case_type = case_type[:200]
case.updated_at = now

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Since the backfill logic has been integrated directly into the main transaction block in parse_case_detail, this helper method is no longer needed and can be removed.


def _save_enrichment(
self,
case_number: str,
Expand Down
Loading