data.py

"""Load and clean the NESO Historic GB Generation Mix dataset.

Cleaning decisions (full reasoning in notebooks/01_eda_cleaning.ipynb):
1. Sort by DATETIME — the raw file has two Dec-2025 rows appended out of order.
2. Verify the series is contiguous half-hourly; fail loudly if NESO ever ships gaps.
3. Keep 2017+ for modelling — the coal era before it is a different grid regime.
"""

from pathlib import Path

import pandas as pd

DATA_URL = (
    "https://api.neso.energy/dataset/88313ae5-94e4-4ddc-a790-593554d8c6b9/"
    "resource/f93d1835-75bc-43e5-84ad-12472b180a98/download/df_fuel_ckan.csv"
)
RAW_PATH = Path(__file__).resolve().parent.parent / "data" / "raw" / "df_fuel_ckan.csv"

MODEL_START = "2017-01-01"  # post coal phase-out regime


def download_raw(path: Path = RAW_PATH) -> Path:
    """Download the raw CSV from NESO if it is not already on disk."""
    if not path.exists():
        path.parent.mkdir(parents=True, exist_ok=True)
        print(f"Downloading NESO dataset to {path} ...")
        pd.read_csv(DATA_URL).to_csv(path, index=False)
    return path


def load_clean(path: Path = RAW_PATH, model_period_only: bool = False) -> pd.DataFrame:
    """Return the cleaned half-hourly dataset, indexed by DATETIME (UTC)."""
    df = pd.read_csv(path, parse_dates=["DATETIME"])

    df = df.sort_values("DATETIME").reset_index(drop=True)
    assert not df["DATETIME"].duplicated().any(), "duplicate timestamps in raw data"

    diffs = df["DATETIME"].diff().dropna()
    gaps = (diffs != pd.Timedelta("30min")).sum()
    assert gaps == 0, f"{gaps} non-half-hourly gaps found — investigate before modelling"

    assert df["CARBON_INTENSITY"].between(0, 1000).all(), "implausible carbon intensity values"

    df = df.set_index("DATETIME")
    if model_period_only:
        df = df.loc[MODEL_START:]
    return df