diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..472d4769 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [2.0.0] - 2026-07-20 + +### Breaking Changes + +- **Removed legacy API endpoints** (`/wtk/*` and `/era5/*`). These routes were deprecated in v1 and have now been removed. + - Migrate to the unified `/v1/{model}/` endpoints. See the [Migration Guide](docs/06-migration.md). + +### Changed + +- API version bumped from `1.0.0` to `2.0.0`. +- Removed orphaned controller files: `wtk_data_controller.py`, `era5_data_controller.py`. + +--- + +## [1.0.0] - Initial release + +- Introduced unified `/v1/{model}/` API endpoints. +- Legacy model-specific routes (`/wtk/*`, `/era5/*`) marked deprecated. +- Supported models: `era5-quantiles`, `era5-timeseries`, `wtk-timeseries`, `ensemble-quantiles`. diff --git a/docs/03-backend.md b/docs/03-backend.md index 41adc740..0a8c90a7 100644 --- a/docs/03-backend.md +++ b/docs/03-backend.md @@ -62,3 +62,15 @@ pytest ## API Documentation When the app is running, visit `/docs` (e.g., `http://localhost:8080/docs`) to see the auto-generated Swagger UI. + +### API Endpoints + +All endpoints are under the `/api/v1/{model}/` prefix. + +| Endpoint | Description | +|---|---| +| `GET /api/v1/{model}/windspeed` | Wind speed data | +| `GET /api/v1/{model}/production` | Energy production estimates | +| `GET /api/v1/{model}/timeseries` | Raw timeseries downloads | + +**Supported models**: `era5-quantiles`, `era5-timeseries`, `wtk-timeseries`, `ensemble-quantiles` diff --git a/docs/06-migration.md b/docs/06-migration.md new file mode 100644 index 00000000..281b66c0 --- /dev/null +++ b/docs/06-migration.md @@ -0,0 +1,34 @@ +# API Migration Guide: Legacy --> v1 + +The legacy model-specific endpoints (`/wtk/*`, `/era5/*`) were removed in API v2.0.0. All functionality is available through the unified v1 API. + +For full endpoint details and parameters, see the **interactive API docs** at `/api/docs` when the app is running. + +## Route Structure + +``` +Legacy: /api/wtk/ → /api/v1/wtk-timeseries/ +Legacy: /api/era5/ → /api/v1/era5-quantiles/ +``` + +## Endpoint Mapping + +| Legacy | v1 Equivalent | +|---|---| +| `GET /api/wtk/windspeed` | `GET /api/v1/wtk-timeseries/windspeed` | +| `GET /api/wtk/energy-production` | `GET /api/v1/wtk-timeseries/production` | +| `GET /api/wtk/download-csv` | `GET /api/v1/wtk-timeseries/timeseries` | +| `POST /api/wtk/download-csv-batch` | `POST /api/v1/wtk-timeseries/timeseries/batch` | +| `GET /api/wtk/nearest-locations` | `GET /api/v1/wtk-timeseries/grid-points` | +| `GET /api/wtk/available-powercurves` | `GET /api/v1/turbines` | +| `GET /api/era5/windspeed` | `GET /api/v1/era5-quantiles/windspeed` | +| `GET /api/era5/production` | `GET /api/v1/era5-quantiles/production` | +| `GET /api/era5/timeseries` | `GET /api/v1/era5-timeseries/timeseries` | +| `POST /api/era5/timeseries/batch` | `POST /api/v1/era5-timeseries/timeseries/batch` | +| `GET /api/era5/grid-points` | `GET /api/v1/era5-quantiles/grid-points` | +| `GET /api/era5/powercurves` | `GET /api/v1/turbines` | + +## Notable Changes + +- **Period** — path-based period (e.g. `/windspeed/{avg_type}`) is now a query parameter: `?period=`. +- **Turbine** — the `powercurve` query parameter is deprecated and the renamed and recommended query parameter is `turbine`. diff --git a/docs/README.md b/docs/README.md index f64221d0..e961970d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ Welcome to the WindWatts documentation. 3. [Backend Guide](03-backend.md) - API development. 4. [Frontend Guide](04-frontend.md) - UI development. 5. [Deployment](05-deployment.md) - Production deployment. +6. [Migration Guide](06-migration.md) - Migrating from legacy API endpoints to v1. ## Contributing diff --git a/windwatts-api/Dockerfile b/windwatts-api/Dockerfile index e59a62ee..eb9bb2ad 100644 --- a/windwatts-api/Dockerfile +++ b/windwatts-api/Dockerfile @@ -19,15 +19,6 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# Download and install windwatts_data.whl from s3 -ARG WINDWATTS_DATA_URL -ARG WINDWATTS_DATA_VERSION=1.0.4 -ARG WINDWATTS_DATA_FILE=windwatts_data-${WINDWATTS_DATA_VERSION}-py3-none-any.whl -ENV WINDWATTS_DATA_URL=${WINDWATTS_DATA_URL} -RUN curl -o /tmp/windwatts_data-${WINDWATTS_DATA_VERSION}-py3-none-any.whl ${WINDWATTS_DATA_URL}${WINDWATTS_DATA_FILE} && \ - pip install --no-cache-dir /tmp/${WINDWATTS_DATA_FILE} && \ - rm /tmp/${WINDWATTS_DATA_FILE} - # Copy the FastAPI application COPY . . diff --git a/windwatts-api/app/config/model_config.py b/windwatts-api/app/config/model_config.py index 952997c8..c5be8d36 100644 --- a/windwatts-api/app/config/model_config.py +++ b/windwatts-api/app/config/model_config.py @@ -75,6 +75,8 @@ "schema": "quantile_yearly", "years": {"full": list(range(2013, 2024)), "sample": [2020, 2021, 2022, 2023]}, "heights": {"windspeed": [30, 40, 50, 60, 80, 100], "winddirection": []}, + "interpolation": True, + "grid": "era5", "grid_info": { "min_lat": 23.402, "min_long": -137.725, @@ -96,6 +98,8 @@ "windspeed": [40, 60, 80, 100, 120, 140, 160, 200], "winddirection": [], }, + "interpolation": True, + "grid": "wtk", "grid_info": { "min_lat": 7.75129, "min_long": -179.99918, @@ -104,7 +108,7 @@ "spatial_resolution": "2 km", "temporal_resolution": "1 hour", }, - "links": ["https://www.nrel.gov/grid/wind-toolkit"], + "links": ["https://www.nlr.gov/grid/wind-toolkit"], "references": [ "Draxl, C., B.M. Hodge, A. Clifton, and J. McCaa. 2015. Overview and Meteorological Validation of the Wind Integration National Dataset Toolkit (Technical Report, NREL/TP-5000-61740). Golden, CO: National Laboratory of the Rockies", 'Draxl, C., B.M. Hodge, A. Clifton, and J. McCaa. 2015. "The Wind Integration National Dataset (WIND) Toolkit." Applied Energy 151: 355366', @@ -116,6 +120,8 @@ "schema": "quantile_atemporal", "years": {"full": list(range(2013, 2024)), "sample": []}, "heights": {"windspeed": [30, 40, 50, 60, 80, 100], "winddirection": []}, + "interpolation": True, + "grid": "era5", "grid_info": { "min_lat": 23.402, "min_long": -137.725, @@ -137,6 +143,8 @@ "windspeed": [10, 30, 40, 50, 60, 80, 100], "winddirection": [10, 100], }, + "interpolation": False, + "grid": "era5", "grid_info": { "min_lat": 23.402, "min_long": -137.725, diff --git a/windwatts-api/app/config/sample_windwatts_data_config.json b/windwatts-api/app/config/sample_windwatts_data_config.json index 1d893c7f..e9e17e7f 100644 --- a/windwatts-api/app/config/sample_windwatts_data_config.json +++ b/windwatts-api/app/config/sample_windwatts_data_config.json @@ -1,10 +1,27 @@ { "region_name": "us-east-1", - "bucket_name": "sample-bucket", - "database": "sample_database", "output_location": "s3://sample-output-location/", "output_bucket": "sample-output-bucket", - "athena_table_name": "sample_table", - "alt_athena_table_name": "sample_alt_table", - "athena_workgroup": "sample_workgroup" -} \ No newline at end of file + "database": "sample_database", + "athena_workgroup": "sample_workgroup", + "sources": { + "wtk-timeseries": { + "bucket_name": "sample-bucket", + "athena_table_name": "sample_table", + "alt_athena_table_name": "sample_alt_table", + "capabilities": { "avg_types": ["all", "annual", "monthly", "hourly"] } + }, + "era5-quantiles": { + "bucket_name": "sample-bucket", + "athena_table_name": "sample_table", + "alt_athena_table_name": "sample_alt_table", + "capabilities": { "avg_types": ["all", "annual"] } + }, + "ensemble-quantiles": { + "bucket_name": "sample-bucket", + "athena_table_name": "sample_table", + "alt_athena_table_name": "sample_alt_table", + "capabilities": { "avg_types": ["all"] } + } + } +} diff --git a/windwatts-api/app/config_manager.py b/windwatts-api/app/config_manager.py index 613c0159..8e652a11 100644 --- a/windwatts-api/app/config_manager.py +++ b/windwatts-api/app/config_manager.py @@ -1,7 +1,6 @@ import os import json import boto3 -import tempfile class ConfigManager: @@ -21,45 +20,34 @@ def __init__( self.local_config_path = local_config_path self.client = boto3.client("secretsmanager", region_name=region_name) - def get_config(self) -> str: + def get_config(self) -> dict: """ Retrieve the secret from AWS Secrets Manager, environment variables, or the local configuration file. - :return: The path to the configuration file. + :return: The dict with config keys. """ # Try to retrieve the secret from AWS Secrets Manager if self.secret_arn: try: response = self.client.get_secret_value(SecretId=self.secret_arn) - secret = response["SecretString"] - config_data = json.loads(secret) - - # Save the secret to a temporary file - temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".json") - temp_file.close() - with open(temp_file.name, "w") as f: - json.dump(config_data, f) - return temp_file.name + return json.loads(response["SecretString"]) except self.client.exceptions.ClientError as e: print(f"Unable to retrieve secret: {e}") # Try to retrieve config from environment variables env_config = self._get_config_from_env() if env_config: - temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".json") - temp_file.close() - with open(temp_file.name, "w") as f: - json.dump(env_config, f) print("Config loaded from environment variables.") - return temp_file.name + return env_config # Fallback: return the path to the local configuration file if self.local_config_path and os.path.exists(self.local_config_path): - print("Local configuration file found.") - return self.local_config_path - else: - raise FileNotFoundError( - "Local configuration file not found and unable to retrieve secret from AWS Secrets Manager or environment variables." - ) + with open(self.local_config_path, "r") as f: + print(f"Loaded config from local path '{self.local_config_path}'") + return json.load(f) + + raise FileNotFoundError( + "Local configuration file not found and unable to retrieve secret from AWS Secrets Manager or environment variables." + ) def _get_config_from_env(self): """ @@ -76,7 +64,7 @@ def _get_config_from_env(self): # Scan for all SOURCES__FIELD_NAME env vars sources = {} prefix = "SOURCES_" - suffixes = ["_BUCKET_NAME", "_ATHENA_TABLE_NAME", "_ALT_ATHENA_TABLE_NAME"] + suffixes = ["_ALT_ATHENA_TABLE_NAME", "_ATHENA_TABLE_NAME", "_BUCKET_NAME"] env = os.environ source_fields = {} for key, value in env.items(): @@ -84,11 +72,12 @@ def _get_config_from_env(self): rest = key[len(prefix) :] for suffix in suffixes: if rest.endswith(suffix): - source = rest[: -len(suffix)].lower() + source = rest[: -len(suffix)].lower().replace("_", "-") field = suffix[1:].lower() # e.g. 'bucket_name' if source not in source_fields: source_fields[source] = {} source_fields[source][field] = value + break # Package the sources with required fields into `sources` for source, fields in source_fields.items(): if "bucket_name" in fields and "athena_table_name" in fields: diff --git a/windwatts-api/app/controllers/era5_data_controller.py b/windwatts-api/app/controllers/era5_data_controller.py deleted file mode 100644 index 00bfe18b..00000000 --- a/windwatts-api/app/controllers/era5_data_controller.py +++ /dev/null @@ -1,586 +0,0 @@ -from typing import List -from fastapi import APIRouter, HTTPException, Path, Query -from fastapi.responses import StreamingResponse -import zipfile -import tempfile -import re -import os -import io - -# commented out the data functions until I can get local athena_config working -from app.config_manager import ConfigManager -from app.data_fetchers.s3_data_fetcher import S3DataFetcher -from app.data_fetchers.athena_data_fetcher import AthenaDataFetcher - -# from app.data_fetchers.database_data_fetcher import DatabaseDataFetcher -from app.data_fetchers.data_fetcher_router import DataFetcherRouter - -# from app.database_manager import DatabaseManager -from app.utils.data_fetcher_utils import format_coordinate, chunker - -from app.power_curve.global_power_curve_manager import power_curve_manager -from app.schemas import ( - WindSpeedResponse, - AvailablePowerCurvesResponse, - EnergyProductionResponse, - NearestLocationsResponse, -) - -router = APIRouter() - -# Create router first, then optionally initialize heavy dependencies unless skipped -data_fetcher_router = DataFetcherRouter() -_skip_data_init = os.environ.get("SKIP_DATA_INIT", "0") == "1" -if not _skip_data_init: - # Initialize ConfigManager - config_manager = ConfigManager( - secret_arn_env_var="WINDWATTS_DATA_CONFIG_SECRET_ARN", - local_config_path="./app/config/windwatts_data_config.json", - ) # replace with YOUR local config path - athena_config = config_manager.get_config() - - # Initialize DataFetchers - # s3_data_fetcher = S3DataFetcher("WINDWATTS_S3_BUCKET_NAME") - athena_data_fetcher_era5 = AthenaDataFetcher( - athena_config=athena_config, source_key="era5" - ) - athena_data_fetcher_ensemble = AthenaDataFetcher( - athena_config=athena_config, source_key="ensemble" - ) - s3_data_fetcher_era5 = S3DataFetcher( - bucket_name="windwatts-era5", - prefix="era5_timeseries", - grid="era5", - s3_key_template="era5", - ) - # db_manager = DatabaseManager() - # db_data_fetcher = DatabaseDataFetcher(db_manager=db_manager) - - # # Initialize DataFetcherRouter and register fetchers - data_fetcher_router = DataFetcherRouter() - # data_fetcher_router.register_fetcher("database", db_data_fetcher) - data_fetcher_router.register_fetcher("s3_era5", s3_data_fetcher_era5) - data_fetcher_router.register_fetcher("athena_era5", athena_data_fetcher_era5) - data_fetcher_router.register_fetcher( - "athena_ensemble", athena_data_fetcher_ensemble - ) - -# Centralized valid avg types dictionary -VALID_AVG_TYPES = { - "athena_era5": { - "wind_speed": ["all", "annual", "none"], - "production": ["all", "summary", "annual", "full", "none"], - }, - "athena_ensemble": { - "wind_speed": ["all", "none"], - "production": ["all", "none"], - }, -} -# YEARS list for the sample data download feature -SAMPLE_YEARS = {"s3_era5": [2020, 2021, 2022, 2023]} - -# YEARS for which we have era5 data in the S3 -ALL_YEARS = {"s3_era5": list(range(2013, 2024))} - -# data_type='era5' -# data_source = "athena_era5" -VALID_SOURCES = {"athena_era5", "athena_ensemble", "s3_era5"} # <-- new -DEFAULT_SOURCE = "athena_era5" - - -# Helper validation functions -def validate_lat(lat: float) -> float: - if not (-90 <= lat <= 90): - raise HTTPException( - status_code=400, detail="Latitude must be between -90 and 90." - ) - return lat - - -def validate_lng(lng: float) -> float: - if not (-180 <= lng <= 180): - raise HTTPException( - status_code=400, detail="Longitude must be between -180 and 180." - ) - return lng - - -def validate_height(height: int) -> int: - if not (0 < height <= 300): - raise HTTPException( - status_code=400, detail="Height must be between 1 and 300 meters." - ) - return height - - -def validate_avg_type(avg_type: str, source: str) -> str: - allowed = VALID_AVG_TYPES[source]["wind_speed"] - if avg_type not in allowed: - raise HTTPException( - status_code=400, - detail=f"Invalid avg_type. Must be one of: {allowed} for {source}.", - ) - return avg_type - - -def validate_production_avg_type(avg_type: str, source: str) -> str: - allowed = VALID_AVG_TYPES[source]["production"] - if avg_type not in allowed: - raise HTTPException( - status_code=400, - detail=f"Invalid time_period. Must be one of: {allowed} for {source}.", - ) - return avg_type - - -def validate_selected_powercurve(selected_powercurve: str) -> str: - # Only allow alphanumeric, dash, underscore, dot - if not re.match(r"^[\w\-.]+$", selected_powercurve): - raise HTTPException(status_code=400, detail="Invalid selected_powercurve name.") - if selected_powercurve not in power_curve_manager.power_curves: - raise HTTPException(status_code=400, detail="Selected power curve not found.") - return selected_powercurve - - -def validate_source(source: str) -> str: - if source not in VALID_SOURCES: - raise HTTPException( - status_code=400, - detail=f"Invalid source for ERA5 data. Must be one of: {sorted(VALID_SOURCES)}.", - ) - return source - - -def validate_year(year: int, source: str) -> int: - if year not in ALL_YEARS[source]: - raise HTTPException( - status_code=400, - detail="Invalid year for ERA5 data. Currently supporting years 2013-2023", - ) - return year - - -def validate_n_neighbor(n_neighbor: int) -> int: - if not 1 <= n_neighbor <= 4: # have to change the limit if needed later on - raise HTTPException( - status_code=400, - detail="Invalid number of neighbors. Currently supporting upto 4 nearest neighbors", - ) - return n_neighbor - - -def _get_windspeed_core( - lat: float, lng: float, height: int, avg_type: str, source: str -): - """ - Core function to retrieve wind speed data from the source database. - Args: - lat (float): Latitude of the location. - lng (float): Longitude of the location. - height (int): Height in meters. - avg_type (str): Type of average to retrieve. Must be one of: global (default), monthly, yearly. - source (str): Source of the data. Must be one of: athena_, s3, database. - """ - lat = validate_lat(lat) - lng = validate_lng(lng) - height = validate_height(height) - source = validate_source(source) - avg_type = validate_avg_type(avg_type, source) - - # Legacy conversion: avg_type -> period for new API - params = {"lat": lat, "lng": lng, "height": height, "period": avg_type} - data = data_fetcher_router.fetch_data(params, key=source) - if data is None: - raise HTTPException(status_code=404, detail="Data not found") - return data - - -@router.get( - "/windspeed/{avg_type}", - summary="Retrieve wind speed with avg type - era5 data", - response_model=WindSpeedResponse, - responses={ - 200: { - "description": "Wind speed data retrieved successfully", - "model": WindSpeedResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def get_windspeed_with_avg_type( - avg_type: str = Path(..., description="Type of average to retrieve."), - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - ensemble: bool = Query( - False, description="If true, use ensemble model (athena_ensemble)." - ), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - if ensemble: - return _get_windspeed_core( - lat, lng, height, avg_type, source="athena_ensemble" - ) - else: - return _get_windspeed_core(lat, lng, height, avg_type, source) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/windspeed", - summary="Retrieve wind speed with default global avg - era5 data", - response_model=WindSpeedResponse, - responses={ - 200: { - "description": "Wind speed data retrieved successfully", - "model": WindSpeedResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def get_windspeed( - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - ensemble: bool = Query( - False, description="If true, use ensemble model (athena_ensemble)." - ), - period: str = Query("all", description="Time period for wind speed calculation."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - if ensemble: - return _get_windspeed_core( - lat, lng, height, period, source="athena_ensemble" - ) - else: - return _get_windspeed_core(lat, lng, height, period, source) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/powercurves", - summary="Fetch all available power curves", - response_model=AvailablePowerCurvesResponse, - responses={ - 200: { - "description": "Available power curves retrieved successfully", - "model": AvailablePowerCurvesResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def fetch_available_powercurves(): - """ - returns available power curves - """ - try: - all_curves = list(power_curve_manager.power_curves.keys()) - prefix = "nlr-reference-" - - def extract_kw(curve_name: str): - # Extracts the kw value from curves, "2.5kW" -> 2.5 - match = re.search(rf"{prefix}([0-9.]+)kW", curve_name) - if match: - return float(match.group(1)) - return float("inf") - - curves = [c for c in all_curves if c.startswith(prefix)] - other_curves = [c for c in all_curves if not c.startswith(prefix)] - - curves_sorted = sorted(curves, key=extract_kw) - other_curves_sorted = sorted(other_curves) - - ordered_curves = curves_sorted + other_curves_sorted - return {"available_power_curves": ordered_curves} - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -def _get_energy_production_core( - lat: float, lng: float, height: int, powercurve: str, period: str, source: str -): - """ - Fetches the global, yearly and monthly energy production and average windspeed for a given location, height, and power curve. - Args: - lat (float): Latitude of the location. - lng (float): Longitude of the location. - height (int): Height in meters. - power_curve(str): Selected powercurve/turbine. - period (str, optional): Time period to retrieve. Must be one of: global, yearly, monthly, all. - source (str): Source of the data. Must be one of: athena, s3, database. - Returns: - A JSON object containing average windspeeds and energy production at specified time period or global energy production when time period is not specified. - """ - lat = validate_lat(lat) - lng = validate_lng(lng) - height = validate_height(height) - selected_powercurve = validate_selected_powercurve(powercurve) - source = validate_source(source) - period = validate_production_avg_type(period, source) - params = {"lat": lat, "lng": lng, "height": height} - df = data_fetcher_router.fetch_raw(params, key=source) - if df is None: - raise HTTPException(status_code=404, detail="Data not found") - - if period == "all": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - return { - "energy_production": summary_avg_energy_production["Average year"][ - "kWh produced" - ] - } - - elif period == "summary": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - return {"summary_avg_energy_production": summary_avg_energy_production} - - elif period == "annual": - yearly_avg_energy_production = ( - power_curve_manager.calculate_yearly_energy_production( - df, height, selected_powercurve - ) - ) - return {"yearly_avg_energy_production": yearly_avg_energy_production} - - elif period == "full": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - yearly_avg_energy_production = ( - power_curve_manager.calculate_yearly_energy_production( - df, height, selected_powercurve - ) - ) - return { - "energy_production": summary_avg_energy_production["Average year"][ - "kWh produced" - ], - "summary_avg_energy_production": summary_avg_energy_production, - "yearly_avg_energy_production": yearly_avg_energy_production, - } - - -@router.get( - "/production/{period}", - summary="Get yearly and monthly energy production estimate and average windspeed for a location at a height with a selected power curve", - response_model=EnergyProductionResponse, - responses={ - 200: { - "description": "Energy production data retrieved successfully", - "model": EnergyProductionResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def energy_production_with_period( - period: str = Path(..., description="Time period for production estimate."), - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - powercurve: str = Query(..., description="Selected power curve name."), - ensemble: bool = Query( - False, description="If true, use ensemble model (athena_ensemble)." - ), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - if ensemble: - return _get_energy_production_core( - lat, lng, height, powercurve, period, source="athena_ensemble" - ) - else: - return _get_energy_production_core( - lat, lng, height, powercurve, period, source - ) - except Exception: - raise HTTPException(status_code=500, detail="Internal server error.") - - -@router.get( - "/production", - summary="Get global energy production estimate for a location at a height with a selected power curve", - response_model=EnergyProductionResponse, - responses={ - 200: { - "description": "Energy production data retrieved successfully", - "model": EnergyProductionResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def energy_production( - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - powercurve: str = Query(..., description="Selected power curve name."), - period: str = Query("all", description="Time period for production estimate."), - ensemble: bool = Query( - False, description="If true, use ensemble model (athena_ensemble)." - ), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - if ensemble: - return _get_energy_production_core( - lat, lng, height, powercurve, period, source="athena_ensemble" - ) - else: - return _get_energy_production_core( - lat, lng, height, powercurve, period, source - ) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -def _download_csv_core(gridIndices: List[str], years: List[int], source: str): - source = validate_source(source) - years = [validate_year(year, source) for year in years] - - params = {"gridIndices": gridIndices, "years": years} - - df = data_fetcher_router.fetch_data(params, key=source) - - if df is None or df.empty: - raise HTTPException( - status_code=404, detail="No data found for the specified parameters" - ) - - return df - - -@router.get( - "/timeseries", - summary="Download csv file for windspeed timeseries for a specific location for certain year(s) with 1 neighbor", -) -def download_timeseries_csv( - gridIndex: str = Query( - ..., description="Grid index with respect to user selected coordinate" - ), - years: List[int] = Query( - SAMPLE_YEARS["s3_era5"], description="years of which the data to download" - ), - source: str = Query("s3_era5", description="Source of the data."), -): - try: - # Getting DataFrame from core function - df = _download_csv_core([gridIndex], years, source) - - # Converting DataFrame to CSV - csv_io = io.StringIO() - df.to_csv(csv_io, index=False) - csv_io.seek(0) - - return StreamingResponse( - iter([csv_io.getvalue()]), media_type="text/csv; charset=utf-8" - ) - - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.post( - "/timeseries/batch", - summary="Download multiple CSVs (one per neighbor) as a streamed ZIP", -) -def download_timeseries_csv_batch( - payload: NearestLocationsResponse, - years: List[int] = Query( - SAMPLE_YEARS["s3_era5"], description="years of which the data to download" - ), - source: str = Query("s3_era5", description="Source of the data."), -): - try: - # Spooled file: stays in memory until threshold, then spills to disk automatically - spooled = tempfile.SpooledTemporaryFile( - max_size=30 * 1024 * 1024, mode="w+b" - ) # 30MB threshold (Each decompressed file is around 5.3 MB) - - with zipfile.ZipFile(spooled, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: - for loc in payload.locations: - df = _download_csv_core([loc.index], years, source) - csv_io = io.StringIO() - df.to_csv(csv_io, index=False) - csv_io.seek(0) - file_name = f"wind_data_{format_coordinate(loc.latitude)}_{format_coordinate(loc.longitude)}.csv" - zf.writestr(file_name, csv_io.getvalue()) - - spooled.seek(0) - - headers = { - "Content-Disposition": f'attachment; filename="wind_data_{len(payload.locations)}_points.zip"' - } - - return StreamingResponse( - chunker(spooled), media_type="application/zip", headers=headers - ) - - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/grid-points", - summary="Find nearest grid points", - response_model=NearestLocationsResponse, - responses={ - 200: {"description": "Nearest locations retrieved successfully"}, - 400: {"description": "Bad request"}, - 500: {"description": "Internal server error"}, - }, -) -def grid_points( - lat: float = Query(..., description="Latitude of the target location."), - lng: float = Query(..., description="Longitude of the target location."), - limit: int = Query(1, description="Number of nearest grid points."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data"), -): - try: - lat = validate_lat(lat) - lng = validate_lng(lng) - limit = validate_n_neighbor(limit) - source = validate_source(source) - - grid_lookup_map = { - "athena_era5": athena_data_fetcher_era5, - } - - fetcher = grid_lookup_map.get(source) - if not fetcher: - raise HTTPException( - status_code=400, - detail=f"Nearest locations lookup not available for source='{source}'", - ) - - # Call find_nearest_locations on the Athena fetcher - result = fetcher.find_nearest_locations(lat=lat, lng=lng, n_neighbors=limit) - - locations = [ - {"index": str(i), "latitude": float(a), "longitude": float(o)} - for i, a, o in result - ] - - return {"locations": locations} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Internal server error: {e}") diff --git a/windwatts-api/app/controllers/wind_data_controller.py b/windwatts-api/app/controllers/wind_data_controller.py index 09ca8fc0..20551bd2 100644 --- a/windwatts-api/app/controllers/wind_data_controller.py +++ b/windwatts-api/app/controllers/wind_data_controller.py @@ -26,6 +26,8 @@ from app.power_curve.global_power_curve_manager import power_curve_manager +from app.spatial.global_spatial_manager import init_spatial, spatial_manager + from app.schemas import ( AvailableTurbinesResponse, RoseRequestPayload, @@ -40,6 +42,7 @@ AvailableModelsResponse, RoseResponse, ProductionRequestPayload, + AthenaConfig, ) router = APIRouter() @@ -58,17 +61,20 @@ secret_arn_env_var="WINDWATTS_DATA_CONFIG_SECRET_ARN", local_config_path="./app/config/windwatts_data_config.json", ) - athena_config = config_manager.get_config() + athena_config = AthenaConfig(**config_manager.get_config()) + + # Initialize spatial + init_spatial() # Initialize Athena data fetchers athena_data_fetchers["era5-quantiles"] = AthenaDataFetcher( - athena_config=athena_config, source_key="era5" + athena_config=athena_config, model_key="era5-quantiles" ) athena_data_fetchers["ensemble-quantiles"] = AthenaDataFetcher( - athena_config=athena_config, source_key="ensemble" + athena_config=athena_config, model_key="ensemble-quantiles" ) athena_data_fetchers["wtk-timeseries"] = AthenaDataFetcher( - athena_config=athena_config, source_key="wtk" + athena_config=athena_config, model_key="wtk-timeseries" ) # Initialize S3 data fetchers @@ -398,19 +404,11 @@ def get_grid_points( try: model = validate_model_exists(model) - # Grid lookup only available via athena - # Use athena fetcher for the specified model - fetcher = athena_data_fetchers.get(model) - - if not fetcher or not hasattr(fetcher, "find_nearest_locations"): - raise HTTPException( - status_code=400, - detail=f"Grid point lookup not available for model '{model}'", - ) - - # Call find_nearest_locations on the fetcher limit = validate_limit(limit) - result = fetcher.find_nearest_locations(lat=lat, lng=lng, n_neighbors=limit) + + result = spatial_manager.find_n_nearest( + lat=lat, lng=lng, model_key=model, n_neighbors=limit + ) locations = [ {"index": str(i), "latitude": float(a), "longitude": float(o)} diff --git a/windwatts-api/app/controllers/wtk_data_controller.py b/windwatts-api/app/controllers/wtk_data_controller.py deleted file mode 100644 index 655179d5..00000000 --- a/windwatts-api/app/controllers/wtk_data_controller.py +++ /dev/null @@ -1,540 +0,0 @@ -from fastapi import APIRouter, HTTPException, Path, Query -import re -import os -from typing import List -import io -from fastapi.responses import StreamingResponse -import zipfile -import tempfile - -# commented out the data functions until I can get local athena_config working -from app.config_manager import ConfigManager -from app.data_fetchers.s3_data_fetcher import S3DataFetcher -from app.data_fetchers.athena_data_fetcher import AthenaDataFetcher - -# from app.data_fetchers.database_data_fetcher import DatabaseDataFetcher -from app.data_fetchers.data_fetcher_router import DataFetcherRouter - -# from app.database_manager import DatabaseManager -from app.utils.data_fetcher_utils import format_coordinate, chunker - -from app.power_curve.global_power_curve_manager import power_curve_manager -from app.schemas import ( - WindSpeedResponse, - AvailablePowerCurvesResponse, - EnergyProductionResponse, - NearestLocationsResponse, -) - -router = APIRouter() - -# Create router first, then optionally initialize heavy dependencies unless skipped -data_fetcher_router = DataFetcherRouter() -_skip_data_init = os.environ.get("SKIP_DATA_INIT", "0") == "1" -if not _skip_data_init: - # Initialize ConfigManager - config_manager = ConfigManager( - secret_arn_env_var="WINDWATTS_DATA_CONFIG_SECRET_ARN", - local_config_path="./app/config/windwatts_data_config.json", - ) # replace with YOUR local config path - athena_config = config_manager.get_config() - - # Initialize DataFetchers - s3_data_fetcher_wtk = S3DataFetcher( - bucket_name="wtk-led", prefix="1224", grid="wtk", s3_key_template="wtk" - ) - athena_data_fetcher_wtk = AthenaDataFetcher( - athena_config=athena_config, source_key="wtk" - ) - # db_manager = DatabaseManager() - # db_data_fetcher = DatabaseDataFetcher(db_manager=db_manager) - - # Register fetchers - # data_fetcher_router.register_fetcher("database", db_data_fetcher) - data_fetcher_router.register_fetcher("s3_wtk", s3_data_fetcher_wtk) - data_fetcher_router.register_fetcher("athena_wtk", athena_data_fetcher_wtk) - -VALID_AVG_TYPES = { - "athena_wtk": { - "wind_speed": ["all", "yearly", "monthly", "hourly", "none"], - "production": ["all", "summary", "yearly", "monthly", "none"], - } -} - -# YEARS list for the sample data download feature -SAMPLE_YEARS = {"s3_wtk": [2018, 2019, 2020]} - -# YEARS for which we have wtk data in the S3 -ALL_YEARS = {"s3_wtk": list(range(2000, 2021))} -# data_type = "wtk" -VALID_SOURCES = {"athena_wtk", "s3_wtk"} # <-- new -DEFAULT_SOURCE = "athena_wtk" - - -# Helper validation functions -def validate_lat(lat: float) -> float: - if not (-90 <= lat <= 90): - raise HTTPException( - status_code=400, detail="Latitude must be between -90 and 90." - ) - return lat - - -def validate_lng(lng: float) -> float: - if not (-180 <= lng <= 180): - raise HTTPException( - status_code=400, detail="Longitude must be between -180 and 180." - ) - return lng - - -def validate_height(height: int) -> int: - if not (0 < height <= 300): - raise HTTPException( - status_code=400, detail="Height must be between 1 and 300 meters." - ) - return height - - -def validate_avg_type(avg_type: str, source: str) -> str: - allowed = VALID_AVG_TYPES[source]["wind_speed"] - if avg_type not in allowed: - raise HTTPException( - status_code=400, - detail=f"Invalid avg_type. Must be one of: {allowed} for {source}.", - ) - return avg_type - - -def validate_production_avg_type(avg_type: str, source: str) -> str: - allowed = VALID_AVG_TYPES[source]["production"] - if avg_type not in allowed: - raise HTTPException( - status_code=400, - detail=f"Invalid time_period. Must be one of: {allowed} for {source}.", - ) - return avg_type - - -def validate_selected_powercurve(selected_powercurve: str) -> str: - if not re.match(r"^[\w\-.]+$", selected_powercurve): - raise HTTPException(status_code=400, detail="Invalid selected_powercurve name.") - if selected_powercurve not in power_curve_manager.power_curves: - raise HTTPException(status_code=400, detail="Selected power curve not found.") - return selected_powercurve - - -def validate_source(source: str) -> str: - if source not in VALID_SOURCES: - raise HTTPException( - status_code=400, - detail=f"Invalid source for WTK data. Must be one of: {sorted(VALID_SOURCES)}.", - ) - return source - - -def validate_year(year: int, source: str) -> int: - if year not in ALL_YEARS[source]: - raise HTTPException( - status_code=400, - detail="Invalid year for WTK data. Currently supporting years 2000-2022", - ) - return year - - -def validate_n_neighbor(n_neighbor: int) -> int: - if not 1 <= n_neighbor <= 4: # have to change the limit if needed later on - raise HTTPException( - status_code=400, - detail="Invalid number of neighbors. Currently supporting upto 4 nearest neighbors", - ) - return n_neighbor - - -def _get_windspeed_core( - lat: float, lng: float, height: int, avg_type: str, source: str -): - """ - Core function to retrieve wind speed data from the source database. - Args: - lat (float): Latitude of the location. - lng (float): Longitude of the location. - height (int): Height in meters. - avg_type (str): Type of average to retrieve. Must be one of: global (default), monthly, yearly. - source (str): Source of the data. Must be one of: athena_, s3, database. - """ - lat = validate_lat(lat) - lng = validate_lng(lng) - height = validate_height(height) - source = validate_source(source) - avg_type = validate_avg_type(avg_type, source) - - params = {"lat": lat, "lng": lng, "height": height, "period": avg_type} - data = data_fetcher_router.fetch_data(params, key=source) - if data is None: - raise HTTPException(status_code=404, detail="Data not found") - return data - - -@router.get( - "/windspeed/{avg_type}", - summary="Retrieve wind speed with avg type - wtk data", - response_model=WindSpeedResponse, - responses={ - 200: { - "description": "Wind speed data retrieved successfully", - "model": WindSpeedResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def get_windspeed_with_avg_type( - avg_type: str = Path(..., description="Type of average to retrieve."), - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - return _get_windspeed_core(lat, lng, height, avg_type, source) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/windspeed", - summary="Retrieve wind speed with default global avg - wtk data", - response_model=WindSpeedResponse, - responses={ - 200: { - "description": "Wind speed data retrieved successfully", - "model": WindSpeedResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def get_windspeed( - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - return _get_windspeed_core(lat, lng, height, "all", source) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/available-powercurves", - summary="Fetch all available power curves", - response_model=AvailablePowerCurvesResponse, - responses={ - 200: { - "description": "Available power curves retrieved successfully", - "model": AvailablePowerCurvesResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def fetch_available_powercurves(): - try: - all_curves = list(power_curve_manager.power_curves.keys()) - prefix = "nlr-reference-" - - def extract_kw(curve_name: str): - import re - - match = re.search(rf"{prefix}([0-9.]+)kW", curve_name) - if match: - return float(match.group(1)) - return float("inf") - - curves = [c for c in all_curves if c.startswith(prefix)] - other_curves = [c for c in all_curves if not c.startswith(prefix)] - curves_sorted = sorted(curves, key=extract_kw) - other_curves_sorted = sorted(other_curves) - ordered_curves = curves_sorted + other_curves_sorted - return {"available_power_curves": ordered_curves} - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -def _get_energy_production_core( - lat: float, - lng: float, - height: int, - selected_powercurve: str, - time_period: str, - source: str, -): - """ - Fetches the global, yearly and monthly energy production and average windspeed for a given location, height, and power curve. - Args: - lat (float): Latitude of the location. - lng (float): Longitude of the location. - height (int): Height in meters. - time_period (str, optional): Time period to retrieve. Must be one of: global, yearly, monthly, all. - source (str): Source of the data. Must be one of: athena, s3, database. - Returns: - A JSON object containing average windspeeds and energy production at specified time period or global energy production when time period is not specified. - """ - lat = validate_lat(lat) - lng = validate_lng(lng) - height = validate_height(height) - selected_powercurve = validate_selected_powercurve(selected_powercurve) - source = validate_source(source) - time_period = validate_production_avg_type(time_period, source) - - params = {"lat": lat, "lng": lng, "height": height} - df = data_fetcher_router.fetch_raw(params, key=source) - if df is None: - raise HTTPException(status_code=404, detail="Data not found") - - if time_period == "all": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - return { - "energy_production": summary_avg_energy_production["Average year"][ - "kWh produced" - ] - } - - elif time_period == "summary": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - return {"summary_avg_energy_production": summary_avg_energy_production} - - elif time_period == "yearly": - yearly_avg_energy_production = ( - power_curve_manager.calculate_yearly_energy_production( - df, height, selected_powercurve - ) - ) - return {"yearly_avg_energy_production": yearly_avg_energy_production} - - elif time_period == "full": - summary_avg_energy_production = ( - power_curve_manager.calculate_energy_production_summary( - df, height, selected_powercurve - ) - ) - yearly_avg_energy_production = ( - power_curve_manager.calculate_yearly_energy_production( - df, height, selected_powercurve - ) - ) - return { - "energy_production": summary_avg_energy_production["Average year"][ - "kWh produced" - ], - "summary_avg_energy_production": summary_avg_energy_production, - "yearly_avg_energy_production": yearly_avg_energy_production, - } - - -@router.get( - "/energy-production/{time_period}", - summary="Get yearly and monthly energy production estimate and average windspeed for a location at a height with a selected power curve", - response_model=EnergyProductionResponse, - responses={ - 200: { - "description": "Energy production data retrieved successfully", - "model": EnergyProductionResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def energy_production_with_period( - time_period: str = Path(..., description="Time period for production estimate."), - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - selected_powercurve: str = Query(..., description="Selected power curve name."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - return _get_energy_production_core( - lat, lng, height, selected_powercurve, time_period, source - ) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/energy-production", - summary="Get global energy production estimate for a location at a height with a selected power curve", - response_model=EnergyProductionResponse, - responses={ - 200: { - "description": "Energy production data retrieved successfully", - "model": EnergyProductionResponse, - }, - 500: {"description": "Internal server error"}, - }, -) -def energy_production( - lat: float = Query(..., description="Latitude of the location."), - lng: float = Query(..., description="Longitude of the location."), - height: int = Query(..., description="Height in meters."), - selected_powercurve: str = Query(..., description="Selected power curve name."), - source: str = Query(DEFAULT_SOURCE, description="Source of the data."), -): - try: - return _get_energy_production_core( - lat, lng, height, selected_powercurve, "all", source - ) - except HTTPException: - raise - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -def _download_csv_core(gridIndices: List[str], years: List[int], source: str): - source = validate_source(source) - years = [validate_year(year, source) for year in years] - - params = {"gridIndices": gridIndices, "years": years} - - df = data_fetcher_router.fetch_data(params, key=source) - - if df is None or df.empty: - raise HTTPException( - status_code=404, detail="No data found for the specified parameters" - ) - - return df - - -@router.get( - "/download-csv", - summary="Download csv file for windspeed for a specific location for certain year(s) with 1 neighbor", -) -def download_csv( - gridIndex: str = Query( - ..., description="Grid index with respect to user selected coordinate" - ), - years: List[int] = Query( - SAMPLE_YEARS["s3_wtk"], description="years of which the data to download" - ), - source: str = Query("s3_wtk", description="Source of the data."), -): - try: - # Getting DataFrame from core function - df = _download_csv_core([gridIndex], years, source) - - # Converting DataFrame to CSV - csv_io = io.StringIO() - df.to_csv(csv_io, index=False) - csv_io.seek(0) - - return StreamingResponse( - iter([csv_io.getvalue()]), media_type="text/csv; charset=utf-8" - ) - - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.post( - "/download-csv-batch", - summary="Download multiple CSVs (one per neighbor) as a streamed ZIP", -) -def download_csv_batch( - payload: NearestLocationsResponse, - years: List[int] = Query( - SAMPLE_YEARS["s3_wtk"], description="years of which the data to download" - ), - source: str = Query("s3_wtk", description="Source of the data."), -): - try: - # Spooled file: stays in memory until threshold, then spills to disk automatically - spooled = tempfile.SpooledTemporaryFile( - max_size=30 * 1024 * 1024, mode="w+b" - ) # 30MB threshold - - with zipfile.ZipFile(spooled, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: - for loc in payload.locations: - df = _download_csv_core([loc.index], years, source) - csv_io = io.StringIO() - df.to_csv(csv_io, index=False) - csv_io.seek(0) - file_name = f"wind_data_{format_coordinate(loc.latitude)}_{format_coordinate(loc.longitude)}.csv" - zf.writestr(file_name, csv_io.getvalue()) - - spooled.seek(0) - - headers = { - "Content-Disposition": f'attachment; filename="wind_data_{len(payload.locations)}_points.zip"' - } - - return StreamingResponse( - chunker(spooled), media_type="application/zip", headers=headers - ) - - except Exception: - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - "/nearest-locations", - summary="Find nearest grid locations", - response_model=NearestLocationsResponse, - responses={ - 200: {"description": "Nearest locations retrieved successfully"}, - 400: {"description": "Bad request"}, - 500: {"description": "Internal server error"}, - }, -) -def nearest_locations( - lat: float = Query(..., description="Latitude of the target location."), - lng: float = Query(..., description="Longitude of the target location."), - n_neighbors: int = Query( - 1, description="Number of nearest grid points.", ge=1, le=4 - ), - source: str = Query(DEFAULT_SOURCE, description="Source of the data"), -): - try: - lat = validate_lat(lat) - lng = validate_lng(lng) - n_neighbors = validate_n_neighbor(n_neighbors) - source = validate_source(source) - - grid_lookup_map = { - "athena_wtk": athena_data_fetcher_wtk, - } - - fetcher = grid_lookup_map.get(source) - if not fetcher: - raise HTTPException( - status_code=400, - detail=f"Grid lookup not available for source='{source}'", - ) - - # Call find_nearest_locations directly on the Athena fetcher - result = fetcher.find_nearest_locations( - lat=lat, lng=lng, n_neighbors=n_neighbors - ) - - locations = [ - {"index": str(i), "latitude": float(a), "longitude": float(o)} - for i, a, o in result - ] - - return {"locations": locations} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Internal server error: {e}") diff --git a/windwatts-api/app/data_fetchers/athena_data_fetcher.py b/windwatts-api/app/data_fetchers/athena_data_fetcher.py index bd4cc19e..fcc8990e 100644 --- a/windwatts-api/app/data_fetchers/athena_data_fetcher.py +++ b/windwatts-api/app/data_fetchers/athena_data_fetcher.py @@ -1,50 +1,67 @@ +import pandas as pd +from collections import OrderedDict +import threading + from .abstract_data_fetcher import AbstractDataFetcher -from windwatts_data import ( - WindwattsWTKClient, - WindwattsERA5Client, - WindwattsEnsembleClient, +from app.spatial.global_spatial_manager import spatial_manager +from app.utils.wind_processing import ( + resolve_heights, + interpolate_windspeed, + aggregate, + aggregate_quantile, ) +from app.config.model_config import MODEL_CONFIG, TEMPORAL_SCHEMAS +from app.utils.athena_query_client import AthenaQueryClient +from app.schemas import AthenaConfig class AthenaDataFetcher(AbstractDataFetcher): - def __init__(self, athena_config: str, source_key: str): + def __init__(self, athena_config: AthenaConfig, model_key: str): """ - Initializes the AthenaDataFetcher with a single source_key like 'wtk', 'era5', or 'era5_bc'. - We infer the base family ('wtk' or 'era5') from the part before the first underscore. + Initializes the AthenaDataFetcher with a single model_key like 'wtk-timeseries', 'era5-quantiles', or 'ensemble-quantiles' with its respective Athena config. Args: - athena_config (str): Path to the Athena configuration file. - source_key (str): Key in the config that specifies which athena source. + athena_config (AthenaConfig): Validated Athena config from ConfigManager. + model_key (str): Key into config["sources"], e.g. "wtk-timeseries", "era5-quantiles", "ensemble-quantiles". Same as MODEL_CONFIG keys. """ - # self.data_type = data_type.lower() - self.source_key = source_key.lower() - self.base_type = self.source_key.split("_", 1)[ - 0 - ] # 'wtk' or 'era5' (from 'era5_bc' too) - - if self.base_type == "wtk": - print(f"Initializing WTK Client with Source Key: {self.source_key}") - self.client = WindwattsWTKClient( - config_path=athena_config, source_key=self.source_key - ) # source_key "wtk" - elif self.base_type == "era5": - print(f"Initializing ERA5 Client with Source Key: {self.source_key}") - self.client = WindwattsERA5Client( - config_path=athena_config, source_key=self.source_key - ) # source_key "era5" or "era5_bc" - elif self.base_type == "ensemble": - print(f"Initializing Ensemble Client with Source Key: {self.source_key}") - self.client = WindwattsEnsembleClient( - config_path=athena_config, source_key=self.source_key - ) # source_key "ensemble" - else: - raise ValueError(f"Unsupported base dataset: {self.base_type}") + print(f"Initializing Athena Data Fetcher for '{model_key}'") + self.model_key = model_key + source = athena_config.sources[model_key] + self.query_client = AthenaQueryClient(athena_config, source) + + self._df_cache: OrderedDict[str, pd.DataFrame] = OrderedDict() + self._df_cache_maxsize = 10 + self._cache_lock = threading.Lock() + + def _schema(self) -> str: + return MODEL_CONFIG[self.model_key]["schema"] + + def _available_heights(self) -> list[int]: + return MODEL_CONFIG[self.model_key]["heights"]["windspeed"] + + def _cache_df(self, grid_idx: str) -> pd.DataFrame: + with self._cache_lock: + if grid_idx in self._df_cache: + self._df_cache.move_to_end(grid_idx) + return self._df_cache[grid_idx].copy() + + df = self.query_client.query(grid_idx) + + with self._cache_lock: + if grid_idx not in self._df_cache: + self._df_cache[grid_idx] = df + if len(self._df_cache) > self._df_cache_maxsize: + self._df_cache.popitem(last=False) + return df.copy() def fetch_data( self, lat: float, lng: float, height: int, period: str = "all" ) -> dict: """ - Fetch aggregated wind data using the configured client. + Fetch aggregated wind data for a location. + Selects only the columns needed for the requested height and period. + Applies interpolation if height is not natively in the dataset. + Routes to the appropriate aggregation strategy (timeseries vs quantile). Args: lat (float): Latitude of the location. @@ -56,28 +73,21 @@ def fetch_data( Returns: dict: Fetched aggregated wind data. - - Raises: - ValueError: If the period is not supported for the selected client. """ - if period == "all": - return self.client.fetch_global_avg_at_height( - lat=lat, long=lng, height=height - ) - elif period == "annual": - return self.client.fetch_yearly_avg_at_height( - lat=lat, long=lng, height=height - ) - elif period == "monthly": - return self.client.fetch_monthly_avg_at_height( - lat=lat, long=lng, height=height - ) - elif period == "hourly": - return self.client.fetch_hourly_avg_at_height( - lat=lat, long=lng, height=height + grid_idx, _, _ = spatial_manager.find_nearest(lat, lng, self.model_key) + height_info = resolve_heights(height, self._available_heights()) + df = self._cache_df(grid_idx) + + if not height_info["exact"]: + df = interpolate_windspeed( + df, height, height_info["lower"], height_info["upper"] ) - else: - raise ValueError(f"Invalid period: {period}") + + schema = self._schema() + if schema in ("quantile_yearly", "quantile_atemporal"): + use_swi = TEMPORAL_SCHEMAS[schema]["processing"]["use_swi"] + return aggregate_quantile(df, height, period, use_swi=use_swi) + return aggregate(df, height, period) def fetch_raw(self, lat: float, lng: float, height: int): """ @@ -91,26 +101,13 @@ def fetch_raw(self, lat: float, lng: float, height: int): Returns: DataFrame: Raw wind data without aggregation. """ - return self.client.fetch_df(lat=lat, long=lng, height=height) + grid_idx, _, _ = spatial_manager.find_nearest(lat, lng, self.model_key) + height_info = resolve_heights(height, self._available_heights()) + df = self._cache_df(grid_idx) - def find_nearest_locations(self, lat: float, lng: float, n_neighbors: int = 1): - """ - Find one or more nearest grid locations (index, latitude, and longitude) to a given coordinate. - - :param lat: Latitude of the target location in decimal degrees. - :type lat: float - :param lng: Longitude of the target location in decimal degrees. - :type lng: float - :param n_neighbors: Number of nearest grid points to return. Defaults to 1. - :type n_neighbors: int - - :return: - - If n_neighbors == 1: a list of single tuple [(index, latitude, longitude)] for the nearest grid point. - - If n_neighbors > 1: a list of tuples, each containing (index, latitude, longitude). - - The list will have length n_neighbors. - :rtype: - :rtype: list[tuple[str, float, float]] - """ - # A list of tuples where each tuple contains: (grid_index, latitude, longitude) - tuples = self.client.find_n_nearest_locations(lat, lng, n_neighbors) - return tuples + if not height_info["exact"]: + df = interpolate_windspeed( + df, height, height_info["lower"], height_info["upper"] + ) + + return df diff --git a/windwatts-api/app/main.py b/windwatts-api/app/main.py index 383d588f..72f04bcc 100644 --- a/windwatts-api/app/main.py +++ b/windwatts-api/app/main.py @@ -4,8 +4,6 @@ from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum -from app.controllers.wtk_data_controller import router as wtk_data_router -from app.controllers.era5_data_controller import router as era5_data_router from app.controllers.wind_data_controller import router as wind_data_router from app.middleware import AuditMiddleware, LoggingMiddleware from app.exception_handlers import log_unhandled_exceptions, log_validation_errors @@ -13,7 +11,7 @@ app = FastAPI( title="WindWatts API", - version="1.0.0", + version="2.0.0", root_path="/api", description=dedent( """ @@ -21,19 +19,16 @@ - Rate limits: tiered - 10 / 100 / 1000 requests per minute per IP. - Base path: `/api` - - Contact: windwatts@nrel.gov + - Contact: windwatts@nlr.gov - ## API Versions + ## API - **v1 (Recommended)**: - - `/v1/{model}/windspeed` - Wind speed data - - `/v1/{model}/production` - Energy production estimates - - `/v1/{model}/timeseries` - Raw timeseries downloads - - Supports models: `era5-quantiles`, `era5-timeseries`, `wtk-timeseries`, `ensemble-quantiles` + - `GET /api/v1/{model}/windspeed` - Wind speed data + - `GET /api/v1/{model}/production` - Energy production estimates + - `GET /api/v1/{model}/timeseries` - Raw timeseries downloads + - Supported models: `era5-quantiles`, `era5-timeseries`, `wtk-timeseries`, `ensemble-quantiles` - **Legacy**: Model-specific endpoints (deprecated) - - `/wtk/*` - WTK-specific endpoints - - `/era5/*` - ERA5-specific endpoints + Full interactive documentation: `/api/docs` Use the endpoints below to retrieve wind resource and production estimates. """ @@ -48,13 +43,9 @@ origins = [ "http://localhost", - "https://windwatts-dev.stratus.nrel.gov", - "https://windwatts-stage.stratus.nrel.gov", - "https://windwatts-prod.stratus.nrel.gov", "https://windwatts-dev.stratus.nlr.gov", "https://windwatts-stage.stratus.nlr.gov", "https://windwatts-prod.stratus.nlr.gov", - "https://windwatts.nrel.gov", "https://windwatts.nlr.gov", ] app.add_middleware( @@ -68,15 +59,6 @@ # API v1 app.include_router(wind_data_router, prefix="/v1", tags=["v1-wind-data"]) -# Legacy routes - Deprecated -# TODO: Remove these routes -app.include_router( - wtk_data_router, prefix="/wtk", tags=["wtk-data (deprecated)"], deprecated=True -) -app.include_router( - era5_data_router, prefix="/era5", tags=["era5-data (deprecated)"], deprecated=True -) - @app.get("/healthcheck", response_model=HealthCheckResponse) def healthcheck(): diff --git a/windwatts-api/app/power_curve/powercurves/nlr-reference-20kW.csv b/windwatts-api/app/power_curve/powercurves/nlr-reference-20kW.csv new file mode 100644 index 00000000..08e2c72e --- /dev/null +++ b/windwatts-api/app/power_curve/powercurves/nlr-reference-20kW.csv @@ -0,0 +1,33 @@ +Wind Speed (m/s),Turbine Output (kW) +3,0.425 +4,1.403 +5,2.82 +6,4.698 +7,7.057 +8,10.003 +9,13.574 +10,17.698 +11,18.169 +12,18.169 +13,18.169 +14,18.169 +15,18.169 +16,18.169 +17,18.169 +18,18.169 +19,18.169 +20,18.169 +21,18.169 +22,18.169 +23,18.169 +24,18.169 +25,18.169 +26,18.169 +27,18.169 +28,18.169 +29,18.169 +30,18.169 +31,18.169 +32,18.169 +33,0 +34,0 diff --git a/windwatts-api/app/schemas.py b/windwatts-api/app/schemas.py index 662e07fd..a7c22a35 100644 --- a/windwatts-api/app/schemas.py +++ b/windwatts-api/app/schemas.py @@ -709,3 +709,27 @@ class ProductionRequestPayload(BaseModel): ] } } + + +class AthenaSourceConfig(BaseModel): + bucket_name: str = Field(..., description="S3 bucket where the dataset lives") + athena_table_name: str = Field( + ..., description="Primary Athena table (partitioned by index)" + ) + alt_athena_table_name: str = Field( + "", description="Alternative table for non-index queries" + ) + capabilities: Optional[Dict[str, List[str]]] = Field( + None, description="Optional capabilities like supported avg_types" + ) + + +class AthenaConfig(BaseModel): + region_name: str = Field("us-west-2", description="AWS region") + output_location: str = Field(..., description="S3 URI for Athena query results") + output_bucket: str = Field(..., description="S3 bucket name for query results") + database: str = Field(..., description="Athena/Glue database name") + athena_workgroup: str = Field(..., description="Athena workgroup name") + sources: Dict[str, AthenaSourceConfig] = Field( + ..., description="Map of model_key to source config" + ) diff --git a/windwatts-api/app/spatial/__init__.py b/windwatts-api/app/spatial/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/windwatts-api/app/spatial/ckdtree_lookup.py b/windwatts-api/app/spatial/ckdtree_lookup.py new file mode 100644 index 00000000..79198414 --- /dev/null +++ b/windwatts-api/app/spatial/ckdtree_lookup.py @@ -0,0 +1,37 @@ +import numpy as np +from scipy.spatial import cKDTree + + +class CKDTreeLookup: + """Nearest-neighbor on a point cloud. For WTK and ERA5.""" + + def __init__(self, index_path: str): + with np.load(index_path) as data: + self._index = data["index"] + self._latitude = data["latitude"] + self._longitude = data["longitude"] + coords = np.column_stack((self._latitude, self._longitude)) + self.tree = cKDTree(coords) + + def find_nearest(self, lat: float, lng: float) -> tuple[str, float, float]: + _, idx = self.tree.query([lat, lng]) + return ( + str(self._index[idx]), + float(self._latitude[idx]), + float(self._longitude[idx]), + ) + + def find_n_nearest( + self, lat: float, lng: float, n_neighbors: int + ) -> list[tuple[str, float, float]]: + _, indices = self.tree.query([lat, lng], k=n_neighbors) + if n_neighbors == 1: + indices = [indices] + return [ + ( + str(self._index[i]), + float(self._latitude[i]), + float(self._longitude[i]), + ) + for i in indices + ] diff --git a/windwatts-api/app/spatial/global_spatial_manager.py b/windwatts-api/app/spatial/global_spatial_manager.py new file mode 100644 index 00000000..c29ec837 --- /dev/null +++ b/windwatts-api/app/spatial/global_spatial_manager.py @@ -0,0 +1,36 @@ +from pathlib import Path +from app.spatial.spatial_manager import SpatialManager +from app.spatial.ckdtree_lookup import CKDTreeLookup +from app.config.model_config import MODEL_CONFIG + +# Singleton +spatial_manager = SpatialManager() + +_GRID_DIR = Path(__file__).parent / "grid_lookup_files" + +_GRID_LOADERS = { + "wtk": lambda: CKDTreeLookup(str(_GRID_DIR / "wtk_location_data.npz")), + "era5": lambda: CKDTreeLookup(str(_GRID_DIR / "era5_location_data.npz")), +} + +_initialized = False + + +def init_spatial(): + "Load grids and register lookups for all models in MODEL_CONFIG" + global _initialized + if _initialized: + return + loaded: dict[str, object] = {} + for model_key, config in MODEL_CONFIG.items(): + grid = config.get("grid") + if not grid: + continue + if grid not in loaded: + loader = _GRID_LOADERS.get(grid) + if loader is None: + raise ValueError(f"No loader for grid '{grid}' (model '{model_key}')") + loaded[grid] = loader() + spatial_manager.register(model_key, loaded[grid]) + print(f"Loaded grid lookup for model '{model_key}' on grid '{grid}'") + _initialized = True diff --git a/windwatts-api/app/spatial/grid_lookup_files/era5_location_data.npz b/windwatts-api/app/spatial/grid_lookup_files/era5_location_data.npz new file mode 100644 index 00000000..e9c64366 Binary files /dev/null and b/windwatts-api/app/spatial/grid_lookup_files/era5_location_data.npz differ diff --git a/windwatts-api/app/spatial/grid_lookup_files/wtk_location_data.npz b/windwatts-api/app/spatial/grid_lookup_files/wtk_location_data.npz new file mode 100644 index 00000000..83ba1ed1 Binary files /dev/null and b/windwatts-api/app/spatial/grid_lookup_files/wtk_location_data.npz differ diff --git a/windwatts-api/app/spatial/spatial_manager.py b/windwatts-api/app/spatial/spatial_manager.py new file mode 100644 index 00000000..295583f9 --- /dev/null +++ b/windwatts-api/app/spatial/spatial_manager.py @@ -0,0 +1,30 @@ +class SpatialManager: + """Manages spatial lookups for all models.""" + + def __init__(self): + self._lookups: dict[str, object] = {} + + def register(self, model_key: str, lookup): + "Register lookup instance for a model key." + self._lookups[model_key] = lookup + + def get_lookup(self, model_key: str): + "Retrieve the lookup for a model key." + lookup = self._lookups.get(model_key) + if lookup is None: + raise ValueError(f"No spatial lookup registered for the '{model_key}'") + return lookup + + def find_nearest( + self, lat: float, lng: float, model_key: str + ) -> tuple[str, float, float]: + return self.get_lookup(model_key).find_nearest(lat, lng) + + def find_n_nearest( + self, lat: float, lng: float, model_key: str, n_neighbors: int = 1 + ) -> list[tuple[str, float, float]]: + return self.get_lookup(model_key).find_n_nearest(lat, lng, n_neighbors) + + @property + def registered_models(self) -> list[str]: + return list(self._lookups.keys()) diff --git a/windwatts-api/app/utils/athena_query_client.py b/windwatts-api/app/utils/athena_query_client.py new file mode 100644 index 00000000..c2ab3622 --- /dev/null +++ b/windwatts-api/app/utils/athena_query_client.py @@ -0,0 +1,71 @@ +import boto3 +import time +import pandas as pd +from io import StringIO + +from botocore.config import Config + +from app.schemas import AthenaConfig, AthenaSourceConfig + + +class AthenaQueryClient: + def __init__(self, config: AthenaConfig, source: AthenaSourceConfig): + self.table = source.athena_table_name + self.alt_table = source.alt_athena_table_name + + self._database = config.database + self._workgroup = config.athena_workgroup + self._output_location = config.output_location + + boto_cfg = Config( + connect_timeout=5, + read_timeout=5, + retries={"max_attempts": 2, "mode": "standard"}, + ) + self._athena = boto3.client( + "athena", region_name=config.region_name, config=boto_cfg + ) + self._s3 = boto3.client("s3", region_name=config.region_name, config=boto_cfg) + + def query(self, grid_idx: str) -> pd.DataFrame: + """Fetch all data for a single grid point.""" + query = f"SELECT * FROM {self.table} WHERE index = '{grid_idx}'" + return self._execute(query) + + def _execute(self, query: str) -> pd.DataFrame: + """Execute an Athena query with 7-day result reuse. Returns DataFrame.""" + execution_id = self._athena.start_query_execution( + QueryString=query, + QueryExecutionContext={"Database": self._database}, + ResultConfiguration={"OutputLocation": self._output_location}, + ResultReuseConfiguration={ + "ResultReuseByAgeConfiguration": { + "Enabled": True, + "MaxAgeInMinutes": 10080, + } + }, + WorkGroup=self._workgroup, + )["QueryExecutionId"] + + start = time.monotonic() + delay = 0.0 + max_wait_seconds = 15 + while True: + resp = self._athena.get_query_execution(QueryExecutionId=execution_id) + state = resp["QueryExecution"]["Status"]["State"] + if state == "SUCCEEDED": + break + if state in ("FAILED", "CANCELLED"): + reason = resp["QueryExecution"]["Status"].get("StateChangeReason", "") + raise RuntimeError(f"Athena query {state}: {reason}") + if time.monotonic() - start > max_wait_seconds: + raise RuntimeError( + f"Athena query timed out after {max_wait_seconds:.0f}s (execution_id={execution_id})" + ) + delay = 0.15 if delay == 0 else min(delay * 2, 3.0) + time.sleep(delay) + + output = resp["QueryExecution"]["ResultConfiguration"]["OutputLocation"] + bucket, key = output.replace("s3://", "").split("/", 1) + obj = self._s3.get_object(Bucket=bucket, Key=key) + return pd.read_csv(StringIO(obj["Body"].read().decode("utf-8"))) diff --git a/windwatts-api/app/utils/validation.py b/windwatts-api/app/utils/validation.py index 935b6fe3..3c7cb0e4 100644 --- a/windwatts-api/app/utils/validation.py +++ b/windwatts-api/app/utils/validation.py @@ -76,11 +76,17 @@ def validate_height(model: str, height: int, height_type: str) -> int: status_code=400, detail=f"Model {model} doesn't support heights for {height_type}.", ) - if height not in valid_heights: + if height in valid_heights: + return height + if not MODEL_CONFIG[model].get("interpolation", False): raise HTTPException( status_code=400, detail=f"Invalid height for {model}. Must be one of: {valid_heights} for {height_type}", ) + # Interpolation check + min_h, max_h = min(valid_heights), max(valid_heights) + if not (min_h <= height <= max_h): + raise HTTPException(400, f"Height must be between {min_h}m and {max_h}m") return height diff --git a/windwatts-api/app/utils/wind_data_core.py b/windwatts-api/app/utils/wind_data_core.py index e14ea413..98e758ee 100644 --- a/windwatts-api/app/utils/wind_data_core.py +++ b/windwatts-api/app/utils/wind_data_core.py @@ -14,6 +14,7 @@ import numpy as np import bisect from app.schemas import PowerCurveData +from app.utils.wind_processing import compute_sectors from app.utils.validation import ( validate_lat, @@ -345,20 +346,6 @@ def get_timeseries_energy_core( return csv_io.getvalue() -def _compute_sectors(n: int): - "Return sector centre bearings (degrees CW from North), sector width in degrees, and sector edges." - sector_width_deg = 360.0 / n - centers = [round(i * sector_width_deg, 2) for i in range(n)] - edges = [ - ( - round((c - 0.5 * sector_width_deg) % 360, 2), - round((c + 0.5 * sector_width_deg) % 360, 2), - ) - for c in centers - ] - return centers, sector_width_deg, edges - - def get_windrose_core( model: str, gridIndices: List[str], @@ -424,7 +411,7 @@ def get_windrose_core( active_wd = wd[~calm_mask] # Divide the compass into equal sectors and assign each active observation to one - sector_centers, sector_width_deg, sector_edges = _compute_sectors(sectors) + sector_centers, sector_width_deg, sector_edges = compute_sectors(sectors) sector_idx = ( np.floor((active_wd + sector_width_deg / 2) % 360 / sector_width_deg) ).astype(int) diff --git a/windwatts-api/app/utils/wind_processing.py b/windwatts-api/app/utils/wind_processing.py new file mode 100644 index 00000000..ed23f6f0 --- /dev/null +++ b/windwatts-api/app/utils/wind_processing.py @@ -0,0 +1,219 @@ +import bisect +import numpy as np +import pandas as pd +from scipy.interpolate import CubicSpline + + +def resolve_heights(target_height: int, available_heights: list[int]) -> dict: + """ + Determine what columns to fetch for a given target height. + + Returns dict with: + exact (bool), columns (list[str]), lower (int|None), upper (int|None) + """ + if target_height in available_heights: + return { + "exact": True, + "columns": [f"windspeed_{target_height}m"], + "lower": None, + "upper": None, + } + + sorted_h = sorted(available_heights) + idx = bisect.bisect_left(sorted_h, target_height) + + if idx == 0 or idx >= len(sorted_h): + raise ValueError( + f"Height {target_height}m outside range [{sorted_h[0]}, {sorted_h[-1]}]m" + ) + + lower, upper = sorted_h[idx - 1], sorted_h[idx] + return { + "exact": False, + "lower": lower, + "upper": upper, + "columns": [f"windspeed_{lower}m", f"windspeed_{upper}m"], + } + + +def _power_law(v_ref: pd.Series, h_target: int, h_ref: int) -> pd.Series: + """Neutral Power Law: V(h) = V_ref * (h / h_ref) ^ (1/7)""" + alpha = 1 / 7 + return v_ref * (h_target / h_ref) ** alpha + + +def interpolate_windspeed_linear( + df: pd.DataFrame, target_height: int, lower: int, upper: int +) -> pd.DataFrame: + """ + Add windspeed_{target_height}m column via linear interpolation. + Returns new DataFrame (does not mutate input). + """ + result = df.copy() + fraction = (target_height - lower) / (upper - lower) + result[f"windspeed_{target_height}m"] = ( + result[f"windspeed_{lower}m"] + + fraction * (result[f"windspeed_{upper}m"] - result[f"windspeed_{lower}m"]) + ).round(2) + return result + + +def interpolate_windspeed_power_law( + df: pd.DataFrame, target_height: int, lower: int, upper: int +) -> pd.DataFrame: + """ + Add windspeed_{target_height}m column via Neutral Power Law interpolation. + + Averages estimates from both bracketing heights because with a fixed + exponent (1/7) the power law curve through one height won't pass exactly + through the other. Averaging uses all available information and reduces + directional bias. + + Returns new DataFrame (does not mutate input). + """ + result = df.copy() + + est_from_lower = _power_law(result[f"windspeed_{lower}m"], target_height, lower) + est_from_upper = _power_law(result[f"windspeed_{upper}m"], target_height, upper) + + result[f"windspeed_{target_height}m"] = ( + 0.5 * (est_from_lower + est_from_upper) + ).round(2) + return result + + +# Default interpolation method +interpolate_windspeed = interpolate_windspeed_power_law + + +def aggregate(df: pd.DataFrame, height: int, period: str) -> dict: + """Aggregate timeseries df by period. Column must exist.""" + col = f"windspeed_{height}m" + + if period == "all": + return {"global_avg": round(float(df[col].mean()), 2)} + + elif period == "annual": + grouped = df.groupby("year")[col].mean().round(2) + return { + "yearly_avg": [{"year": int(y), col: float(v)} for y, v in grouped.items()] + } + + elif period == "monthly": + tmp = df.copy() + tmp["month"] = tmp["mohr"] // 100 + grouped = tmp.groupby("month")[col].mean().round(2) + return { + "monthly_avg": [ + {"month": int(m), col: float(v)} for m, v in grouped.items() + ] + } + + elif period == "hourly": + tmp = df.copy() + tmp["hour"] = tmp["mohr"] % 100 + grouped = tmp.groupby("hour")[col].mean().round(2) + return { + "hourly_avg": [{"hour": int(h), col: float(v)} for h, v in grouped.items()] + } + + raise ValueError(f"Unsupported period: {period}") + + +def aggregate_quantile( + df: pd.DataFrame, height: int, period: str, use_swi: bool = True +) -> dict: + """Aggregate quantile-based data. + + Args: + df: DataFrame with windspeed and probability columns. + height: Hub height in meters. + period: Aggregation period — "all" or "annual". + use_swi: If True, apply SWI smoothing before mean (ERA5). + If False, use simple midpoint formula (Ensemble). + """ + col = f"windspeed_{height}m" + + if period == "all": + if "year" in df.columns: + means = [ + _quantile_mean(g[col].values, g["probability"].values, use_swi) + for _, g in df.groupby("year") + ] + return {"global_avg": round(float(np.mean(means)), 2)} + return { + "global_avg": round( + _quantile_mean(df[col].values, df["probability"].values, use_swi), 2 + ) + } + + elif period == "annual": + return { + "yearly_avg": [ + { + "year": int(y), + col: round( + _quantile_mean(g[col].values, g["probability"].values, use_swi), + 2, + ), + } + for y, g in df.groupby("year") + ] + } + + raise ValueError(f"Unsupported quantile period: {period}") + + +def _quantile_mean(quantiles: np.ndarray, probs: np.ndarray, use_swi: bool) -> float: + """Compute mean from quantiles — SWI for ERA5, simple midpoint for Ensemble.""" + if use_swi: + return estimate_mean_swi(quantiles, probs) + q = np.sort(quantiles) + n = len(q) + return float((q.sum() - 0.5 * (q[0] + q[-1])) / (n - 1)) + + +def estimate_mean_swi( + quantiles: np.ndarray, probs: np.ndarray, M1: int = 1000, M2: int = 501 +) -> float: + """ + Spline-With-Inversion: fit CDF spline on quantiles, invert to get + smooth quantile function, compute mean as average of Q(p). + """ + q = _jitter_nonincreasing(np.asarray(quantiles, dtype=np.float64)) + p = np.asarray(probs, dtype=np.float64) + + dy_start = (p[1] - p[0]) / (q[1] - q[0]) + dy_end = (p[-1] - p[-2]) / (q[-1] - q[-2]) + spline = CubicSpline(q, p, bc_type=((1, dy_start), (1, dy_end))) + + q_smooth = np.linspace(q[0], q[-1], M1) + p_smooth = spline(q_smooth) + + probs_new = np.linspace(0, 1, M2) + diff = np.abs(p_smooth[:, None] - probs_new[None, :]) + q_new = q_smooth[np.argmin(diff, axis=0)] + + return float(np.mean(q_new)) + + +def _jitter_nonincreasing(q: np.ndarray, eps: float = 1e-5) -> np.ndarray: + q = q.copy() + for i in range(1, q.size): + if q[i] <= q[i - 1]: + q[i] = q[i - 1] + eps + return q + + +def compute_sectors(n: int): + "Return sector centre bearings (degrees CW from North), sector width in degrees, and sector edges." + sector_width_deg = 360.0 / n + centers = [round(i * sector_width_deg, 2) for i in range(n)] + edges = [ + ( + round((c - 0.5 * sector_width_deg) % 360, 2), + round((c + 0.5 * sector_width_deg) % 360, 2), + ) + for c in centers + ] + return centers, sector_width_deg, edges diff --git a/windwatts-api/proxy/nginx.conf b/windwatts-api/proxy/nginx.conf index a9d6251c..61c10b9b 100644 --- a/windwatts-api/proxy/nginx.conf +++ b/windwatts-api/proxy/nginx.conf @@ -32,7 +32,7 @@ server { } add_header X-Content-Type-Options "nosniff" always; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' blob: https://maps.googleapis.com https://polyfill.io https://unpkg.com https://cdn.jsdelivr.net; worker-src 'self' blob:; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://cdn.jsdelivr.net; img-src 'self' data: https://maps.googleapis.com https://maps.gstatic.com https://windwatts.nrel.gov https://windwatts.nlr.gov https://www.nrel.gov https://www.nlr.gov; font-src 'self' https://fonts.gstatic.com; connect-src 'self' https://maps.googleapis.com https://maps.gstatic.com; frame-src 'self'; object-src 'none';" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' blob: https://maps.googleapis.com https://polyfill.io https://unpkg.com https://cdn.jsdelivr.net; worker-src 'self' blob:; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://cdn.jsdelivr.net; img-src 'self' data: https://maps.googleapis.com https://maps.gstatic.com https://windwatts.nlr.gov https://www.nlr.gov; font-src 'self' https://fonts.gstatic.com; connect-src 'self' https://maps.googleapis.com https://maps.gstatic.com; frame-src 'self'; object-src 'none';" always; } server_tokens off; \ No newline at end of file diff --git a/windwatts-api/requirements.txt b/windwatts-api/requirements.txt index f2d6a0f2..466b91ee 100644 --- a/windwatts-api/requirements.txt +++ b/windwatts-api/requirements.txt @@ -9,4 +9,5 @@ sqlalchemy psycopg2-binary python-dotenv pydantic-settings -alembic \ No newline at end of file +alembic +geopandas \ No newline at end of file diff --git a/windwatts-api/tests/conftest.py b/windwatts-api/tests/conftest.py index 51194217..c39ce10e 100644 --- a/windwatts-api/tests/conftest.py +++ b/windwatts-api/tests/conftest.py @@ -3,6 +3,7 @@ """ import os +import numpy as np import pandas as pd from unittest.mock import MagicMock, patch @@ -16,235 +17,182 @@ def pytest_configure(config): """ Pytest hook that runs very early, before any test collection. - Patch windwatts_data clients and boto3 to prevent real AWS calls during module imports. + Patch fetchers and boto3 to prevent real AWS calls during module imports. """ global _boto3_patch, _mock_clients, _windwatts_patches - # Add ensemble configuration to environment variables - os.environ["SOURCES_ENSEMBLE_BUCKET_NAME"] = "windwatts-era5" - os.environ["SOURCES_ENSEMBLE_ATHENA_TABLE_NAME"] = "ensemble_101" - os.environ["SOURCES_ENSEMBLE_ALT_ATHENA_TABLE_NAME"] = "" + # Top-level config env vars (needed by ConfigManager._get_config_from_env) + # Use clearly fake values so accidental real AWS calls are impossible + os.environ["REGION_NAME"] = "us-east-1" + os.environ["OUTPUT_LOCATION"] = "s3://test-fake-bucket/" + os.environ["OUTPUT_BUCKET"] = "test-fake-bucket" + os.environ["DATABASE"] = "test_fake_database" + os.environ["ATHENA_WORKGROUP"] = "test_fake_workgroup" + + # Source-specific env vars using model keys + os.environ["SOURCES_ENSEMBLE-QUANTILES_BUCKET_NAME"] = "test-fake-era5" + os.environ["SOURCES_ENSEMBLE-QUANTILES_ATHENA_TABLE_NAME"] = "test_ensemble" + os.environ["SOURCES_ENSEMBLE-QUANTILES_ALT_ATHENA_TABLE_NAME"] = "" + os.environ["SOURCES_ERA5-QUANTILES_BUCKET_NAME"] = "test-fake-era5" + os.environ["SOURCES_ERA5-QUANTILES_ATHENA_TABLE_NAME"] = "test_era5" + os.environ["SOURCES_ERA5-QUANTILES_ALT_ATHENA_TABLE_NAME"] = "" + os.environ["SOURCES_WTK-TIMESERIES_BUCKET_NAME"] = "test-fake-wtk" + os.environ["SOURCES_WTK-TIMESERIES_ATHENA_TABLE_NAME"] = "test_wtk_1224" + os.environ["SOURCES_WTK-TIMESERIES_ALT_ATHENA_TABLE_NAME"] = "" + + # Skip real data initialization (spatial lookups, AWS clients) + os.environ["SKIP_DATA_INIT"] = "1" - # Mock the windwatts_data client classes to return realistic data - def create_mock_windwatts_client(): - """Create a mock windwatts client that returns realistic wind data""" - mock_client = MagicMock() + # Still need boto3 mocks for other AWS services (like secrets manager in config_manager) + mock_s3_client = MagicMock() + mock_athena_client = MagicMock() - # Mock different fetch methods that return data in the format the API expects - mock_client.fetch_global_avg_at_height = MagicMock( - return_value={"global_avg": 8.60} - ) + _mock_clients = { + "athena": mock_athena_client, + "s3": mock_s3_client, + "secretsmanager": MagicMock(), + } + + def mock_boto3_client(service_name, *args, **kwargs): + if service_name in _mock_clients: + return _mock_clients[service_name] + return MagicMock() + + _boto3_patch = patch("boto3.client", side_effect=mock_boto3_client) + _boto3_patch.start() + + +def pytest_collection_finish(session): + """ + After all modules are imported and tests collected, inject mock fetchers + into the controller's module-level dicts (which are empty due to SKIP_DATA_INIT=1). + """ + from app.controllers import wind_data_controller as wdc + from app.spatial.global_spatial_manager import init_spatial + + # Load real spatial lookups (reads local .npz files, no AWS needed) + init_spatial() - mock_client.fetch_yearly_avg_at_height = MagicMock( - return_value={ + mock_athena_fetcher = MagicMock() + + def mock_fetch_data(lat, lng, height, period="all"): + col = f"windspeed_{height}m" + if period == "all": + return {"global_avg": 8.60} + elif period == "annual": + return { "yearly_avg": [ - {"year": 2013, "windspeed_40m": 8.93}, - {"year": 2014, "windspeed_40m": 8.61}, - {"year": 2015, "windspeed_40m": 8.34}, - {"year": 2016, "windspeed_40m": 8.72}, - {"year": 2017, "windspeed_40m": 8.85}, + {"year": 2020, col: 8.50}, + {"year": 2021, col: 8.70}, ] } - ) - - mock_client.fetch_monthly_avg_at_height = MagicMock( - return_value={ + elif period == "monthly": + return { "monthly_avg": [ - {"month": 1, "windspeed_40m": 8.5}, - {"month": 2, "windspeed_40m": 8.7}, + {"month": "Jan", col: 8.5}, + {"month": "Feb", col: 8.7}, ] } - ) - - mock_client.fetch_hourly_avg_at_height = MagicMock( - return_value={ + elif period == "hourly": + return { "hourly_avg": [ - {"hour": 0, "windspeed_40m": 8.5}, - {"hour": 1, "windspeed_40m": 8.6}, + {"hour": 0, col: 8.5}, + {"hour": 1, col: 8.6}, ] } - ) - - # Mock production/energy calculation methods - mock_client.calculate_global_energy = MagicMock(return_value=539072) - - mock_client.calculate_summary_energy = MagicMock( - return_value={ - "Lowest year": { - "year": 2023.0, - "Average wind speed (m/s)": "8.12", - "kWh produced": 504212, - }, - "Average year": { - "year": None, - "Average wind speed (m/s)": "8.60", - "kWh produced": 539072, - }, - "Highest year": { - "year": 2013.0, - "Average wind speed (m/s)": "8.93", - "kWh produced": 579146, - }, - } - ) - - mock_client.calculate_yearly_energy = MagicMock( - return_value={ - "2013": {"Average wind speed (m/s)": "8.93", "kWh produced": 579146}, - "2014": {"Average wind speed (m/s)": "8.61", "kWh produced": 552516}, - "2015": {"Average wind speed (m/s)": "8.34", "kWh produced": 521722}, - } - ) - - mock_client.calculate_monthly_energy = MagicMock( - return_value={ - "Jan": {"Average wind speed, m/s": "8.5", "kWh produced": "45000"}, - "Feb": {"Average wind speed, m/s": "8.7", "kWh produced": "43000"}, - } - ) + return {"global_avg": 8.60} + + mock_athena_fetcher.fetch_data = MagicMock(side_effect=mock_fetch_data) + + # Realistic mock DataFrames for fetch_raw per model schema + n_quantiles = 101 + probs = np.linspace(0.0, 1.0, n_quantiles) + heights_data = { + "windspeed_40m": np.linspace(2.0, 14.0, n_quantiles), + "windspeed_60m": np.linspace(2.5, 15.0, n_quantiles), + "windspeed_80m": np.linspace(3.0, 16.0, n_quantiles), + "windspeed_100m": np.linspace(3.5, 17.0, n_quantiles), + "probability": probs, + } - # Mock the query_athena method for raw queries - def mock_query_athena(query, convert_to_dataframe=True, *args, **kwargs): - if convert_to_dataframe: - return pd.DataFrame( - { - "wind_speed_40m": [8.93, 8.61, 8.34, 8.72, 8.85], - "wind_speed_80m": [9.5, 9.2, 8.9, 9.3, 9.4], - "year": [2013, 2014, 2015, 2016, 2017], - "month": [1, 2, 3, 4, 5], - } - ) - else: - return [ - [ - "wind_speed_30m", - "wind_speed_40m", - "wind_speed_50m", - "wind_speed_60m", - "wind_speed_80m", - "wind_speed_100m", - ] - ] + # ERA5: quantile_yearly — has year column + era5_raw_df = pd.DataFrame({**heights_data, "year": [2020] * n_quantiles}) + + # Ensemble: quantile_atemporal — NO year or mohr columns + ensemble_raw_df = pd.DataFrame(heights_data) + + # WTK: aggregated_mohr — has mohr and year, no probability + n_wtk = 288 # 12 months * 24 hours + wtk_raw_df = pd.DataFrame( + { + "windspeed_40m": np.random.uniform(5, 12, n_wtk), + "windspeed_80m": np.random.uniform(6, 14, n_wtk), + "windspeed_100m": np.random.uniform(7, 15, n_wtk), + "mohr": [m * 100 + h for m in range(1, 13) for h in range(24)], + "year": [2020] * n_wtk, + } + ) - mock_client.query_athena = mock_query_athena - - # Mock find_n_nearest_locations for grid-points endpoint - def mock_find_n_nearest(lat, lon, limit=1): - # Mock locations data - returns tuples of (index, lat, lon) - locations = [ - ("046271", 39.903, -69.97427540107105), - ("046272", 39.904, -69.98), - ("046273", 39.905, -69.99), - ("046274", 39.906, -70.00), - ] - return locations[:limit] - - mock_client.find_n_nearest_locations = MagicMock( - side_effect=mock_find_n_nearest - ) + mock_athena_fetcher.fetch_raw = MagicMock(return_value=era5_raw_df) - # Set available heights (including 10m for legacy WTK endpoints) - mock_client.available_heights = [ - 10, - 30, - 40, - 50, - 60, - 80, - 100, - 120, - 140, - 160, - 200, + def mock_find_n_nearest(lat, lng, n_neighbors=1): + locations = [ + ("046271", 39.903, -69.974), + ("046272", 39.904, -69.98), + ("046273", 39.905, -69.99), + ("046274", 39.906, -70.00), ] - mock_client.column_names = [ - "wind_speed_10m", - "wind_speed_30m", - "wind_speed_40m", - "wind_speed_50m", - "wind_speed_60m", - "wind_speed_80m", - "wind_speed_100m", - "wind_speed_120m", - "wind_speed_140m", - "wind_speed_160m", - "wind_speed_200m", - ] - - return mock_client + return locations[:n_neighbors] - # Create separate mock clients for different data sources - wtk_mock = create_mock_windwatts_client() - era5_mock = create_mock_windwatts_client() - ensemble_mock = create_mock_windwatts_client() - - # WTK returns timeseries data with timestamp/year/month/hour - def mock_fetch_df_wtk(lat, long, height, *args, **kwargs): - timestamps = pd.date_range("2020-01-01", periods=5, freq="h") - return pd.DataFrame( - { - f"windspeed_{height}m": [8.5, 8.7, 8.3, 8.6, 8.4], - "timestamp": timestamps, - "year": timestamps.year, - "month": timestamps.month, - "hour": timestamps.hour, - "mohr": timestamps.month * 100 + timestamps.hour, - } - ) + mock_athena_fetcher.find_nearest_locations = MagicMock( + side_effect=mock_find_n_nearest + ) - wtk_mock.fetch_df = MagicMock(side_effect=mock_fetch_df_wtk) + # Create per-model fetchers with correct DataFrames + mock_era5_fetcher = MagicMock() + mock_era5_fetcher.fetch_data = MagicMock(side_effect=mock_fetch_data) + mock_era5_fetcher.fetch_raw = MagicMock(return_value=era5_raw_df) + mock_era5_fetcher.find_nearest_locations = MagicMock( + side_effect=mock_find_n_nearest + ) - # ERA5 returns quantile data with probability column and year - def mock_fetch_df_era5(lat, long, height, *args, **kwargs): - return pd.DataFrame( - { - f"windspeed_{height}m": [6.2, 7.1, 7.8, 8.5, 9.2, 10.1], - "probability": [0.1, 0.25, 0.5, 0.75, 0.9, 0.95], - "year": [2020, 2020, 2020, 2020, 2020, 2020], - } - ) + mock_ensemble_fetcher = MagicMock() + mock_ensemble_fetcher.fetch_data = MagicMock(side_effect=mock_fetch_data) + mock_ensemble_fetcher.fetch_raw = MagicMock(return_value=ensemble_raw_df) + mock_ensemble_fetcher.find_nearest_locations = MagicMock( + side_effect=mock_find_n_nearest + ) - era5_mock.fetch_df = MagicMock(side_effect=mock_fetch_df_era5) + mock_wtk_fetcher = MagicMock() + mock_wtk_fetcher.fetch_data = MagicMock(side_effect=mock_fetch_data) + mock_wtk_fetcher.fetch_raw = MagicMock(return_value=wtk_raw_df) + mock_wtk_fetcher.find_nearest_locations = MagicMock(side_effect=mock_find_n_nearest) + + # Inject into the controller's module-level dicts + wdc.athena_data_fetchers["era5-quantiles"] = mock_era5_fetcher + wdc.athena_data_fetchers["ensemble-quantiles"] = mock_ensemble_fetcher + wdc.athena_data_fetchers["wtk-timeseries"] = mock_wtk_fetcher + wdc.data_fetcher_router.register_fetcher("athena_era5-quantiles", mock_era5_fetcher) + wdc.data_fetcher_router.register_fetcher( + "athena_ensemble-quantiles", mock_ensemble_fetcher + ) + wdc.data_fetcher_router.register_fetcher("athena_wtk-timeseries", mock_wtk_fetcher) - # Ensemble returns quantile data without year (atemporal) - def mock_fetch_df_ensemble(lat, long, height, *args, **kwargs): - return pd.DataFrame( + # Mock S3 fetcher for timeseries endpoints + mock_s3_fetcher = MagicMock() + mock_s3_fetcher.fetch_data = MagicMock( + return_value=pd.DataFrame( { - f"windspeed_{height}m": [6.5, 7.3, 8.0, 8.7, 9.4, 10.2], - "probability": [0.1, 0.25, 0.5, 0.75, 0.9, 0.95], + "windspeed_40m": [8.5, 8.7, 8.3, 8.6, 8.4], + "windspeed_100m": [10.5, 10.2, 9.9, 10.3, 10.4], + "winddirection_100m": [180, 190, 200, 210, 220], + "time": pd.date_range("2020-01-01", periods=5, freq="h"), } ) - - ensemble_mock.fetch_df = MagicMock(side_effect=mock_fetch_df_ensemble) - - # Patch the windwatts_data client classes with appropriate mocks - wtk_patch = patch("windwatts_data.WindwattsWTKClient", return_value=wtk_mock) - era5_patch = patch("windwatts_data.WindwattsERA5Client", return_value=era5_mock) - ensemble_patch = patch( - "windwatts_data.WindwattsEnsembleClient", return_value=ensemble_mock ) - - _windwatts_patches = [wtk_patch, era5_patch, ensemble_patch] - for p in _windwatts_patches: - p.start() - - # Still need boto3 mocks for other AWS services (like secrets manager in config_manager) - mock_s3_client = MagicMock() - mock_athena_client = MagicMock() - - _mock_clients = { - "athena": mock_athena_client, - "s3": mock_s3_client, - "secretsmanager": MagicMock(), - } - - def mock_boto3_client(service_name, *args, **kwargs): - if service_name in _mock_clients: - return _mock_clients[service_name] - return MagicMock() - - _boto3_patch = patch("boto3.client", side_effect=mock_boto3_client) - _boto3_patch.start() + for model_key in ["era5-timeseries", "wtk-timeseries"]: + wdc.s3_data_fetchers[model_key] = mock_s3_fetcher + wdc.data_fetcher_router.register_fetcher(f"s3_{model_key}", mock_s3_fetcher) def pytest_unconfigure(config): diff --git a/windwatts-api/tests/test_config_manager_env.py b/windwatts-api/tests/test_config_manager_env.py index c0dde694..bfa8befc 100644 --- a/windwatts-api/tests/test_config_manager_env.py +++ b/windwatts-api/tests/test_config_manager_env.py @@ -1,5 +1,6 @@ import os from app.config_manager import ConfigManager +import json # Set required top-level environment variables os.environ["REGION_NAME"] = "us-west-2" @@ -19,10 +20,9 @@ # Instantiate ConfigManager (no secret ARN, no local file) cm = ConfigManager(secret_arn_env_var="DUMMY_SECRET_ARN", local_config_path=None) -# Get config path -config_path = cm.get_config() -print(f"\nGenerated config file path: {config_path}\n") +# Get config (now returns a dict, not a file path) +config = cm.get_config() +print(f"\nGenerated config: {config}\n") -# Print the contents for verification -with open(config_path) as f: - print(f.read()) +# Verify the config dict structure +print(json.dumps(config, indent=2)) diff --git a/windwatts-api/tests/test_v1_api.py b/windwatts-api/tests/test_v1_api.py index c8bb081e..32ef354a 100644 --- a/windwatts-api/tests/test_v1_api.py +++ b/windwatts-api/tests/test_v1_api.py @@ -6,6 +6,7 @@ from fastapi.testclient import TestClient from app.main import app from unittest.mock import patch +import pytest import numpy as np import pandas as pd diff --git a/windwatts-api/tests/test_wind_data_controller.py b/windwatts-api/tests/test_wind_data_controller.py index 142890c9..1cf98245 100644 --- a/windwatts-api/tests/test_wind_data_controller.py +++ b/windwatts-api/tests/test_wind_data_controller.py @@ -1,42 +1,23 @@ from fastapi.testclient import TestClient from app.main import app -# from unittest.mock import patch client = TestClient(app) -# uncomment these when i can get local athena_config working -""" -# this patches the fetch_data method on the data_fetcher_router instance -def test_get_wtk_data_success(): - # Fake data to be returned by the mocked fetch_data call. - fake_data = {"global_avg": 5.5} - # Patch the fetch_data method on the data_fetcher_router instance. - with patch("app.controllers.wind_data_controller.data_fetcher_router.fetch_data", return_value=fake_data): - response = client.get("/wtk-data?lat=40.0&lng=-70.0&height=10&source=athena") - assert response.status_code == 200 - assert response.json() == fake_data -# this patches the fetch_data method on the data_fetcher_router instance -def test_get_wtk_data_failure(): - # Patch the fetch_data method on the data_fetcher_router instance to raise an exception. - with patch("app.controllers.wind_data_controller.data_fetcher_router.fetch_data", side_effect=Exception("Test exception")): - response = client.get("/wtk-data?lat=40.0&lng=-70.0&height=10&source=athena") - assert response.status_code == 500 - assert response.json() == {"detail": "Test exception"} -""" - - -def test_get_wtk_data_success(): - response = client.get( - "/wtk/windspeed?lat=40.0&lng=-70.0&height=10&source=athena_wtk" +def test_legacy_wtk_routes_removed(): + """Verify that legacy /wtk/* routes are no longer served (sunset in API v2.0.0).""" + assert client.get("/wtk/windspeed?lat=40.0&lng=-100.0&height=80").status_code == 404 + assert ( + client.get("/wtk/energy-production?lat=40.0&lng=-100.0&height=80").status_code + == 404 ) - assert response.status_code == 200 - json = response.json() - assert "global_avg" in json + assert client.get("/wtk/nearest-locations?lat=40.0&lng=-100.0").status_code == 404 -def test_get_available_power_curves(): - response = client.get("/wtk/available-powercurves") - assert response.status_code == 200 - json = response.json() - assert "available_power_curves" in json +def test_legacy_era5_routes_removed(): + """Verify that legacy /era5/* routes are no longer served (sunset in API v2.0.0).""" + assert client.get("/era5/windspeed?lat=40.0&lng=-70.0&height=40").status_code == 404 + assert ( + client.get("/era5/production?lat=40.0&lng=-70.0&height=40").status_code == 404 + ) + assert client.get("/era5/grid-points?lat=40.0&lng=-70.0").status_code == 404 diff --git a/windwatts-ui/nginx.conf b/windwatts-ui/nginx.conf index a41d4fb4..58966234 100644 --- a/windwatts-ui/nginx.conf +++ b/windwatts-ui/nginx.conf @@ -5,7 +5,7 @@ server { add_header X-Content-Type-Options "nosniff" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' https://maps.googleapis.com https://maps.gstatic.com https://polyfill.io https://*.googletagmanager.com https://dap.digitalgov.gov https://*.clarity.ms; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https://maps.googleapis.com https://maps.gstatic.com https://windwatts.nrel.gov https://windwatts.nlr.gov https://www.nrel.gov https://www.nlr.gov https://*.google-analytics.com https://*.googletagmanager.com https://*.clarity.ms https://*.bing.com; connect-src 'self' https://maps.googleapis.com https://maps.gstatic.com https://windwatts-dev.stratus.nrel.gov https://windwatts-stage.stratus.nrel.gov https://windwatts-prod.stratus.nrel.gov https://*.google-analytics.com https://*.analytics.google.com https://*.googletagmanager.com https://www.google.com https://*.clarity.ms https://dap.digitalgov.gov; frame-src 'self' https://www.googletagmanager.com; object-src 'none'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; upgrade-insecure-requests;" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' https://maps.googleapis.com https://maps.gstatic.com https://polyfill.io https://*.googletagmanager.com https://dap.digitalgov.gov https://*.clarity.ms; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; font-src 'self' https://fonts.gstatic.com; img-src 'self' data: https://maps.googleapis.com https://maps.gstatic.com https://windwatts.nlr.gov https://www.nlr.gov https://*.google-analytics.com https://*.googletagmanager.com https://*.clarity.ms https://*.bing.com; connect-src 'self' https://maps.googleapis.com https://maps.gstatic.com https://windwatts-dev.stratus.nlr.gov https://windwatts-stage.stratus.nlr.gov https://windwatts-prod.stratus.nlr.gov https://*.google-analytics.com https://*.analytics.google.com https://*.googletagmanager.com https://www.google.com https://*.clarity.ms https://dap.digitalgov.gov; frame-src 'self' https://www.googletagmanager.com; object-src 'none'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; upgrade-insecure-requests;" always; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; location / { diff --git a/windwatts-ui/public/gtm-init.js b/windwatts-ui/public/gtm-init.js index 29986aa7..6b518af8 100644 --- a/windwatts-ui/public/gtm-init.js +++ b/windwatts-ui/public/gtm-init.js @@ -1,10 +1,10 @@ (function (w, d, s, l, i) { w[l] = w[l] || []; - w[l].push({ 'gtm.start': new Date().getTime(), event: 'gtm.js' }); + w[l].push({ "gtm.start": new Date().getTime(), event: "gtm.js" }); var f = d.getElementsByTagName(s)[0], j = d.createElement(s), - dl = l != 'dataLayer' ? '&l=' + l : ''; + dl = l != "dataLayer" ? "&l=" + l : ""; j.async = true; - j.src = 'https://www.googletagmanager.com/gtm.js?id=' + i + dl; + j.src = "https://www.googletagmanager.com/gtm.js?id=" + i + dl; f.parentNode.insertBefore(j, f); -})(window, document, 'script', 'dataLayer', 'GTM-KG7PVKB'); +})(window, document, "script", "dataLayer", "GTM-KG7PVKB"); diff --git a/windwatts-ui/src/components/resultPane/Disclaimer.tsx b/windwatts-ui/src/components/resultPane/Disclaimer.tsx index 060722e9..63d38cb4 100644 --- a/windwatts-ui/src/components/resultPane/Disclaimer.tsx +++ b/windwatts-ui/src/components/resultPane/Disclaimer.tsx @@ -29,7 +29,7 @@ export const Disclaimer = () => { wind installers who may share insights from nearby projects. To access alternative wind models, visit  { - if (dataModel && HUB_HEIGHTS[dataModel]) { - return HUB_HEIGHTS[dataModel]; - } - return HUB_HEIGHTS.default; - }, [dataModel]); + const { values: availableHeights, interpolation: interpolable } = + useMemo(() => { + if (dataModel && HUB_HEIGHTS[dataModel]) { + return HUB_HEIGHTS[dataModel]; + } + return HUB_HEIGHTS.default; + }, [dataModel]); + + const modelMin = Math.min(...availableHeights); + const modelMax = Math.max(...availableHeights); + + const [inputValue, setInputValue] = useState(String(hubHeight)); - // ensure slider compatibility with model switching and available heights changes + // keep text field in sync when hubHeight changes externally (slider, model switch) useEffect(() => { - if (!availableHeights.includes(hubHeight)) { - // if current height not available, set to the closest available height - const closestHeight = availableHeights.reduce((prev, curr) => - Math.abs(curr - hubHeight) < Math.abs(prev - hubHeight) ? curr : prev - ); - setHubHeight(closestHeight); - } - }, [availableHeights, hubHeight, setHubHeight]); + setInputValue(String(hubHeight)); + }, [hubHeight]); + + // clamp or snap on model switch / available heights change + useEffect(() => { + const resolved = resolveHubHeight( + hubHeight, + availableHeights, + interpolable + ); + if (resolved !== hubHeight) setHubHeight(resolved); + }, [availableHeights, interpolable, hubHeight, setHubHeight]); const hubHeightMarks = availableHeights.map((value: number) => ({ - value: value, + value, label: `${value}m`, })); - const handleHubHeightChange = ( - _: Event, - newHubHeight: number | number[] | null - ) => { - if (newHubHeight !== null && typeof newHubHeight === "number") { - setHubHeight(newHubHeight); + const handleSliderChange = (_: Event, newValue: number | number[]) => { + if (typeof newValue === "number") { + setHubHeight(newValue); } }; + const handleInputChange = (e: React.ChangeEvent) => { + setInputValue(e.target.value); + }; + + const commitInput = () => { + const parsed = parseInt(inputValue, 10); + if (!isNaN(parsed)) { + const resolvedHubHeight = resolveHubHeight( + parsed, + availableHeights, + interpolable + ); + setHubHeight(resolvedHubHeight); + } else { + setInputValue(String(hubHeight)); + } + }; + + const handleInputKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter") commitInput(); + }; + const customCurve = resolveCustomCurve(turbine, customCurves); const turbineInfo = TURBINE_DATA[turbine]; - const minHeight = customCurve?.minHeight ?? turbineInfo?.minHeight; - const maxHeight = customCurve?.maxHeight ?? turbineInfo?.maxHeight; - const hasHeightRange = minHeight !== undefined && maxHeight !== undefined; + const turbineMinHeight = customCurve?.minHeight ?? turbineInfo?.minHeight; + const turbineMaxHeight = customCurve?.maxHeight ?? turbineInfo?.maxHeight; + const hasHeightRange = + turbineMinHeight !== undefined && turbineMaxHeight !== undefined; const heightRangeInfo = turbineInfo?.info ?? ""; const isHeightInRange: boolean = hasHeightRange - ? hubHeight >= minHeight! && hubHeight <= maxHeight! + ? hubHeight >= turbineMinHeight! && hubHeight <= turbineMaxHeight! : true; const validationColor: "primary" | "success" | "warning" = hasHeightRange @@ -76,50 +107,76 @@ export function HubHeightSettings() { Hub Height - - Choose nearest hub height (m): - - {hasHeightRange && ( - + Set hub height: + m, + }, + htmlInput: { + min: modelMin, + max: modelMax, + step: 1, + "aria-label": "hub height input", + }, }} - > - - - - {isHeightInRange ? "Within" : "Outside"} recommended range - ( - {minHeight}m - {maxHeight}m) - - - {heightRangeInfo && ( - - - - - - )} - - + sx={{ width: 100 }} + /> + + + + `${value}m`} + step={interpolable ? 1 : null} + marks={hubHeightMarks} + min={modelMin} + max={modelMax} + color={validationColor} + /> + + + {interpolable && ( + + * Values between marks are interpolated (no extrapolation). + )} - `${value}m`} - step={step} - marks={hubHeightMarks} - min={Math.min(...availableHeights)} - max={Math.max(...availableHeights)} - color={validationColor} - /> + {hasHeightRange && ( + + + * Recommended range: {turbineMinHeight}m - {turbineMaxHeight}m + + {heightRangeInfo && ( + + + + + + )} + + )} ); } diff --git a/windwatts-ui/src/components/settings/LossAssumptionSettings.tsx b/windwatts-ui/src/components/settings/LossAssumptionSettings.tsx index 8a04cc26..9d34bf6f 100644 --- a/windwatts-ui/src/components/settings/LossAssumptionSettings.tsx +++ b/windwatts-ui/src/components/settings/LossAssumptionSettings.tsx @@ -44,7 +44,7 @@ export function LossAssumptionSettings() { Loss Assumption - Set an energy percent loss (17% recommended). + Set an energy percent loss (17% recommended): = { - "era5-quantiles": { values: [30, 40, 50, 60, 80, 100], interpolation: null }, - "wtk-timeseries": { values: [40, 60, 80, 100, 120, 140], interpolation: 10 }, + "era5-quantiles": { values: [30, 40, 50, 60, 80, 100], interpolation: true }, + "wtk-timeseries": { + values: [40, 60, 80, 100, 120, 140], + interpolation: true, + }, "ensemble-quantiles": { values: [30, 40, 50, 60, 80, 100], - interpolation: null, + interpolation: true, }, - "era5-timeseries": { values: [30, 40, 50, 60, 80, 100], interpolation: null }, - default: { values: [40, 60, 80, 100], interpolation: null }, + "era5-timeseries": { values: [30, 40, 50, 60, 80, 100], interpolation: true }, + default: { values: [40, 60, 80, 100], interpolation: false }, }; diff --git a/windwatts-ui/src/constants/turbines.ts b/windwatts-ui/src/constants/turbines.ts index 7f52bad8..3e7de3d8 100644 --- a/windwatts-ui/src/constants/turbines.ts +++ b/windwatts-ui/src/constants/turbines.ts @@ -21,9 +21,20 @@ export const TURBINE_DATA: Record = { info: "Residential reference turbines (0-20 kW): system size 2.5 kW, rotor diameter 2.2m, allowable hub heights 20, 30, 40 m (Lantz et al., 2016). Also see https://doi.org/10.2172/1333625", // Source: Lantz et al. (2016) // Lantz, E., Sigrin, B., Gleason, M., Preus, R., & Baring-Gould, I. (2016). Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects (NREL/TP--6A20-67337, 1333625; p. NREL/TP--6A20-67337, 1333625). https://doi.org/10.2172/1333625 - // For more information, refer to https://atb.nrel.gov/electricity/2024b/distributed_wind + // For more information, refer to https://atb.nlr.gov/electricity/2024b/distributed_wind // Note: residential turbines (0 - 20 kW): system size 2.5 kW, rotor diameter 2.2m, allowable hub heights 20, 30, 40 m }, + "nlr-reference-20kW": { + label: "NLR Reference 20kW", + minHeight: 30, + maxHeight: 50, + info: "Distributed wind reference turbine released by NLR: rated power 20 kW, rotor diameter 12.6m, hub height 31.03m. Aeroelastic model: https://github.com/NatLabRockies/DistributedWindReferenceTurbines. Also, residential reference turbines (2-20 kW): allowable hub heights 30, 40, 50 m (Lantz et al., 2016), see https://doi.org/10.2172/1333625", + // Source: NLR Distributed Wind Reference Turbines aeroelastic model repository and Lantz et al. (2016) + // https://github.com/NatLabRockies/DistributedWindReferenceTurbines, accessed 2026-08-03. + // Lantz, E., Sigrin, B., Gleason, M., Preus, R., & Baring-Gould, I. (2016). Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects (NREL/TP--6A20-67337, 1333625; p. NREL/TP--6A20-67337, 1333625). https://doi.org/10.2172/1333625 + // For more information, refer to https://atb.nlr.gov/electricity/2024b/distributed_wind + // Note: residential turbines (2 - 20 kW): allowable hub heights 30, 40, 50 m + }, "nlr-reference-100kW": { label: "NLR Reference 100kW", minHeight: 40, @@ -31,7 +42,7 @@ export const TURBINE_DATA: Record = { info: "Commercial reference turbines (20-100 kW): system size 100 kW, rotor diameter 13.8m, allowable hub heights 40, 50 m (Lantz et al., 2016). Also see https://doi.org/10.2172/1333625", // Source: Lantz et al. (2016) // Lantz, E., Sigrin, B., Gleason, M., Preus, R., & Baring-Gould, I. (2016). Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects (NREL/TP--6A20-67337, 1333625; p. NREL/TP--6A20-67337, 1333625). https://doi.org/10.2172/1333625 - // For more information, refer to https://atb.nrel.gov/electricity/2024b/distributed_wind + // For more information, refer to https://atb.nlr.gov/electricity/2024b/distributed_wind // Note: commercial turbines (20-100 kW): system size 100 kW, rotor diameter 13.8m, allowable hub heights 40, 50 m }, "nlr-reference-250kW": { @@ -41,7 +52,7 @@ export const TURBINE_DATA: Record = { info: "Mid-size reference turbines (100 kW - 1 MW): system size 250 kW, rotor diameter 21.9m, allowable hub heights 50 m (Lantz et al., 2016). Also see https://doi.org/10.2172/1333625", // Source: Lantz et al. (2016) // Lantz, E., Sigrin, B., Gleason, M., Preus, R., & Baring-Gould, I. (2016). Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects (NREL/TP--6A20-67337, 1333625; p. NREL/TP--6A20-67337, 1333625). https://doi.org/10.2172/1333625 - // For more information, refer to https://atb.nrel.gov/electricity/2024b/distributed_wind + // For more information, refer to https://atb.nlr.gov/electricity/2024b/distributed_wind // Note: mid-size turbines (100 kW - 1 MW): system size 250 kW, rotor diameter 21.9m, allowable hub heights 50 m }, "nlr-reference-2000kW": { @@ -51,7 +62,7 @@ export const TURBINE_DATA: Record = { maxHeight: 80, // Source: Lantz et al. (2016) // Lantz, E., Sigrin, B., Gleason, M., Preus, R., & Baring-Gould, I. (2016). Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects (NREL/TP--6A20-67337, 1333625; p. NREL/TP--6A20-67337, 1333625). https://doi.org/10.2172/1333625 - // For more information, refer to https://atb.nrel.gov/electricity/2024b/distributed_wind + // For more information, refer to https://atb.nlr.gov/electricity/2024b/distributed_wind // Note: large-size turbines (> 1 MW): system size 1 MW, rotor diameter 43.7m, allowable hub heights 50, 80 m }, "bergey-excel-15": { @@ -183,6 +194,7 @@ export const VALID_TURBINES = Object.keys(TURBINE_DATA); /** Fallback turbine list used when the API has not yet responded. */ export const DEFAULT_TURBINES = [ "nlr-reference-2.5kW", + "nlr-reference-20kW", "nlr-reference-100kW", "nlr-reference-250kW", "nlr-reference-2000kW", diff --git a/windwatts-ui/src/types/Heights.ts b/windwatts-ui/src/types/Heights.ts index 449c2cc2..5652cb0b 100644 --- a/windwatts-ui/src/types/Heights.ts +++ b/windwatts-ui/src/types/Heights.ts @@ -1,4 +1,4 @@ export interface Heights { values: number[]; - interpolation: number | null; + interpolation: boolean; } diff --git a/windwatts-ui/src/utils/turbine.ts b/windwatts-ui/src/utils/turbine.ts index f643f961..ba6adfaa 100644 --- a/windwatts-ui/src/utils/turbine.ts +++ b/windwatts-ui/src/utils/turbine.ts @@ -51,6 +51,25 @@ export const parseOptionalHeight = (value: string): number | undefined => { return trimmed ? Number(trimmed) : undefined; }; +/** + * Resolves a raw hub height value to a valid value for the given model. + * - Interpolable model: clamps to [min, max] — no extrapolation. + * - Non-interpolable model: clamps then snaps to the nearest available height. + */ +export const resolveHubHeight = ( + value: number, + availableHeights: number[], + interpolable: boolean +): number => { + const min = Math.min(...availableHeights); + const max = Math.max(...availableHeights); + const clamped = Math.max(min, Math.min(max, value)); + if (interpolable) return clamped; + return availableHeights.reduce((prev, curr) => + Math.abs(curr - clamped) < Math.abs(prev - clamped) ? curr : prev + ); +}; + /** * Validates an optional hub height range. * Returns an error message string, or null when the values are valid.