diff --git a/.env.example b/.env.example index 21a74f820e..bf34e41d5e 100644 --- a/.env.example +++ b/.env.example @@ -70,6 +70,17 @@ EIA_API_KEY= FRED_API_KEY= +# ------ Air Quality Intelligence (Railway seed) ------ + +# OpenAQ API v3 (required for scripts/seed-health-air-quality.mjs) +# Register at: https://docs.openaq.org/using-the-api/api-key +OPENAQ_API_KEY= + +# WAQI API (optional supplement for additional city/station coverage) +# Register at: https://aqicn.org/data-platform/token/ +WAQI_API_KEY= + + # ------ Aviation Intelligence (Vercel) ------ # AviationStack (live flight data, airport flights, carrier ops) diff --git a/api/health.js b/api/health.js index b3d528c11a..fba55462d0 100644 --- a/api/health.js +++ b/api/health.js @@ -10,6 +10,7 @@ const BOOTSTRAP_KEYS = { sectors: 'market:sectors:v1', etfFlows: 'market:etf-flows:v1', climateAnomalies: 'climate:anomalies:v2', + climateAirQuality: 'climate:air-quality:v1', co2Monitoring: 'climate:co2-monitoring:v1', wildfires: 'wildfire:fires:v1', marketQuotes: 'market:stocks-bootstrap:v1', @@ -74,6 +75,7 @@ const BOOTSTRAP_KEYS = { euFsi: 'economic:fsi-eu:v1', shippingStress: 'supply_chain:shipping_stress:v1', diseaseOutbreaks: 'health:disease-outbreaks:v1', + healthAirQuality: 'health:air-quality:v1', socialVelocity: 'intelligence:social:reddit:v1', vpdTrackerRealtime: 'health:vpd-tracker:realtime:v1', vpdTrackerHistorical: 'health:vpd-tracker:historical:v1', @@ -130,6 +132,7 @@ const SEED_META = { wildfires: { key: 'seed-meta:wildfire:fires', maxStaleMin: 360 }, // FIRMS NRT resets at midnight UTC; new-day data takes 3-6h to accumulate outages: { key: 'seed-meta:infra:outages', maxStaleMin: 30 }, climateAnomalies: { key: 'seed-meta:climate:anomalies', maxStaleMin: 240 }, // runs as independent Railway cron (0 */2 * * *); 240 = 2x interval + climateAirQuality:{ key: 'seed-meta:health:air-quality', maxStaleMin: 180 }, // hourly cron; 180 = 3x interval — shares meta key with healthAirQuality (same seeder run) climateZoneNormals: { key: 'seed-meta:climate:zone-normals', maxStaleMin: 89280 }, // monthly cron on the 1st; 62d = 2x 31-day cadence co2Monitoring: { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 4320 }, // daily cron at 06:00 UTC; 72h tolerates two missed runs climateNews: { key: 'seed-meta:climate:news-intelligence', maxStaleMin: 90 }, // relay loop every 30min; 90 = 3× interval @@ -225,6 +228,7 @@ const SEED_META = { newsThreatSummary: { key: 'seed-meta:news:threat-summary', maxStaleMin: 60 }, // relay classify every ~20min; 60min = 3x interval shippingStress: { key: 'seed-meta:supply_chain:shipping_stress', maxStaleMin: 45 }, // relay loop every 15min; 45 = 3x interval (was 30 = 2×, too tight on relay hiccup) diseaseOutbreaks: { key: 'seed-meta:health:disease-outbreaks', maxStaleMin: 2880 }, // daily seed; 2880 = 48h = 2x interval + healthAirQuality: { key: 'seed-meta:health:air-quality', maxStaleMin: 180 }, // hourly cron; 180 = 3x interval for shared health/climate seed socialVelocity: { key: 'seed-meta:intelligence:social-reddit', maxStaleMin: 30 }, // relay loop every 10min; 30 = 3x interval (was 20 = equals retry window, too tight) vpdTrackerRealtime: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // daily seed (0 2 * * *); 2880min = 48h = 2x interval vpdTrackerHistorical: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // shares seed-meta key with vpdTrackerRealtime (same run) diff --git a/api/seed-health.js b/api/seed-health.js index 03b45030b0..972cad9046 100644 --- a/api/seed-health.js +++ b/api/seed-health.js @@ -59,6 +59,7 @@ const SEED_DOMAINS = { 'thermal:escalation': { key: 'seed-meta:thermal:escalation', intervalMin: 180 }, 'radiation:observations': { key: 'seed-meta:radiation:observations', intervalMin: 15 }, 'sanctions:pressure': { key: 'seed-meta:sanctions:pressure', intervalMin: 360 }, + 'health:air-quality': { key: 'seed-meta:health:air-quality', intervalMin: 60 }, // hourly cron (shared seeder writes health + climate keys) 'economic:grocery-basket': { key: 'seed-meta:economic:grocery-basket', intervalMin: 5040 }, // weekly seed; intervalMin = maxStaleMin / 2 'economic:bigmac': { key: 'seed-meta:economic:bigmac', intervalMin: 5040 }, // weekly seed; intervalMin = maxStaleMin / 2 }; diff --git a/docs/api/ClimateService.openapi.json b/docs/api/ClimateService.openapi.json index 763acdfe86..a52bcc79ff 100644 --- a/docs/api/ClimateService.openapi.json +++ b/docs/api/ClimateService.openapi.json @@ -1 +1 @@ -{"components":{"schemas":{"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal in millimeters.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"ClimateNewsItem":{"description":"ClimateNewsItem represents a single climate/environment news article.","properties":{"id":{"description":"Unique identifier (URL hash + publish timestamp).","type":"string"},"publishedAt":{"description":"Publication time as Unix epoch milliseconds.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"sourceName":{"description":"Source publication name.","type":"string"},"summary":{"description":"Short summary/description (max 300 chars in seed pipeline).","type":"string"},"title":{"description":"Article headline.","type":"string"},"url":{"description":"Canonical article URL.","type":"string"}},"type":"object"},"Co2DataPoint":{"properties":{"anomaly":{"description":"Year-over-year delta vs same calendar month, in ppm.","format":"double","type":"number"},"month":{"type":"string"},"ppm":{"format":"double","type":"number"}},"type":"object"},"Co2Monitoring":{"properties":{"annualGrowthRate":{"format":"double","type":"number"},"currentPpm":{"format":"double","type":"number"},"measuredAt":{"format":"int64","type":"string"},"methanePpb":{"format":"double","type":"number"},"monthlyAverage":{"format":"double","type":"number"},"nitrousOxidePpb":{"format":"double","type":"number"},"preIndustrialBaseline":{"format":"double","type":"number"},"station":{"type":"string"},"trend12m":{"items":{"$ref":"#/components/schemas/Co2DataPoint"},"type":"array"},"yearAgoPpm":{"format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetCo2MonitoringRequest":{"type":"object"},"GetCo2MonitoringResponse":{"properties":{"monitoring":{"$ref":"#/components/schemas/Co2Monitoring"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"ListClimateNewsRequest":{"type":"object"},"ListClimateNewsResponse":{"properties":{"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"items":{"items":{"$ref":"#/components/schemas/ClimateNewsItem"},"type":"array"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/get-co2-monitoring":{"get":{"description":"GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data.","operationId":"GetCo2Monitoring","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetCo2MonitoringResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetCo2Monitoring","tags":["ClimateService"]}},"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}},"/api/climate/v1/list-climate-news":{"get":{"description":"ListClimateNews retrieves latest climate/environment intelligence headlines from seeded RSS feeds.","operationId":"ListClimateNews","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateNewsResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateNews","tags":["ClimateService"]}}}} \ No newline at end of file +{"components":{"schemas":{"AirQualityStation":{"properties":{"aqi":{"format":"int32","type":"integer"},"city":{"type":"string"},"countryCode":{"type":"string"},"lat":{"format":"double","type":"number"},"lng":{"format":"double","type":"number"},"measuredAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"pm25":{"format":"double","type":"number"},"pollutant":{"type":"string"},"riskLevel":{"type":"string"},"source":{"type":"string"}},"type":"object"},"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal in millimeters.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"ClimateNewsItem":{"description":"ClimateNewsItem represents a single climate/environment news article.","properties":{"id":{"description":"Unique identifier (URL hash + publish timestamp).","type":"string"},"publishedAt":{"description":"Publication time as Unix epoch milliseconds.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"sourceName":{"description":"Source publication name.","type":"string"},"summary":{"description":"Short summary/description (max 300 chars in seed pipeline).","type":"string"},"title":{"description":"Article headline.","type":"string"},"url":{"description":"Canonical article URL.","type":"string"}},"type":"object"},"Co2DataPoint":{"properties":{"anomaly":{"description":"Year-over-year delta vs same calendar month, in ppm.","format":"double","type":"number"},"month":{"type":"string"},"ppm":{"format":"double","type":"number"}},"type":"object"},"Co2Monitoring":{"properties":{"annualGrowthRate":{"format":"double","type":"number"},"currentPpm":{"format":"double","type":"number"},"measuredAt":{"format":"int64","type":"string"},"methanePpb":{"format":"double","type":"number"},"monthlyAverage":{"format":"double","type":"number"},"nitrousOxidePpb":{"format":"double","type":"number"},"preIndustrialBaseline":{"format":"double","type":"number"},"station":{"type":"string"},"trend12m":{"items":{"$ref":"#/components/schemas/Co2DataPoint"},"type":"array"},"yearAgoPpm":{"format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetCo2MonitoringRequest":{"type":"object"},"GetCo2MonitoringResponse":{"properties":{"monitoring":{"$ref":"#/components/schemas/Co2Monitoring"}},"type":"object"},"ListAirQualityDataRequest":{"type":"object"},"ListAirQualityDataResponse":{"properties":{"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"stations":{"items":{"$ref":"#/components/schemas/AirQualityStation"},"type":"array"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"ListClimateNewsRequest":{"type":"object"},"ListClimateNewsResponse":{"properties":{"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"items":{"items":{"$ref":"#/components/schemas/ClimateNewsItem"},"type":"array"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/get-co2-monitoring":{"get":{"description":"GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data.","operationId":"GetCo2Monitoring","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetCo2MonitoringResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetCo2Monitoring","tags":["ClimateService"]}},"/api/climate/v1/list-air-quality-data":{"get":{"description":"ListAirQualityData retrieves recent PM2.5 station data from the shared air-quality seed.","operationId":"ListAirQualityData","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListAirQualityDataResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListAirQualityData","tags":["ClimateService"]}},"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}},"/api/climate/v1/list-climate-news":{"get":{"description":"ListClimateNews retrieves latest climate/environment intelligence headlines from seeded RSS feeds.","operationId":"ListClimateNews","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateNewsResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateNews","tags":["ClimateService"]}}}} \ No newline at end of file diff --git a/docs/api/ClimateService.openapi.yaml b/docs/api/ClimateService.openapi.yaml index bbd0598e03..d01783f7aa 100644 --- a/docs/api/ClimateService.openapi.yaml +++ b/docs/api/ClimateService.openapi.yaml @@ -75,6 +75,32 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /api/climate/v1/list-air-quality-data: + get: + tags: + - ClimateService + summary: ListAirQualityData + description: ListAirQualityData retrieves recent PM2.5 station data from the shared air-quality seed. + operationId: ListAirQualityData + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ListAirQualityDataResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /api/climate/v1/list-climate-news: get: tags: @@ -293,6 +319,48 @@ components: type: number format: double description: Year-over-year delta vs same calendar month, in ppm. + ListAirQualityDataRequest: + type: object + ListAirQualityDataResponse: + type: object + properties: + stations: + type: array + items: + $ref: '#/components/schemas/AirQualityStation' + fetchedAt: + type: integer + format: int64 + description: 'Warning: Values > 2^53 may lose precision in JavaScript' + AirQualityStation: + type: object + properties: + city: + type: string + countryCode: + type: string + lat: + type: number + format: double + lng: + type: number + format: double + pm25: + type: number + format: double + aqi: + type: integer + format: int32 + riskLevel: + type: string + pollutant: + type: string + measuredAt: + type: integer + format: int64 + description: 'Warning: Values > 2^53 may lose precision in JavaScript' + source: + type: string ListClimateNewsRequest: type: object ListClimateNewsResponse: diff --git a/docs/api/HealthService.openapi.json b/docs/api/HealthService.openapi.json index 3c301fe38a..b0349ecf1a 100644 --- a/docs/api/HealthService.openapi.json +++ b/docs/api/HealthService.openapi.json @@ -1 +1 @@ -{"components":{"schemas":{"DiseaseOutbreakItem":{"description":"DiseaseOutbreakItem represents a single disease outbreak event.","properties":{"alertLevel":{"description":"Alert level: \"watch\" | \"warning\" | \"alert\".","type":"string"},"cases":{"description":"Case count if reported by source (0 = unknown).","format":"int32","type":"integer"},"countryCode":{"description":"ISO2 country code when known.","type":"string"},"disease":{"description":"Disease or outbreak name.","type":"string"},"id":{"description":"Unique identifier (URL-derived).","type":"string"},"lat":{"description":"Precise latitude from source (overrides country centroid on map when non-zero).","format":"double","type":"number"},"lng":{"description":"Precise longitude from source (overrides country centroid on map when non-zero).","format":"double","type":"number"},"location":{"description":"Affected country or region.","type":"string"},"publishedAt":{"description":"Unix epoch milliseconds when published.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"sourceName":{"description":"Source name (e.g., \"WHO\", \"ProMED\", \"HealthMap\").","type":"string"},"sourceUrl":{"description":"Source URL.","type":"string"},"summary":{"description":"Short description from the source.","type":"string"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"ListDiseaseOutbreaksRequest":{"type":"object"},"ListDiseaseOutbreaksResponse":{"properties":{"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"outbreaks":{"items":{"$ref":"#/components/schemas/DiseaseOutbreakItem"},"type":"array"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"HealthService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/health/v1/list-disease-outbreaks":{"get":{"description":"ListDiseaseOutbreaks returns recent WHO/ProMED disease outbreak alerts.","operationId":"ListDiseaseOutbreaks","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListDiseaseOutbreaksResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListDiseaseOutbreaks","tags":["HealthService"]}}}} \ No newline at end of file +{"components":{"schemas":{"AirQualityAlert":{"properties":{"aqi":{"format":"int32","type":"integer"},"city":{"type":"string"},"countryCode":{"type":"string"},"lat":{"format":"double","type":"number"},"lng":{"format":"double","type":"number"},"measuredAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"pm25":{"format":"double","type":"number"},"pollutant":{"type":"string"},"riskLevel":{"type":"string"},"source":{"type":"string"}},"type":"object"},"DiseaseOutbreakItem":{"description":"DiseaseOutbreakItem represents a single disease outbreak event.","properties":{"alertLevel":{"description":"Alert level: \"watch\" | \"warning\" | \"alert\".","type":"string"},"cases":{"description":"Case count if reported by source (0 = unknown).","format":"int32","type":"integer"},"countryCode":{"description":"ISO2 country code when known.","type":"string"},"disease":{"description":"Disease or outbreak name.","type":"string"},"id":{"description":"Unique identifier (URL-derived).","type":"string"},"lat":{"description":"Precise latitude from source (overrides country centroid on map when non-zero).","format":"double","type":"number"},"lng":{"description":"Precise longitude from source (overrides country centroid on map when non-zero).","format":"double","type":"number"},"location":{"description":"Affected country or region.","type":"string"},"publishedAt":{"description":"Unix epoch milliseconds when published.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"sourceName":{"description":"Source name (e.g., \"WHO\", \"ProMED\", \"HealthMap\").","type":"string"},"sourceUrl":{"description":"Source URL.","type":"string"},"summary":{"description":"Short description from the source.","type":"string"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"ListAirQualityAlertsRequest":{"type":"object"},"ListAirQualityAlertsResponse":{"properties":{"alerts":{"items":{"$ref":"#/components/schemas/AirQualityAlert"},"type":"array"},"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"}},"type":"object"},"ListDiseaseOutbreaksRequest":{"type":"object"},"ListDiseaseOutbreaksResponse":{"properties":{"fetchedAt":{"description":"Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"outbreaks":{"items":{"$ref":"#/components/schemas/DiseaseOutbreakItem"},"type":"array"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"HealthService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/health/v1/list-air-quality-alerts":{"get":{"description":"ListAirQualityAlerts returns recent PM2.5 stations with AQI-derived health risk.","operationId":"ListAirQualityAlerts","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListAirQualityAlertsResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListAirQualityAlerts","tags":["HealthService"]}},"/api/health/v1/list-disease-outbreaks":{"get":{"description":"ListDiseaseOutbreaks returns recent WHO/ProMED disease outbreak alerts.","operationId":"ListDiseaseOutbreaks","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListDiseaseOutbreaksResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListDiseaseOutbreaks","tags":["HealthService"]}}}} \ No newline at end of file diff --git a/docs/api/HealthService.openapi.yaml b/docs/api/HealthService.openapi.yaml index 9931945143..d79bbee7d1 100644 --- a/docs/api/HealthService.openapi.yaml +++ b/docs/api/HealthService.openapi.yaml @@ -29,6 +29,32 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /api/health/v1/list-air-quality-alerts: + get: + tags: + - HealthService + summary: ListAirQualityAlerts + description: ListAirQualityAlerts returns recent PM2.5 stations with AQI-derived health risk. + operationId: ListAirQualityAlerts + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ListAirQualityAlertsResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' components: schemas: Error: @@ -119,3 +145,45 @@ components: format: int32 description: Case count if reported by source (0 = unknown). description: DiseaseOutbreakItem represents a single disease outbreak event. + ListAirQualityAlertsRequest: + type: object + ListAirQualityAlertsResponse: + type: object + properties: + alerts: + type: array + items: + $ref: '#/components/schemas/AirQualityAlert' + fetchedAt: + type: integer + format: int64 + description: 'Warning: Values > 2^53 may lose precision in JavaScript' + AirQualityAlert: + type: object + properties: + city: + type: string + countryCode: + type: string + lat: + type: number + format: double + lng: + type: number + format: double + pm25: + type: number + format: double + aqi: + type: integer + format: int32 + riskLevel: + type: string + pollutant: + type: string + measuredAt: + type: integer + format: int64 + description: 'Warning: Values > 2^53 may lose precision in JavaScript' + source: + type: string diff --git a/docs/climate-variant-full.md b/docs/climate-variant-full.md index 2e31cbd84f..77bd98d2c9 100644 --- a/docs/climate-variant-full.md +++ b/docs/climate-variant-full.md @@ -135,7 +135,7 @@ message ClimateDisaster { **Sources:** -- **OpenAQ API v3** (no key): `https://api.openaq.org/v3/locations?limit=2000¶meters=pm25` +- **OpenAQ API v3** (`OPENAQ_API_KEY`): `https://api.openaq.org/v3/locations?limit=1000¶meters_id=2` - Measurements: PM2.5, PM10, O3, NO2, CO, SO2, BC - 12,000+ stations - **WAQI API** (`WAQI_API_KEY`): city aggregates + dominant pollutant @@ -318,14 +318,14 @@ Replace current entry in `api/mcp.ts`: | Service | Key Name | Free Tier | |---------|----------|-----------| | WAQI (air quality) | `WAQI_API_KEY` | 1000 req/day | -| OpenAQ | None | Free | +| OpenAQ v3 (air quality) | `OPENAQ_API_KEY` | Required by current API docs | | NOAA GML | None | Free | | NSIDC | None | Free | | ReliefWeb API | None | Free | | RSS feeds (all) | None | Public | | Copernicus CDS | `CDS_API_KEY` | Free (registration required) — only needed for CAMS/ERA5 advanced queries | -**Only 1-2 new API keys required.** WAQI is optional (OpenAQ alone is sufficient). CDS key is optional (enhances but not required). +**OpenAQ now requires `OPENAQ_API_KEY`.** `WAQI_API_KEY` is still optional, and `CDS_API_KEY` is only needed for CAMS/ERA5 advanced queries. --- @@ -371,7 +371,7 @@ climate: { 2. **`seed-co2-monitoring.mjs`** — NOAA GML text file parsing, no key, 30min effort, high impact (single most important climate number) 3. **`seed-climate-news.mjs`** — RSS aggregation, no key, fast win 4. **`seed-climate-disasters.mjs`** — ReliefWeb API (no key) + reuse GDACS from natural seeder -5. **`seed-health-air-quality.mjs`** — OpenAQ (no key), writes both `health:air-quality:v1` and `climate:air-quality:v1` +5. **`seed-health-air-quality.mjs`** — OpenAQ (`OPENAQ_API_KEY`), writes both `health:air-quality:v1` and `climate:air-quality:v1` 6. **`seed-climate-ocean-ice.mjs`** — NSIDC CSV parsing (no key), daily data 7. **`seed-climate-zone-normals.mjs`** — one-time + monthly refresh, feeds anomaly baseline 8. **Proto + handler additions** for each new RPC diff --git a/docs/health-variant-full.md b/docs/health-variant-full.md index dd91376770..9783d0fe40 100644 --- a/docs/health-variant-full.md +++ b/docs/health-variant-full.md @@ -99,7 +99,7 @@ message VaccinationCoverageItem { **Sources:** -- **OpenAQ API v3** (no key for basic): `https://api.openaq.org/v3/locations?limit=1000¶meters=pm25&bbox={bbox}` +- **OpenAQ API v3** (`OPENAQ_API_KEY`): `https://api.openaq.org/v3/locations?limit=1000¶meters_id=2&bbox={bbox}` - Readings: `https://api.openaq.org/v3/sensors/{id}/measurements/daily` - 12,000+ stations globally, free tier sufficient - **WAQI (World Air Quality Index)** — city-level aggregation: `https://api.waqi.info/map/bounds/?latlng={bbox}&token={key}` @@ -295,13 +295,13 @@ health: { | Service | Key Name | Free Tier | |---------|----------|-----------| | WAQI (air quality) | `WAQI_API_KEY` | 1000 req/day (sufficient for hourly city aggregation) | -| OpenAQ | None required | Free, rate limit 60 req/min | +| OpenAQ v3 (air quality) | `OPENAQ_API_KEY` | Required by current API docs | | WHO GHO API | None required | Free, public | | Our World in Data | None required | Free, public CSV | | Nextstrain | None required | Free, public JSON | | RSS feeds (all) | None required | Public | -**Only 1 new API key required (WAQI)** — everything else is keyless. +**At least 1 new API key is required (`OPENAQ_API_KEY`)**. `WAQI_API_KEY` remains optional; the seed still works with OpenAQ alone. --- @@ -311,7 +311,7 @@ health: { 2. **`seed-health-news.mjs`** — pure RSS aggregation, no key needed, fast win 3. **`seed-pathogen-surveillance.mjs`** — Nextstrain JSON + WHO WER RSS 4. **`seed-epidemic-trends.mjs`** — WHO GHO API (no key, daily data) -5. **`seed-health-air-quality.mjs`** — OpenAQ (no key) + WAQI (1 key) +5. **`seed-health-air-quality.mjs`** — OpenAQ (`OPENAQ_API_KEY`) + optional WAQI 6. **`seed-vaccination-coverage.mjs`** — WHO immunization API (weekly, lowest priority) 7. **Proto + handler additions** for each new RPC 8. **MCP tool registration** `get_health_data` diff --git a/proto/worldmonitor/climate/v1/list_air_quality_data.proto b/proto/worldmonitor/climate/v1/list_air_quality_data.proto new file mode 100644 index 0000000000..1303a5b251 --- /dev/null +++ b/proto/worldmonitor/climate/v1/list_air_quality_data.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package worldmonitor.climate.v1; + +import "sebuf/http/annotations.proto"; + +message AirQualityStation { + string city = 1; + string country_code = 2; + double lat = 3; + double lng = 4; + double pm25 = 5; + int32 aqi = 6; + string risk_level = 7; + string pollutant = 8; + int64 measured_at = 9 [(sebuf.http.int64_encoding) = INT64_ENCODING_NUMBER]; + string source = 10; +} + +message ListAirQualityDataRequest {} + +message ListAirQualityDataResponse { + repeated AirQualityStation stations = 1; + int64 fetched_at = 2 [(sebuf.http.int64_encoding) = INT64_ENCODING_NUMBER]; +} diff --git a/proto/worldmonitor/climate/v1/service.proto b/proto/worldmonitor/climate/v1/service.proto index 70efa2823e..e982043fe7 100644 --- a/proto/worldmonitor/climate/v1/service.proto +++ b/proto/worldmonitor/climate/v1/service.proto @@ -4,6 +4,7 @@ package worldmonitor.climate.v1; import "sebuf/http/annotations.proto"; import "worldmonitor/climate/v1/get_co2_monitoring.proto"; +import "worldmonitor/climate/v1/list_air_quality_data.proto"; import "worldmonitor/climate/v1/list_climate_anomalies.proto"; import "worldmonitor/climate/v1/list_climate_news.proto"; @@ -21,6 +22,11 @@ service ClimateService { option (sebuf.http.config) = {path: "/get-co2-monitoring", method: HTTP_METHOD_GET}; } + // ListAirQualityData retrieves recent PM2.5 station data from the shared air-quality seed. + rpc ListAirQualityData(ListAirQualityDataRequest) returns (ListAirQualityDataResponse) { + option (sebuf.http.config) = {path: "/list-air-quality-data", method: HTTP_METHOD_GET}; + } + // ListClimateNews retrieves latest climate/environment intelligence headlines from seeded RSS feeds. rpc ListClimateNews(ListClimateNewsRequest) returns (ListClimateNewsResponse) { option (sebuf.http.config) = {path: "/list-climate-news", method: HTTP_METHOD_GET}; diff --git a/proto/worldmonitor/health/v1/list_air_quality_alerts.proto b/proto/worldmonitor/health/v1/list_air_quality_alerts.proto new file mode 100644 index 0000000000..be2f94fbdf --- /dev/null +++ b/proto/worldmonitor/health/v1/list_air_quality_alerts.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package worldmonitor.health.v1; + +import "sebuf/http/annotations.proto"; + +message AirQualityAlert { + string city = 1; + string country_code = 2; + double lat = 3; + double lng = 4; + double pm25 = 5; + int32 aqi = 6; + string risk_level = 7; + string pollutant = 8; + int64 measured_at = 9 [(sebuf.http.int64_encoding) = INT64_ENCODING_NUMBER]; + string source = 10; +} + +message ListAirQualityAlertsRequest {} + +message ListAirQualityAlertsResponse { + repeated AirQualityAlert alerts = 1; + int64 fetched_at = 2 [(sebuf.http.int64_encoding) = INT64_ENCODING_NUMBER]; +} diff --git a/proto/worldmonitor/health/v1/service.proto b/proto/worldmonitor/health/v1/service.proto index faf03a1f22..9678b5a531 100644 --- a/proto/worldmonitor/health/v1/service.proto +++ b/proto/worldmonitor/health/v1/service.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package worldmonitor.health.v1; import "sebuf/http/annotations.proto"; +import "worldmonitor/health/v1/list_air_quality_alerts.proto"; import "worldmonitor/health/v1/list_disease_outbreaks.proto"; service HealthService { @@ -12,4 +13,9 @@ service HealthService { rpc ListDiseaseOutbreaks(ListDiseaseOutbreaksRequest) returns (ListDiseaseOutbreaksResponse) { option (sebuf.http.config) = {path: "/list-disease-outbreaks", method: HTTP_METHOD_GET}; } + + // ListAirQualityAlerts returns recent PM2.5 stations with AQI-derived health risk. + rpc ListAirQualityAlerts(ListAirQualityAlertsRequest) returns (ListAirQualityAlertsResponse) { + option (sebuf.http.config) = {path: "/list-air-quality-alerts", method: HTTP_METHOD_GET}; + } } diff --git a/scripts/run-seeders.sh b/scripts/run-seeders.sh index f079786b05..840f0e6a21 100755 --- a/scripts/run-seeders.sh +++ b/scripts/run-seeders.sh @@ -22,7 +22,7 @@ if [ -f "$OVERRIDE" ]; then | sed 's/^\s*//' \ | sed 's/: */=/' \ | sed "s/[\"']//g" \ - | grep -E '^(NASA_FIRMS|GROQ|AISSTREAM|FRED|FINNHUB|EIA|ACLED_ACCESS_TOKEN|ACLED_EMAIL|ACLED_PASSWORD|CLOUDFLARE|AVIATIONSTACK|OPENROUTER_API_KEY|LLM_API_URL|LLM_API_KEY|LLM_MODEL|OLLAMA_API_URL|OLLAMA_MODEL)' \ + | grep -E '^(NASA_FIRMS|GROQ|AISSTREAM|FRED|FINNHUB|EIA|ACLED_ACCESS_TOKEN|ACLED_EMAIL|ACLED_PASSWORD|CLOUDFLARE|AVIATIONSTACK|OPENAQ_API_KEY|WAQI_API_KEY|OPENROUTER_API_KEY|LLM_API_URL|LLM_API_KEY|LLM_MODEL|OLLAMA_API_URL|OLLAMA_MODEL)' \ | sed 's/^/export /' > "$_env_tmp" . "$_env_tmp" rm -f "$_env_tmp" diff --git a/scripts/seed-health-air-quality.mjs b/scripts/seed-health-air-quality.mjs new file mode 100644 index 0000000000..2381c5de6b --- /dev/null +++ b/scripts/seed-health-air-quality.mjs @@ -0,0 +1,598 @@ +#!/usr/bin/env node + +import { + acquireLockSafely, + CHROME_UA, + extendExistingTtl, + getRedisCredentials, + loadEnvFile, + logSeedResult, + releaseLock, + verifySeedKey, + withRetry, +} from './_seed-utils.mjs'; + +loadEnvFile(import.meta.url); + +export const HEALTH_AIR_QUALITY_KEY = 'health:air-quality:v1'; +export const CLIMATE_AIR_QUALITY_KEY = 'climate:air-quality:v1'; +export const CACHE_TTL = 10800; // 3h — 3× the 1h cron cadence (gold standard: TTL ≥ 3× interval) +export const AIR_QUALITY_WINDOW_MS = 2 * 60 * 60 * 1000; +export const OPENAQ_META_KEY = 'seed-meta:health:air-quality'; +export const CLIMATE_META_KEY = 'seed-meta:climate:air-quality'; +export const OPENAQ_SOURCE_VERSION = 'openaq-v3-pm25-waqi-optional-v2'; + +const OPENAQ_LOCATIONS_URL = 'https://api.openaq.org/v3/locations'; +const OPENAQ_PM25_LATEST_URL = 'https://api.openaq.org/v3/parameters/2/latest'; +const OPENAQ_PAGE_LIMIT = 1000; +const OPENAQ_MAX_PAGES = 20; +// Worst case: 2 OpenAQ calls × 20 pages × (30s timeout × 3 attempts) ≈ 3600s +const AIR_QUALITY_LOCK_TTL_MS = 3_600_000; + +// The product only exposes four buckets, so EPA's sensitive/unhealthy/very-unhealthy +// bands are collapsed into a single "unhealthy" level. +const EPA_PM25_BREAKPOINTS = [ + { cLow: 0.0, cHigh: 12.0, iLow: 0, iHigh: 50 }, + { cLow: 12.1, cHigh: 35.4, iLow: 51, iHigh: 100 }, + { cLow: 35.5, cHigh: 55.4, iLow: 101, iHigh: 150 }, + { cLow: 55.5, cHigh: 150.4, iLow: 151, iHigh: 200 }, + { cLow: 150.5, cHigh: 250.4, iLow: 201, iHigh: 300 }, + { cLow: 250.5, cHigh: 350.4, iLow: 301, iHigh: 400 }, + { cLow: 350.5, cHigh: 500.4, iLow: 401, iHigh: 500 }, +]; + +const WAQI_WORLD_TILES = [ + '-55,-180,0,-60', + '-55,-60,0,60', + '-55,60,0,180', + '0,-180,55,-60', + '0,-60,55,60', + '0,60,55,180', +]; + +class SeedConfigurationError extends Error { + constructor(message) { + super(message); + this.name = 'SeedConfigurationError'; + this.code = 'SEED_CONFIGURATION_ERROR'; + this.retryable = false; + } +} + +function toFiniteNumber(value) { + const numeric = typeof value === 'number' ? value : Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function trimString(value) { + return typeof value === 'string' ? value.trim() : ''; +} + +function normalizeCountryCode(value) { + const code = trimString(value).toUpperCase(); + return /^[A-Z]{2}$/.test(code) ? code : ''; +} + +function toEpochMs(value) { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (value instanceof Date) return value.getTime(); + if (typeof value !== 'string' || !value.trim()) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function roundTo(value, decimals = 1) { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +function truncatePm25(value) { + return Math.floor(value * 10) / 10; +} + +export function computeUsAqiFromPm25(pm25) { + const numeric = toFiniteNumber(pm25); + if (numeric == null || numeric < 0) return 0; + const concentration = Math.min(truncatePm25(numeric), 500.4); + const breakpoint = EPA_PM25_BREAKPOINTS.find(({ cHigh }) => concentration <= cHigh) ?? EPA_PM25_BREAKPOINTS.at(-1); + const ratio = (breakpoint.iHigh - breakpoint.iLow) / (breakpoint.cHigh - breakpoint.cLow); + return Math.max(0, Math.min(500, Math.round((ratio * (concentration - breakpoint.cLow)) + breakpoint.iLow))); +} + +export function classifyRiskLevel(aqi) { + const numeric = Math.max(0, Math.min(500, Math.round(toFiniteNumber(aqi) ?? 0))); + if (numeric <= 50) return 'good'; + if (numeric <= 100) return 'moderate'; + if (numeric <= 300) return 'unhealthy'; + return 'hazardous'; +} + +function isFreshMeasurement(measuredAt, nowMs = Date.now()) { + return Number.isFinite(measuredAt) && measuredAt >= (nowMs - AIR_QUALITY_WINDOW_MS) && measuredAt <= (nowMs + 5 * 60 * 1000); +} + +function pickLocationName(location) { + return trimString(location?.locality) + || trimString(location?.city) + || trimString(location?.name) + || normalizeCountryCode(location?.country?.code) + || 'Unknown'; +} + +function pickCoordinates(primary, fallback) { + const lat = toFiniteNumber(primary?.latitude ?? primary?.lat) ?? toFiniteNumber(fallback?.latitude ?? fallback?.lat); + const lng = toFiniteNumber(primary?.longitude ?? primary?.lng) ?? toFiniteNumber(fallback?.longitude ?? fallback?.lng); + if (lat == null || lng == null) return null; + return { lat: roundTo(lat, 4), lng: roundTo(lng, 4) }; +} + +export function buildOpenAqLocationIndex(locations = []) { + const index = new Map(); + for (const location of locations) { + const id = toFiniteNumber(location?.id); + if (id == null) continue; + index.set(id, { + city: pickLocationName(location), + countryCode: normalizeCountryCode(location?.country?.code), + coordinates: pickCoordinates(location?.coordinates), + }); + } + return index; +} + +function buildLocationMetadata(result, locationIndex) { + const locationId = toFiniteNumber( + result?.locationsId + ?? result?.locationId + ?? result?.location?.id, + ); + const indexed = locationId != null ? locationIndex.get(locationId) : null; + const inlineLocation = result?.location ?? null; + const city = indexed?.city || pickLocationName(inlineLocation); + const countryCode = indexed?.countryCode || normalizeCountryCode(inlineLocation?.country?.code); + const coordinates = pickCoordinates(result?.coordinates, indexed?.coordinates ?? inlineLocation?.coordinates); + if (!city || !coordinates) return null; + return { locationId: locationId ?? null, city, countryCode, coordinates }; +} + +export function buildOpenAqStations(locations = [], latestMeasurements = [], nowMs = Date.now()) { + const locationIndex = buildOpenAqLocationIndex(locations); + const latestByLocation = new Map(); + + for (const result of latestMeasurements) { + const pm25 = toFiniteNumber(result?.value); + if (pm25 == null || pm25 < 0) continue; + + const measuredAt = toEpochMs(result?.datetime?.utc ?? result?.datetime?.local ?? result?.date?.utc ?? result?.date?.local); + if (!isFreshMeasurement(measuredAt, nowMs)) continue; + + const metadata = buildLocationMetadata(result, locationIndex); + if (!metadata) continue; + + const pollutant = trimString(result?.parameter?.name) || trimString(result?.parameter) || 'pm25'; + const normalizedPm25 = roundTo(pm25, 1); + const aqi = computeUsAqiFromPm25(normalizedPm25); + const station = { + city: metadata.city, + countryCode: metadata.countryCode, + lat: metadata.coordinates.lat, + lng: metadata.coordinates.lng, + pm25: normalizedPm25, + aqi, + riskLevel: classifyRiskLevel(aqi), + pollutant, + measuredAt, + source: 'OpenAQ', + }; + const dedupeKey = metadata.locationId ?? `${station.city}:${station.lat}:${station.lng}`; + const previous = latestByLocation.get(dedupeKey); + if (!previous || station.measuredAt > previous.measuredAt || (station.measuredAt === previous.measuredAt && station.pm25 > previous.pm25)) { + latestByLocation.set(dedupeKey, station); + } + } + + return [...latestByLocation.values()].sort((left, right) => right.aqi - left.aqi || right.measuredAt - left.measuredAt); +} + +function extractCountryCodeFromName(name) { + const match = trimString(name).match(/\b([A-Z]{2})\b$/); + return match ? normalizeCountryCode(match[1]) : ''; +} + +export function buildWaqiStations(entries = [], nowMs = Date.now()) { + const stations = []; + for (const entry of entries) { + const pm25 = toFiniteNumber(entry?.iaqi?.pm25?.v ?? entry?.pm25); + const lat = toFiniteNumber(entry?.lat); + const lng = toFiniteNumber(entry?.lon); + const aqi = toFiniteNumber(entry?.aqi); + const stationName = trimString(entry?.station?.name); + const measuredAt = toEpochMs(entry?.station?.time); + if (pm25 == null || lat == null || lng == null || aqi == null || !stationName || !isFreshMeasurement(measuredAt, nowMs)) continue; + + stations.push({ + city: stationName.split(',')[0]?.trim() || stationName, + countryCode: extractCountryCodeFromName(stationName), + lat: roundTo(lat, 4), + lng: roundTo(lng, 4), + pm25: roundTo(pm25, 1), + aqi: Math.max(0, Math.min(500, Math.round(aqi))), + riskLevel: classifyRiskLevel(aqi), + pollutant: trimString(entry?.dominentpol) || 'pm25', + measuredAt, + source: 'WAQI', + }); + } + return stations; +} + +function isNormalizedAirQualityStation(station) { + return Boolean( + trimString(station?.city) + && toFiniteNumber(station?.lat) != null + && toFiniteNumber(station?.lng) != null + && toFiniteNumber(station?.aqi) != null + && toEpochMs(station?.measuredAt) != null, + ); +} + +function normalizeSupplementalStations({ waqiStations = [], waqiEntries = [], nowMs = Date.now() }) { + const normalizedStations = Array.isArray(waqiStations) + ? waqiStations.filter(isNormalizedAirQualityStation) + : []; + + if (!Array.isArray(waqiEntries) || waqiEntries.length === 0) { + return normalizedStations; + } + + // `buildAirQualityPayload()` now accepts pre-normalized `waqiStations`. + // Keep `waqiEntries` as a backward-compatible alias for raw WAQI API payloads. + const legacyStations = waqiEntries.some(isNormalizedAirQualityStation) + ? waqiEntries.filter(isNormalizedAirQualityStation) + : buildWaqiStations(waqiEntries, nowMs); + + return [...normalizedStations, ...legacyStations]; +} + +function stationIdentity(station) { + return [ + trimString(station.city).toLowerCase(), + normalizeCountryCode(station.countryCode).toLowerCase(), + roundTo(station.lat, 2), + roundTo(station.lng, 2), + ].join('|'); +} + +export function mergeAirQualityStations(primaryStations = [], secondaryStations = []) { + const merged = new Map(); + for (const station of primaryStations) { + if (!isNormalizedAirQualityStation(station)) continue; + merged.set(stationIdentity(station), station); + } + for (const station of secondaryStations) { + if (!isNormalizedAirQualityStation(station)) continue; + const key = stationIdentity(station); + if (!merged.has(key)) merged.set(key, station); + } + return [...merged.values()].sort((left, right) => right.aqi - left.aqi || right.measuredAt - left.measuredAt); +} + +function toOutputStation(station) { + return { + city: station.city, + country_code: station.countryCode, + lat: station.lat, + lng: station.lng, + pm25: station.pm25, + aqi: station.aqi, + risk_level: station.riskLevel, + pollutant: station.pollutant, + measured_at: station.measuredAt, + source: station.source, + }; +} + +export function buildOpenAqHeaders(apiKey = process.env.OPENAQ_API_KEY) { + const trimmedKey = trimString(apiKey); + if (!trimmedKey) { + throw new SeedConfigurationError('Missing OPENAQ_API_KEY — OpenAQ v3 requests now require X-API-Key'); + } + return { + Accept: 'application/json', + 'User-Agent': CHROME_UA, + 'X-API-Key': trimmedKey, + }; +} + +function isConfigurationError(error) { + return error instanceof SeedConfigurationError || error?.code === 'SEED_CONFIGURATION_ERROR'; +} + +async function fetchJson(url, label, headers = {}) { + const response = await fetch(url, { + headers: { + Accept: 'application/json', + 'User-Agent': CHROME_UA, + ...headers, + }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${label}: HTTP ${response.status} ${body.slice(0, 200)}`.trim()); + } + return response.json(); +} + +function buildUrl(baseUrl, params) { + const url = new URL(baseUrl); + for (const [key, value] of Object.entries(params)) { + if (value == null || value === '') continue; + url.searchParams.set(key, String(value)); + } + return url.toString(); +} + +async function fetchOpenAqLocationsPage(page) { + const headers = buildOpenAqHeaders(); + const url = buildUrl(OPENAQ_LOCATIONS_URL, { + limit: OPENAQ_PAGE_LIMIT, + page, + parameters_id: 2, + sort_order: 'desc', + }); + return await withRetry(() => fetchJson(url, `OpenAQ locations page ${page}`, headers), 2, 1_000); +} + +async function fetchOpenAqLatestPage(page) { + const headers = buildOpenAqHeaders(); + const url = buildUrl(OPENAQ_PM25_LATEST_URL, { + limit: OPENAQ_PAGE_LIMIT, + page, + }); + return withRetry(() => fetchJson(url, `OpenAQ latest page ${page}`, headers), 2, 1_000); +} + +async function fetchPagedResults(fetchPage, label) { + const results = []; + let expectedFound = 0; + + for (let page = 1; page <= OPENAQ_MAX_PAGES; page++) { + const payload = await fetchPage(page); + const pageResults = Array.isArray(payload?.results) ? payload.results : []; + results.push(...pageResults); + + const found = toFiniteNumber(payload?.meta?.found); + const effectiveLimit = toFiniteNumber(payload?.meta?.limit) ?? OPENAQ_PAGE_LIMIT; + if (found != null && found > 0) expectedFound = found; + + if (pageResults.length < effectiveLimit) break; + if (expectedFound > 0 && results.length >= expectedFound) break; + } + + if (results.length === 0) { + throw new Error(`${label}: no results returned`); + } + + return results; +} + +async function fetchWaqiStations(nowMs) { + const apiKey = trimString(process.env.WAQI_API_KEY); + if (!apiKey) { + console.log(' [AIR] WAQI_API_KEY missing; skipping WAQI supplement'); + return []; + } + + const entries = []; + for (const bbox of WAQI_WORLD_TILES) { + const url = buildUrl('https://api.waqi.info/map/bounds/', { latlng: bbox, token: apiKey }); + try { + const payload = await withRetry(() => fetchJson(url, `WAQI ${bbox}`), 1, 1_000); + if (payload?.status === 'ok' && Array.isArray(payload.data)) { + entries.push(...payload.data); + } + } catch (error) { + console.warn(` [AIR] WAQI tile ${bbox} failed: ${error?.message ?? error}`); + } + } + + return buildWaqiStations(entries, nowMs); +} + +export function buildAirQualityPayload({ + locations = [], + latestMeasurements = [], + waqiStations = [], + waqiEntries = [], + nowMs = Date.now(), +} = {}) { + const openAqStations = buildOpenAqStations(locations, latestMeasurements, nowMs); + const supplementalStations = normalizeSupplementalStations({ waqiStations, waqiEntries, nowMs }); + const mergedStations = mergeAirQualityStations(openAqStations, supplementalStations); + return { + stations: mergedStations.map(toOutputStation), + fetchedAt: nowMs, + }; +} + +export async function fetchAirQualityPayload(nowMs = Date.now()) { + const [locations, latestMeasurements, waqiStations] = await Promise.all([ + fetchPagedResults(fetchOpenAqLocationsPage, 'OpenAQ locations'), + fetchPagedResults(fetchOpenAqLatestPage, 'OpenAQ latest'), + fetchWaqiStations(nowMs).catch((error) => { + console.warn(` [AIR] WAQI supplement failed: ${error?.message ?? error}`); + return []; + }), + ]); + + const payload = buildAirQualityPayload({ + locations, + latestMeasurements, + waqiStations, + nowMs, + }); + + if (!payload.stations.length) { + throw new Error('No fresh PM2.5 stations found in the last 2 hours'); + } + + return payload; +} + +export function validateAirQualityPayload(payload) { + return Array.isArray(payload?.stations) && payload.stations.length > 0; +} + +export function buildMirrorWriteCommands(payload, ttlSeconds, fetchedAt = Date.now(), sourceVersion = OPENAQ_SOURCE_VERSION) { + const payloadJson = JSON.stringify(payload); + const recordCount = payload?.stations?.length ?? 0; + const metaTtl = 86400 * 7; + const healthMeta = JSON.stringify({ fetchedAt, recordCount, sourceVersion }); + const climateMeta = JSON.stringify({ fetchedAt, recordCount, sourceVersion }); + return [ + ['SET', HEALTH_AIR_QUALITY_KEY, payloadJson, 'EX', String(ttlSeconds)], + ['SET', CLIMATE_AIR_QUALITY_KEY, payloadJson, 'EX', String(ttlSeconds)], + ['SET', OPENAQ_META_KEY, healthMeta, 'EX', String(metaTtl)], + ['SET', CLIMATE_META_KEY, climateMeta, 'EX', String(metaTtl)], + ]; +} + +async function redisPipeline(commands) { + const { url, token } = getRedisCredentials(); + const response = await fetch(`${url}/pipeline`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(commands), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Redis pipeline failed: HTTP ${response.status} — ${body.slice(0, 200)}`); + } + return response.json(); +} + +async function publishMirroredPayload(payload) { + const fetchedAt = Date.now(); + const commands = buildMirrorWriteCommands(payload, CACHE_TTL, fetchedAt, OPENAQ_SOURCE_VERSION); + await redisPipeline(commands); + return { + fetchedAt, + payloadBytes: Buffer.byteLength(JSON.stringify(payload), 'utf8'), + recordCount: payload?.stations?.length ?? 0, + }; +} + +async function verifyMirroredKeys() { + const [healthPayload, climatePayload] = await Promise.all([ + verifySeedKey(HEALTH_AIR_QUALITY_KEY), + verifySeedKey(CLIMATE_AIR_QUALITY_KEY), + ]); + return Boolean(healthPayload && climatePayload); +} + +async function fetchAirQualityPayloadWithRetry(maxRetries = 2, delayMs = 1_000) { + let lastError; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fetchAirQualityPayload(); + } catch (error) { + lastError = error; + if (isConfigurationError(error) || attempt >= maxRetries) break; + const wait = delayMs * 2 ** attempt; + const cause = error?.cause ? ` (cause: ${error.cause.message || error.cause.code || error.cause})` : ''; + console.warn(` Retry ${attempt + 1}/${maxRetries} in ${wait}ms: ${error?.message ?? error}${cause}`); + await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + throw lastError; +} + +async function main() { + const domain = 'health'; + const resource = 'air-quality'; + const startMs = Date.now(); + const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + + console.log(`=== ${domain}:${resource} Seed ===`); + console.log(` Run ID: ${runId}`); + console.log(` Keys: ${HEALTH_AIR_QUALITY_KEY}, ${CLIMATE_AIR_QUALITY_KEY}`); + + // Each OpenAQ branch can walk up to 20 pages sequentially with per-request timeouts. + // Keep the lock well above the realistic worst-case runtime to avoid overlapping cron runs. + const lockResult = await acquireLockSafely(`${domain}:${resource}`, runId, AIR_QUALITY_LOCK_TTL_MS, { + label: `${domain}:${resource}`, + }); + if (lockResult.skipped) process.exit(0); + if (!lockResult.locked) { + console.log(' SKIPPED: another seed run in progress'); + process.exit(0); + } + + let payload; + try { + payload = await fetchAirQualityPayloadWithRetry(); + } catch (error) { + await releaseLock(`${domain}:${resource}`, runId); + const durationMs = Date.now() - startMs; + const cause = error?.cause ? ` (cause: ${error.cause.message || error.cause.code || error.cause})` : ''; + console.error(` FETCH FAILED: ${error?.message ?? error}${cause}`); + await extendExistingTtl([ + HEALTH_AIR_QUALITY_KEY, + CLIMATE_AIR_QUALITY_KEY, + OPENAQ_META_KEY, + CLIMATE_META_KEY, + ], CACHE_TTL).catch(() => {}); + if (isConfigurationError(error)) { + console.log(`\n=== Fatal configuration error (${Math.round(durationMs)}ms) ===`); + process.exit(1); + } + console.log(`\n=== Failed gracefully (${Math.round(durationMs)}ms) ===`); + process.exit(0); + } + + if (!validateAirQualityPayload(payload)) { + await releaseLock(`${domain}:${resource}`, runId); + await extendExistingTtl([ + HEALTH_AIR_QUALITY_KEY, + CLIMATE_AIR_QUALITY_KEY, + OPENAQ_META_KEY, + CLIMATE_META_KEY, + ], CACHE_TTL).catch(() => {}); + console.log(' SKIPPED: validation failed (empty data)'); + process.exit(0); + } + + try { + const publishResult = await publishMirroredPayload(payload); + const durationMs = Date.now() - startMs; + logSeedResult(domain, publishResult.recordCount, durationMs, { + payloadBytes: publishResult.payloadBytes, + mirroredKeys: 2, + }); + + const verified = await verifyMirroredKeys().catch(() => false); + if (verified) { + console.log(' Verified: both Redis keys present'); + } else { + console.warn(` WARNING: verification read returned null for one or more mirror keys (${HEALTH_AIR_QUALITY_KEY}, ${CLIMATE_AIR_QUALITY_KEY})`); + } + + console.log(`\n=== Done (${Math.round(durationMs)}ms) ===`); + await releaseLock(`${domain}:${resource}`, runId); + process.exit(0); + } catch (error) { + await releaseLock(`${domain}:${resource}`, runId); + throw error; + } +} + +const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^file:\/\//, '')); +if (isMain) { + main().catch((error) => { + const cause = error?.cause ? ` (cause: ${error.cause.message || error.cause.code || error.cause})` : ''; + console.error('FATAL:', `${error?.message ?? error}${cause}`); + process.exit(1); + }); +} diff --git a/server/_shared/air-quality-stations.ts b/server/_shared/air-quality-stations.ts new file mode 100644 index 0000000000..e21ed776e3 --- /dev/null +++ b/server/_shared/air-quality-stations.ts @@ -0,0 +1,75 @@ +type LooseRecord = Record; + +export interface AirQualityStationRecord { + city: string; + countryCode: string; + lat: number; + lng: number; + pm25: number; + aqi: number; + riskLevel: string; + pollutant: string; + measuredAt: number; + source: string; +} + +function asRecord(value: unknown): LooseRecord | null { + return value != null && typeof value === 'object' ? (value as LooseRecord) : null; +} + +function asString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +function asNumber(value: unknown): number | null { + const numeric = typeof value === 'number' ? value : Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function pickKey(record: LooseRecord, snakeKey: string, camelKey: string): unknown { + if (record[snakeKey] != null) return record[snakeKey]; + return record[camelKey]; +} + +export function normalizeAirQualityStation(value: unknown): AirQualityStationRecord | null { + const record = asRecord(value); + if (!record) return null; + + const city = asString(record.city); + const lat = asNumber(record.lat); + const lng = asNumber(record.lng); + const pm25 = asNumber(record.pm25); + const aqi = asNumber(record.aqi); + const measuredAt = asNumber(pickKey(record, 'measured_at', 'measuredAt')); + + if (!city || lat == null || lng == null || pm25 == null || aqi == null || measuredAt == null) { + return null; + } + + return { + city, + countryCode: asString(pickKey(record, 'country_code', 'countryCode')), + lat, + lng, + pm25, + aqi: Math.max(0, Math.min(500, Math.round(aqi))), + riskLevel: asString(pickKey(record, 'risk_level', 'riskLevel')), + pollutant: asString(record.pollutant) || 'pm25', + measuredAt: Math.round(measuredAt), + source: asString(record.source), + }; +} + +export function normalizeAirQualityStations(value: unknown): AirQualityStationRecord[] { + if (!Array.isArray(value)) return []; + return value + .map((entry) => normalizeAirQualityStation(entry)) + .filter((entry): entry is AirQualityStationRecord => entry != null); +} + +export function normalizeAirQualityFetchedAt(value: unknown): number { + const record = asRecord(value); + if (!record) return 0; + const numeric = asNumber(pickKey(record, 'fetched_at', 'fetchedAt')); + return numeric == null ? 0 : Math.round(numeric); +} diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index e5a0048c83..e18d1c9a5a 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -6,9 +6,11 @@ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; export const CLIMATE_ANOMALIES_KEY = 'climate:anomalies:v2'; +export const CLIMATE_AIR_QUALITY_KEY = 'climate:air-quality:v1'; export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; export const CLIMATE_CO2_MONITORING_KEY = 'climate:co2-monitoring:v1'; export const CLIMATE_NEWS_KEY = 'climate:news-intelligence:v1'; +export const HEALTH_AIR_QUALITY_KEY = 'health:air-quality:v1'; /** * Static cache keys for the bootstrap endpoint. diff --git a/server/gateway.ts b/server/gateway.ts index 48417966ae..825849bd1d 100644 --- a/server/gateway.ts +++ b/server/gateway.ts @@ -108,6 +108,7 @@ const RPC_CACHE_TIER: Record = { '/api/intelligence/v1/get-gdelt-topic-timeline': 'medium', '/api/climate/v1/list-climate-anomalies': 'static', '/api/climate/v1/get-co2-monitoring': 'static', + '/api/climate/v1/list-air-quality-data': 'fast', '/api/climate/v1/list-climate-news': 'slow', '/api/sanctions/v1/list-sanctions-pressure': 'static', '/api/sanctions/v1/lookup-sanction-entity': 'no-store', @@ -203,6 +204,7 @@ const RPC_CACHE_TIER: Record = { '/api/economic/v1/get-economic-stress': 'slow', '/api/supply-chain/v1/get-shipping-stress': 'medium', '/api/health/v1/list-disease-outbreaks': 'slow', + '/api/health/v1/list-air-quality-alerts': 'fast', '/api/intelligence/v1/get-social-velocity': 'fast', }; diff --git a/server/worldmonitor/climate/v1/handler.ts b/server/worldmonitor/climate/v1/handler.ts index 63872b7369..aa259ae6ff 100644 --- a/server/worldmonitor/climate/v1/handler.ts +++ b/server/worldmonitor/climate/v1/handler.ts @@ -1,11 +1,13 @@ import type { ClimateServiceHandler } from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; import { getCo2Monitoring } from './get-co2-monitoring'; +import { listAirQualityData } from './list-air-quality-data'; import { listClimateAnomalies } from './list-climate-anomalies'; import { listClimateNews } from './list-climate-news'; export const climateHandler: ClimateServiceHandler = { getCo2Monitoring, + listAirQualityData, listClimateAnomalies, listClimateNews, }; diff --git a/server/worldmonitor/climate/v1/list-air-quality-data.ts b/server/worldmonitor/climate/v1/list-air-quality-data.ts new file mode 100644 index 0000000000..1cc356d8d3 --- /dev/null +++ b/server/worldmonitor/climate/v1/list-air-quality-data.ts @@ -0,0 +1,25 @@ +import type { + ClimateServiceHandler, + ListAirQualityDataRequest, + ListAirQualityDataResponse, + ServerContext, +} from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; + +import { + normalizeAirQualityFetchedAt, + normalizeAirQualityStations, +} from '../../../_shared/air-quality-stations'; +import { CLIMATE_AIR_QUALITY_KEY } from '../../../_shared/cache-keys'; +import { getCachedJson } from '../../../_shared/redis'; + +export const listAirQualityData: ClimateServiceHandler['listAirQualityData'] = async ( + _ctx: ServerContext, + _req: ListAirQualityDataRequest, +): Promise => { + const payload = (await getCachedJson(CLIMATE_AIR_QUALITY_KEY, true)) as Record | null; + const sourceStations = payload?.stations ?? payload?.alerts; + return { + stations: normalizeAirQualityStations(sourceStations), + fetchedAt: normalizeAirQualityFetchedAt(payload), + }; +}; diff --git a/server/worldmonitor/health/v1/handler.ts b/server/worldmonitor/health/v1/handler.ts index b32ddf883c..db6376004b 100644 --- a/server/worldmonitor/health/v1/handler.ts +++ b/server/worldmonitor/health/v1/handler.ts @@ -1,7 +1,9 @@ import type { HealthServiceHandler } from '../../../../src/generated/server/worldmonitor/health/v1/service_server'; +import { listAirQualityAlerts } from './list-air-quality-alerts'; import { listDiseaseOutbreaks } from './list-disease-outbreaks'; export const healthHandler: HealthServiceHandler = { + listAirQualityAlerts, listDiseaseOutbreaks, }; diff --git a/server/worldmonitor/health/v1/list-air-quality-alerts.ts b/server/worldmonitor/health/v1/list-air-quality-alerts.ts new file mode 100644 index 0000000000..83dd39a8c5 --- /dev/null +++ b/server/worldmonitor/health/v1/list-air-quality-alerts.ts @@ -0,0 +1,27 @@ +import type { + AirQualityAlert, + HealthServiceHandler, + ListAirQualityAlertsRequest, + ListAirQualityAlertsResponse, + ServerContext, +} from '../../../../src/generated/server/worldmonitor/health/v1/service_server'; + +import { + normalizeAirQualityFetchedAt, + normalizeAirQualityStations, +} from '../../../_shared/air-quality-stations'; +import { HEALTH_AIR_QUALITY_KEY } from '../../../_shared/cache-keys'; +import { getCachedJson } from '../../../_shared/redis'; + +export const listAirQualityAlerts: HealthServiceHandler['listAirQualityAlerts'] = async ( + _ctx: ServerContext, + _req: ListAirQualityAlertsRequest, +): Promise => { + const payload = (await getCachedJson(HEALTH_AIR_QUALITY_KEY, true)) as Record | null; + const sourceStations = payload?.stations ?? payload?.alerts; + const alerts = normalizeAirQualityStations(sourceStations) as AirQualityAlert[]; + return { + alerts, + fetchedAt: normalizeAirQualityFetchedAt(payload), + }; +}; diff --git a/src/generated/client/worldmonitor/climate/v1/service_client.ts b/src/generated/client/worldmonitor/climate/v1/service_client.ts index ad5d8878e2..585f744d4f 100644 --- a/src/generated/client/worldmonitor/climate/v1/service_client.ts +++ b/src/generated/client/worldmonitor/climate/v1/service_client.ts @@ -59,6 +59,27 @@ export interface Co2DataPoint { anomaly: number; } +export interface ListAirQualityDataRequest { +} + +export interface ListAirQualityDataResponse { + stations: AirQualityStation[]; + fetchedAt: number; +} + +export interface AirQualityStation { + city: string; + countryCode: string; + lat: number; + lng: number; + pm25: number; + aqi: number; + riskLevel: string; + pollutant: string; + measuredAt: number; + source: string; +} + export interface ListClimateNewsRequest { } @@ -178,6 +199,29 @@ export class ClimateServiceClient { return await resp.json() as GetCo2MonitoringResponse; } + async listAirQualityData(req: ListAirQualityDataRequest, options?: ClimateServiceCallOptions): Promise { + let path = "/api/climate/v1/list-air-quality-data"; + const url = this.baseURL + path; + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as ListAirQualityDataResponse; + } + async listClimateNews(req: ListClimateNewsRequest, options?: ClimateServiceCallOptions): Promise { let path = "/api/climate/v1/list-climate-news"; const url = this.baseURL + path; diff --git a/src/generated/client/worldmonitor/health/v1/service_client.ts b/src/generated/client/worldmonitor/health/v1/service_client.ts index 291fd6d4cd..eca7dd4919 100644 --- a/src/generated/client/worldmonitor/health/v1/service_client.ts +++ b/src/generated/client/worldmonitor/health/v1/service_client.ts @@ -25,6 +25,27 @@ export interface DiseaseOutbreakItem { cases: number; } +export interface ListAirQualityAlertsRequest { +} + +export interface ListAirQualityAlertsResponse { + alerts: AirQualityAlert[]; + fetchedAt: number; +} + +export interface AirQualityAlert { + city: string; + countryCode: string; + lat: number; + lng: number; + pm25: number; + aqi: number; + riskLevel: string; + pollutant: string; + measuredAt: number; + source: string; +} + export interface FieldViolation { field: string; description: string; @@ -96,6 +117,29 @@ export class HealthServiceClient { return await resp.json() as ListDiseaseOutbreaksResponse; } + async listAirQualityAlerts(req: ListAirQualityAlertsRequest, options?: HealthServiceCallOptions): Promise { + let path = "/api/health/v1/list-air-quality-alerts"; + const url = this.baseURL + path; + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as ListAirQualityAlertsResponse; + } + private async handleError(resp: Response): Promise { const body = await resp.text(); if (resp.status === 400) { diff --git a/src/generated/server/worldmonitor/climate/v1/service_server.ts b/src/generated/server/worldmonitor/climate/v1/service_server.ts index b77f4505e3..df0c120339 100644 --- a/src/generated/server/worldmonitor/climate/v1/service_server.ts +++ b/src/generated/server/worldmonitor/climate/v1/service_server.ts @@ -59,6 +59,27 @@ export interface Co2DataPoint { anomaly: number; } +export interface ListAirQualityDataRequest { +} + +export interface ListAirQualityDataResponse { + stations: AirQualityStation[]; + fetchedAt: number; +} + +export interface AirQualityStation { + city: string; + countryCode: string; + lat: number; + lng: number; + pm25: number; + aqi: number; + riskLevel: string; + pollutant: string; + measuredAt: number; + source: string; +} + export interface ListClimateNewsRequest { } @@ -127,6 +148,7 @@ export interface RouteDescriptor { export interface ClimateServiceHandler { listClimateAnomalies(ctx: ServerContext, req: ListClimateAnomaliesRequest): Promise; getCo2Monitoring(ctx: ServerContext, req: GetCo2MonitoringRequest): Promise; + listAirQualityData(ctx: ServerContext, req: ListAirQualityDataRequest): Promise; listClimateNews(ctx: ServerContext, req: ListClimateNewsRequest): Promise; } @@ -221,6 +243,43 @@ export function createClimateServiceRoutes( } }, }, + { + method: "GET", + path: "/api/climate/v1/list-air-quality-data", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const body = {} as ListAirQualityDataRequest; + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.listAirQualityData(ctx, body); + return new Response(JSON.stringify(result as ListAirQualityDataResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, { method: "GET", path: "/api/climate/v1/list-climate-news", diff --git a/src/generated/server/worldmonitor/health/v1/service_server.ts b/src/generated/server/worldmonitor/health/v1/service_server.ts index 6010988abb..ec886874b7 100644 --- a/src/generated/server/worldmonitor/health/v1/service_server.ts +++ b/src/generated/server/worldmonitor/health/v1/service_server.ts @@ -25,6 +25,27 @@ export interface DiseaseOutbreakItem { cases: number; } +export interface ListAirQualityAlertsRequest { +} + +export interface ListAirQualityAlertsResponse { + alerts: AirQualityAlert[]; + fetchedAt: number; +} + +export interface AirQualityAlert { + city: string; + countryCode: string; + lat: number; + lng: number; + pm25: number; + aqi: number; + riskLevel: string; + pollutant: string; + measuredAt: number; + source: string; +} + export interface FieldViolation { field: string; description: string; @@ -71,6 +92,7 @@ export interface RouteDescriptor { export interface HealthServiceHandler { listDiseaseOutbreaks(ctx: ServerContext, req: ListDiseaseOutbreaksRequest): Promise; + listAirQualityAlerts(ctx: ServerContext, req: ListAirQualityAlertsRequest): Promise; } export function createHealthServiceRoutes( @@ -115,6 +137,43 @@ export function createHealthServiceRoutes( } }, }, + { + method: "GET", + path: "/api/health/v1/list-air-quality-alerts", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const body = {} as ListAirQualityAlertsRequest; + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.listAirQualityAlerts(ctx, body); + return new Response(JSON.stringify(result as ListAirQualityAlertsResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, ]; } diff --git a/src/services/climate-air-quality.ts b/src/services/climate-air-quality.ts new file mode 100644 index 0000000000..c46a3de364 --- /dev/null +++ b/src/services/climate-air-quality.ts @@ -0,0 +1,19 @@ +import { getRpcBaseUrl } from '@/services/rpc-client'; +import { + ClimateServiceClient, + type AirQualityStation, + type ListAirQualityDataResponse, +} from '@/generated/client/worldmonitor/climate/v1/service_client'; + +export type { AirQualityStation, ListAirQualityDataResponse }; + +const client = new ClimateServiceClient(getRpcBaseUrl(), { fetch: (...args) => globalThis.fetch(...args) }); +const emptyClimateAirQuality: ListAirQualityDataResponse = { stations: [], fetchedAt: 0 }; + +export async function fetchClimateAirQuality(): Promise { + try { + return await client.listAirQualityData({}); + } catch { + return emptyClimateAirQuality; + } +} diff --git a/src/services/health-air-quality.ts b/src/services/health-air-quality.ts new file mode 100644 index 0000000000..634242c29d --- /dev/null +++ b/src/services/health-air-quality.ts @@ -0,0 +1,19 @@ +import { getRpcBaseUrl } from '@/services/rpc-client'; +import { + HealthServiceClient, + type AirQualityAlert, + type ListAirQualityAlertsResponse, +} from '@/generated/client/worldmonitor/health/v1/service_client'; + +export type { AirQualityAlert, ListAirQualityAlertsResponse }; + +const client = new HealthServiceClient(getRpcBaseUrl(), { fetch: (...args) => globalThis.fetch(...args) }); +const emptyAirQualityAlerts: ListAirQualityAlertsResponse = { alerts: [], fetchedAt: 0 }; + +export async function fetchHealthAirQuality(): Promise { + try { + return await client.listAirQualityAlerts({}); + } catch { + return emptyAirQualityAlerts; + } +} diff --git a/tests/air-quality-seed.test.mjs b/tests/air-quality-seed.test.mjs new file mode 100644 index 0000000000..bb83b9e70e --- /dev/null +++ b/tests/air-quality-seed.test.mjs @@ -0,0 +1,231 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + buildOpenAqHeaders, + buildMirrorWriteCommands, + buildAirQualityPayload, + buildOpenAqStations, + buildWaqiStations, + CLIMATE_AIR_QUALITY_KEY, + CLIMATE_META_KEY, + classifyRiskLevel, + computeUsAqiFromPm25, + HEALTH_AIR_QUALITY_KEY, + OPENAQ_META_KEY, + mergeAirQualityStations, +} from '../scripts/seed-health-air-quality.mjs'; + +describe('air quality AQI helpers', () => { + it('maps PM2.5 concentrations onto EPA AQI breakpoints', () => { + assert.equal(computeUsAqiFromPm25(12.0), 50); + assert.equal(computeUsAqiFromPm25(35.4), 100); + assert.equal(computeUsAqiFromPm25(55.4), 150); + assert.equal(computeUsAqiFromPm25(250.5), 301); + }); + + it('collapses AQI values into the requested risk buckets', () => { + assert.equal(classifyRiskLevel(25), 'good'); + assert.equal(classifyRiskLevel(90), 'moderate'); + assert.equal(classifyRiskLevel(220), 'unhealthy'); + assert.equal(classifyRiskLevel(350), 'hazardous'); + }); + + it('requires an OpenAQ API key when building request headers', () => { + assert.throws(() => buildOpenAqHeaders(''), /OPENAQ_API_KEY/); + assert.deepEqual(buildOpenAqHeaders('test-key'), { + Accept: 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', + 'X-API-Key': 'test-key', + }); + }); +}); + +describe('air quality payload assembly', () => { + it('filters stale measurements and keeps the freshest reading per location', () => { + const nowMs = Date.UTC(2026, 3, 3, 12, 0, 0); + const stations = buildOpenAqStations( + [ + { + id: 101, + locality: 'Delhi', + country: { code: 'IN' }, + coordinates: { latitude: 28.61, longitude: 77.21 }, + }, + { + id: 202, + locality: 'Paris', + country: { code: 'FR' }, + coordinates: { latitude: 48.85, longitude: 2.35 }, + }, + ], + [ + { + locationsId: 101, + value: 82.4, + datetime: { utc: new Date(nowMs - (10 * 60 * 1000)).toISOString() }, + coordinates: { latitude: 28.61, longitude: 77.21 }, + parameter: { name: 'pm25' }, + }, + { + locationsId: 101, + value: 45.2, + datetime: { utc: new Date(nowMs - (40 * 60 * 1000)).toISOString() }, + coordinates: { latitude: 28.61, longitude: 77.21 }, + parameter: { name: 'pm25' }, + }, + { + locationsId: 202, + value: 18.7, + datetime: { utc: new Date(nowMs - (3 * 60 * 60 * 1000)).toISOString() }, + coordinates: { latitude: 48.85, longitude: 2.35 }, + parameter: { name: 'pm25' }, + }, + ], + nowMs, + ); + + assert.equal(stations.length, 1); + assert.equal(stations[0].city, 'Delhi'); + assert.equal(stations[0].countryCode, 'IN'); + assert.equal(stations[0].aqi, computeUsAqiFromPm25(82.4)); + assert.equal(stations[0].riskLevel, 'unhealthy'); + }); + + it('parses WAQI entries when PM2.5 and timestamps are present', () => { + const nowMs = Date.UTC(2026, 3, 3, 12, 0, 0); + const stations = buildWaqiStations( + [ + { + lat: 25.2, + lon: 55.27, + aqi: '180', + dominentpol: 'pm25', + iaqi: { pm25: { v: 74.1 } }, + station: { + name: 'Dubai, AE', + time: new Date(nowMs - (20 * 60 * 1000)).toISOString(), + }, + }, + ], + nowMs, + ); + + assert.equal(stations.length, 1); + assert.equal(stations[0].city, 'Dubai'); + assert.equal(stations[0].countryCode, 'AE'); + assert.equal(stations[0].source, 'WAQI'); + }); + + it('merges OpenAQ and WAQI stations without duplicating identical locations', () => { + const openAqStations = [ + { city: 'Paris', countryCode: 'FR', lat: 48.8566, lng: 2.3522, pm25: 18, aqi: 64, riskLevel: 'moderate', pollutant: 'pm25', measuredAt: 1000, source: 'OpenAQ' }, + ]; + const waqiStations = [ + { city: 'Paris', countryCode: 'FR', lat: 48.8571, lng: 2.3519, pm25: 20, aqi: 68, riskLevel: 'moderate', pollutant: 'pm25', measuredAt: 1100, source: 'WAQI' }, + { city: 'Dubai', countryCode: 'AE', lat: 25.2048, lng: 55.2708, pm25: 50, aqi: 137, riskLevel: 'unhealthy', pollutant: 'pm25', measuredAt: 1200, source: 'WAQI' }, + ]; + + const merged = mergeAirQualityStations(openAqStations, waqiStations); + + assert.equal(merged.length, 2); + assert.equal(merged[0].city, 'Dubai'); + }); + + it('builds the final payload with fetchedAt and sorted stations', () => { + const nowMs = Date.UTC(2026, 3, 3, 12, 0, 0); + const payload = buildAirQualityPayload({ + locations: [ + { + id: 11, + locality: 'Lahore', + country: { code: 'PK' }, + coordinates: { latitude: 31.52, longitude: 74.36 }, + }, + ], + latestMeasurements: [ + { + locationsId: 11, + value: 145.6, + datetime: { utc: new Date(nowMs - (15 * 60 * 1000)).toISOString() }, + coordinates: { latitude: 31.52, longitude: 74.36 }, + parameter: { name: 'pm25' }, + }, + ], + waqiStations: [], + nowMs, + }); + + assert.equal(payload.fetchedAt, nowMs); + assert.equal(payload.stations.length, 1); + assert.equal(payload.stations[0].city, 'Lahore'); + assert.equal(payload.stations[0].country_code, 'PK'); + assert.equal(payload.stations[0].risk_level, 'unhealthy'); + assert.equal(typeof payload.stations[0].measured_at, 'number'); + assert.equal('riskLevel' in payload.stations[0], false); + }); + + it('normalizes legacy raw waqiEntries before merging them into the payload', () => { + const nowMs = Date.UTC(2026, 3, 3, 12, 0, 0); + const payload = buildAirQualityPayload({ + locations: [], + latestMeasurements: [], + waqiEntries: [ + { + lat: 25.2, + lon: 55.27, + aqi: '180', + dominentpol: 'pm25', + iaqi: { pm25: { v: 74.1 } }, + station: { + name: 'Dubai, AE', + time: new Date(nowMs - (20 * 60 * 1000)).toISOString(), + }, + }, + ], + nowMs, + }); + + assert.equal(payload.fetchedAt, nowMs); + assert.equal(payload.stations.length, 1); + assert.equal(payload.stations[0].city, 'Dubai'); + assert.equal(payload.stations[0].country_code, 'AE'); + assert.equal(payload.stations[0].risk_level, 'unhealthy'); + assert.equal(typeof payload.stations[0].measured_at, 'number'); + assert.equal('riskLevel' in payload.stations[0], false); + }); + + it('builds one Redis pipeline containing both mirrored keys and both seed-meta keys', () => { + const payload = { + stations: [ + { + city: 'Delhi', + country_code: 'IN', + lat: 28.61, + lng: 77.21, + pm25: 80.4, + aqi: 164, + risk_level: 'unhealthy', + pollutant: 'pm25', + measured_at: 123, + source: 'OpenAQ', + }, + ], + fetchedAt: 456, + }; + + const commands = buildMirrorWriteCommands(payload, 3600, 789, 'source-v1'); + + assert.equal(commands.length, 4); + assert.deepEqual(commands.map((command) => command[1]), [ + HEALTH_AIR_QUALITY_KEY, + CLIMATE_AIR_QUALITY_KEY, + OPENAQ_META_KEY, + CLIMATE_META_KEY, + ]); + assert.equal(commands[0][4], '3600'); + assert.equal(commands[1][4], '3600'); + assert.match(String(commands[2][2]), /"recordCount":1/); + assert.match(String(commands[3][2]), /"sourceVersion":"source-v1"/); + }); +});