From 3602084789b29580533d1d813cbbcbadcfecee39 Mon Sep 17 00:00:00 2001 From: fayez bast Date: Mon, 30 Mar 2026 03:02:42 +0300 Subject: [PATCH 1/4] feat(climate): add WMO normals seeding and CO2 monitoring --- api/bootstrap.js | 4 +- api/health.js | 5 + api/mcp.ts | 4 +- api/seed-health.js | 2 + docs/api/ClimateService.openapi.json | 2 +- docs/api/ClimateService.openapi.yaml | 79 ++++++- .../climate/v1/climate_anomaly.proto | 2 +- .../climate/v1/co2_monitoring.proto | 22 ++ .../climate/v1/get_co2_monitoring.proto | 12 + proto/worldmonitor/climate/v1/service.proto | 6 + scripts/_climate-zones.mjs | 41 ++++ scripts/_open-meteo-archive.mjs | 78 ++++++ scripts/seed-climate-anomalies.mjs | 186 +++++++++------ scripts/seed-climate-zone-normals.mjs | 149 ++++++++++++ scripts/seed-co2-monitoring.mjs | 206 ++++++++++++++++ server/_shared/cache-keys.ts | 5 +- server/gateway.ts | 1 + .../climate/v1/get-co2-monitoring.ts | 22 ++ server/worldmonitor/climate/v1/handler.ts | 2 + .../climate/v1/list-climate-anomalies.ts | 2 +- .../worldmonitor/climate/v1/service_client.ts | 49 ++++ .../worldmonitor/climate/v1/service_server.ts | 64 +++++ src/services/climate/index.ts | 62 ++++- tests/bootstrap.test.mjs | 8 +- tests/climate-seeds.test.mjs | 222 ++++++++++++++++++ 25 files changed, 1149 insertions(+), 86 deletions(-) create mode 100644 proto/worldmonitor/climate/v1/co2_monitoring.proto create mode 100644 proto/worldmonitor/climate/v1/get_co2_monitoring.proto create mode 100644 scripts/_climate-zones.mjs create mode 100644 scripts/_open-meteo-archive.mjs create mode 100644 scripts/seed-climate-zone-normals.mjs create mode 100644 scripts/seed-co2-monitoring.mjs create mode 100644 server/worldmonitor/climate/v1/get-co2-monitoring.ts create mode 100644 tests/climate-seeds.test.mjs diff --git a/api/bootstrap.js b/api/bootstrap.js index 0ab6aaa3ea..0050750830 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -26,6 +26,8 @@ const BOOTSTRAP_CACHE_KEYS = { minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', climateAnomalies: 'climate:anomalies:v1', + climateZoneNormals: 'climate:zone-normals:v1', + co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', crossSourceSignals: 'intelligence:cross-source-signals:v1', @@ -85,7 +87,7 @@ const BOOTSTRAP_CACHE_KEYS = { const SLOW_KEYS = new Set([ 'bisPolicy', 'bisExchange', 'bisCredit', 'minerals', 'giving', - 'sectors', 'etfFlows', 'wildfires', 'climateAnomalies', + 'sectors', 'etfFlows', 'wildfires', 'climateAnomalies', 'climateZoneNormals', 'co2Monitoring', 'radiationWatch', 'thermalEscalation', 'crossSourceSignals', 'cyberThreats', 'techReadiness', 'progressData', 'renewableEnergy', 'naturalEvents', diff --git a/api/health.js b/api/health.js index b37f13177a..e368aba5a4 100644 --- a/api/health.js +++ b/api/health.js @@ -10,6 +10,8 @@ const BOOTSTRAP_KEYS = { sectors: 'market:sectors:v1', etfFlows: 'market:etf-flows:v1', climateAnomalies: 'climate:anomalies:v1', + climateZoneNormals: 'climate:zone-normals:v1', + co2Monitoring: 'climate:co2-monitoring:v1', wildfires: 'wildfire:fires:v1', marketQuotes: 'market:stocks-bootstrap:v1', commodityQuotes: 'market:commodities-bootstrap:v1', @@ -84,6 +86,7 @@ const STANDALONE_KEYS = { bisPolicy: 'economic:bis:policy:v1', bisExchange: 'economic:bis:eer:v1', bisCredit: 'economic:bis:credit:v1', + climateZoneNormals: 'climate:zone-normals:v1', shippingRates: 'supply_chain:shipping:v2', chokepoints: 'supply_chain:chokepoints:v4', minerals: 'supply_chain:minerals:v2', @@ -127,6 +130,8 @@ const SEED_META = { wildfires: { key: 'seed-meta:wildfire:fires', maxStaleMin: 360 }, // FIRMS NRT resets at midnight UTC; new-day data takes 3-6h to accumulate outages: { key: 'seed-meta:infra:outages', maxStaleMin: 30 }, climateAnomalies: { key: 'seed-meta:climate:anomalies', maxStaleMin: 120 }, // runs as independent Railway cron (0 */2 * * *) + climateZoneNormals: { key: 'seed-meta:climate:zone-normals', maxStaleMin: 89280 }, // monthly cron on the 1st; 62d = 2x 31-day cadence + co2Monitoring: { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 2880 }, // daily cron at 06:00 UTC; 48h tolerates one missed run unrestEvents: { key: 'seed-meta:unrest:events', maxStaleMin: 120 }, // 45min cron; 120 = 2h grace (was 75 = 30min buffer, too tight) cyberThreats: { key: 'seed-meta:cyber:threats', maxStaleMin: 240 }, // 2h interval; 240min = 2x interval cryptoQuotes: { key: 'seed-meta:market:crypto', maxStaleMin: 30 }, diff --git a/api/mcp.ts b/api/mcp.ts index d5789b8126..a196c32805 100644 --- a/api/mcp.ts +++ b/api/mcp.ts @@ -180,9 +180,9 @@ const TOOL_REGISTRY: ToolDef[] = [ }, { name: 'get_climate_data', - description: 'Climate anomalies (Open-Meteo temperature/precipitation deviations), weather alerts, and natural environmental events from NASA EONET.', + description: 'Climate anomalies, NOAA atmospheric greenhouse gas monitoring, weather alerts, and natural environmental events from WorldMonitor climate feeds.', inputSchema: { type: 'object', properties: {}, required: [] }, - _cacheKeys: ['climate:anomalies:v1', 'weather:alerts:v1'], + _cacheKeys: ['climate:anomalies:v1', 'climate:co2-monitoring:v1', 'weather:alerts:v1'], _seedMetaKey: 'seed-meta:climate:anomalies', _maxStaleMin: 120, }, diff --git a/api/seed-health.js b/api/seed-health.js index 7fd6ea59b4..4d41eab764 100644 --- a/api/seed-health.js +++ b/api/seed-health.js @@ -12,6 +12,8 @@ const SEED_DOMAINS = { 'wildfire:fires': { key: 'seed-meta:wildfire:fires', intervalMin: 60 }, 'infra:outages': { key: 'seed-meta:infra:outages', intervalMin: 15 }, 'climate:anomalies': { key: 'seed-meta:climate:anomalies', intervalMin: 60 }, + 'climate:zone-normals': { key: 'seed-meta:climate:zone-normals', intervalMin: 44640 }, + 'climate:co2-monitoring': { key: 'seed-meta:climate:co2-monitoring', intervalMin: 1440 }, // Phase 2 — Parameterized endpoints 'unrest:events': { key: 'seed-meta:unrest:events', intervalMin: 15 }, 'cyber:threats': { key: 'seed-meta:cyber:threats', intervalMin: 240 }, diff --git a/docs/api/ClimateService.openapi.json b/docs/api/ClimateService.openapi.json index 281d57c8e1..c0f51f403c 100644 --- a/docs/api/ClimateService.openapi.json +++ b/docs/api/ClimateService.openapi.json @@ -1 +1 @@ -{"components":{"schemas":{"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal as a percentage.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}}}} \ No newline at end of file +{"components":{"schemas":{"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal in millimeters.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"Co2DataPoint":{"properties":{"anomaly":{"format":"double","type":"number"},"month":{"type":"string"},"ppm":{"format":"double","type":"number"}},"type":"object"},"Co2Monitoring":{"properties":{"annualGrowthRate":{"format":"double","type":"number"},"currentPpm":{"format":"double","type":"number"},"measuredAt":{"format":"int64","type":"string"},"methanePpb":{"format":"double","type":"number"},"monthlyAverage":{"format":"double","type":"number"},"nitrousOxidePpb":{"format":"double","type":"number"},"preIndustrialBaseline":{"format":"double","type":"number"},"station":{"type":"string"},"trend12m":{"items":{"$ref":"#/components/schemas/Co2DataPoint"},"type":"array"},"yearAgoPpm":{"format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetCo2MonitoringRequest":{"type":"object"},"GetCo2MonitoringResponse":{"properties":{"monitoring":{"$ref":"#/components/schemas/Co2Monitoring"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/get-co2-monitoring":{"get":{"description":"GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data.","operationId":"GetCo2Monitoring","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetCo2MonitoringResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetCo2Monitoring","tags":["ClimateService"]}},"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}}}} \ No newline at end of file diff --git a/docs/api/ClimateService.openapi.yaml b/docs/api/ClimateService.openapi.yaml index fadfe6b047..be737aa4aa 100644 --- a/docs/api/ClimateService.openapi.yaml +++ b/docs/api/ClimateService.openapi.yaml @@ -49,6 +49,32 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /api/climate/v1/get-co2-monitoring: + get: + tags: + - ClimateService + summary: GetCo2Monitoring + description: GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data. + operationId: GetCo2Monitoring + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/GetCo2MonitoringResponse' + "400": + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + default: + description: Error response + content: + application/json: + schema: + $ref: '#/components/schemas/Error' components: schemas: Error: @@ -129,7 +155,7 @@ components: precipDelta: type: number format: double - description: Precipitation deviation from normal as a percentage. + description: Precipitation deviation from normal in millimeters. severity: type: string enum: @@ -189,3 +215,54 @@ components: format: int32 description: Total count of items matching the query, if known. Zero if the total is unknown. description: PaginationResponse contains pagination metadata returned alongside list results. + GetCo2MonitoringRequest: + type: object + GetCo2MonitoringResponse: + type: object + properties: + monitoring: + $ref: '#/components/schemas/Co2Monitoring' + Co2Monitoring: + type: object + properties: + currentPpm: + type: number + format: double + yearAgoPpm: + type: number + format: double + annualGrowthRate: + type: number + format: double + preIndustrialBaseline: + type: number + format: double + monthlyAverage: + type: number + format: double + trend12m: + type: array + items: + $ref: '#/components/schemas/Co2DataPoint' + methanePpb: + type: number + format: double + nitrousOxidePpb: + type: number + format: double + measuredAt: + type: string + format: int64 + station: + type: string + Co2DataPoint: + type: object + properties: + month: + type: string + ppm: + type: number + format: double + anomaly: + type: number + format: double diff --git a/proto/worldmonitor/climate/v1/climate_anomaly.proto b/proto/worldmonitor/climate/v1/climate_anomaly.proto index 599bf04244..1a72b0abef 100644 --- a/proto/worldmonitor/climate/v1/climate_anomaly.proto +++ b/proto/worldmonitor/climate/v1/climate_anomaly.proto @@ -17,7 +17,7 @@ message ClimateAnomaly { worldmonitor.core.v1.GeoCoordinates location = 2; // Temperature deviation from normal in degrees Celsius. double temp_delta = 3; - // Precipitation deviation from normal as a percentage. + // Precipitation deviation from normal in millimeters. double precip_delta = 4; // Severity classification of the anomaly. AnomalySeverity severity = 5; diff --git a/proto/worldmonitor/climate/v1/co2_monitoring.proto b/proto/worldmonitor/climate/v1/co2_monitoring.proto new file mode 100644 index 0000000000..4d82a2d179 --- /dev/null +++ b/proto/worldmonitor/climate/v1/co2_monitoring.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package worldmonitor.climate.v1; + +message Co2Monitoring { + double current_ppm = 1; + double year_ago_ppm = 2; + double annual_growth_rate = 3; + double pre_industrial_baseline = 4; + double monthly_average = 5; + repeated Co2DataPoint trend_12m = 6; + double methane_ppb = 7; + double nitrous_oxide_ppb = 8; + int64 measured_at = 9; + string station = 10; +} + +message Co2DataPoint { + string month = 1; + double ppm = 2; + double anomaly = 3; +} diff --git a/proto/worldmonitor/climate/v1/get_co2_monitoring.proto b/proto/worldmonitor/climate/v1/get_co2_monitoring.proto new file mode 100644 index 0000000000..ff15f7d6db --- /dev/null +++ b/proto/worldmonitor/climate/v1/get_co2_monitoring.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package worldmonitor.climate.v1; + +import "sebuf/http/annotations.proto"; +import "worldmonitor/climate/v1/co2_monitoring.proto"; + +message GetCo2MonitoringRequest {} + +message GetCo2MonitoringResponse { + Co2Monitoring monitoring = 1; +} diff --git a/proto/worldmonitor/climate/v1/service.proto b/proto/worldmonitor/climate/v1/service.proto index 30731143be..dedb0042fa 100644 --- a/proto/worldmonitor/climate/v1/service.proto +++ b/proto/worldmonitor/climate/v1/service.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package worldmonitor.climate.v1; import "sebuf/http/annotations.proto"; +import "worldmonitor/climate/v1/get_co2_monitoring.proto"; import "worldmonitor/climate/v1/list_climate_anomalies.proto"; // ClimateService provides APIs for climate anomaly data sourced from Open-Meteo. @@ -13,4 +14,9 @@ service ClimateService { rpc ListClimateAnomalies(ListClimateAnomaliesRequest) returns (ListClimateAnomaliesResponse) { option (sebuf.http.config) = {path: "/list-climate-anomalies", method: HTTP_METHOD_GET}; } + + // GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data. + rpc GetCo2Monitoring(GetCo2MonitoringRequest) returns (GetCo2MonitoringResponse) { + option (sebuf.http.config) = {path: "/get-co2-monitoring", method: HTTP_METHOD_GET}; + } } diff --git a/scripts/_climate-zones.mjs b/scripts/_climate-zones.mjs new file mode 100644 index 0000000000..7b4e9b61e1 --- /dev/null +++ b/scripts/_climate-zones.mjs @@ -0,0 +1,41 @@ +export const CLIMATE_ZONES = [ + { name: 'Ukraine', lat: 48.4, lon: 31.2 }, + { name: 'Middle East', lat: 33.0, lon: 44.0 }, + { name: 'Sahel', lat: 14.0, lon: 0.0 }, + { name: 'Horn of Africa', lat: 8.0, lon: 42.0 }, + { name: 'South Asia', lat: 25.0, lon: 78.0 }, + { name: 'California', lat: 36.8, lon: -119.4 }, + { name: 'Amazon', lat: -3.4, lon: -60.0 }, + { name: 'Australia', lat: -25.0, lon: 134.0 }, + { name: 'Mediterranean', lat: 38.0, lon: 20.0 }, + { name: 'Taiwan Strait', lat: 24.0, lon: 120.0 }, + { name: 'Myanmar', lat: 19.8, lon: 96.7 }, + { name: 'Central Africa', lat: 4.0, lon: 22.0 }, + { name: 'Southern Africa', lat: -25.0, lon: 28.0 }, + { name: 'Central Asia', lat: 42.0, lon: 65.0 }, + { name: 'Caribbean', lat: 19.0, lon: -72.0 }, + { name: 'Arctic', lat: 70.0, lon: 0.0 }, + { name: 'Greenland', lat: 72.0, lon: -42.0 }, + { name: 'Western Antarctic Ice Sheet', lat: -78.0, lon: -100.0 }, + { name: 'Tibetan Plateau', lat: 31.0, lon: 91.0 }, + { name: 'Congo Basin', lat: -1.0, lon: 24.0 }, + { name: 'Coral Triangle', lat: -5.0, lon: 128.0 }, + { name: 'North Atlantic', lat: 55.0, lon: -30.0 }, +]; + +export const REQUIRED_CLIMATE_ZONE_NAMES = [ + 'Arctic', + 'Greenland', + 'Western Antarctic Ice Sheet', + 'Tibetan Plateau', + 'Congo Basin', + 'Coral Triangle', + 'North Atlantic', +]; + +export const MIN_CLIMATE_ZONE_COUNT = Math.ceil(CLIMATE_ZONES.length * 2 / 3); + +export function hasRequiredClimateZones(items, getName = (item) => item?.zone ?? item?.name) { + const present = new Set(items.map((item) => getName(item)).filter(Boolean)); + return REQUIRED_CLIMATE_ZONE_NAMES.every((name) => present.has(name)); +} diff --git a/scripts/_open-meteo-archive.mjs b/scripts/_open-meteo-archive.mjs new file mode 100644 index 0000000000..8dd880df83 --- /dev/null +++ b/scripts/_open-meteo-archive.mjs @@ -0,0 +1,78 @@ +import { CHROME_UA, sleep } from './_seed-utils.mjs'; + +export function chunkItems(items, size) { + const chunks = []; + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)); + } + return chunks; +} + +export function normalizeArchiveBatchResponse(payload) { + return Array.isArray(payload) ? payload : [payload]; +} + +function parseRetryAfterMs(value) { + if (!value) return null; + + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds > 0) { + return seconds * 1000; + } + + const retryAt = Date.parse(value); + if (Number.isFinite(retryAt)) { + return Math.max(retryAt - Date.now(), 1000); + } + + return null; +} + +export async function fetchOpenMeteoArchiveBatch(zones, opts) { + const { + startDate, + endDate, + daily, + timezone = 'UTC', + timeoutMs = 30_000, + maxRetries = 3, + retryBaseMs = 2_000, + label = zones.map((zone) => zone.name).join(', '), + } = opts; + + const params = new URLSearchParams({ + latitude: zones.map((zone) => String(zone.lat)).join(','), + longitude: zones.map((zone) => String(zone.lon)).join(','), + start_date: startDate, + end_date: endDate, + daily: daily.join(','), + timezone, + }); + const url = `https://archive-api.open-meteo.com/v1/archive?${params.toString()}`; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(timeoutMs), + }); + + if (resp.ok) { + const data = normalizeArchiveBatchResponse(await resp.json()); + if (data.length !== zones.length) { + throw new Error(`Open-Meteo batch size mismatch for ${label}: expected ${zones.length}, got ${data.length}`); + } + return data; + } + + if (resp.status === 429 && attempt < maxRetries) { + const retryMs = parseRetryAfterMs(resp.headers.get('retry-after')) ?? (retryBaseMs * 2 ** attempt); + console.log(` [OPEN_METEO] 429 for ${label}; retrying batch in ${Math.round(retryMs / 1000)}s`); + await sleep(retryMs); + continue; + } + + throw new Error(`Open-Meteo ${resp.status} for ${label}`); + } + + throw new Error(`Open-Meteo retries exhausted for ${label}`); +} diff --git a/scripts/seed-climate-anomalies.mjs b/scripts/seed-climate-anomalies.mjs index c9e500f0ed..03de6bac3b 100755 --- a/scripts/seed-climate-anomalies.mjs +++ b/scripts/seed-climate-anomalies.mjs @@ -1,87 +1,85 @@ #!/usr/bin/env node -import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs'; +import { loadEnvFile, runSeed, sleep, verifySeedKey } from './_seed-utils.mjs'; +import { CLIMATE_ZONES, MIN_CLIMATE_ZONE_COUNT, hasRequiredClimateZones } from './_climate-zones.mjs'; +import { chunkItems, fetchOpenMeteoArchiveBatch } from './_open-meteo-archive.mjs'; +import { CLIMATE_ZONE_NORMALS_KEY } from './seed-climate-zone-normals.mjs'; loadEnvFile(import.meta.url); const CANONICAL_KEY = 'climate:anomalies:v1'; const CACHE_TTL = 10800; // 3h - -const ZONES = [ - { name: 'Ukraine', lat: 48.4, lon: 31.2 }, - { name: 'Middle East', lat: 33.0, lon: 44.0 }, - { name: 'Sahel', lat: 14.0, lon: 0.0 }, - { name: 'Horn of Africa', lat: 8.0, lon: 42.0 }, - { name: 'South Asia', lat: 25.0, lon: 78.0 }, - { name: 'California', lat: 36.8, lon: -119.4 }, - { name: 'Amazon', lat: -3.4, lon: -60.0 }, - { name: 'Australia', lat: -25.0, lon: 134.0 }, - { name: 'Mediterranean', lat: 38.0, lon: 20.0 }, - { name: 'Taiwan Strait', lat: 24.0, lon: 120.0 }, - { name: 'Myanmar', lat: 19.8, lon: 96.7 }, - { name: 'Central Africa', lat: 4.0, lon: 22.0 }, - { name: 'Southern Africa', lat: -25.0, lon: 28.0 }, - { name: 'Central Asia', lat: 42.0, lon: 65.0 }, - { name: 'Caribbean', lat: 19.0, lon: -72.0 }, -]; +const ANOMALY_BATCH_SIZE = 8; +const ANOMALY_BATCH_DELAY_MS = 750; +const PRECIP_MODERATE_THRESHOLD = 6; +const PRECIP_EXTREME_THRESHOLD = 12; +const PRECIP_MIXED_THRESHOLD = 3; +const TEMP_TO_PRECIP_RATIO = 3; function avg(arr) { - return arr.length ? arr.reduce((s, v) => s + v, 0) / arr.length : 0; + return arr.length ? arr.reduce((sum, value) => sum + value, 0) / arr.length : 0; +} + +function round(value, decimals = 1) { + const scale = 10 ** decimals; + return Math.round(value * scale) / scale; } function classifySeverity(tempDelta, precipDelta) { const absTemp = Math.abs(tempDelta); const absPrecip = Math.abs(precipDelta); - if (absTemp >= 5 || absPrecip >= 80) return 'ANOMALY_SEVERITY_EXTREME'; - if (absTemp >= 3 || absPrecip >= 40) return 'ANOMALY_SEVERITY_MODERATE'; + if (absTemp >= 5 || absPrecip >= PRECIP_EXTREME_THRESHOLD) return 'ANOMALY_SEVERITY_EXTREME'; + if (absTemp >= 3 || absPrecip >= PRECIP_MODERATE_THRESHOLD) return 'ANOMALY_SEVERITY_MODERATE'; return 'ANOMALY_SEVERITY_NORMAL'; } function classifyType(tempDelta, precipDelta) { const absTemp = Math.abs(tempDelta); const absPrecip = Math.abs(precipDelta); - if (absTemp >= absPrecip / 20) { - if (tempDelta > 0 && precipDelta < -20) return 'ANOMALY_TYPE_MIXED'; + if (absTemp >= absPrecip / TEMP_TO_PRECIP_RATIO) { + if (tempDelta > 0 && precipDelta < -PRECIP_MIXED_THRESHOLD) return 'ANOMALY_TYPE_MIXED'; if (tempDelta > 3) return 'ANOMALY_TYPE_WARM'; if (tempDelta < -3) return 'ANOMALY_TYPE_COLD'; } - if (precipDelta > 40) return 'ANOMALY_TYPE_WET'; - if (precipDelta < -40) return 'ANOMALY_TYPE_DRY'; + if (precipDelta > PRECIP_MODERATE_THRESHOLD) return 'ANOMALY_TYPE_WET'; + if (precipDelta < -PRECIP_MODERATE_THRESHOLD) return 'ANOMALY_TYPE_DRY'; if (tempDelta > 0) return 'ANOMALY_TYPE_WARM'; return 'ANOMALY_TYPE_COLD'; } -async function fetchZone(zone, startDate, endDate) { - const url = `https://archive-api.open-meteo.com/v1/archive?latitude=${zone.lat}&longitude=${zone.lon}&start_date=${startDate}&end_date=${endDate}&daily=temperature_2m_mean,precipitation_sum&timezone=UTC`; - - const resp = await fetch(url, { - headers: { 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(20_000), - }); - if (!resp.ok) throw new Error(`Open-Meteo ${resp.status} for ${zone.name}`); - - const data = await resp.json(); - - const rawTemps = data.daily?.temperature_2m_mean ?? []; - const rawPrecips = data.daily?.precipitation_sum ?? []; - const temps = []; - const precips = []; - for (let i = 0; i < rawTemps.length; i++) { - if (rawTemps[i] != null && rawPrecips[i] != null) { - temps.push(rawTemps[i]); - precips.push(rawPrecips[i]); +export function indexZoneNormals(payload) { + const index = new Map(); + for (const zone of payload?.normals ?? []) { + for (const month of zone?.months ?? []) { + index.set(`${zone.zone}:${month.month}`, month); } } + return index; +} - if (temps.length < 14) return null; +export function buildClimateAnomaly(zone, daily, monthlyNormal) { + const observations = []; + const times = daily?.time ?? []; + const temps = daily?.temperature_2m_mean ?? []; + const precips = daily?.precipitation_sum ?? []; + + for (let i = 0; i < times.length; i++) { + const time = times[i]; + const temp = temps[i]; + const precip = precips[i]; + if (typeof time !== 'string' || temp == null || precip == null) continue; + observations.push({ + date: time, + temp: Number(temp), + precip: Number(precip), + }); + } - const recentTemps = temps.slice(-7); - const baselineTemps = temps.slice(0, -7); - const recentPrecips = precips.slice(-7); - const baselinePrecips = precips.slice(0, -7); + if (observations.length < 7) return null; - const tempDelta = Math.round((avg(recentTemps) - avg(baselineTemps)) * 10) / 10; - const precipDelta = Math.round((avg(recentPrecips) - avg(baselinePrecips)) * 10) / 10; + const recent = observations.slice(-7); + const tempDelta = round(avg(recent.map((entry) => entry.temp)) - monthlyNormal.tempMean); + const precipDelta = round(avg(recent.map((entry) => entry.precip)) - monthlyNormal.precipMean); return { zone: zone.name, @@ -90,44 +88,86 @@ async function fetchZone(zone, startDate, endDate) { precipDelta, severity: classifySeverity(tempDelta, precipDelta), type: classifyType(tempDelta, precipDelta), - period: `${startDate} to ${endDate}`, + period: `${recent[0].date} to ${recent.at(-1).date}`, }; } -async function fetchClimateAnomalies() { - const endDate = new Date().toISOString().slice(0, 10); - const startDate = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); +export function buildClimateAnomalyFromResponse(zone, payload, normalsIndex) { + const latestDate = payload?.daily?.time?.filter((value) => typeof value === 'string').at(-1); + if (!latestDate) return null; + const month = Number(latestDate.slice(5, 7)); + const monthlyNormal = normalsIndex.get(`${zone.name}:${month}`); + if (!monthlyNormal) throw new Error(`Missing monthly normal for ${zone.name} month ${month}`); + + return buildClimateAnomaly(zone, payload.daily, monthlyNormal); +} + +export function buildClimateAnomaliesFromBatch(zones, batchPayloads, normalsIndex) { + return zones + .map((zone, index) => buildClimateAnomalyFromResponse(zone, batchPayloads[index], normalsIndex)) + .filter((anomaly) => anomaly != null); +} + +function toIsoDate(date) { + return date.toISOString().slice(0, 10); +} + +export async function fetchClimateAnomalies() { + const normalsPayload = await verifySeedKey(CLIMATE_ZONE_NORMALS_KEY).catch(() => null); + if (!normalsPayload?.normals?.length) { + throw new Error(`Missing ${CLIMATE_ZONE_NORMALS_KEY} baseline; run seed-climate-zone-normals.mjs first`); + } + const normalsIndex = indexZoneNormals(normalsPayload); + + const endDate = toIsoDate(new Date()); + const startDate = toIsoDate(new Date(Date.now() - 21 * 24 * 60 * 60 * 1000)); const anomalies = []; let failures = 0; - for (const zone of ZONES) { + for (const batch of chunkItems(CLIMATE_ZONES, ANOMALY_BATCH_SIZE)) { try { - const result = await fetchZone(zone, startDate, endDate); - if (result != null) anomalies.push(result); + const payloads = await fetchOpenMeteoArchiveBatch(batch, { + startDate, + endDate, + daily: ['temperature_2m_mean', 'precipitation_sum'], + timeoutMs: 20_000, + maxRetries: 4, + retryBaseMs: 3_000, + label: `anomalies batch (${batch.map((zone) => zone.name).join(', ')})`, + }); + anomalies.push(...buildClimateAnomaliesFromBatch(batch, payloads, normalsIndex)); } catch (err) { console.log(` [CLIMATE] ${err?.message ?? err}`); - failures++; + failures += batch.length; } - await new Promise((r) => setTimeout(r, 200)); + await sleep(ANOMALY_BATCH_DELAY_MS); } - const MIN_ZONES = Math.ceil(ZONES.length * 2 / 3); - if (anomalies.length < MIN_ZONES) { - throw new Error(`Only ${anomalies.length}/${ZONES.length} zones returned data (${failures} errors) — skipping write to preserve previous Redis data`); + if (anomalies.length < MIN_CLIMATE_ZONE_COUNT) { + throw new Error(`Only ${anomalies.length}/${CLIMATE_ZONES.length} zones returned data (${failures} errors) — skipping write to preserve previous Redis data`); + } + if (!hasRequiredClimateZones(anomalies, (zone) => zone.zone)) { + throw new Error('Missing one or more required climate-specific anomalies'); } return { anomalies, pagination: undefined }; } function validate(data) { - return Array.isArray(data?.anomalies) && data.anomalies.length >= Math.ceil(ZONES.length * 2 / 3); + return Array.isArray(data?.anomalies) + && data.anomalies.length >= MIN_CLIMATE_ZONE_COUNT + && hasRequiredClimateZones(data.anomalies, (zone) => zone.zone); } -runSeed('climate', 'anomalies', CANONICAL_KEY, fetchClimateAnomalies, { - validateFn: validate, - ttlSeconds: CACHE_TTL, - sourceVersion: 'open-meteo-archive-30d', -}).catch((err) => { - const _cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + _cause); - process.exit(1); -}); +const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^file:\/\//, '')); +if (isMain) { + runSeed('climate', 'anomalies', CANONICAL_KEY, fetchClimateAnomalies, { + validateFn: validate, + ttlSeconds: CACHE_TTL, + sourceVersion: 'open-meteo-archive-wmo-1991-2020-v1', + }).catch((err) => { + const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; + console.error('FATAL:', (err.message || err) + cause); + process.exit(1); + }); +} diff --git a/scripts/seed-climate-zone-normals.mjs b/scripts/seed-climate-zone-normals.mjs new file mode 100644 index 0000000000..dec7dde71e --- /dev/null +++ b/scripts/seed-climate-zone-normals.mjs @@ -0,0 +1,149 @@ +#!/usr/bin/env node + +import { loadEnvFile, runSeed, sleep } from './_seed-utils.mjs'; +import { CLIMATE_ZONES, MIN_CLIMATE_ZONE_COUNT, hasRequiredClimateZones } from './_climate-zones.mjs'; +import { chunkItems, fetchOpenMeteoArchiveBatch } from './_open-meteo-archive.mjs'; + +loadEnvFile(import.meta.url); + +export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; +const NORMALS_TTL = 90 * 24 * 60 * 60; // 90 days +const NORMALS_START = '1991-01-01'; +const NORMALS_END = '2020-12-31'; +const NORMALS_BATCH_SIZE = 2; +const NORMALS_BATCH_DELAY_MS = 3_000; + +function round(value, decimals = 2) { + const scale = 10 ** decimals; + return Math.round(value * scale) / scale; +} + +function average(values) { + return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0; +} + +export function computeMonthlyNormals(daily) { + const dailyBucketByYearMonth = new Map(); + for (let month = 1; month <= 12; month++) { + dailyBucketByYearMonth.set(month, []); + } + + const times = daily?.time ?? []; + const temps = daily?.temperature_2m_mean ?? []; + const precips = daily?.precipitation_sum ?? []; + + for (let i = 0; i < times.length; i++) { + const time = times[i]; + const temp = temps[i]; + const precip = precips[i]; + if (typeof time !== 'string' || temp == null || precip == null) continue; + const year = Number(time.slice(0, 4)); + const month = Number(time.slice(5, 7)); + if (!Number.isInteger(year) || !Number.isInteger(month) || month < 1 || month > 12) continue; + const key = `${year}-${String(month).padStart(2, '0')}`; + const bucket = dailyBucketByYearMonth.get(month); + const existing = bucket.find((entry) => entry.key === key); + if (existing) { + existing.temps.push(Number(temp)); + existing.precips.push(Number(precip)); + continue; + } + bucket.push({ + key, + temps: [Number(temp)], + precips: [Number(precip)], + }); + } + + return Array.from(dailyBucketByYearMonth.entries()) + .map(([month, bucket]) => { + const monthlyMeans = bucket + .map((entry) => ({ + tempMean: average(entry.temps), + precipMean: average(entry.precips), + })) + .filter((entry) => Number.isFinite(entry.tempMean) && Number.isFinite(entry.precipMean)); + + return { + month, + sampleCount: monthlyMeans.length, + tempMean: round(average(monthlyMeans.map((entry) => entry.tempMean))), + precipMean: round(average(monthlyMeans.map((entry) => entry.precipMean))), + }; + }) + .filter((entry) => entry.sampleCount > 0 && Number.isFinite(entry.tempMean) && Number.isFinite(entry.precipMean)) + .map(({ sampleCount: _sampleCount, ...entry }) => entry); +} + +export function buildZoneNormalsFromBatch(zones, batchPayloads) { + return zones.map((zone, index) => { + const data = batchPayloads[index]; + const months = computeMonthlyNormals(data?.daily); + if (months.length !== 12) { + throw new Error(`Open-Meteo normals incomplete for ${zone.name}: expected 12 months, got ${months.length}`); + } + + return { + zone: zone.name, + location: { latitude: zone.lat, longitude: zone.lon }, + months, + }; + }); +} + +export async function fetchClimateZoneNormals() { + const normals = []; + let failures = 0; + + for (const batch of chunkItems(CLIMATE_ZONES, NORMALS_BATCH_SIZE)) { + try { + const payloads = await fetchOpenMeteoArchiveBatch(batch, { + startDate: NORMALS_START, + endDate: NORMALS_END, + daily: ['temperature_2m_mean', 'precipitation_sum'], + timeoutMs: 30_000, + maxRetries: 4, + retryBaseMs: 5_000, + label: `normals batch (${batch.map((zone) => zone.name).join(', ')})`, + }); + normals.push(...buildZoneNormalsFromBatch(batch, payloads)); + } catch (err) { + console.log(` [CLIMATE_NORMALS] ${err?.message ?? err}`); + failures += batch.length; + } + await sleep(NORMALS_BATCH_DELAY_MS); + } + + if (normals.length < MIN_CLIMATE_ZONE_COUNT) { + throw new Error(`Only ${normals.length}/${CLIMATE_ZONES.length} zones returned normals (${failures} errors)`); + } + if (!hasRequiredClimateZones(normals, (zone) => zone.zone)) { + throw new Error('Missing one or more required climate-specific zone normals'); + } + + return { + referencePeriod: '1991-2020', + fetchedAt: Date.now(), + normals, + }; +} + +function validate(data) { + return Array.isArray(data?.normals) + && data.normals.length >= MIN_CLIMATE_ZONE_COUNT + && hasRequiredClimateZones(data.normals, (zone) => zone.zone) + && data.normals.every((zone) => Array.isArray(zone?.months) && zone.months.length === 12); +} + +const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^file:\/\//, '')); +if (isMain) { + runSeed('climate', 'zone-normals', CLIMATE_ZONE_NORMALS_KEY, fetchClimateZoneNormals, { + validateFn: validate, + ttlSeconds: NORMALS_TTL, + sourceVersion: 'open-meteo-wmo-1991-2020-v1', + }).catch((err) => { + const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; + console.error('FATAL:', (err.message || err) + cause); + process.exit(1); + }); +} diff --git a/scripts/seed-co2-monitoring.mjs b/scripts/seed-co2-monitoring.mjs new file mode 100644 index 0000000000..c8b25381b4 --- /dev/null +++ b/scripts/seed-co2-monitoring.mjs @@ -0,0 +1,206 @@ +#!/usr/bin/env node + +import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs'; + +loadEnvFile(import.meta.url); + +export const CO2_MONITORING_KEY = 'climate:co2-monitoring:v1'; +const CACHE_TTL = 86400; +const PRE_INDUSTRIAL_BASELINE = 280.0; +const STATION = 'Mauna Loa, Hawaii'; + +const NOAA_URLS = { + dailyCo2: 'https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_daily_mlo.txt', + monthlyCo2: 'https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_mm_mlo.txt', + annualCo2Global: 'https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_annmean_gl.txt', + methaneMonthly: 'https://gml.noaa.gov/webdata/ccgg/trends/ch4/ch4_mm_gl.txt', + nitrousMonthly: 'https://gml.noaa.gov/webdata/ccgg/trends/n2o/n2o_mm_gl.txt', +}; + +function toEpochMs(year, month, day = 1) { + return Date.UTC(year, month - 1, day); +} + +function isValidMeasurement(value) { + return Number.isFinite(value) && value > 0; +} + +function formatMonth(year, month) { + return `${year}-${String(month).padStart(2, '0')}`; +} + +export function parseNoaaRows(text) { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => line.split(/\s+/)); +} + +export function parseCo2DailyRows(text) { + return parseNoaaRows(text) + .map((cols) => ({ + year: Number(cols[0]), + month: Number(cols[1]), + day: Number(cols[2]), + average: Number(cols[4]), + })) + .filter((row) => Number.isInteger(row.year) && Number.isInteger(row.month) && Number.isInteger(row.day) && isValidMeasurement(row.average)) + .sort((a, b) => toEpochMs(a.year, a.month, a.day) - toEpochMs(b.year, b.month, b.day)); +} + +export function parseCo2MonthlyRows(text) { + return parseNoaaRows(text) + .map((cols) => ({ + year: Number(cols[0]), + month: Number(cols[1]), + average: Number(cols[3]), + })) + .filter((row) => Number.isInteger(row.year) && Number.isInteger(row.month) && isValidMeasurement(row.average)) + .sort((a, b) => toEpochMs(a.year, a.month) - toEpochMs(b.year, b.month)); +} + +export function parseAnnualCo2Rows(text) { + return parseNoaaRows(text) + .map((cols) => ({ + year: Number(cols[0]), + mean: Number(cols[1]), + })) + .filter((row) => Number.isInteger(row.year) && isValidMeasurement(row.mean)) + .sort((a, b) => a.year - b.year); +} + +export function parseGlobalMonthlyPpbRows(text) { + return parseNoaaRows(text) + .map((cols) => ({ + year: Number(cols[0]), + month: Number(cols[1]), + average: Number(cols[3]), + })) + .filter((row) => Number.isInteger(row.year) && Number.isInteger(row.month) && isValidMeasurement(row.average)) + .sort((a, b) => toEpochMs(a.year, a.month) - toEpochMs(b.year, b.month)); +} + +function findClosestPriorYearValue(rows, latest) { + const exact = rows.find((row) => row.year === latest.year - 1 && row.month === latest.month && row.day === latest.day); + if (exact) return exact.average; + + const targetTime = toEpochMs(latest.year - 1, latest.month, latest.day); + const candidates = rows.filter((row) => row.year === latest.year - 1); + if (!candidates.length) return 0; + + const closest = candidates.reduce((best, row) => { + if (!best) return row; + const bestDelta = Math.abs(toEpochMs(best.year, best.month, best.day) - targetTime); + const rowDelta = Math.abs(toEpochMs(row.year, row.month, row.day) - targetTime); + if (rowDelta < bestDelta) return row; + if (rowDelta === bestDelta && toEpochMs(row.year, row.month, row.day) < toEpochMs(best.year, best.month, best.day)) { + return row; + } + return best; + }, null); + + return closest?.average ?? 0; +} + +export function buildTrend12m(monthlyRows) { + const byMonth = new Map(monthlyRows.map((row) => [formatMonth(row.year, row.month), row.average])); + return monthlyRows.slice(-12).map((row) => { + const prior = byMonth.get(formatMonth(row.year - 1, row.month)); + return { + month: formatMonth(row.year, row.month), + ppm: row.average, + anomaly: prior ? Math.round((row.average - prior) * 100) / 100 : 0, + }; + }); +} + +function findMonthlyAverageForLatestDaily(monthlyRows, latestDaily) { + const exact = monthlyRows.findLast((row) => row.year === latestDaily.year && row.month === latestDaily.month); + if (exact) return exact.average; + + const targetTime = toEpochMs(latestDaily.year, latestDaily.month); + const prior = monthlyRows.filter((row) => toEpochMs(row.year, row.month) <= targetTime).at(-1); + return prior?.average ?? 0; +} + +export function buildCo2MonitoringPayload({ dailyRows, monthlyRows, annualRows, methaneRows, nitrousRows }) { + const latestDaily = dailyRows.at(-1); + const monthlyAverage = latestDaily ? findMonthlyAverageForLatestDaily(monthlyRows, latestDaily) : 0; + const latestMethane = methaneRows.at(-1); + const latestNitrous = nitrousRows.at(-1); + const latestAnnual = annualRows.at(-1); + const previousAnnual = annualRows.at(-2); + + if (!latestDaily || !latestMethane || !latestNitrous || !latestAnnual || !previousAnnual || monthlyRows.length < 12 || monthlyAverage <= 0) { + throw new Error('Insufficient NOAA GML data to build CO2 monitoring payload'); + } + + return { + monitoring: { + currentPpm: latestDaily.average, + yearAgoPpm: findClosestPriorYearValue(dailyRows, latestDaily), + annualGrowthRate: Math.round((latestAnnual.mean - previousAnnual.mean) * 100) / 100, + preIndustrialBaseline: PRE_INDUSTRIAL_BASELINE, + monthlyAverage, + trend12m: buildTrend12m(monthlyRows), + methanePpb: latestMethane.average, + nitrousOxidePpb: latestNitrous.average, + measuredAt: String(toEpochMs(latestDaily.year, latestDaily.month, latestDaily.day)), + station: STATION, + }, + }; +} + +async function fetchText(url) { + const resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA, Accept: 'text/plain' }, + signal: AbortSignal.timeout(20_000), + }); + if (!resp.ok) throw new Error(`NOAA GML ${resp.status} for ${url}`); + return resp.text(); +} + +export async function fetchCo2Monitoring() { + const [dailyText, monthlyText, annualText, methaneText, nitrousText] = await Promise.all([ + fetchText(NOAA_URLS.dailyCo2), + fetchText(NOAA_URLS.monthlyCo2), + fetchText(NOAA_URLS.annualCo2Global), + fetchText(NOAA_URLS.methaneMonthly), + fetchText(NOAA_URLS.nitrousMonthly), + ]); + + return buildCo2MonitoringPayload({ + dailyRows: parseCo2DailyRows(dailyText), + monthlyRows: parseCo2MonthlyRows(monthlyText), + annualRows: parseAnnualCo2Rows(annualText), + methaneRows: parseGlobalMonthlyPpbRows(methaneText), + nitrousRows: parseGlobalMonthlyPpbRows(nitrousText), + }); +} + +function validate(data) { + const annualGrowthRate = data?.monitoring?.annualGrowthRate; + return data?.monitoring?.currentPpm > 0 + && data?.monitoring?.yearAgoPpm > 0 + && Number.isFinite(annualGrowthRate) + && data?.monitoring?.monthlyAverage > 0 + && data?.monitoring?.methanePpb > 0 + && data?.monitoring?.nitrousOxidePpb > 0 + && Array.isArray(data?.monitoring?.trend12m) + && data.monitoring.trend12m.length === 12; +} + +const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^file:\/\//, '')); +if (isMain) { + runSeed('climate', 'co2-monitoring', CO2_MONITORING_KEY, fetchCo2Monitoring, { + validateFn: validate, + ttlSeconds: CACHE_TTL, + recordCount: (data) => data?.monitoring?.trend12m?.length ?? 0, + sourceVersion: 'noaa-gml-co2-ch4-n2o-v1', + }).catch((err) => { + const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; + console.error('FATAL:', (err.message || err) + cause); + process.exit(1); + }); +} diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index fb48505afd..4c612c01cb 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -5,6 +5,7 @@ */ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; +export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; /** * Static cache keys for the bootstrap endpoint. @@ -28,6 +29,8 @@ export const BOOTSTRAP_CACHE_KEYS: Record = { minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', climateAnomalies: 'climate:anomalies:v1', + climateZoneNormals: 'climate:zone-normals:v1', + co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', crossSourceSignals: 'intelligence:cross-source-signals:v1', @@ -88,7 +91,7 @@ export const BOOTSTRAP_TIERS: Record = { minerals: 'slow', giving: 'slow', sectors: 'slow', progressData: 'slow', renewableEnergy: 'slow', etfFlows: 'slow', shippingRates: 'fast', wildfires: 'slow', - climateAnomalies: 'slow', sanctionsPressure: 'slow', radiationWatch: 'slow', thermalEscalation: 'slow', crossSourceSignals: 'slow', cyberThreats: 'slow', techReadiness: 'slow', + climateAnomalies: 'slow', climateZoneNormals: 'slow', co2Monitoring: 'slow', sanctionsPressure: 'slow', radiationWatch: 'slow', thermalEscalation: 'slow', crossSourceSignals: 'slow', cyberThreats: 'slow', techReadiness: 'slow', theaterPosture: 'fast', naturalEvents: 'slow', cryptoQuotes: 'slow', gulfQuotes: 'slow', stablecoinMarkets: 'slow', unrestEvents: 'slow', ucdpEvents: 'slow', techEvents: 'slow', diff --git a/server/gateway.ts b/server/gateway.ts index 799700b3ac..52c74364df 100644 --- a/server/gateway.ts +++ b/server/gateway.ts @@ -107,6 +107,7 @@ const RPC_CACHE_TIER: Record = { '/api/intelligence/v1/get-country-intel-brief': 'static', '/api/intelligence/v1/get-gdelt-topic-timeline': 'medium', '/api/climate/v1/list-climate-anomalies': 'static', + '/api/climate/v1/get-co2-monitoring': 'static', '/api/sanctions/v1/list-sanctions-pressure': 'static', '/api/sanctions/v1/lookup-sanction-entity': 'no-store', '/api/radiation/v1/list-radiation-observations': 'slow', diff --git a/server/worldmonitor/climate/v1/get-co2-monitoring.ts b/server/worldmonitor/climate/v1/get-co2-monitoring.ts new file mode 100644 index 0000000000..0f1c1f8998 --- /dev/null +++ b/server/worldmonitor/climate/v1/get-co2-monitoring.ts @@ -0,0 +1,22 @@ +import type { + ClimateServiceHandler, + ServerContext, + GetCo2MonitoringRequest, + GetCo2MonitoringResponse, +} from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; + +import { getCachedJson } from '../../../_shared/redis'; + +const SEED_CACHE_KEY = 'climate:co2-monitoring:v1'; + +export const getCo2Monitoring: ClimateServiceHandler['getCo2Monitoring'] = async ( + _ctx: ServerContext, + _req: GetCo2MonitoringRequest, +): Promise => { + try { + const cached = await getCachedJson(SEED_CACHE_KEY, true); + return (cached as GetCo2MonitoringResponse | null) ?? {}; + } catch { + return {}; + } +}; diff --git a/server/worldmonitor/climate/v1/handler.ts b/server/worldmonitor/climate/v1/handler.ts index 40e51c0f0d..ef0f9b186f 100644 --- a/server/worldmonitor/climate/v1/handler.ts +++ b/server/worldmonitor/climate/v1/handler.ts @@ -1,7 +1,9 @@ import type { ClimateServiceHandler } from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; +import { getCo2Monitoring } from './get-co2-monitoring'; import { listClimateAnomalies } from './list-climate-anomalies'; export const climateHandler: ClimateServiceHandler = { + getCo2Monitoring, listClimateAnomalies, }; diff --git a/server/worldmonitor/climate/v1/list-climate-anomalies.ts b/server/worldmonitor/climate/v1/list-climate-anomalies.ts index c7647392ba..93f53e0b25 100644 --- a/server/worldmonitor/climate/v1/list-climate-anomalies.ts +++ b/server/worldmonitor/climate/v1/list-climate-anomalies.ts @@ -1,6 +1,6 @@ /** * ListClimateAnomalies RPC -- reads seeded climate data from Railway seed cache. - * All external Open-Meteo API calls happen in seed-climate.mjs on Railway. + * All external Open-Meteo API calls happen in the climate seed scripts on Railway. */ import type { diff --git a/src/generated/client/worldmonitor/climate/v1/service_client.ts b/src/generated/client/worldmonitor/climate/v1/service_client.ts index dfdd369e0f..659b4a25d4 100644 --- a/src/generated/client/worldmonitor/climate/v1/service_client.ts +++ b/src/generated/client/worldmonitor/climate/v1/service_client.ts @@ -33,6 +33,32 @@ export interface PaginationResponse { totalCount: number; } +export interface GetCo2MonitoringRequest { +} + +export interface GetCo2MonitoringResponse { + monitoring?: Co2Monitoring; +} + +export interface Co2Monitoring { + currentPpm: number; + yearAgoPpm: number; + annualGrowthRate: number; + preIndustrialBaseline: number; + monthlyAverage: number; + trend12m: Co2DataPoint[]; + methanePpb: number; + nitrousOxidePpb: number; + measuredAt: string; + station: string; +} + +export interface Co2DataPoint { + month: string; + ppm: number; + anomaly: number; +} + export type AnomalySeverity = "ANOMALY_SEVERITY_UNSPECIFIED" | "ANOMALY_SEVERITY_NORMAL" | "ANOMALY_SEVERITY_MODERATE" | "ANOMALY_SEVERITY_EXTREME"; export type AnomalyType = "ANOMALY_TYPE_UNSPECIFIED" | "ANOMALY_TYPE_WARM" | "ANOMALY_TYPE_COLD" | "ANOMALY_TYPE_WET" | "ANOMALY_TYPE_DRY" | "ANOMALY_TYPE_MIXED"; @@ -112,6 +138,29 @@ export class ClimateServiceClient { return await resp.json() as ListClimateAnomaliesResponse; } + async getCo2Monitoring(req: GetCo2MonitoringRequest, options?: ClimateServiceCallOptions): Promise { + let path = "/api/climate/v1/get-co2-monitoring"; + const url = this.baseURL + path; + + const headers: Record = { + "Content-Type": "application/json", + ...this.defaultHeaders, + ...options?.headers, + }; + + const resp = await this.fetchFn(url, { + method: "GET", + headers, + signal: options?.signal, + }); + + if (!resp.ok) { + return this.handleError(resp); + } + + return await resp.json() as GetCo2MonitoringResponse; + } + private async handleError(resp: Response): Promise { const body = await resp.text(); if (resp.status === 400) { diff --git a/src/generated/server/worldmonitor/climate/v1/service_server.ts b/src/generated/server/worldmonitor/climate/v1/service_server.ts index 5465e283a4..97e0793a6d 100644 --- a/src/generated/server/worldmonitor/climate/v1/service_server.ts +++ b/src/generated/server/worldmonitor/climate/v1/service_server.ts @@ -33,6 +33,32 @@ export interface PaginationResponse { totalCount: number; } +export interface GetCo2MonitoringRequest { +} + +export interface GetCo2MonitoringResponse { + monitoring?: Co2Monitoring; +} + +export interface Co2Monitoring { + currentPpm: number; + yearAgoPpm: number; + annualGrowthRate: number; + preIndustrialBaseline: number; + monthlyAverage: number; + trend12m: Co2DataPoint[]; + methanePpb: number; + nitrousOxidePpb: number; + measuredAt: string; + station: string; +} + +export interface Co2DataPoint { + month: string; + ppm: number; + anomaly: number; +} + export type AnomalySeverity = "ANOMALY_SEVERITY_UNSPECIFIED" | "ANOMALY_SEVERITY_NORMAL" | "ANOMALY_SEVERITY_MODERATE" | "ANOMALY_SEVERITY_EXTREME"; export type AnomalyType = "ANOMALY_TYPE_UNSPECIFIED" | "ANOMALY_TYPE_WARM" | "ANOMALY_TYPE_COLD" | "ANOMALY_TYPE_WET" | "ANOMALY_TYPE_DRY" | "ANOMALY_TYPE_MIXED"; @@ -83,6 +109,7 @@ export interface RouteDescriptor { export interface ClimateServiceHandler { listClimateAnomalies(ctx: ServerContext, req: ListClimateAnomaliesRequest): Promise; + getCo2Monitoring(ctx: ServerContext, req: GetCo2MonitoringRequest): Promise; } export function createClimateServiceRoutes( @@ -139,6 +166,43 @@ export function createClimateServiceRoutes( } }, }, + { + method: "GET", + path: "/api/climate/v1/get-co2-monitoring", + handler: async (req: Request): Promise => { + try { + const pathParams: Record = {}; + const body = {} as GetCo2MonitoringRequest; + + const ctx: ServerContext = { + request: req, + pathParams, + headers: Object.fromEntries(req.headers.entries()), + }; + + const result = await handler.getCo2Monitoring(ctx, body); + return new Response(JSON.stringify(result as GetCo2MonitoringResponse), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } catch (err: unknown) { + if (err instanceof ValidationError) { + return new Response(JSON.stringify({ violations: err.violations }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + if (options?.onError) { + return options.onError(err, req); + } + const message = err instanceof Error ? err.message : String(err); + return new Response(JSON.stringify({ message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } + }, + }, ]; } diff --git a/src/services/climate/index.ts b/src/services/climate/index.ts index 866b4f77df..28be8544f5 100644 --- a/src/services/climate/index.ts +++ b/src/services/climate/index.ts @@ -2,8 +2,11 @@ import { getRpcBaseUrl } from '@/services/rpc-client'; import { ClimateServiceClient, type ClimateAnomaly as ProtoClimateAnomaly, + type Co2DataPoint as ProtoCo2DataPoint, + type Co2Monitoring as ProtoCo2Monitoring, type AnomalySeverity as ProtoAnomalySeverity, type AnomalyType as ProtoAnomalyType, + type GetCo2MonitoringResponse, type ListClimateAnomaliesResponse, } from '@/generated/client/worldmonitor/climate/v1/service_client'; import { createCircuitBreaker } from '@/utils'; @@ -26,7 +29,7 @@ export interface ClimateAnomaly { */ tempDelta: number; /** - * The precipitation deviation from the historical average, measured in millimeters (mm). + * The precipitation deviation from the historical average, measured in millimeters. */ precipDelta: number; severity: 'normal' | 'moderate' | 'extreme'; @@ -39,10 +42,31 @@ export interface ClimateFetchResult { anomalies: ClimateAnomaly[]; } +export interface Co2DataPoint { + month: string; + ppm: number; + anomaly: number; +} + +export interface Co2Monitoring { + currentPpm: number; + yearAgoPpm: number; + annualGrowthRate: number; + preIndustrialBaseline: number; + monthlyAverage: number; + trend12m: Co2DataPoint[]; + methanePpb: number; + nitrousOxidePpb: number; + measuredAt: Date; + station: string; +} + const client = new ClimateServiceClient(getRpcBaseUrl(), { fetch: (...args) => globalThis.fetch(...args) }); const breaker = createCircuitBreaker({ name: 'Climate Anomalies', cacheTtlMs: 20 * 60 * 1000, persistCache: true }); +const co2Breaker = createCircuitBreaker({ name: 'CO2 Monitoring', cacheTtlMs: 6 * 60 * 60 * 1000, persistCache: true }); const emptyClimateFallback: ListClimateAnomaliesResponse = { anomalies: [] }; +const emptyCo2Fallback: GetCo2MonitoringResponse = {}; export async function fetchClimateAnomalies(): Promise { const hydrated = getHydratedData('climateAnomalies') as ListClimateAnomaliesResponse | undefined; @@ -60,6 +84,19 @@ export async function fetchClimateAnomalies(): Promise { return { ok: true, anomalies }; } +export async function fetchCo2Monitoring(): Promise { + const hydrated = getHydratedData('co2Monitoring') as GetCo2MonitoringResponse | undefined; + if (hydrated?.monitoring) { + return toDisplayCo2Monitoring(hydrated.monitoring); + } + + const response = await co2Breaker.execute(async () => { + return client.getCo2Monitoring({}); + }, emptyCo2Fallback, { shouldCache: (result) => Boolean(result.monitoring?.currentPpm) }); + + return response.monitoring ? toDisplayCo2Monitoring(response.monitoring) : null; +} + // Presentation helpers (used by ClimateAnomalyPanel) export function getSeverityIcon(anomaly: ClimateAnomaly): string { switch (anomaly.type) { @@ -91,6 +128,29 @@ function toDisplayAnomaly(proto: ProtoClimateAnomaly): ClimateAnomaly { }; } +function toDisplayCo2Monitoring(proto: ProtoCo2Monitoring): Co2Monitoring { + return { + currentPpm: proto.currentPpm, + yearAgoPpm: proto.yearAgoPpm, + annualGrowthRate: proto.annualGrowthRate, + preIndustrialBaseline: proto.preIndustrialBaseline, + monthlyAverage: proto.monthlyAverage, + trend12m: (proto.trend12m ?? []).map(toDisplayCo2Point), + methanePpb: proto.methanePpb, + nitrousOxidePpb: proto.nitrousOxidePpb, + measuredAt: new Date(Number(proto.measuredAt ?? 0)), + station: proto.station, + }; +} + +function toDisplayCo2Point(proto: ProtoCo2DataPoint): Co2DataPoint { + return { + month: proto.month, + ppm: proto.ppm, + anomaly: proto.anomaly, + }; +} + function mapSeverity(s: ProtoAnomalySeverity): ClimateAnomaly['severity'] { switch (s) { case 'ANOMALY_SEVERITY_EXTREME': return 'extreme'; diff --git a/tests/bootstrap.test.mjs b/tests/bootstrap.test.mjs index 72454d152b..6b99abca69 100644 --- a/tests/bootstrap.test.mjs +++ b/tests/bootstrap.test.mjs @@ -23,7 +23,7 @@ describe('Bootstrap cache key registry', () => { const extractKeys = (src) => { const block = src.match(/BOOTSTRAP_CACHE_KEYS[^=]*=\s*\{([^}]+)\}/); if (!block) return {}; - const re = /(\w+):\s+'([a-z_-]+(?::[a-z_-]+)+:v\d+)'/g; + const re = /(\w+):\s+'([a-z0-9_-]+(?::[a-z0-9_-]+)+:v\d+)'/g; const keys = {}; let m; while ((m = re.exec(block[1])) !== null) keys[m[1]] = m[2]; @@ -48,7 +48,7 @@ describe('Bootstrap cache key registry', () => { keys.push(m[1]); } for (const key of keys) { - assert.match(key, /^[a-z_-]+(?::[a-z_-]+)+:v\d+$/, `Cache key "${key}" does not match expected pattern`); + assert.match(key, /^[a-z0-9_-]+(?::[a-z0-9_-]+)+:v\d+$/, `Cache key "${key}" does not match expected pattern`); } }); @@ -236,7 +236,7 @@ describe('Bootstrap key hydration coverage', () => { it('every bootstrap key has a getHydratedData consumer in src/', () => { const bootstrapSrc = readFileSync(join(root, 'api', 'bootstrap.js'), 'utf-8'); const block = bootstrapSrc.match(/BOOTSTRAP_CACHE_KEYS\s*=\s*\{([^}]+)\}/); - const keyRe = /(\w+):\s+'[a-z_]+(?::[a-z_-]+)+:v\d+'/g; + const keyRe = /(\w+):\s+'[a-z0-9_-]+(?::[a-z0-9_-]+)+:v\d+'/g; const keys = []; let m; while ((m = keyRe.exec(block[1])) !== null) keys.push(m[1]); @@ -253,7 +253,7 @@ describe('Bootstrap key hydration coverage', () => { const allSrc = srcFiles.map(f => readFileSync(f, 'utf-8')).join('\n'); // Keys with planned but not-yet-wired consumers - const PENDING_CONSUMERS = new Set(['chokepointTransits', 'correlationCards', 'euGasStorage']); + const PENDING_CONSUMERS = new Set(['chokepointTransits', 'climateZoneNormals', 'correlationCards', 'euGasStorage']); for (const key of keys) { if (PENDING_CONSUMERS.has(key)) continue; assert.ok( diff --git a/tests/climate-seeds.test.mjs b/tests/climate-seeds.test.mjs new file mode 100644 index 0000000000..75acc9b459 --- /dev/null +++ b/tests/climate-seeds.test.mjs @@ -0,0 +1,222 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { computeMonthlyNormals, buildZoneNormalsFromBatch } from '../scripts/seed-climate-zone-normals.mjs'; +import { hasRequiredClimateZones } from '../scripts/_climate-zones.mjs'; +import { + buildClimateAnomaly, + buildClimateAnomaliesFromBatch, + indexZoneNormals, +} from '../scripts/seed-climate-anomalies.mjs'; +import { + buildCo2MonitoringPayload, + parseCo2DailyRows, + parseCo2MonthlyRows, + parseAnnualCo2Rows, + parseGlobalMonthlyPpbRows, +} from '../scripts/seed-co2-monitoring.mjs'; + +describe('climate zone normals', () => { + it('aggregates per-year monthly means into calendar-month normals', () => { + const normals = computeMonthlyNormals({ + time: ['1991-01-01', '1991-01-02', '1991-02-01', '1992-01-01'], + temperature_2m_mean: [10, 14, 20, 16], + precipitation_sum: [2, 6, 1, 4], + }); + + assert.equal(normals.length, 2); + assert.equal(normals[0].month, 1); + assert.equal(normals[0].tempMean, 14); + assert.equal(normals[0].precipMean, 4); + assert.equal(normals[1].month, 2); + assert.equal(normals[1].tempMean, 20); + assert.equal(normals[1].precipMean, 1); + }); + + it('maps multi-location archive responses back to their zones', () => { + const zones = [ + { name: 'Zone A', lat: 1, lon: 2 }, + { name: 'Zone B', lat: 3, lon: 4 }, + ]; + const months = Array.from({ length: 12 }, (_, index) => index + 1); + const payloads = [ + { + daily: { + time: months.map((month) => `1991-${String(month).padStart(2, '0')}-01`), + temperature_2m_mean: months.map((month) => month), + precipitation_sum: months.map((month) => month + 0.5), + }, + }, + { + daily: { + time: months.map((month) => `1991-${String(month).padStart(2, '0')}-01`), + temperature_2m_mean: months.map((month) => month + 10), + precipitation_sum: months.map((month) => month + 20), + }, + }, + ]; + + const normals = buildZoneNormalsFromBatch(zones, payloads); + + assert.equal(normals.length, 2); + assert.equal(normals[0].zone, 'Zone A'); + assert.equal(normals[1].zone, 'Zone B'); + assert.equal(normals[0].months[0].tempMean, 1); + assert.equal(normals[1].months[0].tempMean, 11); + }); + + it('requires the new climate-specific zones to be present', () => { + assert.equal(hasRequiredClimateZones([ + { zone: 'Arctic' }, + { zone: 'Greenland' }, + { zone: 'Western Antarctic Ice Sheet' }, + { zone: 'Tibetan Plateau' }, + { zone: 'Congo Basin' }, + { zone: 'Coral Triangle' }, + { zone: 'North Atlantic' }, + ], (zone) => zone.zone), true); + + assert.equal(hasRequiredClimateZones([ + { zone: 'Arctic' }, + { zone: 'Greenland' }, + ], (zone) => zone.zone), false); + }); +}); + +describe('climate anomalies', () => { + it('uses stored monthly normals instead of a rolling 30-day baseline', () => { + const normalsIndex = indexZoneNormals({ + normals: [ + { + zone: 'Test Zone', + months: [ + { month: 3, tempMean: 10, precipMean: 2 }, + ], + }, + ], + }); + + const anomaly = buildClimateAnomaly( + { name: 'Test Zone', lat: 1, lon: 2 }, + { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [15, 15, 15, 15, 15, 15, 15], + precipitation_sum: [1, 1, 1, 1, 1, 1, 1], + }, + normalsIndex.get('Test Zone:3'), + ); + + assert.equal(anomaly.tempDelta, 5); + assert.equal(anomaly.precipDelta, -1); + assert.equal(anomaly.severity, 'ANOMALY_SEVERITY_EXTREME'); + assert.equal(anomaly.type, 'ANOMALY_TYPE_WARM'); + }); + + it('maps batched archive payloads back to the correct zones', () => { + const zones = [ + { name: 'Zone A', lat: 1, lon: 2 }, + { name: 'Zone B', lat: 3, lon: 4 }, + ]; + const normalsIndex = indexZoneNormals({ + normals: [ + { zone: 'Zone A', months: [{ month: 3, tempMean: 10, precipMean: 2 }] }, + { zone: 'Zone B', months: [{ month: 3, tempMean: 20, precipMean: 5 }] }, + ], + }); + const payloads = [ + { + daily: { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [12, 12, 12, 12, 12, 12, 12], + precipitation_sum: [1, 1, 1, 1, 1, 1, 1], + }, + }, + { + daily: { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [25, 25, 25, 25, 25, 25, 25], + precipitation_sum: [9, 9, 9, 9, 9, 9, 9], + }, + }, + ]; + + const anomalies = buildClimateAnomaliesFromBatch(zones, payloads, normalsIndex); + + assert.equal(anomalies.length, 2); + assert.equal(anomalies[0].zone, 'Zone A'); + assert.equal(anomalies[0].tempDelta, 2); + assert.equal(anomalies[1].zone, 'Zone B'); + assert.equal(anomalies[1].tempDelta, 5); + assert.equal(anomalies[1].precipDelta, 4); + }); + + it('classifies wet precipitation anomalies with calibrated daily thresholds', () => { + const anomaly = buildClimateAnomaly( + { name: 'Wet Zone', lat: 1, lon: 2 }, + { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [10, 10, 10, 10, 10, 10, 10], + precipitation_sum: [8, 8, 8, 8, 8, 8, 8], + }, + { month: 3, tempMean: 10, precipMean: 1 }, + ); + + assert.equal(anomaly.tempDelta, 0); + assert.equal(anomaly.precipDelta, 7); + assert.equal(anomaly.severity, 'ANOMALY_SEVERITY_MODERATE'); + assert.equal(anomaly.type, 'ANOMALY_TYPE_WET'); + }); +}); + +describe('co2 monitoring seed', () => { + it('parses NOAA text tables and computes monitoring metrics', () => { + const dailyRows = parseCo2DailyRows(` +# comment +2024 03 28 2024.240 -999.99 0 0 0 +2025 03 28 2025.238 424.10 424.10 424.10 1 +2026 03 28 2026.238 427.55 427.55 427.55 1 +`); + const monthlyLines = ['# comment']; + const monthlyValues = [ + ['2024-05', 420.0], ['2024-06', 420.1], ['2024-07', 420.2], ['2024-08', 420.3], + ['2024-09', 420.4], ['2024-10', 420.5], ['2024-11', 420.6], ['2024-12', 420.7], + ['2025-01', 420.8], ['2025-02', 420.9], ['2025-03', 421.0], ['2025-04', 421.1], + ['2025-05', 422.0], ['2025-06', 422.1], ['2025-07', 422.2], ['2025-08', 422.3], + ['2025-09', 422.4], ['2025-10', 422.5], ['2025-11', 422.6], ['2025-12', 422.7], + ['2026-01', 422.8], ['2026-02', 422.9], ['2026-03', 423.0], ['2026-04', 423.1], + ]; + for (const [month, value] of monthlyValues) { + const [year, monthNum] = month.split('-'); + monthlyLines.push(`${year} ${monthNum} ${year}.${monthNum} ${value.toFixed(2)} ${value.toFixed(2)} 30 0.12 0.08`); + } + const monthlyRows = parseCo2MonthlyRows(monthlyLines.join('\n')); + const annualRows = parseAnnualCo2Rows(` +# comment +2024 422.79 0.10 +2025 425.64 0.09 +`); + const methaneRows = parseGlobalMonthlyPpbRows(` +# comment +2026 03 2026.208 1934.49 0.50 1933.80 0.48 +`); + const nitrousRows = parseGlobalMonthlyPpbRows(` +# comment +2026 03 2026.208 337.62 0.12 337.40 0.11 +`); + + const payload = buildCo2MonitoringPayload({ dailyRows, monthlyRows, annualRows, methaneRows, nitrousRows }); + + assert.equal(payload.monitoring.currentPpm, 427.55); + assert.equal(payload.monitoring.yearAgoPpm, 424.1); + assert.equal(payload.monitoring.annualGrowthRate, 2.85); + assert.equal(payload.monitoring.preIndustrialBaseline, 280); + assert.equal(payload.monitoring.monthlyAverage, 423); + assert.equal(payload.monitoring.station, 'Mauna Loa, Hawaii'); + assert.equal(payload.monitoring.trend12m.length, 12); + assert.equal(payload.monitoring.trend12m[0].month, '2025-05'); + assert.equal(payload.monitoring.trend12m.at(-1).month, '2026-04'); + assert.equal(payload.monitoring.trend12m.at(-1).anomaly, 2); + assert.equal(payload.monitoring.methanePpb, 1934.49); + assert.equal(payload.monitoring.nitrousOxidePpb, 337.62); + }); +}); From 339094402fbfda283e334c8124f52872f8e9a2e1 Mon Sep 17 00:00:00 2001 From: fayez bast Date: Mon, 30 Mar 2026 14:07:20 +0300 Subject: [PATCH 2/4] fix(climate): skip missing normals per-zone and align anomaly tooltip copy --- scripts/seed-climate-anomalies.mjs | 7 ++- src/locales/en.json | 2 +- tests/climate-seeds.test.mjs | 74 ++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/scripts/seed-climate-anomalies.mjs b/scripts/seed-climate-anomalies.mjs index 03de6bac3b..42be60b0b1 100755 --- a/scripts/seed-climate-anomalies.mjs +++ b/scripts/seed-climate-anomalies.mjs @@ -11,6 +11,8 @@ const CANONICAL_KEY = 'climate:anomalies:v1'; const CACHE_TTL = 10800; // 3h const ANOMALY_BATCH_SIZE = 8; const ANOMALY_BATCH_DELAY_MS = 750; +// Daily precipitation deltas are in mm/day (Open-Meteo daily precipitation_sum). +// Thresholds were calibrated against ERA5-style daily precipitation distributions. const PRECIP_MODERATE_THRESHOLD = 6; const PRECIP_EXTREME_THRESHOLD = 12; const PRECIP_MIXED_THRESHOLD = 3; @@ -97,7 +99,10 @@ export function buildClimateAnomalyFromResponse(zone, payload, normalsIndex) { if (!latestDate) return null; const month = Number(latestDate.slice(5, 7)); const monthlyNormal = normalsIndex.get(`${zone.name}:${month}`); - if (!monthlyNormal) throw new Error(`Missing monthly normal for ${zone.name} month ${month}`); + if (!monthlyNormal) { + console.warn(` [CLIMATE] Missing monthly normal for ${zone.name} month ${month}; skipping zone`); + return null; + } return buildClimateAnomaly(zone, payload.daily, monthlyNormal); } diff --git a/src/locales/en.json b/src/locales/en.json index 1d083ce2bd..2826696da7 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1713,7 +1713,7 @@ "moderate": "MODERATE", "normal": "NORMAL" }, - "infoTooltip": "Climate Anomaly Monitor Temperature and precipitation deviations from 30-day baseline. Data from Open-Meteo (ERA5 reanalysis).
  • Extreme: >5°C or >80mm/day deviation
  • Moderate: >3°C or >40mm/day deviation
Monitors 15 conflict/disaster-prone zones." + "infoTooltip": "Climate Anomaly Monitor 7-day temperature and precipitation anomalies versus 1991-2020 monthly normals. Data from Open-Meteo (ERA5 reanalysis).
  • Extreme: >5°C or >12mm/day anomaly
  • Moderate: >3°C or >6mm/day anomaly
Tracks climate-sensitive zones for sustained departures from WMO baselines." }, "newsPanel": { "close": "Close", diff --git a/tests/climate-seeds.test.mjs b/tests/climate-seeds.test.mjs index 75acc9b459..551ada34b0 100644 --- a/tests/climate-seeds.test.mjs +++ b/tests/climate-seeds.test.mjs @@ -33,6 +33,17 @@ describe('climate zone normals', () => { assert.equal(normals[1].precipMean, 1); }); + it('drops months that have zero samples', () => { + const normals = computeMonthlyNormals({ + time: ['1991-01-01'], + temperature_2m_mean: [10], + precipitation_sum: [2], + }); + + assert.equal(normals.length, 1); + assert.equal(normals[0].month, 1); + }); + it('maps multi-location archive responses back to their zones', () => { const zones = [ { name: 'Zone A', lat: 1, lon: 2 }, @@ -65,6 +76,36 @@ describe('climate zone normals', () => { assert.equal(normals[1].months[0].tempMean, 11); }); + it('skips zones with incomplete monthly normals but keeps other zones in the batch', () => { + const zones = [ + { name: 'Zone A', lat: 1, lon: 2 }, + { name: 'Zone B', lat: 3, lon: 4 }, + ]; + const fullMonths = Array.from({ length: 12 }, (_, index) => index + 1); + const shortMonths = Array.from({ length: 11 }, (_, index) => index + 1); + const payloads = [ + { + daily: { + time: fullMonths.map((month) => `1991-${String(month).padStart(2, '0')}-01`), + temperature_2m_mean: fullMonths.map((month) => month), + precipitation_sum: fullMonths.map((month) => month + 0.5), + }, + }, + { + daily: { + time: shortMonths.map((month) => `1991-${String(month).padStart(2, '0')}-01`), + temperature_2m_mean: shortMonths.map((month) => month + 10), + precipitation_sum: shortMonths.map((month) => month + 20), + }, + }, + ]; + + const normals = buildZoneNormalsFromBatch(zones, payloads); + + assert.equal(normals.length, 1); + assert.equal(normals[0].zone, 'Zone A'); + }); + it('requires the new climate-specific zones to be present', () => { assert.equal(hasRequiredClimateZones([ { zone: 'Arctic' }, @@ -150,6 +191,39 @@ describe('climate anomalies', () => { assert.equal(anomalies[1].precipDelta, 4); }); + it('skips zones missing monthly normals without failing the whole batch', () => { + const zones = [ + { name: 'Zone A', lat: 1, lon: 2 }, + { name: 'Zone B', lat: 3, lon: 4 }, + ]; + const normalsIndex = indexZoneNormals({ + normals: [ + { zone: 'Zone A', months: [{ month: 3, tempMean: 10, precipMean: 2 }] }, + ], + }); + const payloads = [ + { + daily: { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [12, 12, 12, 12, 12, 12, 12], + precipitation_sum: [1, 1, 1, 1, 1, 1, 1], + }, + }, + { + daily: { + time: ['2026-03-01', '2026-03-02', '2026-03-03', '2026-03-04', '2026-03-05', '2026-03-06', '2026-03-07'], + temperature_2m_mean: [25, 25, 25, 25, 25, 25, 25], + precipitation_sum: [9, 9, 9, 9, 9, 9, 9], + }, + }, + ]; + + const anomalies = buildClimateAnomaliesFromBatch(zones, payloads, normalsIndex); + + assert.equal(anomalies.length, 1); + assert.equal(anomalies[0].zone, 'Zone A'); + }); + it('classifies wet precipitation anomalies with calibrated daily thresholds', () => { const anomaly = buildClimateAnomaly( { name: 'Wet Zone', lat: 1, lon: 2 }, From 7bfcb330aaa869b330c30977bc78d3bbc8a03fff Mon Sep 17 00:00:00 2001 From: fayez bast Date: Mon, 30 Mar 2026 15:32:07 +0300 Subject: [PATCH 3/4] fix(climate): remove normals from bootstrap and harden health/cache key wiring --- api/bootstrap.js | 3 +-- api/health.js | 1 - scripts/seed-climate-zone-normals.mjs | 14 +++++++++----- server/_shared/cache-keys.ts | 4 ++-- .../climate/v1/get-co2-monitoring.ts | 5 ++--- tests/bootstrap.test.mjs | 19 ++++++++++++++++++- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/api/bootstrap.js b/api/bootstrap.js index 0050750830..07c6faf861 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -26,7 +26,6 @@ const BOOTSTRAP_CACHE_KEYS = { minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', climateAnomalies: 'climate:anomalies:v1', - climateZoneNormals: 'climate:zone-normals:v1', co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', @@ -87,7 +86,7 @@ const BOOTSTRAP_CACHE_KEYS = { const SLOW_KEYS = new Set([ 'bisPolicy', 'bisExchange', 'bisCredit', 'minerals', 'giving', - 'sectors', 'etfFlows', 'wildfires', 'climateAnomalies', 'climateZoneNormals', 'co2Monitoring', + 'sectors', 'etfFlows', 'wildfires', 'climateAnomalies', 'co2Monitoring', 'radiationWatch', 'thermalEscalation', 'crossSourceSignals', 'cyberThreats', 'techReadiness', 'progressData', 'renewableEnergy', 'naturalEvents', diff --git a/api/health.js b/api/health.js index e368aba5a4..b351680c2c 100644 --- a/api/health.js +++ b/api/health.js @@ -10,7 +10,6 @@ const BOOTSTRAP_KEYS = { sectors: 'market:sectors:v1', etfFlows: 'market:etf-flows:v1', climateAnomalies: 'climate:anomalies:v1', - climateZoneNormals: 'climate:zone-normals:v1', co2Monitoring: 'climate:co2-monitoring:v1', wildfires: 'wildfire:fires:v1', marketQuotes: 'market:stocks-bootstrap:v1', diff --git a/scripts/seed-climate-zone-normals.mjs b/scripts/seed-climate-zone-normals.mjs index dec7dde71e..ae91b1a1f7 100644 --- a/scripts/seed-climate-zone-normals.mjs +++ b/scripts/seed-climate-zone-normals.mjs @@ -7,6 +7,7 @@ import { chunkItems, fetchOpenMeteoArchiveBatch } from './_open-meteo-archive.mj loadEnvFile(import.meta.url); export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; +// Keep the previous baseline available across monthly cron gaps; health.js enforces freshness separately. const NORMALS_TTL = 90 * 24 * 60 * 60; // 90 days const NORMALS_START = '1991-01-01'; const NORMALS_END = '2020-12-31'; @@ -76,18 +77,19 @@ export function computeMonthlyNormals(daily) { } export function buildZoneNormalsFromBatch(zones, batchPayloads) { - return zones.map((zone, index) => { + return zones.flatMap((zone, index) => { const data = batchPayloads[index]; const months = computeMonthlyNormals(data?.daily); if (months.length !== 12) { - throw new Error(`Open-Meteo normals incomplete for ${zone.name}: expected 12 months, got ${months.length}`); + console.warn(` [CLIMATE_NORMALS] Open-Meteo normals incomplete for ${zone.name}: expected 12 months, got ${months.length}`); + return []; } - return { + return [{ zone: zone.name, location: { latitude: zone.lat, longitude: zone.lon }, months, - }; + }]; }); } @@ -106,7 +108,9 @@ export async function fetchClimateZoneNormals() { retryBaseMs: 5_000, label: `normals batch (${batch.map((zone) => zone.name).join(', ')})`, }); - normals.push(...buildZoneNormalsFromBatch(batch, payloads)); + const batchNormals = buildZoneNormalsFromBatch(batch, payloads); + normals.push(...batchNormals); + failures += Math.max(0, batch.length - batchNormals.length); } catch (err) { console.log(` [CLIMATE_NORMALS] ${err?.message ?? err}`); failures += batch.length; diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index 4c612c01cb..52b3ae109a 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -6,6 +6,7 @@ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; +export const CLIMATE_CO2_MONITORING_KEY = 'climate:co2-monitoring:v1'; /** * Static cache keys for the bootstrap endpoint. @@ -29,7 +30,6 @@ export const BOOTSTRAP_CACHE_KEYS: Record = { minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', climateAnomalies: 'climate:anomalies:v1', - climateZoneNormals: 'climate:zone-normals:v1', co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', @@ -91,7 +91,7 @@ export const BOOTSTRAP_TIERS: Record = { minerals: 'slow', giving: 'slow', sectors: 'slow', progressData: 'slow', renewableEnergy: 'slow', etfFlows: 'slow', shippingRates: 'fast', wildfires: 'slow', - climateAnomalies: 'slow', climateZoneNormals: 'slow', co2Monitoring: 'slow', sanctionsPressure: 'slow', radiationWatch: 'slow', thermalEscalation: 'slow', crossSourceSignals: 'slow', cyberThreats: 'slow', techReadiness: 'slow', + climateAnomalies: 'slow', co2Monitoring: 'slow', sanctionsPressure: 'slow', radiationWatch: 'slow', thermalEscalation: 'slow', crossSourceSignals: 'slow', cyberThreats: 'slow', techReadiness: 'slow', theaterPosture: 'fast', naturalEvents: 'slow', cryptoQuotes: 'slow', gulfQuotes: 'slow', stablecoinMarkets: 'slow', unrestEvents: 'slow', ucdpEvents: 'slow', techEvents: 'slow', diff --git a/server/worldmonitor/climate/v1/get-co2-monitoring.ts b/server/worldmonitor/climate/v1/get-co2-monitoring.ts index 0f1c1f8998..cb36eb4582 100644 --- a/server/worldmonitor/climate/v1/get-co2-monitoring.ts +++ b/server/worldmonitor/climate/v1/get-co2-monitoring.ts @@ -5,16 +5,15 @@ import type { GetCo2MonitoringResponse, } from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; +import { CLIMATE_CO2_MONITORING_KEY } from '../../../_shared/cache-keys'; import { getCachedJson } from '../../../_shared/redis'; -const SEED_CACHE_KEY = 'climate:co2-monitoring:v1'; - export const getCo2Monitoring: ClimateServiceHandler['getCo2Monitoring'] = async ( _ctx: ServerContext, _req: GetCo2MonitoringRequest, ): Promise => { try { - const cached = await getCachedJson(SEED_CACHE_KEY, true); + const cached = await getCachedJson(CLIMATE_CO2_MONITORING_KEY, true); return (cached as GetCo2MonitoringResponse | null) ?? {}; } catch { return {}; diff --git a/tests/bootstrap.test.mjs b/tests/bootstrap.test.mjs index 6b99abca69..83cf2a4b48 100644 --- a/tests/bootstrap.test.mjs +++ b/tests/bootstrap.test.mjs @@ -253,7 +253,7 @@ describe('Bootstrap key hydration coverage', () => { const allSrc = srcFiles.map(f => readFileSync(f, 'utf-8')).join('\n'); // Keys with planned but not-yet-wired consumers - const PENDING_CONSUMERS = new Set(['chokepointTransits', 'climateZoneNormals', 'correlationCards', 'euGasStorage']); + const PENDING_CONSUMERS = new Set(['chokepointTransits', 'correlationCards', 'euGasStorage']); for (const key of keys) { if (PENDING_CONSUMERS.has(key)) continue; assert.ok( @@ -264,6 +264,23 @@ describe('Bootstrap key hydration coverage', () => { }); }); +describe('Health key registries', () => { + it('does not duplicate Redis keys across BOOTSTRAP_KEYS and STANDALONE_KEYS', () => { + const healthSrc = readFileSync(join(root, 'api', 'health.js'), 'utf-8'); + const extractValues = (name) => { + const block = healthSrc.match(new RegExp(`${name}\\s*=\\s*\\{([\\s\\S]*?)\\n\\};`)); + if (!block) return []; + return [...block[1].matchAll(/:\s+'([^']+)'/g)].map((m) => m[1]); + }; + + const bootstrap = new Set(extractValues('BOOTSTRAP_KEYS')); + const standalone = new Set(extractValues('STANDALONE_KEYS')); + const overlap = [...bootstrap].filter((key) => standalone.has(key)); + + assert.deepEqual(overlap, [], `health.js duplicates keys across registries: ${overlap.join(', ')}`); + }); +}); + describe('Bootstrap tier definitions', () => { const bootstrapSrc = readFileSync(join(root, 'api', 'bootstrap.js'), 'utf-8'); const cacheKeysSrc = readFileSync(join(root, 'server', '_shared', 'cache-keys.ts'), 'utf-8'); From 5f920772f0a5c173381261602c6b30ee6d6efbd1 Mon Sep 17 00:00:00 2001 From: fayez bast Date: Wed, 1 Apr 2026 20:13:27 +0300 Subject: [PATCH 4/4] feat(climate): version anomaly cache to v2, harden seed freshness, and align CO2/normal baselines --- api/bootstrap.js | 2 +- api/health.js | 4 +- api/mcp.ts | 65 ++++++++++--- api/seed-health.js | 2 +- docs/api/ClimateService.openapi.json | 2 +- docs/api/ClimateService.openapi.yaml | 1 + docs/architecture.mdx | 2 +- docs/climate-variant-full.md | 18 ++-- docs/strategic-risk.mdx | 2 +- .../climate/v1/co2_monitoring.proto | 1 + scripts/_open-meteo-archive.mjs | 32 +++++-- scripts/seed-climate-anomalies.mjs | 9 +- scripts/seed-climate-zone-normals.mjs | 17 ++-- scripts/seed-co2-monitoring.mjs | 2 +- server/_shared/cache-keys.ts | 3 +- .../climate/v1/list-climate-anomalies.ts | 5 +- .../intelligence/v1/get-risk-scores.ts | 3 +- src/services/climate/index.ts | 6 +- tests/climate-seeds.test.mjs | 93 +++++++++++++++++++ tests/mcp.test.mjs | 42 +++++++++ 20 files changed, 253 insertions(+), 58 deletions(-) diff --git a/api/bootstrap.js b/api/bootstrap.js index 07c6faf861..6394bf9578 100644 --- a/api/bootstrap.js +++ b/api/bootstrap.js @@ -25,7 +25,7 @@ const BOOTSTRAP_CACHE_KEYS = { chokepointTransits: 'supply_chain:chokepoint_transits:v1', minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', - climateAnomalies: 'climate:anomalies:v1', + climateAnomalies: 'climate:anomalies:v2', co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', diff --git a/api/health.js b/api/health.js index b351680c2c..eb5b062e33 100644 --- a/api/health.js +++ b/api/health.js @@ -9,7 +9,7 @@ const BOOTSTRAP_KEYS = { outages: 'infra:outages:v1', sectors: 'market:sectors:v1', etfFlows: 'market:etf-flows:v1', - climateAnomalies: 'climate:anomalies:v1', + climateAnomalies: 'climate:anomalies:v2', co2Monitoring: 'climate:co2-monitoring:v1', wildfires: 'wildfire:fires:v1', marketQuotes: 'market:stocks-bootstrap:v1', @@ -130,7 +130,7 @@ const SEED_META = { outages: { key: 'seed-meta:infra:outages', maxStaleMin: 30 }, climateAnomalies: { key: 'seed-meta:climate:anomalies', maxStaleMin: 120 }, // runs as independent Railway cron (0 */2 * * *) climateZoneNormals: { key: 'seed-meta:climate:zone-normals', maxStaleMin: 89280 }, // monthly cron on the 1st; 62d = 2x 31-day cadence - co2Monitoring: { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 2880 }, // daily cron at 06:00 UTC; 48h tolerates one missed run + co2Monitoring: { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 4320 }, // daily cron at 06:00 UTC; 72h tolerates two missed runs unrestEvents: { key: 'seed-meta:unrest:events', maxStaleMin: 120 }, // 45min cron; 120 = 2h grace (was 75 = 30min buffer, too tight) cyberThreats: { key: 'seed-meta:cyber:threats', maxStaleMin: 240 }, // 2h interval; 240min = 2x interval cryptoQuotes: { key: 'seed-meta:market:crypto', maxStaleMin: 30 }, diff --git a/api/mcp.ts b/api/mcp.ts index a196c32805..5beb142ab0 100644 --- a/api/mcp.ts +++ b/api/mcp.ts @@ -48,11 +48,17 @@ interface BaseToolDef { inputSchema: { type: string; properties: Record; required: string[] }; } +interface FreshnessCheck { + key: string; + maxStaleMin: number; +} + // Cache-read tool: reads one or more Redis keys and returns them with staleness info. interface CacheToolDef extends BaseToolDef { _cacheKeys: string[]; _seedMetaKey: string; _maxStaleMin: number; + _freshnessChecks?: FreshnessCheck[]; _execute?: never; } @@ -61,6 +67,7 @@ interface RpcToolDef extends BaseToolDef { _cacheKeys?: never; _seedMetaKey?: never; _maxStaleMin?: never; + _freshnessChecks?: never; _execute: (params: Record, base: string, apiKey: string) => Promise; } @@ -180,11 +187,16 @@ const TOOL_REGISTRY: ToolDef[] = [ }, { name: 'get_climate_data', - description: 'Climate anomalies, NOAA atmospheric greenhouse gas monitoring, weather alerts, and natural environmental events from WorldMonitor climate feeds.', + description: 'Climate anomalies, NOAA atmospheric greenhouse gas monitoring (CO2 ppm, methane ppb, N2O ppb, Mauna Loa 12-month trend), weather alerts, and natural environmental events from WorldMonitor climate feeds.', inputSchema: { type: 'object', properties: {}, required: [] }, - _cacheKeys: ['climate:anomalies:v1', 'climate:co2-monitoring:v1', 'weather:alerts:v1'], - _seedMetaKey: 'seed-meta:climate:anomalies', - _maxStaleMin: 120, + _cacheKeys: ['climate:anomalies:v2', 'climate:co2-monitoring:v1', 'weather:alerts:v1'], + _seedMetaKey: 'seed-meta:climate:co2-monitoring', + _maxStaleMin: 2880, + _freshnessChecks: [ + { key: 'seed-meta:climate:anomalies', maxStaleMin: 120 }, + { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 2880 }, + { key: 'seed-meta:weather:alerts', maxStaleMin: 45 }, + ], }, { name: 'get_infrastructure_status', @@ -673,21 +685,46 @@ function rpcError(id: unknown, code: number, message: string): Response { return jsonResponse({ jsonrpc: '2.0', id: id ?? null, error: { code, message } }, 200); } +export function evaluateFreshness(checks: FreshnessCheck[], metas: unknown[], now = Date.now()): { cached_at: string | null; stale: boolean } { + let stale = false; + let oldestFetchedAt = Number.POSITIVE_INFINITY; + let hasAnyValidMeta = false; + let hasAllValidMeta = true; + + for (const [i, check] of checks.entries()) { + const meta = metas[i]; + const fetchedAt = meta && typeof meta === 'object' && 'fetchedAt' in meta + ? Number((meta as { fetchedAt: unknown }).fetchedAt) + : Number.NaN; + + if (!Number.isFinite(fetchedAt) || fetchedAt <= 0) { + hasAllValidMeta = false; + stale = true; + continue; + } + + hasAnyValidMeta = true; + oldestFetchedAt = Math.min(oldestFetchedAt, fetchedAt); + stale ||= (now - fetchedAt) / 60_000 > check.maxStaleMin; + } + + return { + cached_at: hasAnyValidMeta && hasAllValidMeta ? new Date(oldestFetchedAt).toISOString() : null, + stale, + }; +} + // --------------------------------------------------------------------------- // Tool execution // --------------------------------------------------------------------------- async function executeTool(tool: CacheToolDef): Promise<{ cached_at: string | null; stale: boolean; data: Record }> { const reads = tool._cacheKeys.map(k => readJsonFromUpstash(k)); - const metaRead = readJsonFromUpstash(tool._seedMetaKey); - const [results, meta] = await Promise.all([Promise.all(reads), metaRead]); - - let cached_at: string | null = null; - let stale = true; - if (meta && typeof meta === 'object' && 'fetchedAt' in meta) { - const fetchedAt = (meta as { fetchedAt: number }).fetchedAt; - cached_at = new Date(fetchedAt).toISOString(); - stale = (Date.now() - fetchedAt) / 60_000 > tool._maxStaleMin; - } + const freshnessChecks = tool._freshnessChecks?.length + ? tool._freshnessChecks + : [{ key: tool._seedMetaKey, maxStaleMin: tool._maxStaleMin }]; + const metaReads = freshnessChecks.map((check) => readJsonFromUpstash(check.key)); + const [results, metas] = await Promise.all([Promise.all(reads), Promise.all(metaReads)]); + const { cached_at, stale } = evaluateFreshness(freshnessChecks, metas); const data: Record = {}; // Walk backward through ':'-delimited segments, skipping non-informative suffixes diff --git a/api/seed-health.js b/api/seed-health.js index 4d41eab764..c1bc0d56bb 100644 --- a/api/seed-health.js +++ b/api/seed-health.js @@ -13,7 +13,7 @@ const SEED_DOMAINS = { 'infra:outages': { key: 'seed-meta:infra:outages', intervalMin: 15 }, 'climate:anomalies': { key: 'seed-meta:climate:anomalies', intervalMin: 60 }, 'climate:zone-normals': { key: 'seed-meta:climate:zone-normals', intervalMin: 44640 }, - 'climate:co2-monitoring': { key: 'seed-meta:climate:co2-monitoring', intervalMin: 1440 }, + 'climate:co2-monitoring': { key: 'seed-meta:climate:co2-monitoring', intervalMin: 2160 }, // Phase 2 — Parameterized endpoints 'unrest:events': { key: 'seed-meta:unrest:events', intervalMin: 15 }, 'cyber:threats': { key: 'seed-meta:cyber:threats', intervalMin: 240 }, diff --git a/docs/api/ClimateService.openapi.json b/docs/api/ClimateService.openapi.json index c0f51f403c..738839738d 100644 --- a/docs/api/ClimateService.openapi.json +++ b/docs/api/ClimateService.openapi.json @@ -1 +1 @@ -{"components":{"schemas":{"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal in millimeters.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"Co2DataPoint":{"properties":{"anomaly":{"format":"double","type":"number"},"month":{"type":"string"},"ppm":{"format":"double","type":"number"}},"type":"object"},"Co2Monitoring":{"properties":{"annualGrowthRate":{"format":"double","type":"number"},"currentPpm":{"format":"double","type":"number"},"measuredAt":{"format":"int64","type":"string"},"methanePpb":{"format":"double","type":"number"},"monthlyAverage":{"format":"double","type":"number"},"nitrousOxidePpb":{"format":"double","type":"number"},"preIndustrialBaseline":{"format":"double","type":"number"},"station":{"type":"string"},"trend12m":{"items":{"$ref":"#/components/schemas/Co2DataPoint"},"type":"array"},"yearAgoPpm":{"format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetCo2MonitoringRequest":{"type":"object"},"GetCo2MonitoringResponse":{"properties":{"monitoring":{"$ref":"#/components/schemas/Co2Monitoring"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/get-co2-monitoring":{"get":{"description":"GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data.","operationId":"GetCo2Monitoring","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetCo2MonitoringResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetCo2Monitoring","tags":["ClimateService"]}},"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}}}} \ No newline at end of file +{"components":{"schemas":{"ClimateAnomaly":{"description":"ClimateAnomaly represents a temperature or precipitation deviation from historical norms.\n Sourced from Open-Meteo / ERA5 reanalysis data.","properties":{"location":{"$ref":"#/components/schemas/GeoCoordinates"},"period":{"description":"Time period covered (e.g., \"2024-W03\", \"2024-01\").","minLength":1,"type":"string"},"precipDelta":{"description":"Precipitation deviation from normal in millimeters.","format":"double","type":"number"},"severity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"tempDelta":{"description":"Temperature deviation from normal in degrees Celsius.","format":"double","type":"number"},"type":{"description":"AnomalyType represents the type of climate anomaly.\n Maps to existing TS union: 'warm' | 'cold' | 'wet' | 'dry' | 'mixed'.","enum":["ANOMALY_TYPE_UNSPECIFIED","ANOMALY_TYPE_WARM","ANOMALY_TYPE_COLD","ANOMALY_TYPE_WET","ANOMALY_TYPE_DRY","ANOMALY_TYPE_MIXED"],"type":"string"},"zone":{"description":"Climate zone name (e.g., \"Northern Europe\", \"Sahel\").","minLength":1,"type":"string"}},"required":["zone","period"],"type":"object"},"Co2DataPoint":{"properties":{"anomaly":{"description":"Year-over-year delta vs same calendar month, in ppm.","format":"double","type":"number"},"month":{"type":"string"},"ppm":{"format":"double","type":"number"}},"type":"object"},"Co2Monitoring":{"properties":{"annualGrowthRate":{"format":"double","type":"number"},"currentPpm":{"format":"double","type":"number"},"measuredAt":{"format":"int64","type":"string"},"methanePpb":{"format":"double","type":"number"},"monthlyAverage":{"format":"double","type":"number"},"nitrousOxidePpb":{"format":"double","type":"number"},"preIndustrialBaseline":{"format":"double","type":"number"},"station":{"type":"string"},"trend12m":{"items":{"$ref":"#/components/schemas/Co2DataPoint"},"type":"array"},"yearAgoPpm":{"format":"double","type":"number"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetCo2MonitoringRequest":{"type":"object"},"GetCo2MonitoringResponse":{"properties":{"monitoring":{"$ref":"#/components/schemas/Co2Monitoring"}},"type":"object"},"ListClimateAnomaliesRequest":{"description":"ListClimateAnomaliesRequest specifies filters for retrieving climate anomaly data.","properties":{"cursor":{"description":"Cursor for next page.","type":"string"},"minSeverity":{"description":"AnomalySeverity represents the severity of a climate anomaly.\n Maps to existing TS union: 'normal' | 'moderate' | 'extreme'.","enum":["ANOMALY_SEVERITY_UNSPECIFIED","ANOMALY_SEVERITY_NORMAL","ANOMALY_SEVERITY_MODERATE","ANOMALY_SEVERITY_EXTREME"],"type":"string"},"pageSize":{"description":"Maximum items per page (1-100).","format":"int32","type":"integer"}},"type":"object"},"ListClimateAnomaliesResponse":{"description":"ListClimateAnomaliesResponse contains the list of climate anomalies.","properties":{"anomalies":{"items":{"$ref":"#/components/schemas/ClimateAnomaly"},"type":"array"},"pagination":{"$ref":"#/components/schemas/PaginationResponse"}},"type":"object"},"PaginationResponse":{"description":"PaginationResponse contains pagination metadata returned alongside list results.","properties":{"nextCursor":{"description":"Cursor for fetching the next page. Empty string indicates no more pages.","type":"string"},"totalCount":{"description":"Total count of items matching the query, if known. Zero if the total is unknown.","format":"int32","type":"integer"}},"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}}},"info":{"title":"ClimateService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/climate/v1/get-co2-monitoring":{"get":{"description":"GetCo2Monitoring retrieves seeded NOAA greenhouse gas monitoring data.","operationId":"GetCo2Monitoring","responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GetCo2MonitoringResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"GetCo2Monitoring","tags":["ClimateService"]}},"/api/climate/v1/list-climate-anomalies":{"get":{"description":"ListClimateAnomalies retrieves temperature and precipitation anomalies from ERA5 data.","operationId":"ListClimateAnomalies","parameters":[{"description":"Maximum items per page (1-100).","in":"query","name":"page_size","required":false,"schema":{"format":"int32","type":"integer"}},{"description":"Cursor for next page.","in":"query","name":"cursor","required":false,"schema":{"type":"string"}},{"description":"Optional filter by anomaly severity.","in":"query","name":"min_severity","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListClimateAnomaliesResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidationError"}}},"description":"Validation error"},"default":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}},"description":"Error response"}},"summary":"ListClimateAnomalies","tags":["ClimateService"]}}}} \ No newline at end of file diff --git a/docs/api/ClimateService.openapi.yaml b/docs/api/ClimateService.openapi.yaml index be737aa4aa..c343e49827 100644 --- a/docs/api/ClimateService.openapi.yaml +++ b/docs/api/ClimateService.openapi.yaml @@ -266,3 +266,4 @@ components: anomaly: type: number format: double + description: Year-over-year delta vs same calendar month, in ppm. diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 46e2684c4e..7a73307326 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -225,7 +225,7 @@ The `SmartPollLoop` is the core refresh orchestration primitive used by all data | `seed-cyber-threats` | Feodo, URLhaus, C2Intel, OTX, AbuseIPDB | 10 min | `cyber:threats-bootstrap:v2` | | `seed-internet-outages` | Cloudflare Radar | 5 min | `infra:outages:v1` | | `seed-fire-detections` | NASA FIRMS VIIRS | 10 min | `wildfire:fires:v1` | -| `seed-climate-anomalies` | Open-Meteo ERA5 | 15 min | `climate:anomalies:v1` | +| `seed-climate-anomalies` | Open-Meteo ERA5 | 15 min | `climate:anomalies:v2` | | `seed-natural-events` | USGS + GDACS + NASA EONET | 10 min | `natural:events:v1` | | `seed-airport-delays` | FAA + AviationStack + ICAO NOTAM | 10 min | `aviation:delays-bootstrap:v1` | | `seed-insights` | Groq LLM world brief + top stories | 10 min | `news:insights:v1` | diff --git a/docs/climate-variant-full.md b/docs/climate-variant-full.md index 15a0ac722d..2e31cbd84f 100644 --- a/docs/climate-variant-full.md +++ b/docs/climate-variant-full.md @@ -5,7 +5,7 @@ | Component | Status | |-----------|--------| | Proto RPCs | 1 — `ListClimateAnomalies` | -| Redis keys | 1 — `climate:anomalies:v1` | +| Redis keys | 1 — `climate:anomalies:v2` | | Seed scripts | 1 — `seed-climate-anomalies.mjs` | | MCP tool | `get_climate_data` — bundled with `weather:alerts:v1` | | Hostname variant | Not configured | @@ -39,7 +39,7 @@ - Coral Triangle (-5°S, 128°E) — reef bleaching proxy (sea temp) - North Atlantic (55°N, -30°W) — AMOC slowdown signal -**No change to cache key `climate:anomalies:v1`** — fix in place. +**Bump cache key to `climate:anomalies:v2`** to avoid stale `%`-based precipitation anomalies being misread as millimeters. ### Layer 2: CO2 & Greenhouse Gas Monitoring (NEW) @@ -66,7 +66,7 @@ **Redis key:** `climate:co2-monitoring:v1` **Seed script:** `seed-co2-monitoring.mjs` -**Cache TTL:** 86400 (24h — NOAA updates daily with ~2 day lag) +**Cache TTL:** 259200 (72h — 3x daily interval gold standard) **Proto RPC:** `GetCo2Monitoring` ```proto @@ -85,7 +85,7 @@ message Co2Monitoring { message Co2DataPoint { string month = 1; // "YYYY-MM" double ppm = 2; - double anomaly = 3; // vs same month previous year + double anomaly = 3; // year-over-year delta vs same calendar month, in ppm } ``` @@ -225,8 +225,8 @@ message IceTrendPoint { | Script | Interval | Key | TTL | |--------|----------|-----|-----| -| `seed-climate-anomalies.mjs` | Every 3h (existing, fix baseline) | `climate:anomalies:v1` | 3h | -| `seed-co2-monitoring.mjs` | Daily 06:00 UTC | `climate:co2-monitoring:v1` | 24h | +| `seed-climate-anomalies.mjs` | Every 3h (existing, fix baseline) | `climate:anomalies:v2` | 3h | +| `seed-co2-monitoring.mjs` | Daily 06:00 UTC | `climate:co2-monitoring:v1` | 72h | | `seed-climate-disasters.mjs` | Every 6h | `climate:disasters:v1` | 6h | | `seed-health-air-quality.mjs` | Every 1h (shared) | `climate:air-quality:v1` | 1h | | `seed-climate-ocean-ice.mjs` | Daily 08:00 UTC | `climate:ocean-ice:v1` | 24h | @@ -299,15 +299,15 @@ Replace current entry in `api/mcp.ts`: required: [], }, _cacheKeys: [ - 'climate:anomalies:v1', + 'climate:anomalies:v2', 'climate:co2-monitoring:v1', 'climate:disasters:v1', 'climate:air-quality:v1', 'climate:ocean-ice:v1', 'climate:news-intelligence:v1', ], - _seedMetaKey: 'seed-meta:climate:anomalies', - _maxStaleMin: 120, + _seedMetaKey: 'seed-meta:climate:co2-monitoring', + _maxStaleMin: 2880, } ``` diff --git a/docs/strategic-risk.mdx b/docs/strategic-risk.mdx index c0dc52763b..cc79952c73 100644 --- a/docs/strategic-risk.mdx +++ b/docs/strategic-risk.mdx @@ -230,7 +230,7 @@ The top countries contribute most heavily, with diminishing influence for lower- | ACLED | Fetched live via API | Protests, riots, battles, explosions, civilian violence, fatalities | | UCDP | `conflict:ucdp-events:v1` | War/minor conflict floors | | Outages | `infra:outages:v1` | Unrest outage boost (TOTAL/MAJOR/PARTIAL severity) | -| Climate | `climate:anomalies:v1` | Climate severity boost | +| Climate | `climate:anomalies:v2` | Climate severity boost | | Cyber | `cyber:threats-bootstrap:v2` | Cyber threat count boost | | Fires | `wildfire:fires:v1` | Wildfire count boost | | GPS Jamming | `intelligence:gpsjam:v2` | Security score (high/medium hex levels) | diff --git a/proto/worldmonitor/climate/v1/co2_monitoring.proto b/proto/worldmonitor/climate/v1/co2_monitoring.proto index 4d82a2d179..7e3b55bc38 100644 --- a/proto/worldmonitor/climate/v1/co2_monitoring.proto +++ b/proto/worldmonitor/climate/v1/co2_monitoring.proto @@ -18,5 +18,6 @@ message Co2Monitoring { message Co2DataPoint { string month = 1; double ppm = 2; + // Year-over-year delta vs same calendar month, in ppm. double anomaly = 3; } diff --git a/scripts/_open-meteo-archive.mjs b/scripts/_open-meteo-archive.mjs index 8dd880df83..7775c61b52 100644 --- a/scripts/_open-meteo-archive.mjs +++ b/scripts/_open-meteo-archive.mjs @@ -1,5 +1,8 @@ import { CHROME_UA, sleep } from './_seed-utils.mjs'; +const MAX_RETRY_AFTER_MS = 60_000; +const RETRYABLE_STATUSES = new Set([429, 503]); + export function chunkItems(items, size) { const chunks = []; for (let i = 0; i < items.length; i += size) { @@ -12,17 +15,17 @@ export function normalizeArchiveBatchResponse(payload) { return Array.isArray(payload) ? payload : [payload]; } -function parseRetryAfterMs(value) { +export function parseRetryAfterMs(value) { if (!value) return null; const seconds = Number(value); if (Number.isFinite(seconds) && seconds > 0) { - return seconds * 1000; + return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS); } const retryAt = Date.parse(value); if (Number.isFinite(retryAt)) { - return Math.max(retryAt - Date.now(), 1000); + return Math.min(Math.max(retryAt - Date.now(), 1000), MAX_RETRY_AFTER_MS); } return null; @@ -51,10 +54,21 @@ export async function fetchOpenMeteoArchiveBatch(zones, opts) { const url = `https://archive-api.open-meteo.com/v1/archive?${params.toString()}`; for (let attempt = 0; attempt <= maxRetries; attempt++) { - const resp = await fetch(url, { - headers: { 'User-Agent': CHROME_UA }, - signal: AbortSignal.timeout(timeoutMs), - }); + let resp; + try { + resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA }, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (attempt < maxRetries) { + const retryMs = retryBaseMs * 2 ** attempt; + console.log(` [OPEN_METEO] ${err?.message ?? err} for ${label}; retrying batch in ${Math.round(retryMs / 1000)}s`); + await sleep(retryMs); + continue; + } + throw err; + } if (resp.ok) { const data = normalizeArchiveBatchResponse(await resp.json()); @@ -64,9 +78,9 @@ export async function fetchOpenMeteoArchiveBatch(zones, opts) { return data; } - if (resp.status === 429 && attempt < maxRetries) { + if (RETRYABLE_STATUSES.has(resp.status) && attempt < maxRetries) { const retryMs = parseRetryAfterMs(resp.headers.get('retry-after')) ?? (retryBaseMs * 2 ** attempt); - console.log(` [OPEN_METEO] 429 for ${label}; retrying batch in ${Math.round(retryMs / 1000)}s`); + console.log(` [OPEN_METEO] ${resp.status} for ${label}; retrying batch in ${Math.round(retryMs / 1000)}s`); await sleep(retryMs); continue; } diff --git a/scripts/seed-climate-anomalies.mjs b/scripts/seed-climate-anomalies.mjs index 42be60b0b1..24857df70b 100755 --- a/scripts/seed-climate-anomalies.mjs +++ b/scripts/seed-climate-anomalies.mjs @@ -7,7 +7,7 @@ import { CLIMATE_ZONE_NORMALS_KEY } from './seed-climate-zone-normals.mjs'; loadEnvFile(import.meta.url); -const CANONICAL_KEY = 'climate:anomalies:v1'; +const CANONICAL_KEY = 'climate:anomalies:v2'; const CACHE_TTL = 10800; // 3h const ANOMALY_BATCH_SIZE = 8; const ANOMALY_BATCH_DELAY_MS = 750; @@ -118,9 +118,14 @@ function toIsoDate(date) { } export async function fetchClimateAnomalies() { + // ## First Deploy + // The anomaly cron depends on the monthly normals cache. Seed + // `node scripts/seed-climate-zone-normals.mjs` once before enabling the + // anomaly cron in a fresh environment, otherwise every 2h anomaly run will + // fail until the monthly normals cron executes on the 1st of the month. const normalsPayload = await verifySeedKey(CLIMATE_ZONE_NORMALS_KEY).catch(() => null); if (!normalsPayload?.normals?.length) { - throw new Error(`Missing ${CLIMATE_ZONE_NORMALS_KEY} baseline; run seed-climate-zone-normals.mjs first`); + throw new Error(`Missing ${CLIMATE_ZONE_NORMALS_KEY} baseline; run node scripts/seed-climate-zone-normals.mjs before enabling the anomaly cron`); } const normalsIndex = indexZoneNormals(normalsPayload); diff --git a/scripts/seed-climate-zone-normals.mjs b/scripts/seed-climate-zone-normals.mjs index ae91b1a1f7..aec0e67871 100644 --- a/scripts/seed-climate-zone-normals.mjs +++ b/scripts/seed-climate-zone-normals.mjs @@ -8,7 +8,7 @@ loadEnvFile(import.meta.url); export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; // Keep the previous baseline available across monthly cron gaps; health.js enforces freshness separately. -const NORMALS_TTL = 90 * 24 * 60 * 60; // 90 days +const NORMALS_TTL = 95 * 24 * 60 * 60; // 95 days = >3x a 31-day monthly interval const NORMALS_START = '1991-01-01'; const NORMALS_END = '2020-12-31'; const NORMALS_BATCH_SIZE = 2; @@ -26,7 +26,7 @@ function average(values) { export function computeMonthlyNormals(daily) { const dailyBucketByYearMonth = new Map(); for (let month = 1; month <= 12; month++) { - dailyBucketByYearMonth.set(month, []); + dailyBucketByYearMonth.set(month, new Map()); } const times = daily?.time ?? []; @@ -43,14 +43,13 @@ export function computeMonthlyNormals(daily) { if (!Number.isInteger(year) || !Number.isInteger(month) || month < 1 || month > 12) continue; const key = `${year}-${String(month).padStart(2, '0')}`; const bucket = dailyBucketByYearMonth.get(month); - const existing = bucket.find((entry) => entry.key === key); + const existing = bucket.get(key); if (existing) { existing.temps.push(Number(temp)); existing.precips.push(Number(precip)); continue; } - bucket.push({ - key, + bucket.set(key, { temps: [Number(temp)], precips: [Number(precip)], }); @@ -58,22 +57,22 @@ export function computeMonthlyNormals(daily) { return Array.from(dailyBucketByYearMonth.entries()) .map(([month, bucket]) => { - const monthlyMeans = bucket + const monthlyMeans = Array.from(bucket.values()) .map((entry) => ({ tempMean: average(entry.temps), precipMean: average(entry.precips), })) .filter((entry) => Number.isFinite(entry.tempMean) && Number.isFinite(entry.precipMean)); + if (monthlyMeans.length === 0) return null; + return { month, - sampleCount: monthlyMeans.length, tempMean: round(average(monthlyMeans.map((entry) => entry.tempMean))), precipMean: round(average(monthlyMeans.map((entry) => entry.precipMean))), }; }) - .filter((entry) => entry.sampleCount > 0 && Number.isFinite(entry.tempMean) && Number.isFinite(entry.precipMean)) - .map(({ sampleCount: _sampleCount, ...entry }) => entry); + .filter((entry) => entry != null && Number.isFinite(entry.tempMean) && Number.isFinite(entry.precipMean)); } export function buildZoneNormalsFromBatch(zones, batchPayloads) { diff --git a/scripts/seed-co2-monitoring.mjs b/scripts/seed-co2-monitoring.mjs index c8b25381b4..059a780e68 100644 --- a/scripts/seed-co2-monitoring.mjs +++ b/scripts/seed-co2-monitoring.mjs @@ -5,7 +5,7 @@ import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs'; loadEnvFile(import.meta.url); export const CO2_MONITORING_KEY = 'climate:co2-monitoring:v1'; -const CACHE_TTL = 86400; +const CACHE_TTL = 259200; // 72h = 3x daily interval (gold standard) const PRE_INDUSTRIAL_BASELINE = 280.0; const STATION = 'Mauna Loa, Hawaii'; diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index 52b3ae109a..7b4a3a6171 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -5,6 +5,7 @@ */ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; +export const CLIMATE_ANOMALIES_KEY = 'climate:anomalies:v2'; export const CLIMATE_ZONE_NORMALS_KEY = 'climate:zone-normals:v1'; export const CLIMATE_CO2_MONITORING_KEY = 'climate:co2-monitoring:v1'; @@ -29,7 +30,7 @@ export const BOOTSTRAP_CACHE_KEYS: Record = { chokepointTransits: 'supply_chain:chokepoint_transits:v1', minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v1', - climateAnomalies: 'climate:anomalies:v1', + climateAnomalies: 'climate:anomalies:v2', co2Monitoring: 'climate:co2-monitoring:v1', radiationWatch: 'radiation:observations:v1', thermalEscalation: 'thermal:escalation:v1', diff --git a/server/worldmonitor/climate/v1/list-climate-anomalies.ts b/server/worldmonitor/climate/v1/list-climate-anomalies.ts index 93f53e0b25..2f0dadbc07 100644 --- a/server/worldmonitor/climate/v1/list-climate-anomalies.ts +++ b/server/worldmonitor/climate/v1/list-climate-anomalies.ts @@ -11,15 +11,14 @@ import type { } from '../../../../src/generated/server/worldmonitor/climate/v1/service_server'; import { getCachedJson } from '../../../_shared/redis'; - -const SEED_CACHE_KEY = 'climate:anomalies:v1'; +import { CLIMATE_ANOMALIES_KEY } from '../../../_shared/cache-keys'; export const listClimateAnomalies: ClimateServiceHandler['listClimateAnomalies'] = async ( _ctx: ServerContext, _req: ListClimateAnomaliesRequest, ): Promise => { try { - const result = await getCachedJson(SEED_CACHE_KEY, true) as ListClimateAnomaliesResponse | null; + const result = await getCachedJson(CLIMATE_ANOMALIES_KEY, true) as ListClimateAnomaliesResponse | null; return { anomalies: result?.anomalies || [], pagination: undefined }; } catch { return { anomalies: [], pagination: undefined }; diff --git a/server/worldmonitor/intelligence/v1/get-risk-scores.ts b/server/worldmonitor/intelligence/v1/get-risk-scores.ts index f2ce5be428..1f35cae4ea 100644 --- a/server/worldmonitor/intelligence/v1/get-risk-scores.ts +++ b/server/worldmonitor/intelligence/v1/get-risk-scores.ts @@ -9,6 +9,7 @@ import type { } from '../../../../src/generated/server/worldmonitor/intelligence/v1/service_server'; import { getCachedJson, setCachedJson, cachedFetchJsonWithMeta } from '../../../_shared/redis'; +import { CLIMATE_ANOMALIES_KEY } from '../../../_shared/cache-keys'; import { TIER1_COUNTRIES } from './_shared'; import { fetchAcledCached } from '../../../_shared/acled'; @@ -247,7 +248,7 @@ async function fetchAuxiliarySources(): Promise { const [ucdpRaw, outagesRaw, climateRaw, cyberRaw, firesRaw, gpsRaw, iranRaw, orefRaw, advisoriesRaw, displacementRaw, insightsRaw, threatSummaryRaw] = await Promise.all([ getCachedJson('conflict:ucdp-events:v1', true).catch(() => null), getCachedJson('infra:outages:v1', true).catch(() => null), - getCachedJson('climate:anomalies:v1', true).catch(() => null), + getCachedJson(CLIMATE_ANOMALIES_KEY, true).catch(() => null), getCachedJson('cyber:threats-bootstrap:v2', true).catch(() => null), getCachedJson('wildfire:fires:v1', true).catch(() => null), getCachedJson('intelligence:gpsjam:v2', true).catch(() => null), diff --git a/src/services/climate/index.ts b/src/services/climate/index.ts index 28be8544f5..47e6b03eb7 100644 --- a/src/services/climate/index.ts +++ b/src/services/climate/index.ts @@ -45,6 +45,7 @@ export interface ClimateFetchResult { export interface Co2DataPoint { month: string; ppm: number; + // Year-over-year delta vs the same calendar month, in ppm. anomaly: number; } @@ -57,7 +58,7 @@ export interface Co2Monitoring { trend12m: Co2DataPoint[]; methanePpb: number; nitrousOxidePpb: number; - measuredAt: Date; + measuredAt?: Date; station: string; } @@ -129,6 +130,7 @@ function toDisplayAnomaly(proto: ProtoClimateAnomaly): ClimateAnomaly { } function toDisplayCo2Monitoring(proto: ProtoCo2Monitoring): Co2Monitoring { + const measuredAt = Number(proto.measuredAt); return { currentPpm: proto.currentPpm, yearAgoPpm: proto.yearAgoPpm, @@ -138,7 +140,7 @@ function toDisplayCo2Monitoring(proto: ProtoCo2Monitoring): Co2Monitoring { trend12m: (proto.trend12m ?? []).map(toDisplayCo2Point), methanePpb: proto.methanePpb, nitrousOxidePpb: proto.nitrousOxidePpb, - measuredAt: new Date(Number(proto.measuredAt ?? 0)), + measuredAt: Number.isFinite(measuredAt) && measuredAt > 0 ? new Date(measuredAt) : undefined, station: proto.station, }; } diff --git a/tests/climate-seeds.test.mjs b/tests/climate-seeds.test.mjs index 551ada34b0..d9dd9210f8 100644 --- a/tests/climate-seeds.test.mjs +++ b/tests/climate-seeds.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { computeMonthlyNormals, buildZoneNormalsFromBatch } from '../scripts/seed-climate-zone-normals.mjs'; import { hasRequiredClimateZones } from '../scripts/_climate-zones.mjs'; +import { fetchOpenMeteoArchiveBatch, parseRetryAfterMs } from '../scripts/_open-meteo-archive.mjs'; import { buildClimateAnomaly, buildClimateAnomaliesFromBatch, @@ -294,3 +295,95 @@ describe('co2 monitoring seed', () => { assert.equal(payload.monitoring.nitrousOxidePpb, 337.62); }); }); + +describe('open-meteo archive helper', () => { + it('caps oversized Retry-After values', () => { + assert.equal(parseRetryAfterMs('86400'), 60_000); + }); + + it('retries transient fetch errors', async () => { + const originalFetch = globalThis.fetch; + let attempts = 0; + + try { + globalThis.fetch = async () => { + attempts += 1; + if (attempts === 1) { + throw new TypeError('fetch failed'); + } + + return new Response(JSON.stringify({ + daily: { + time: ['2026-03-01'], + temperature_2m_mean: [12], + precipitation_sum: [1], + }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + + const result = await fetchOpenMeteoArchiveBatch( + [{ name: 'Retry Zone', lat: 1, lon: 2 }], + { + startDate: '2026-03-01', + endDate: '2026-03-01', + daily: ['temperature_2m_mean', 'precipitation_sum'], + maxRetries: 1, + retryBaseMs: 0, + label: 'network retry test', + }, + ); + + assert.equal(attempts, 2); + assert.equal(result.length, 1); + assert.equal(result[0].daily.time[0], '2026-03-01'); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('retries transient 503 responses', async () => { + const originalFetch = globalThis.fetch; + let attempts = 0; + + try { + globalThis.fetch = async () => { + attempts += 1; + if (attempts === 1) { + return new Response('busy', { status: 503 }); + } + + return new Response(JSON.stringify({ + daily: { + time: ['2026-03-01'], + temperature_2m_mean: [12], + precipitation_sum: [1], + }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + + const result = await fetchOpenMeteoArchiveBatch( + [{ name: 'Retry Zone', lat: 1, lon: 2 }], + { + startDate: '2026-03-01', + endDate: '2026-03-01', + daily: ['temperature_2m_mean', 'precipitation_sum'], + maxRetries: 1, + retryBaseMs: 0, + label: 'retry test', + }, + ); + + assert.equal(attempts, 2); + assert.equal(result.length, 1); + assert.equal(result[0].daily.time[0], '2026-03-01'); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/tests/mcp.test.mjs b/tests/mcp.test.mjs index 496a183060..1416f02c65 100644 --- a/tests/mcp.test.mjs +++ b/tests/mcp.test.mjs @@ -28,6 +28,7 @@ function initBody(id = 1) { } let handler; +let evaluateFreshness; describe('api/mcp.ts — PRO MCP Server', () => { beforeEach(async () => { @@ -38,6 +39,7 @@ describe('api/mcp.ts — PRO MCP Server', () => { const mod = await import(`../api/mcp.ts?t=${Date.now()}`); handler = mod.default; + evaluateFreshness = mod.evaluateFreshness; }); afterEach(() => { @@ -159,6 +161,46 @@ describe('api/mcp.ts — PRO MCP Server', () => { assert.ok('data' in data, 'data field must be present'); }); + it('evaluateFreshness marks bundled data stale when any required source meta is missing', () => { + const now = Date.UTC(2026, 3, 1, 12, 0, 0); + const freshness = evaluateFreshness( + [ + { key: 'seed-meta:climate:anomalies', maxStaleMin: 120 }, + { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 2880 }, + { key: 'seed-meta:weather:alerts', maxStaleMin: 45 }, + ], + [ + { fetchedAt: now - 30 * 60_000 }, + { fetchedAt: now - 60 * 60_000 }, + null, + ], + now, + ); + + assert.equal(freshness.stale, true); + assert.equal(freshness.cached_at, null); + }); + + it('evaluateFreshness stays fresh only when every required source meta is within its threshold', () => { + const now = Date.UTC(2026, 3, 1, 12, 0, 0); + const freshness = evaluateFreshness( + [ + { key: 'seed-meta:climate:anomalies', maxStaleMin: 120 }, + { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 2880 }, + { key: 'seed-meta:weather:alerts', maxStaleMin: 45 }, + ], + [ + { fetchedAt: now - 30 * 60_000 }, + { fetchedAt: now - 24 * 60 * 60_000 }, + { fetchedAt: now - 15 * 60_000 }, + ], + now, + ); + + assert.equal(freshness.stale, false); + assert.equal(freshness.cached_at, new Date(now - 24 * 60 * 60_000).toISOString()); + }); + // --- Rate limiting --- it('returns JSON-RPC -32029 when rate limited', async () => {