From 4dd8f948ada7be187b04ed3d5b0c0afea69e7e11 Mon Sep 17 00:00:00 2001 From: Michael Bisbjerg Date: Sat, 14 Mar 2026 16:07:52 +0100 Subject: [PATCH 1/2] Centralize timestamp parsing across integration --- .../wattplan/adapter_auto.py | 11 ++- src/custom_components/wattplan/config_flow.py | 34 +++------ src/custom_components/wattplan/coordinator.py | 14 ++-- .../wattplan/datetime_utils.py | 18 +++++ src/custom_components/wattplan/repairs.py | 9 ++- src/custom_components/wattplan/sensor.py | 10 +-- .../wattplan/source_provider.py | 31 +++++---- .../integration/test_source_flow_modifiers.py | 59 ++++++++++++++++ tests/integration/test_source_provider.py | 69 +++++++++++++++++++ 9 files changed, 188 insertions(+), 67 deletions(-) create mode 100644 src/custom_components/wattplan/datetime_utils.py diff --git a/src/custom_components/wattplan/adapter_auto.py b/src/custom_components/wattplan/adapter_auto.py index 27c72e3..db20311 100644 --- a/src/custom_components/wattplan/adapter_auto.py +++ b/src/custom_components/wattplan/adapter_auto.py @@ -8,6 +8,8 @@ import re from typing import Any +from .datetime_utils import parse_datetime_like + @dataclass(frozen=True, slots=True) class AdapterAutoDetectResult: @@ -66,13 +68,8 @@ def iter_candidate_lists(root: Any, prefix: str = "") -> list[tuple[str, list[An def _coerce_timestamp(value: Any) -> datetime | None: - """Return parsed datetime when the value looks like an ISO timestamp.""" - if not isinstance(value, str): - return None - try: - return datetime.fromisoformat(value) - except ValueError: - return None + """Return parsed datetime when the value looks like a timestamp.""" + return parse_datetime_like(value) def _coerce_decimal(value: Any) -> float | None: diff --git a/src/custom_components/wattplan/config_flow.py b/src/custom_components/wattplan/config_flow.py index bdd290e..7ab61ea 100644 --- a/src/custom_components/wattplan/config_flow.py +++ b/src/custom_components/wattplan/config_flow.py @@ -101,6 +101,7 @@ SUBENTRY_TYPE_COMFORT, SUBENTRY_TYPE_OPTIONAL, ) +from .datetime_utils import parse_datetime_like from .forecast_provider import ForecastProvider from .source_pipeline import build_source_base_provider, build_source_value_provider from .source_provider import ( @@ -140,12 +141,7 @@ def _format_coverage_datetime( value: str | datetime, timezone_name: str | None ) -> str: """Format coverage datetimes in the Home Assistant local timezone.""" - parsed = value - if isinstance(value, str): - try: - parsed = datetime.fromisoformat(value) - except ValueError: - return value + parsed = parse_datetime_like(value) if not isinstance(parsed, datetime): return str(value) try: @@ -480,20 +476,14 @@ def _built_in_history_coverage( rows: list[datetime] = [] for row in debug.get("raw_history_states", []): - last_changed = row.get("last_changed") - if isinstance(last_changed, str): - try: - rows.append(datetime.fromisoformat(last_changed)) - except ValueError: - continue + last_changed = parse_datetime_like(row.get("last_changed")) + if last_changed is not None: + rows.append(last_changed) if not rows: for row in debug.get("raw_statistics_rows", []): - started = row.get("start") - if isinstance(started, str): - try: - rows.append(datetime.fromisoformat(started)) - except ValueError: - continue + started = parse_datetime_like(row.get("start")) + if started is not None: + rows.append(started) if not rows: return start_at, start_at, 0.0 coverage_start = min(rows) @@ -524,12 +514,8 @@ def _summarize_payload_coverage( for point in payload: if not isinstance(point, dict): continue - stamp = point.get(time_key) - if not isinstance(stamp, str): - continue - try: - point_dt = datetime.fromisoformat(stamp) - except ValueError: + point_dt = parse_datetime_like(point.get(time_key)) + if point_dt is None: continue point_dt = floor_to_slot( point_dt.astimezone(UTC) diff --git a/src/custom_components/wattplan/coordinator.py b/src/custom_components/wattplan/coordinator.py index 1502219..c2fd197 100644 --- a/src/custom_components/wattplan/coordinator.py +++ b/src/custom_components/wattplan/coordinator.py @@ -65,6 +65,7 @@ SUBENTRY_TYPE_COMFORT, SUBENTRY_TYPE_OPTIONAL, ) +from .datetime_utils import parse_datetime_like from .historical_on_off_provider import HistoricalOnOffProvider from .optimizer import OptimizationParams, optimize from .source_fixup import SourceFixupProvider, SourceHealthKind @@ -103,15 +104,10 @@ def _snapshot_schema_id() -> str: def _parse_datetime(value: Any) -> datetime | None: """Parse a datetime-like restore value.""" - if isinstance(value, datetime): - return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC) - if isinstance(value, str): - try: - parsed = datetime.fromisoformat(value) - except ValueError: - return None - return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC) - return None + parsed = parse_datetime_like(value) + if parsed is None: + return None + return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC) SCHEDULE_OFFSET = timedelta(seconds=2) diff --git a/src/custom_components/wattplan/datetime_utils.py b/src/custom_components/wattplan/datetime_utils.py new file mode 100644 index 0000000..062db37 --- /dev/null +++ b/src/custom_components/wattplan/datetime_utils.py @@ -0,0 +1,18 @@ +"""Datetime parsing helpers shared across WattPlan.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + + +def parse_datetime_like(value: Any) -> datetime | None: + """Return a datetime for native datetimes or ISO-8601 strings.""" + if isinstance(value, datetime): + return value + if not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value) + except ValueError: + return None diff --git a/src/custom_components/wattplan/repairs.py b/src/custom_components/wattplan/repairs.py index 67f91fe..a894943 100644 --- a/src/custom_components/wattplan/repairs.py +++ b/src/custom_components/wattplan/repairs.py @@ -11,6 +11,7 @@ from homeassistant.helpers import issue_registry as ir from .const import DOMAIN +from .datetime_utils import parse_datetime_like from .source_issues import ( _covered_hours, source_display_name, @@ -111,11 +112,9 @@ async def async_create_fix_flow( available_count = int(data.get("available_count", 0)) required_count = int(data.get("required_count", 0)) slot_minutes = int(entry.data.get("slot_minutes", 60)) if entry else 60 - if expires_at := data.get("expires_at"): - expires_dt = datetime.fromisoformat(str(expires_at)).astimezone() - expires_local = datetime.fromisoformat(str(expires_at)).astimezone().strftime( - "%Y-%m-%d %H:%M %Z" - ) + if expires_at := parse_datetime_like(data.get("expires_at")): + expires_dt = expires_at.astimezone() + expires_local = expires_dt.strftime("%Y-%m-%d %H:%M %Z") now_local = datetime.now(tz=expires_dt.tzinfo) total_minutes = max(int((expires_dt - now_local).total_seconds() // 60), 0) hours, minutes = divmod(total_minutes, 60) diff --git a/src/custom_components/wattplan/sensor.py b/src/custom_components/wattplan/sensor.py index 4c843aa..012d034 100644 --- a/src/custom_components/wattplan/sensor.py +++ b/src/custom_components/wattplan/sensor.py @@ -33,6 +33,7 @@ SUBENTRY_TYPE_OPTIONAL, ) from .coordinator import CoordinatorSnapshot, WattPlanCoordinator +from .datetime_utils import parse_datetime_like SUBOPTIMAL_REASON_DESCRIPTIONS: dict[str, str] = { "battery_min_unmet": ( @@ -72,14 +73,7 @@ def _entry_device_info(config_entry: ConfigEntry) -> DeviceInfo: def _as_datetime(value: Any) -> datetime | None: """Convert a dynamic value to datetime when possible.""" - if isinstance(value, datetime): - return value - if isinstance(value, str): - try: - return datetime.fromisoformat(value) - except ValueError: - return None - return None + return parse_datetime_like(value) class WattPlanCoordinatorSensor(CoordinatorEntity[WattPlanCoordinator], SensorEntity): diff --git a/src/custom_components/wattplan/source_provider.py b/src/custom_components/wattplan/source_provider.py index 2e129a5..bf400bd 100644 --- a/src/custom_components/wattplan/source_provider.py +++ b/src/custom_components/wattplan/source_provider.py @@ -61,6 +61,7 @@ SOURCE_MODE_TEMPLATE, ) from .forecast_provider import ForecastProvider +from .datetime_utils import parse_datetime_like from .source_types import SourceProvider, SourceProviderError, SourceWindow CONF_WATTPLAN_ENTITY_ID = "entity_id" @@ -613,7 +614,7 @@ def _object_payload_to_points( start_value = point.get(time_key) numeric_value = point.get(value_key) - if not isinstance(start_value, str): + if not isinstance(start_value, (str, datetime)): if strict: raise SourceProviderError( "source_parse", @@ -623,11 +624,17 @@ def _object_payload_to_points( continue try: - start_dt = datetime.fromisoformat(start_value) + start_dt = parse_datetime_like(start_value) + if start_dt is None: + raise ValueError value = float(numeric_value) except (TypeError, ValueError) as err: if strict: - field_name = time_key if not isinstance(start_value, str) else value_key + field_name = ( + value_key + if isinstance(start_value, (str, datetime)) + else time_key + ) raise SourceProviderError( "source_parse", ( @@ -725,16 +732,15 @@ def _object_values( start_value = point.get(time_key) numeric_value = point.get(value_key) - if not isinstance(start_value, str): + if not isinstance(start_value, (str, datetime)): raise SourceProviderError( "source_parse", f"{self._source_name} point {index + 1} missing `{time_key}`", details={"source": self._source_name, "index": index, "key": time_key}, ) - try: - start_dt = datetime.fromisoformat(start_value) - except ValueError as err: + start_dt = parse_datetime_like(start_value) + if start_dt is None: raise SourceProviderError( "source_parse", ( @@ -746,7 +752,7 @@ def _object_values( "index": index, "value": start_value, }, - ) from err + ) try: value = float(numeric_value) @@ -1198,13 +1204,10 @@ def _energy_payload_slots( slot_delta = timedelta(minutes=window.slot_minutes) max_slot = 0 for point in payload: - stamp = point.get("start") - if not isinstance(stamp, str): - continue - try: - point_start = self._as_utc(datetime.fromisoformat(stamp)) - except ValueError: + point_start = parse_datetime_like(point.get("start")) + if point_start is None: continue + point_start = self._as_utc(point_start) if self._clamp_mode == CLAMP_MODE_NEAREST: slot_index = self._nearest_slot_index(point_start, start_at, slot_delta) diff --git a/tests/integration/test_source_flow_modifiers.py b/tests/integration/test_source_flow_modifiers.py index 476ba71..4b228a1 100644 --- a/tests/integration/test_source_flow_modifiers.py +++ b/tests/integration/test_source_flow_modifiers.py @@ -621,6 +621,65 @@ async def test_config_flow_failed_entity_auto_detect_previews_usable_providers( ) +async def test_config_flow_auto_detect_accepts_native_datetime_attributes( + hass: HomeAssistant, +) -> None: + """Entity auto-detect should accept attribute rows with native datetimes.""" + start = datetime.now(tz=UTC).replace(minute=0, second=0, microsecond=0) + hass.states.async_set( + "sensor.native_datetime_prices", + "ok", + { + "prices": [ + {"start": start, "price": 1.0}, + {"start": start + timedelta(hours=1), "price": 2.0}, + {"start": start + timedelta(hours=2), "price": 3.0}, + ] + }, + ) + + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": config_entries.SOURCE_USER} + ) + result = await hass.config_entries.flow.async_configure(result["flow_id"], {}) + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + { + CONF_NAME: "Native datetime auto detect", + CONF_SLOT_MINUTES: "60", + CONF_HOURS_TO_PLAN: "12", + }, + ) + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + {CONF_SOURCE_MODE: SOURCE_MODE_ENTITY_ADAPTER}, + ) + assert result["step_id"] == "source_price_adapter" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], + { + "entity_id": ["sensor.native_datetime_prices"], + CONF_ADAPTER_TYPE: ADAPTER_TYPE_AUTO_DETECT, + SECTION_SOURCE_MANUAL: { + CONF_NAME: "", + "time_key": "", + "value_key": "", + }, + CONF_FIXUP_PROFILE: FIXUP_PROFILE_REPAIR, + }, + ) + + assert result["type"] is FlowResultType.FORM + assert result["step_id"] == "source_review" + assert result["errors"] == {} + assert ( + "✅ Looks usable. Found forecast data in `prices` " + "using `start` for time and `price` for value." + in result["description_placeholders"]["diagnostic_text"] + ) + + async def test_options_flow_auto_detects_service_adapter( hass: HomeAssistant, ) -> None: diff --git a/tests/integration/test_source_provider.py b/tests/integration/test_source_provider.py index 8aed6ea..639d163 100644 --- a/tests/integration/test_source_provider.py +++ b/tests/integration/test_source_provider.py @@ -443,6 +443,75 @@ async def test_merged_provider_tolerates_one_empty_entity_provider( assert "produced 0 usable points" in caplog.text +async def test_entity_adapter_accepts_native_datetime_attribute_values( + hass: HomeAssistant, +) -> None: + """Entity object adapters should accept native datetime attribute values.""" + start = datetime(2026, 3, 14, 0, 0, tzinfo=UTC) + hass.states.async_set( + "sensor.native_datetime_prices", + "ok", + { + "prices": [ + {"start": start, "price": 1.0}, + {"start": start.replace(hour=1), "price": 2.0}, + {"start": start.replace(hour=2), "price": 3.0}, + {"start": start.replace(hour=3), "price": 4.0}, + ] + }, + ) + + provider = build_source_base_provider( + hass, + source_key="import_price", + source_config={ + CONF_SOURCE_MODE: SOURCE_MODE_ENTITY_ADAPTER, + "entity_id": "sensor.native_datetime_prices", + CONF_ADAPTER_TYPE: ADAPTER_TYPE_ATTRIBUTE_OBJECTS, + CONF_NAME: "prices", + "time_key": "start", + "value_key": "price", + }, + ) + + values = await provider.async_values( + SourceWindow( + start_at=start, + slot_minutes=60, + slots=4, + ) + ) + + assert values == [1.0, 2.0, 3.0, 4.0] + + +async def test_auto_detect_entity_adapter_accepts_native_datetime_attributes( + hass: HomeAssistant, +) -> None: + """Auto-detect should treat native datetime values as timestamp-like.""" + start = datetime(2026, 3, 14, 0, 0, tzinfo=UTC) + hass.states.async_set( + "sensor.native_datetime_prices", + "ok", + { + "prices": [ + {"start": start, "price": 1.0}, + {"start": start.replace(hour=1), "price": 2.0}, + ] + }, + ) + + detected = await async_auto_detect_entity_adapter( + hass, + ["sensor.native_datetime_prices"], + ) + + assert len(detected) == 1 + assert detected[0].root_key == "prices" + assert detected[0].time_key == "start" + assert detected[0].value_key == "price" + + async def test_service_adapter_provider_returns_values(hass: HomeAssistant) -> None: """Service adapter should return nested service response values.""" From b628a614a503c61b5ec81b6f68342dcf8a10e490 Mon Sep 17 00:00:00 2001 From: Michael Bisbjerg Date: Sat, 14 Mar 2026 17:19:36 +0100 Subject: [PATCH 2/2] Normalize battery hold charge source --- .../wattplan/optimizer/mpc_power_optimizer.py | 17 +++- src/custom_components/wattplan/sensor.py | 18 +++- tests/integration/test_integration_runtime.py | 89 +++++++++++++++++++ tests/optimizer/test_optimizer_scenarios.py | 16 ++++ 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/src/custom_components/wattplan/optimizer/mpc_power_optimizer.py b/src/custom_components/wattplan/optimizer/mpc_power_optimizer.py index a464803..98dd201 100644 --- a/src/custom_components/wattplan/optimizer/mpc_power_optimizer.py +++ b/src/custom_components/wattplan/optimizer/mpc_power_optimizer.py @@ -1169,6 +1169,18 @@ def _optional_entity_options(entity, grid_import_prices, baseline_net_import): ] +def _battery_schedule_charge_source(result, battery_index: int, timeslot: int) -> int: + """Return a normalized charge source bitmask for one battery schedule slot.""" + battery_state = int(result["battery_states"][battery_index, timeslot]) + if battery_state == 0: + return 0 + + return int( + (1 if result["battery_charge_grid"][battery_index, timeslot] > EPSILON else 0) + | (2 if result["battery_charge_pv"][battery_index, timeslot] > EPSILON else 0) + ) + + def optimize_internal(normalized: CalculationInput): total_steps = normalized.total_steps grid_import_prices = normalized.grid_import_prices @@ -1266,9 +1278,8 @@ def optimize_internal(normalized: CalculationInput): "state": battery_state_name[ int(result["battery_states"][i, t]) ], - "charge_source": int( - (1 if result["battery_charge_grid"][i, t] > EPSILON else 0) - | (2 if result["battery_charge_pv"][i, t] > EPSILON else 0) + "charge_source": _battery_schedule_charge_source( + result, i, t ), "level": float(result["battery_levels"][i, t + 1]), } diff --git a/src/custom_components/wattplan/sensor.py b/src/custom_components/wattplan/sensor.py index 012d034..9fcd291 100644 --- a/src/custom_components/wattplan/sensor.py +++ b/src/custom_components/wattplan/sensor.py @@ -50,6 +50,13 @@ ), } +BATTERY_CHARGE_SOURCE_LABELS: dict[str, str] = { + "n": "(N)one", + "g": "(G)rid", + "p": "(P)V", + "gp": "(G)rid and (P)V", +} + def _subentry_slug(subentry: Any) -> str: """Return slug for subentry naming.""" @@ -76,6 +83,11 @@ def _as_datetime(value: Any) -> datetime | None: return parse_datetime_like(value) +def _friendly_charge_source_label(charge_source: str) -> str: + """Return a user-facing charge source label for compact planner codes.""" + return BATTERY_CHARGE_SOURCE_LABELS.get(charge_source, charge_source) + + class WattPlanCoordinatorSensor(CoordinatorEntity[WattPlanCoordinator], SensorEntity): """Base WattPlan sensor backed by coordinator snapshots.""" @@ -281,7 +293,11 @@ def extra_state_attributes(self) -> dict[str, str] | None: attrs["next_action"] = next_action if self._group == "batteries" and (charge_source := data.get("charge_source")): - attrs["charge_source"] = str(charge_source) + charge_source_code = str(charge_source) + attrs["charge_source"] = charge_source_code + attrs["charge_source_friendly"] = _friendly_charge_source_label( + charge_source_code + ) return attrs or None diff --git a/tests/integration/test_integration_runtime.py b/tests/integration/test_integration_runtime.py index 93bc214..c88d73e 100644 --- a/tests/integration/test_integration_runtime.py +++ b/tests/integration/test_integration_runtime.py @@ -333,10 +333,99 @@ async def test_full_runtime_optimize_and_emit_once(hass: HomeAssistant) -> None: battery_action = hass.states.get("sensor.home_battery_action") assert battery_action is not None + assert battery_action.attributes["charge_source"] == "g" + assert battery_action.attributes["charge_source_friendly"] == "(G)rid" assert battery_action.attributes["next_action"] == "hold" assert "next_action_timestamp" in battery_action.attributes +async def test_battery_action_sensor_exposes_friendly_combined_charge_source( + hass: HomeAssistant, +) -> None: + """Battery action sensor should expose a friendly label for source codes.""" + entry = MockConfigEntry( + domain=DOMAIN, + title="Home", + data={ + CONF_NAME: "Home", + CONF_SLOT_MINUTES: 60, + CONF_HOURS_TO_PLAN: 4, + CONF_SOURCES: { + CONF_SOURCE_IMPORT_PRICE: { + CONF_SOURCE_MODE: SOURCE_MODE_TEMPLATE, + CONF_TEMPLATE: "{{ [0.2, 0.25, 0.3, 0.35] }}", + }, + CONF_SOURCE_USAGE: { + CONF_SOURCE_MODE: SOURCE_MODE_TEMPLATE, + CONF_TEMPLATE: "{{ [1.0, 1.1, 1.0, 0.9] }}", + }, + CONF_SOURCE_PV: { + CONF_SOURCE_MODE: SOURCE_MODE_TEMPLATE, + CONF_TEMPLATE: "{{ [0.0, 0.2, 0.3, 0.1] }}", + }, + }, + }, + options={ + CONF_PLANNING_ENABLED: False, + CONF_ACTION_EMISSION_ENABLED: False, + }, + subentries_data=[ + config_entries.ConfigSubentryData( + subentry_id="battery_sub", + subentry_type=SUBENTRY_TYPE_BATTERY, + title="battery", + unique_id="battery:battery", + data={ + CONF_NAME: "battery", + CONF_SOC_SOURCE: "sensor.battery_soc", + CONF_CAPACITY_KWH: 10.0, + CONF_MINIMUM_KWH: 1.0, + CONF_MAX_CHARGE_KW: 3.0, + CONF_MAX_DISCHARGE_KW: 3.0, + CONF_CHARGE_EFFICIENCY: 0.9, + CONF_DISCHARGE_EFFICIENCY: 0.9, + CONF_CAN_CHARGE_FROM_GRID: True, + CONF_CAN_CHARGE_FROM_PV: True, + }, + ) + ], + ) + entry.add_to_hass(hass) + + hass.states.async_set("sensor.battery_soc", "5.0") + + with patch("custom_components.wattplan.coordinator.optimize") as optimize_mock: + optimize_mock.return_value = { + **_fake_optimize(None), + "entities": [ + { + "name": "battery", + "type": "battery", + "schedule": [ + {"state": "charge", "charge_source": 3, "level": 5.2}, + {"state": "hold", "charge_source": 0, "level": 5.2}, + {"state": "hold", "charge_source": 0, "level": 5.2}, + {"state": "hold", "charge_source": 0, "level": 5.2}, + ], + } + ], + "optional_entity_options": [], + } + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + + await hass.services.async_call( + DOMAIN, SERVICE_RUN_OPTIMIZE_NOW, {}, blocking=True + ) + await hass.async_block_till_done() + + battery_action = hass.states.get("sensor.home_battery_action") + assert battery_action is not None + assert battery_action.state == "charge" + assert battery_action.attributes["charge_source"] == "gp" + assert battery_action.attributes["charge_source_friendly"] == "(G)rid and (P)V" + + async def test_restore_snapshot_on_startup(hass: HomeAssistant) -> None: """Restore the serialized coordinator snapshot so entities keep their last plan.""" entry = MockConfigEntry( diff --git a/tests/optimizer/test_optimizer_scenarios.py b/tests/optimizer/test_optimizer_scenarios.py index efbe65e..697a25c 100644 --- a/tests/optimizer/test_optimizer_scenarios.py +++ b/tests/optimizer/test_optimizer_scenarios.py @@ -501,6 +501,22 @@ def test_feed_in_prices_shift_pv_charging_to_lower_export_value_slots(): ) +def test_hold_state_always_serializes_no_charge_source(): + result = { + "battery_states": optimizer.np.asarray([[0, 1, 2]], dtype=optimizer.np.float64), + "battery_charge_grid": optimizer.np.asarray( + [[1.0, 1.0, 0.0]], dtype=optimizer.np.float64 + ), + "battery_charge_pv": optimizer.np.asarray( + [[1.0, 0.0, 1.0]], dtype=optimizer.np.float64 + ), + } + + assert optimizer._battery_schedule_charge_source(result, 0, 0) == 0 + assert optimizer._battery_schedule_charge_source(result, 0, 1) == 1 + assert optimizer._battery_schedule_charge_source(result, 0, 2) == 2 + + def test_live_grid_export_benchmark_scenario_uses_real_15min_stromligning_values(): # Live Home Assistant data captured on 2026-03-09 in Europe/Copenhagen. # Strømligning is native 15-minute price data. Deye daily energy totals are