diff --git a/docs/command-contract.md b/docs/command-contract.md index 49e17786..648535bf 100644 --- a/docs/command-contract.md +++ b/docs/command-contract.md @@ -871,7 +871,8 @@ customization: optional embedded `query.metric_map` (overridden by `--metric-map-file` on duplicate keys). When a dashboard carries a `gnetId` that matches a bundled curated pack (e.g. Redis 763 / 11835 / 14091, Redis Enterprise 18405, Redis - Cloud 18406, Node Exporter Full 1860), the + Cloud 18406, Node Exporter Full 1860, MySQL Overview 7362, PostgreSQL + Database 9628, PostgreSQL Exporter Quickstart 14114), the pack is merged in automatically beneath the user `--rules-file` so the user always wins on collision. Pass `--no-curated-packs` to skip all curated packs and use only the base rule pack. @@ -907,7 +908,9 @@ Two further gates still push panels to ES|QL even when the target supports limits can still degrade individual panels to ES|QL even on Kibana 9.5+. 2. **Curated pack / construct limits** — pack overrides and unsupported PromQL shapes can still emit ES|QL. Pass `--no-curated-packs` to isolate the core - translator. + translator. Same-metric Grafana fallbacks (`rate(M[$interval]) or irate(M[5m])`) + collapse to the left operand before the native path, matching ES|QL + translation. True set-union `or` / `and` / `unless` still degrades. Construct-level unsupported cases can still degrade or require manual review. Datadog accepts `--translation-mode` for CLI parity, but it is a no-op because diff --git a/docs/sources/grafana.md b/docs/sources/grafana.md index e5584b6f..7dd11c50 100644 --- a/docs/sources/grafana.md +++ b/docs/sources/grafana.md @@ -122,7 +122,27 @@ field-caps discovery (`--es-url`) to resolve the metric's actual field name; without it they no-op rather than guess. Metrics listed in the pack's `live_optional_metrics` that field-caps proved absent are stripped from the hand-written override so the rest of the panel can still render; those -omissions are not reported as pack gaps. +omissions are not reported as pack gaps. `kibana_type_override` forces the +Lens chart type when the curated query shape does not match the Grafana panel +(for example a stacked CPU graph emitted as overlay lines). `drop_time_from` +strips Grafana's panel `timeFrom` so the override follows the dashboard time +picker; use it when a pinned window (commonly 24h hourly bars) renders empty +in Lens on mixed `metrics-*` even though `_query` returns rows. +`panel.layout_overrides` can also set `title` to rename a section or leaf +panel after translation (Grafana's empty first row becomes Kibana +"Section 1"; a pack can rename it to "Overview"). Grafana 5 singlestat +panels store units on the panel root (`format: bytes` / `s` / `percent`); +those map to Lens bytes, duration, and `%` formats. Helm-flavored community +dashboards (PostgreSQL Database 9628) may also ship a pack `plugin.py` that +rewrites `query_result()` Instance variables to `label_values()` so Kibana +still gets a populate query after the unused `release` / `namespace` cascade +is dropped. Native PROMQL also strips `ignored_labels` matchers (so a Helm +`release` filter cannot bind a kernel `release` field from mixed `metrics-*` +and empty the panel). The PostgreSQL Exporter Quickstart (14114) plugin +rewrites Instance from Prometheus `up{job=~"postgres.*"}` to +`label_values(pg_up, instance)` — Elastic prometheus_native scrapes store +exporter health as `pg_up`, not scrape `up` — and drops the unused `$job` +control so native PROMQL panels are not left with an empty Instance param. The console pipeline is **5 stages**, not 7: `[1/5] Extracting dashboards`, `[2/5] Translating dashboards`, `[3/5] Verification-packet ES|QL validation`, diff --git a/observability_migration/adapters/source/datadog/generate.py b/observability_migration/adapters/source/datadog/generate.py index e60e6582..cfc15df4 100644 --- a/observability_migration/adapters/source/datadog/generate.py +++ b/observability_migration/adapters/source/datadog/generate.py @@ -971,12 +971,11 @@ def _build_esql_panel( for field in keep_fields ] else: - if widget.widget_type == "change" and "value" not in metrics: + if widget.widget_type == "change": # A change widget ranks by the computed delta `value` (the SORT - # key), which STATS-based inference misses because it is an EVAL - # alias. Surface it as the leading metric so the ranked delta — - # the whole point of the widget — is actually displayed. - metrics = ["value", *metrics] + # key). Keep it as the leading metric even when shape inference + # already includes the EVAL alias among the STATS columns. + metrics = ["value", *[m for m in metrics if m != "value"]] if metrics: esql_block["metrics"] = [ _metric_config(widget, result, m) diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/fidelity_manifest.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/fidelity_manifest.yaml new file mode 100644 index 00000000..7165424d --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/fidelity_manifest.yaml @@ -0,0 +1,52 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Fidelity manifest — Grafana dashboard 14114 (PostgreSQL Exporter Quickstart) +# https://grafana.com/grafana/dashboards/14114-postgres-overview/ +# +# PERFECT = same information as Grafana on postgres_exporter. +# APPROXIMATE = documented delta (lifetime cache-hit ratio, includeAll All). + +schema_version: 1 +gnet_id: 14114 +gnet_revision: 1 +dashboard_title: "PostgreSQL Exporter Quickstart and Dashboard" +maintainer: "Grafana Labs" + +panels: + - title: "Rows" + fidelity: PERFECT + notes: "irate() of tup_fetched/returned/inserted/updated/deleted; ES|QL fusion of five targets." + - title: "QPS" + fidelity: PERFECT + notes: "Native PROMQL sum(irate(commit)) + sum(irate(rollback))." + - title: "Buffers" + fidelity: PERFECT + notes: > + Source names pg_stat_bgwriter_buffers_* without OpenMetrics _total. + Pack metric_map (and live suffix alias) retargets to *_total on + postgres_exporter v0.15. ES|QL IRATE of the five series. + - title: "Conflicts/Deadlocks" + fidelity: PERFECT + notes: "rate() of deadlocks and conflicts counters." + - title: "Cache hit ratio" + fidelity: APPROXIMATE + notes: "Grafana divides lifetime blks_hit / (blks_read + blks_hit) without rate(). Native PROMQL preserves that lifetime ratio." + - title: "Number of active connections" + fidelity: APPROXIMATE + notes: > + Native PROMQL of pg_stat_database_numbackends returns the gauge series. + Grafana legend {{__name__}} GROKs native PROMQL _timeseries, which does + not carry __name__, so Kibana shows "(null)" in the legend. The line is + the connection count. + +summary: + total_panels: 6 + perfect: 4 + approximate: 2 + not_feasible: 0 + overall_fidelity: HIGH + known_gaps: + - "Grafana $job is unused in panel queries; pack drops the Kibana control" + - "Instance populate query rewritten from Prometheus up to pg_up" + - "instance/db includeAll All is a single-select .* token in Kibana" diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/pack.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/pack.yaml new file mode 100644 index 00000000..f80860b6 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/pack.yaml @@ -0,0 +1,88 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Curated pack — Grafana dashboard 14114 +# https://grafana.com/grafana/dashboards/14114-postgres-overview/ +# +# Source: Grafana Labs "PostgreSQL Exporter Quickstart and Dashboard" +# (postgres_exporter mixin / Prometheus scrape) +# Revision 1. Panels: 6 (5 graph + 1 singlestat). +# +# Engine vs pack split (do not duplicate engine work here): +# Engine: OpenMetrics ``_total`` suffix alias against live field-caps +# (Buffers without ``--es-url`` still needs the pack map below); +# native PROMQL for QPS / cache-hit / numbackends; ES|QL fusion +# for multi-target Rows / Buffers / Conflicts. +# Pack: mixin Instance is ``label_values(up{job=~"postgres.*"}, instance)`` +# but Elastic prometheus_native scrapes store postgres health as +# ``pg_up``, not Prometheus ``up`` (that series is redis-only in +# mixed ``metrics-*``). plugin.py rewrites Instance to +# ``label_values(pg_up, instance)`` and drops the unused ``job`` +# control (no panel binds ``$job``). ``metric_map`` renames the +# five pre-OpenMetrics bgwriter counters so offline ES|QL matches +# postgres_exporter v0.15. + +query: + metrics_dataset_filter: "prometheus" + + label_rewrites: + instance: labels.instance + job: labels.job + datname: labels.datname + db: labels.datname + + metric_kinds: + pg_stat_database_tup_fetched: counter + pg_stat_database_tup_returned: counter + pg_stat_database_tup_inserted: counter + pg_stat_database_tup_updated: counter + pg_stat_database_tup_deleted: counter + pg_stat_database_xact_commit: counter + pg_stat_database_xact_rollback: counter + pg_stat_database_deadlocks: counter + pg_stat_database_conflicts: counter + pg_stat_database_blks_hit: counter + pg_stat_database_blks_read: counter + pg_stat_bgwriter_buffers_alloc: counter + pg_stat_bgwriter_buffers_backend: counter + pg_stat_bgwriter_buffers_backend_fsync: counter + pg_stat_bgwriter_buffers_clean: counter + pg_stat_bgwriter_buffers_checkpoint: counter + pg_stat_bgwriter_buffers_alloc_total: counter + pg_stat_bgwriter_buffers_backend_total: counter + pg_stat_bgwriter_buffers_backend_fsync_total: counter + pg_stat_bgwriter_buffers_clean_total: counter + pg_stat_bgwriter_buffers_checkpoint_total: counter + pg_stat_database_numbackends: gauge + pg_up: gauge + + # metric_map targets are emitted VERBATIM (the field profile prefix is NOT + # prepended). See docs/command-contract.md, "metric_map targets are verbatim". + metric_map: + pg_stat_bgwriter_buffers_alloc: metrics.pg_stat_bgwriter_buffers_alloc_total + pg_stat_bgwriter_buffers_backend: metrics.pg_stat_bgwriter_buffers_backend_total + pg_stat_bgwriter_buffers_backend_fsync: metrics.pg_stat_bgwriter_buffers_backend_fsync_total + pg_stat_bgwriter_buffers_clean: metrics.pg_stat_bgwriter_buffers_clean_total + pg_stat_bgwriter_buffers_checkpoint: metrics.pg_stat_bgwriter_buffers_checkpoint_total + + label_candidates: + instance: + - labels.instance + - service.instance.id + - host.name + job: + - labels.job + - service.name + datname: + - labels.datname + - datname + db: + - labels.datname + - datname + +controls: + field_overrides: + instance: labels.instance + db: labels.datname + datname: labels.datname + job: labels.job diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/plugin.py b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/plugin.py new file mode 100644 index 00000000..b6df8171 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_14114_postgres_exporter_quickstart/plugin.py @@ -0,0 +1,36 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 + +"""Grafana 14114 (PostgreSQL Exporter Quickstart) curated pack plugin. + +Revision 1's Instance variable is ``label_values(up{job=~"postgres.*"}, instance)``. +Prometheus ``up`` is not stored for postgres_exporter on typical Elastic +prometheus_native scrapes (``pg_up`` is). The unused ``job`` template never +appears in panel PromQL, so emitting it only adds an incompatible default +(``postgres`` vs ``postgres_exporter``). +""" + + +_PACK_NAME = "grafana_14114_postgres_exporter_quickstart" + + +def register(api): + @api["variable_translators"].register("grafana_14114_pg_up_instance", priority=5) + def rewrite_instance_and_drop_job(context): + pack = getattr(context, "rule_pack", None) + if getattr(pack, "_curated_pack_name", "") != _PACK_NAME: + return None + variable = context.variable or {} + name = str(variable.get("name") or "") + query_text = context.query_text or str(variable.get("query") or "") + compact = query_text.replace(" ", "").lower() + if name == "job": + context.handled = True + return f"skipped unused job variable {name}" + if name == "instance" and "label_values(up" in compact: + rewritten = "label_values(pg_up, instance)" + context.query_text = rewritten + context.variable = dict(variable) + context.variable["query"] = rewritten + return None + return None diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/fidelity_manifest.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/fidelity_manifest.yaml new file mode 100644 index 00000000..d47e58ba --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/fidelity_manifest.yaml @@ -0,0 +1,123 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Fidelity manifest — Grafana dashboard 7362 (MySQL Overview) +# https://grafana.com/grafana/dashboards/7362-mysql-overview/ +# +# PERFECT = same information as Grafana on mysqld_exporter + node_exporter. +# APPROXIMATE = documented delta (MySQL 8 replacements, $interval, host split). + +schema_version: 1 +gnet_id: 7362 +gnet_revision: 5 +dashboard_title: "MySQL Overview" + +panels: + - title: "MySQL Uptime" + fidelity: PERFECT + notes: "Native PROMQL LAST of mysql_global_status_uptime. Grafana 5 singlestat format s maps to a duration tile." + - title: "Current QPS" + fidelity: APPROXIMATE + notes: "rate() or irate() prefers rate; Grafana $interval is inlined to TBUCKET. Requires untyped queries pinned as counter." + - title: "InnoDB Buffer Pool Size" + fidelity: PERFECT + notes: "Native PROMQL LAST of innodb_buffer_pool_size. Grafana 5 singlestat format bytes maps to a bytes tile." + - title: "Buffer Pool Size of Total RAM" + fidelity: APPROXIMATE + notes: "Cross-instance ratio when mysqld and node_exporter use different instance labels. TS LAST_OVER_TIME of both gauges over the dashboard picker (FROM over mixed metrics-* timed out as N/A in Lens). Grafana 5 singlestat format percent maps to a % suffix." + - title: "MySQL Connections" + fidelity: PERFECT + notes: "" + - title: "MySQL Client Thread Activity" + fidelity: APPROXIMATE + notes: "max_over_time($interval) or 5m prefers the left window." + - title: "MySQL Questions" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Thread Cache" + fidelity: APPROXIMATE + notes: "threads_created rate requires counter typing." + - title: "MySQL Temporary Objects" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Select Types" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Sorts" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Slow Queries" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Aborted Connections" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Table Locks" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Network Traffic" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Network Usage Hourly" + fidelity: APPROXIMATE + notes: "Grafana pins a 24h panel window with 1h increase() bars. Kibana follows the dashboard time picker (mixed metrics-* Lens 24h override rendered empty despite rows). RATE over TBUCKET(20)." + - title: "MySQL Internal Memory Overview" + fidelity: APPROXIMATE + notes: "Query cache / additional mem pool / TokuDB series are live_optional and omitted on MySQL 8." + - title: "Top Command Counters" + fidelity: APPROXIMATE + notes: "topk(rate or irate) prefers rate; $interval inlined." + - title: "Top Command Counters Hourly" + fidelity: APPROXIMATE + notes: "Grafana 24h/1h increase() bars; Kibana follows the dashboard picker with RATE by labels.command. No hard top-k cut." + - title: "MySQL Handlers" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Transaction Handlers" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "Process States" + fidelity: PERFECT + notes: "Pack maps mysql_info_schema_threads → mysql_info_schema_processlist_threads and groups by labels.state." + - title: "Top Process States Hourly" + fidelity: APPROXIMATE + notes: "Grafana 24h topk(5, avg_over_time[1h]); Kibana follows the dashboard picker with AVG_OVER_TIME by state and no hard top-5 cut." + - title: "MySQL Query Cache Memory" + fidelity: APPROXIMATE + notes: "Query cache removed in MySQL 8. Kibana panel title is InnoDB Buffer Pool Pages; shows InnoDB buffer-pool data/free/dirty pages instead of an empty cache gauge." + - title: "MySQL Query Cache Activity" + fidelity: APPROXIMATE + notes: "Query cache removed in MySQL 8. Kibana panel title is InnoDB Buffer Pool Activity; shows InnoDB buffer-pool hit ratio plus data read/write rates." + - title: "MySQL File Openings" + fidelity: APPROXIMATE + notes: "rate/irate pair; $interval inlined." + - title: "MySQL Open Files" + fidelity: PERFECT + notes: "" + - title: "MySQL Table Open Cache Status" + fidelity: APPROXIMATE + notes: "rate/irate pair plus hit-ratio; $interval inlined." + - title: "MySQL Open Tables" + fidelity: PERFECT + notes: "" + - title: "MySQL Table Definition Cache" + fidelity: APPROXIMATE + notes: "opened_table_definitions rate requires counter typing." + - title: "I/O Activity" + fidelity: APPROXIMATE + notes: "Host filter dropped: mysql_up instance does not match node_exporter instance. Shows node_exporter pgpgin/pgpgout." + - title: "Memory Distribution" + fidelity: APPROXIMATE + notes: "Host filter dropped for the same instance split. Uses node_exporter 0.16 *_bytes names." + - title: "CPU Usage / Load" + fidelity: APPROXIMATE + notes: "Grafana per-mode stacked clamp_max(rate or irate)*100 plus Load on a second axis is not_feasible. Kibana shows overall non-idle busy % and load1 as overlay lines, without the mysql host filter. Load 1m is on the right axis without a % suffix." + - title: "Disk Latency" + fidelity: APPROXIMATE + notes: "Linux read/write latency ratio without AWS RDS fallback; host filter dropped." + - title: "Network Traffic" + fidelity: APPROXIMATE + notes: "node_exporter receive/transmit rates without RDS OSMetrics fallback; host filter dropped. Distinct from MySQL Network Traffic." + - title: "Swap Activity" + fidelity: APPROXIMATE + notes: "Host filter dropped; pswpin/pswpout * 4096." diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/pack.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/pack.yaml new file mode 100644 index 00000000..c078d572 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_7362_mysql_overview/pack.yaml @@ -0,0 +1,282 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Curated pack — Grafana dashboard 7362 +# https://grafana.com/grafana/dashboards/7362-mysql-overview/ +# +# Source: Percona Monitoring and Management "MySQL Overview" +# (mysqld_exporter + node_exporter Prometheus scrape) +# Revision 5. Panels: 36 renderable. +# +# Engine vs pack split (do not duplicate engine work here): +# Engine: mixed ``rate(node) or irate(node) or rdsosmetrics/aws_rds`` chains +# drop live-absent cloud fallbacks; ``sum((rate/rate) or …)`` prefers +# the Linux disk ratio when AWS RDS metrics are absent. +# Pack: mysqld_exporter untyped status counters, processlist rename, +# MySQL 8 query-cache replacements, System Chart host-label split +# (mysql instance ≠ node_exporter instance). + +query: + metrics_dataset_filter: "prometheus" + + label_rewrites: + instance: labels.instance + job: labels.job + state: labels.state + handler: labels.handler + command: labels.command + + # mysqld_exporter publishes these as ``# TYPE untyped``. Elasticsearch infers + # gauge from the missing ``_total`` suffix, so RATE() 400s at render time. + # Pin counter here (and in the lab scraper) so ``rate()``/``increase()`` stay + # RATE()/Increase, matching PromQL. + metric_kinds: + mysql_global_status_queries: counter + mysql_global_status_questions: counter + mysql_global_status_threads_created: counter + mysql_global_status_created_tmp_tables: counter + mysql_global_status_created_tmp_disk_tables: counter + mysql_global_status_created_tmp_files: counter + mysql_global_status_select_full_join: counter + mysql_global_status_select_full_range_join: counter + mysql_global_status_select_range: counter + mysql_global_status_select_range_check: counter + mysql_global_status_select_scan: counter + mysql_global_status_sort_rows: counter + mysql_global_status_sort_range: counter + mysql_global_status_sort_merge_passes: counter + mysql_global_status_sort_scan: counter + mysql_global_status_slow_queries: counter + mysql_global_status_aborted_connects: counter + mysql_global_status_aborted_clients: counter + mysql_global_status_table_locks_immediate: counter + mysql_global_status_table_locks_waited: counter + mysql_global_status_bytes_received: counter + mysql_global_status_bytes_sent: counter + mysql_global_status_opened_files: counter + mysql_global_status_opened_tables: counter + mysql_global_status_table_open_cache_hits: counter + mysql_global_status_table_open_cache_misses: counter + mysql_global_status_table_open_cache_overflows: counter + mysql_global_status_opened_table_definitions: counter + mysql_global_status_commands_total: counter + mysql_global_status_handlers_total: counter + mysql_global_status_innodb_buffer_pool_read_requests: counter + mysql_global_status_innodb_buffer_pool_reads: counter + mysql_global_status_innodb_data_reads: counter + mysql_global_status_innodb_data_writes: counter + mysql_global_status_qcache_hits: counter + mysql_global_status_qcache_inserts: counter + mysql_global_status_qcache_not_cached: counter + mysql_global_status_qcache_lowmem_prunes: counter + node_vmstat_pgpgin: counter + node_vmstat_pgpgout: counter + node_vmstat_pswpin: counter + node_vmstat_pswpout: counter + node_cpu_seconds_total: counter + node_disk_read_time_seconds_total: counter + node_disk_reads_completed_total: counter + node_disk_write_time_seconds_total: counter + node_disk_writes_completed_total: counter + node_network_receive_bytes_total: counter + node_network_transmit_bytes_total: counter + mysql_global_status_uptime: gauge + mysql_global_variables_innodb_buffer_pool_size: gauge + mysql_global_status_threads_connected: gauge + mysql_global_status_threads_running: gauge + mysql_global_status_threads_cached: gauge + mysql_global_status_max_used_connections: gauge + mysql_global_variables_max_connections: gauge + mysql_global_variables_thread_cache_size: gauge + mysql_info_schema_processlist_threads: gauge + mysql_info_schema_threads: gauge + node_load1: gauge + node_memory_MemTotal_bytes: gauge + + # metric_map targets are emitted VERBATIM (no field-profile prefix). + metric_map: + mysql_info_schema_threads: metrics.mysql_info_schema_processlist_threads + + live_optional_metrics: + - mysql_global_variables_innodb_additional_mem_pool_size + - mysql_global_variables_query_cache_size + - mysql_global_status_qcache_free_memory + - mysql_global_status_qcache_hits + - mysql_global_status_qcache_inserts + - mysql_global_status_qcache_not_cached + - mysql_global_status_qcache_lowmem_prunes + - mysql_global_status_qcache_queries_in_cache + - mysql_global_variables_tokudb_cache_size + - mysql_info_schema_threads + - rdsosmetrics_network_rx + - rdsosmetrics_network_tx + - aws_rds_read_latency_average + - aws_rds_write_latency_average + - node_cpu_seconds_total_average + + label_candidates: + instance: + - labels.instance + - service.instance.id + - host.name + job: + - labels.job + - service.name + state: + - labels.state + - state + handler: + - labels.handler + - handler + command: + - labels.command + - command + +controls: + field_overrides: + host: labels.instance + +panel: + query_overrides: + - title_match: "Buffer Pool Size of Total RAM" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | STATS pool = MAX(LAST_OVER_TIME({{metric:mysql_global_variables_innodb_buffer_pool_size:gauge}})), ram = MAX(LAST_OVER_TIME({{metric:node_memory_MemTotal_bytes:gauge}})) + | EVAL computed_value = CASE(ram > 0, ((pool * 100) / ram), NULL) + | KEEP computed_value + status_override: migrated + - title_match: "MySQL Network Usage Hourly" + drop_time_from: true + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_global_status_bytes_received:counter}} IS NOT NULL OR {{metric:mysql_global_status_bytes_sent:counter}} IS NOT NULL + | STATS Received = SUM(RATE({{metric:mysql_global_status_bytes_received:counter}})), Sent = SUM(RATE({{metric:mysql_global_status_bytes_sent:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | SORT time_bucket ASC + status_override: migrated + - title_match: "Top Command Counters Hourly" + drop_time_from: true + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_global_status_commands_total:counter}} IS NOT NULL + | STATS Commands = SUM(RATE({{metric:mysql_global_status_commands_total:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend), labels.command + | WHERE Commands > 0 + | SORT time_bucket ASC + status_override: migrated + - title_match: "Process States" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_info_schema_processlist_threads:gauge}} IS NOT NULL + | STATS process_threads = MAX(LAST_OVER_TIME({{metric:mysql_info_schema_processlist_threads:gauge}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend), labels.state + | SORT time_bucket ASC + status_override: migrated + - title_match: "Top Process States Hourly" + drop_time_from: true + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_info_schema_processlist_threads:gauge}} IS NOT NULL + | STATS process_threads = MAX(AVG_OVER_TIME({{metric:mysql_info_schema_processlist_threads:gauge}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend), labels.state + | SORT time_bucket ASC + status_override: migrated + # Query cache was removed in MySQL 8. Show InnoDB buffer-pool pages instead. + - title_match: "MySQL Query Cache Memory" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_global_status_buffer_pool_pages:gauge}} IS NOT NULL + | STATS pages = MAX(LAST_OVER_TIME({{metric:mysql_global_status_buffer_pool_pages:gauge}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend), labels.state + | SORT time_bucket ASC + status_override: migrated + - title_match: "MySQL Query Cache Activity" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (?host == "" OR ({{label:instance}} RLIKE ?host OR ({{label:instance}} IS NULL AND "" RLIKE ?host))) + | WHERE {{metric:mysql_global_status_innodb_buffer_pool_read_requests:counter}} IS NOT NULL OR {{metric:mysql_global_status_innodb_buffer_pool_reads:counter}} IS NOT NULL OR {{metric:mysql_global_status_innodb_data_reads:counter}} IS NOT NULL OR {{metric:mysql_global_status_innodb_data_writes:counter}} IS NOT NULL + | STATS requests = SUM(RATE({{metric:mysql_global_status_innodb_buffer_pool_read_requests:counter}})), reads = SUM(RATE({{metric:mysql_global_status_innodb_buffer_pool_reads:counter}})), data_reads = SUM(RATE({{metric:mysql_global_status_innodb_data_reads:counter}})), data_writes = SUM(RATE({{metric:mysql_global_status_innodb_data_writes:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | EVAL Hit_ratio_pct = CASE(requests > 0, ((1 - (reads / requests)) * 100), NULL) + | KEEP time_bucket, Hit_ratio_pct, data_reads, data_writes + | SORT time_bucket ASC + status_override: migrated + # PMM assumes mysqld and node_exporter share ``instance``. Typical scrapes + # use mysql:3306 vs node:9100, and ``host`` is sourced from mysql_up only. + # Drop the host matcher so System Charts light up from node_exporter. + - title_match: "CPU Usage / Load" + kibana_type_override: line + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE {{metric:node_cpu_seconds_total:counter}} IS NOT NULL OR {{metric:node_load1:gauge}} IS NOT NULL + | STATS Load_1m = MAX(LAST_OVER_TIME({{metric:node_load1:gauge}})), non_idle = SUM(CASE(labels.mode != "idle", RATE({{metric:node_cpu_seconds_total:counter}}), NULL)), cpu_cores = COUNT_DISTINCT(CASE({{metric:node_cpu_seconds_total:counter}} IS NOT NULL, labels.cpu, NULL)) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | EVAL CPU_busy_pct = CASE(cpu_cores > 0, ((non_idle * 100) / cpu_cores), NULL), `Load 1m` = Load_1m + | KEEP time_bucket, CPU_busy_pct, `Load 1m` + | SORT time_bucket ASC + status_override: migrated + - title_match: "Disk Latency" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (labels.device IS NULL OR labels.device NOT RLIKE "dm-.*") + | WHERE {{metric:node_disk_read_time_seconds_total:counter}} IS NOT NULL OR {{metric:node_disk_reads_completed_total:counter}} IS NOT NULL OR {{metric:node_disk_write_time_seconds_total:counter}} IS NOT NULL OR {{metric:node_disk_writes_completed_total:counter}} IS NOT NULL + | STATS Read = SUM((RATE({{metric:node_disk_read_time_seconds_total:counter}}) / RATE({{metric:node_disk_reads_completed_total:counter}}))), Write = SUM((RATE({{metric:node_disk_write_time_seconds_total:counter}}) / RATE({{metric:node_disk_writes_completed_total:counter}}))) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | SORT time_bucket ASC + status_override: migrated + - title_match: "Network Traffic" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE (labels.device IS NULL OR labels.device != "lo") + | WHERE {{metric:node_network_receive_bytes_total:counter}} IS NOT NULL OR {{metric:node_network_transmit_bytes_total:counter}} IS NOT NULL + | STATS Inbound = SUM(RATE({{metric:node_network_receive_bytes_total:counter}})), Outbound = SUM(RATE({{metric:node_network_transmit_bytes_total:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | SORT time_bucket ASC + status_override: migrated + - title_match: "I/O Activity" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE {{metric:node_vmstat_pgpgin:counter}} IS NOT NULL OR {{metric:node_vmstat_pgpgout:counter}} IS NOT NULL + | STATS pgpgin = SUM(RATE({{metric:node_vmstat_pgpgin:counter}})), pgpgout = SUM(RATE({{metric:node_vmstat_pgpgout:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | EVAL Pages_in = pgpgin * 1024, Pages_out = pgpgout * 1024 + | KEEP time_bucket, Pages_in, Pages_out + | SORT time_bucket ASC + status_override: migrated + - title_match: "Memory Distribution" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE {{metric:node_memory_MemTotal_bytes:gauge}} IS NOT NULL OR {{metric:node_memory_MemFree_bytes:gauge}} IS NOT NULL OR {{metric:node_memory_Buffers_bytes:gauge}} IS NOT NULL OR {{metric:node_memory_Cached_bytes:gauge}} IS NOT NULL + | STATS Total = MAX(LAST_OVER_TIME({{metric:node_memory_MemTotal_bytes:gauge}})), Free = MAX(LAST_OVER_TIME({{metric:node_memory_MemFree_bytes:gauge}})), Buffers = MAX(LAST_OVER_TIME({{metric:node_memory_Buffers_bytes:gauge}})), Cached = MAX(LAST_OVER_TIME({{metric:node_memory_Cached_bytes:gauge}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | EVAL Used = Total - (Free + Buffers + Cached) + | KEEP time_bucket, Used, Free, Buffers, Cached + | SORT time_bucket ASC + status_override: migrated + - title_match: "Swap Activity" + esql_query: | + TS metrics-* + | WHERE @timestamp >= ?_tstart AND @timestamp <= ?_tend + | WHERE {{metric:node_vmstat_pswpin:counter}} IS NOT NULL OR {{metric:node_vmstat_pswpout:counter}} IS NOT NULL + | STATS pswpin = SUM(RATE({{metric:node_vmstat_pswpin:counter}})), pswpout = SUM(RATE({{metric:node_vmstat_pswpout:counter}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | EVAL Swap_in = pswpin * 4096, Swap_out = pswpout * 4096 + | KEEP time_bucket, Swap_in, Swap_out + | SORT time_bucket ASC + status_override: migrated + layout_overrides: + # Grafana's first row is untitled; Kibana requires a section name. + - title_match: "Section 1" + title: "Overview" + # Query cache was removed in MySQL 8; Kibana shows InnoDB buffer pool. + - title_match: "Query Cache" + title: "InnoDB Buffer Pool" + - title_match: "MySQL Query Cache Memory" + title: "InnoDB Buffer Pool Pages" + - title_match: "MySQL Query Cache Activity" + title: "InnoDB Buffer Pool Activity" diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/fidelity_manifest.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/fidelity_manifest.yaml new file mode 100644 index 00000000..80988bc1 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/fidelity_manifest.yaml @@ -0,0 +1,112 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Fidelity manifest — Grafana dashboard 9628 (PostgreSQL Database) +# https://grafana.com/grafana/dashboards/9628-postgresql-database/ +# +# PERFECT = same information as Grafana on postgres_exporter. +# APPROXIMATE = documented delta (Helm labels, exporter process metrics, +# missing postmaster start, Grafana rate() on gauges, cumulative tuple graphs). + +schema_version: 1 +gnet_id: 9628 +gnet_revision: 1 +dashboard_title: "PostgreSQL Database" + +panels: + - title: "Version" + fidelity: APPROXIMATE + notes: "Grafana singlestat valueName=name shows {{short_version}}; Kibana metric shows 1 broken down by labels.short_version." + - title: "Start Time" + fidelity: APPROXIMATE + notes: "pg_postmaster_start_time_seconds is absent on postgres_exporter v0.15. Pack uses process_start_time_seconds (exporter process) and DATE_DIFF seconds-ago instead of Grafana dateTimeFromNow." + - title: "Current fetch data" + fidelity: APPROXIMATE + notes: "Grafana SUMs the tup_fetched counter as a gauge (lifetime tuples) with format decbytes. Native PROMQL preserves that." + - title: "Current insert data" + fidelity: APPROXIMATE + notes: "Lifetime tup_inserted counter shown as a gauge, matching Grafana." + - title: "Current update data" + fidelity: APPROXIMATE + notes: "Lifetime tup_updated counter shown as a gauge, matching Grafana." + - title: "Max Connections" + fidelity: PERFECT + notes: "Native PROMQL LAST of pg_settings_max_connections." + - title: "Average CPU Usage" + fidelity: APPROXIMATE + notes: "process_cpu_seconds_total is the exporter process, not Postgres. Grafana y-axis format s on rate()*1000 is kept." + - title: "Average Memory Usage" + fidelity: APPROXIMATE + notes: "Grafana rate() on RSS/VMS gauges is invalid in Elasticsearch. Pack plots LAST_OVER_TIME of the gauges (exporter process memory)." + - title: "Open File Descriptors" + fidelity: PERFECT + notes: "Native PROMQL of process_open_fds (exporter process, same as Grafana)." + - title: "Shared Buffers" + fidelity: PERFECT + notes: "" + - title: "Effective Cache" + fidelity: PERFECT + notes: "" + - title: "Maintenance Work Mem" + fidelity: PERFECT + notes: "" + - title: "Work Mem" + fidelity: PERFECT + notes: "" + - title: "Max WAL Size" + fidelity: PERFECT + notes: "" + - title: "Random Page Cost" + fidelity: PERFECT + notes: "" + - title: "Seq Page Cost" + fidelity: PERFECT + notes: "" + - title: "Max Worker Processes" + fidelity: PERFECT + notes: "" + - title: "Max Parallel Workers" + fidelity: PERFECT + notes: "" + - title: "Active sessions" + fidelity: PERFECT + notes: "Native PROMQL of pg_stat_activity_count{state=\"active\"} != 0." + - title: "Transactions" + fidelity: APPROXIMATE + notes: "irate() of commit/rollback counters; Grafana $datname multi-select becomes MV_CONTAINS. Native path may fall back to ES|QL IRATE." + - title: "Update data" + fidelity: APPROXIMATE + notes: "Grafana graphs the cumulative tup_updated counter with != 0 (not a rate). Empty until the database has at least one update." + - title: "Fetch data (SELECT)" + fidelity: APPROXIMATE + notes: "Cumulative tup_fetched != 0, matching Grafana." + - title: "Insert data" + fidelity: APPROXIMATE + notes: "Cumulative tup_inserted != 0, matching Grafana. Empty until the first insert." + - title: "Lock tables" + fidelity: PERFECT + notes: "Native PROMQL of pg_locks_count{mode=~$mode} != 0." + - title: "Return data" + fidelity: APPROXIMATE + notes: "Cumulative tup_returned != 0, matching Grafana." + - title: "Idle sessions" + fidelity: PERFECT + notes: "" + - title: "Delete data" + fidelity: APPROXIMATE + notes: "Cumulative tup_deleted != 0, matching Grafana. Empty until the first delete." + - title: "Cache Hit Rate" + fidelity: APPROXIMATE + notes: "Grafana divides lifetime blks_hit / (blks_read + blks_hit) without rate(). Native PROMQL preserves that lifetime ratio." + - title: "Buffers (bgwriter)" + fidelity: PERFECT + notes: "irate() of pg_stat_bgwriter_buffers_*_total counters." + - title: "Conflicts/Deadlocks" + fidelity: PERFECT + notes: "irate() of conflicts and deadlocks counters." + - title: "Temp File (Bytes)" + fidelity: PERFECT + notes: "irate() of pg_stat_database_temp_bytes." + - title: "Checkpoint Stats" + fidelity: PERFECT + notes: "irate() of checkpoint write/sync time; Grafana legend text is the long HELP string." diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/pack.yaml b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/pack.yaml new file mode 100644 index 00000000..52acd1a8 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/pack.yaml @@ -0,0 +1,143 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 +# +# Curated pack — Grafana dashboard 9628 +# https://grafana.com/grafana/dashboards/9628-postgresql-database/ +# +# Source: "PostgreSQL Database" (postgres_exporter Prometheus scrape) +# Revision 1. Panels: 32 renderable + 3 named rows. +# +# Engine vs pack split (do not duplicate engine work here): +# Engine: Grafana 5 singlestat root ``format``, native PROMQL for bare +# gauges and top-level ``!= 0`` comparisons, ``irate()`` on typed +# counters, query_result() skip (no Kibana populate equivalent). +# Pack: Helm ``release`` / ``kubernetes_namespace`` labels that this +# dashboard's revision 1 filters on but typical scrapes do not +# store; postgres_exporter v0.15 dropped +# ``pg_postmaster_start_time_seconds``; Grafana ``rate()`` on +# process memory gauges; ``query_result(pg_up{release=...})`` +# rewritten to ``label_values(pg_up, instance)`` so Instance is +# a real Kibana control. + +query: + metrics_dataset_filter: "prometheus" + + ignored_labels: + - release + - kubernetes_namespace + + label_rewrites: + instance: labels.instance + job: labels.job + datname: labels.datname + state: labels.state + mode: labels.mode + short_version: labels.short_version + + metric_kinds: + pg_stat_database_tup_fetched: counter + pg_stat_database_tup_inserted: counter + pg_stat_database_tup_updated: counter + pg_stat_database_tup_returned: counter + pg_stat_database_tup_deleted: counter + pg_stat_database_xact_commit: counter + pg_stat_database_xact_rollback: counter + pg_stat_database_blks_hit: counter + pg_stat_database_blks_read: counter + pg_stat_database_conflicts: counter + pg_stat_database_deadlocks: counter + pg_stat_database_temp_bytes: counter + pg_stat_bgwriter_buffers_backend_total: counter + pg_stat_bgwriter_buffers_alloc_total: counter + pg_stat_bgwriter_buffers_backend_fsync_total: counter + pg_stat_bgwriter_buffers_checkpoint_total: counter + pg_stat_bgwriter_buffers_clean_total: counter + pg_stat_bgwriter_checkpoint_write_time_total: counter + pg_stat_bgwriter_checkpoint_sync_time_total: counter + process_cpu_seconds_total: counter + pg_static: gauge + pg_up: gauge + pg_settings_max_connections: gauge + pg_settings_shared_buffers_bytes: gauge + pg_settings_effective_cache_size_bytes: gauge + pg_settings_maintenance_work_mem_bytes: gauge + pg_settings_work_mem_bytes: gauge + pg_settings_max_wal_size_bytes: gauge + pg_settings_random_page_cost: gauge + pg_settings_seq_page_cost: gauge + pg_settings_max_worker_processes: gauge + pg_settings_max_parallel_workers: gauge + pg_stat_activity_count: gauge + pg_locks_count: gauge + process_resident_memory_bytes: gauge + process_virtual_memory_bytes: gauge + process_open_fds: gauge + process_start_time_seconds: gauge + pg_postmaster_start_time_seconds: gauge + + live_optional_metrics: + - pg_postmaster_start_time_seconds + + label_candidates: + instance: + - labels.instance + - service.instance.id + - host.name + job: + - labels.job + - service.name + datname: + - labels.datname + - datname + state: + - labels.state + - state + mode: + - labels.mode + - mode + short_version: + - labels.short_version + - short_version + +controls: + field_overrides: + instance: labels.instance + datname: labels.datname + mode: labels.mode + +panel: + query_overrides: + # Grafana singlestat valueName=name + legend {{short_version}} shows the + # version label, not the numeric 1 on pg_static. + - title_match: "Version" + kibana_type_override: metric + esql_query: | + TS metrics-* + | WHERE (?instance == "" OR ({{label:instance}} RLIKE ?instance OR ({{label:instance}} IS NULL AND "" RLIKE ?instance))) + | WHERE {{metric:pg_static:gauge}} IS NOT NULL + | STATS computed_value = MAX(LAST_OVER_TIME({{metric:pg_static:gauge}})) BY labels.short_version + | KEEP labels.short_version, computed_value + status_override: migrated + # postgres_exporter v0.15 no longer exports pg_postmaster_start_time_seconds. + # process_start_time_seconds is the exporter process start (same scrape + # target); DATE_DIFF yields seconds-ago for the duration-style tile. + - title_match: "Start Time" + esql_query: | + TS metrics-* + | WHERE (?instance == "" OR ({{label:instance}} RLIKE ?instance OR ({{label:instance}} IS NULL AND "" RLIKE ?instance))) + | WHERE {{metric:process_start_time_seconds:gauge}} IS NOT NULL + | STATS start = MAX(LAST_OVER_TIME({{metric:process_start_time_seconds:gauge}})) + | EVAL computed_value = DATE_DIFF("seconds", TO_DATETIME(start * 1000), NOW()) + | KEEP computed_value + status_override: migrated + # Grafana rate() on RSS/VMS gauges 400s in ES|QL RATE() and is the wrong + # aggregation even in Prometheus. Plot current gauge values. + - title_match: "Average Memory Usage" + esql_query: | + TS metrics-* + | WHERE (?instance == "" OR ({{label:instance}} RLIKE ?instance OR ({{label:instance}} IS NULL AND "" RLIKE ?instance))) + | WHERE {{metric:process_resident_memory_bytes:gauge}} IS NOT NULL OR {{metric:process_virtual_memory_bytes:gauge}} IS NOT NULL + | STATS Resident_Mem = AVG(LAST_OVER_TIME({{metric:process_resident_memory_bytes:gauge}})), Virtual_Mem = AVG(LAST_OVER_TIME({{metric:process_virtual_memory_bytes:gauge}})) BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) + | KEEP time_bucket, Resident_Mem, Virtual_Mem + | SORT time_bucket ASC + status_override: migrated diff --git a/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/plugin.py b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/plugin.py new file mode 100644 index 00000000..207e67a0 --- /dev/null +++ b/observability_migration/adapters/source/grafana/curated_packs/grafana_9628_postgresql_database/plugin.py @@ -0,0 +1,37 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. +# SPDX-License-Identifier: Elastic-2.0 + +"""Grafana 9628 (PostgreSQL Database) curated pack plugin. + +Revision 1's Instance / Namespace / Release variables are Helm +``query_result()`` helpers. Kibana has no equivalent populate query, so the +engine skips them and the dashboard would have no Instance control. Rewrite +Instance to ``label_values(pg_up, instance)`` (the label the regex already +extracted) and drop the unused Helm cascade parents. +""" + + +_PACK_NAME = "grafana_9628_postgresql_database" + + +def register(api): + @api["variable_translators"].register("grafana_9628_helm_query_result", priority=5) + def rewrite_helm_query_result(context): + pack = getattr(context, "rule_pack", None) + if getattr(pack, "_curated_pack_name", "") != _PACK_NAME: + return None + variable = context.variable or {} + name = str(variable.get("name") or "") + query_text = context.query_text or str(variable.get("query") or "") + if "query_result(" not in query_text.lower(): + return None + if name in {"namespace", "release"}: + context.handled = True + return f"skipped helm-only query_result variable {name}" + if name == "instance": + rewritten = "label_values(pg_up, instance)" + context.query_text = rewritten + context.variable = dict(variable) + context.variable["query"] = rewritten + return None + return None diff --git a/observability_migration/adapters/source/grafana/curated_packs/registry.yaml b/observability_migration/adapters/source/grafana/curated_packs/registry.yaml index 84128250..38381d5b 100644 --- a/observability_migration/adapters/source/grafana/curated_packs/registry.yaml +++ b/observability_migration/adapters/source/grafana/curated_packs/registry.yaml @@ -65,3 +65,30 @@ packs: gnet_revision: 4 dashboard_sha256: "bd67264fd0d73f633a67b3b8f9d4e5f2e527c569e0e946e8f511168922b18b96" description: "Redis helm stable/redis-ha (oliver006/redis_exporter) — counter/gauge classification, label map" + + - gnet_id: 7362 + name: grafana_7362_mysql_overview + title_hint: "MySQL Overview" + tags_hint: ["Percona", "MySQL"] + path: grafana_7362_mysql_overview + gnet_revision: 5 + dashboard_sha256: "6d5c16e29a314253f7c6b3b4b4b35f0e993815c1f6f4cf4b92bf443af3ad8106" + description: "Percona MySQL Overview — mysqld_exporter counter typing, processlist rename, MySQL 8 query-cache replacements" + + - gnet_id: 9628 + name: grafana_9628_postgresql_database + title_hint: "PostgreSQL Database" + tags_hint: ["postgres", "db", "stats"] + path: grafana_9628_postgresql_database + gnet_revision: 1 + dashboard_sha256: "521bac34065bcadedf255fc384550ae0528e9a74e91b4475e583b6afe9c39cc7" + description: "PostgreSQL Database — Helm release ignore, memory LAST_OVER_TIME, exporter start-time stand-in" + + - gnet_id: 14114 + name: grafana_14114_postgres_exporter_quickstart + title_hint: "PostgreSQL Exporter Quickstart and Dashboard" + tags_hint: ["postgres"] + path: grafana_14114_postgres_exporter_quickstart + gnet_revision: 1 + dashboard_sha256: "5f90614f7ac6eebb6d7dbb784da58d1ef6602fe7c3eb135aa3055a34ad686042" + description: "PostgreSQL Exporter mixin — pg_up Instance populate, unused job drop, bgwriter _total map" diff --git a/observability_migration/adapters/source/grafana/extension_schema.py b/observability_migration/adapters/source/grafana/extension_schema.py index 51aeb0c5..bd037ea0 100644 --- a/observability_migration/adapters/source/grafana/extension_schema.py +++ b/observability_migration/adapters/source/grafana/extension_schema.py @@ -68,6 +68,10 @@ class PanelQueryOverrideModel(_StrictModel): # Optional Lens presentation override when the curated ES|QL shape does # not match the Grafana panel type (e.g. multi-value bargauge → datatable). kibana_type_override: str | None = None + # Drop Grafana ``timeFrom`` so the panel follows the dashboard picker. + # Used when a pinned window (e.g. 24h hourly bars) renders empty in Lens + # on mixed ``metrics-*`` despite the same query returning rows via ``_query``. + drop_time_from: bool = False class PanelPositionOverrideModel(_StrictModel): @@ -82,6 +86,7 @@ class PanelSizeOverrideModel(_StrictModel): class PanelLayoutOverrideModel(_StrictModel): title_match: str + title: str | None = None position: PanelPositionOverrideModel = Field(default_factory=PanelPositionOverrideModel) size: PanelSizeOverrideModel = Field(default_factory=PanelSizeOverrideModel) collapsed: bool | None = None diff --git a/observability_migration/adapters/source/grafana/panels.py b/observability_migration/adapters/source/grafana/panels.py index 92473b61..d3ae2964 100644 --- a/observability_migration/adapters/source/grafana/panels.py +++ b/observability_migration/adapters/source/grafana/panels.py @@ -91,6 +91,7 @@ _split_top_level_csv, _summary_mode_from_metadata, _unique_safe_alias, + collapse_or_for_native_promql, grafana_template_var_name, substitute_grafana_range_macros, substitute_scalar_template_vars, @@ -1137,11 +1138,19 @@ def _native_esql_panel_spec(query, kibana_type, promql_expr=None, panel=None, return None if kibana_type == "metric": if metric_fields and len(metric_fields) > 1: - return None + if "computed_value" in metric_fields: + metric_col = "computed_value" + metric_fields = ["computed_value"] + else: + return None return _build_esql_metric_panel(query, metric_col=metric_col) if kibana_type == "gauge": if metric_fields and len(metric_fields) > 1: - return None + if "computed_value" in metric_fields: + metric_col = "computed_value" + metric_fields = ["computed_value"] + else: + return None return _build_esql_gauge_panel(query, metric_col=metric_col, panel=panel) if kibana_type in ("line", "bar", "area"): if not xy_by_cols: @@ -1559,6 +1568,11 @@ def rewrite_selector(selector_text): changed = True return ", ".join(parts) if changed else selector_text + return _map_promql_brace_selectors(expr, rewrite_selector) + + +def _map_promql_brace_selectors(expr, rewrite_selector): + """Rewrite the contents of every top-level ``{...}`` PromQL selector.""" pieces = [] start = 0 idx = 0 @@ -1594,6 +1608,41 @@ def rewrite_selector(selector_text): return "".join(pieces) +def _strip_ignored_promql_label_matchers(expr, ignored_labels): + """Drop selector matchers whose label is in the rule-pack ignore list. + + Native PROMQL keeps Prometheus label names inside ``{}``. Pack + ``ignored_labels`` already omit those filters from ES|QL WHERE clauses, but + without this strip the native path still emits ``{release=~?release}``. + Kibana then synthesizes a Release control from mixed ``metrics-*`` (often a + kernel ``release`` field) and every panel that still binds ``?release`` + goes empty. + """ + drop = { + str(name).strip() + for name in (ignored_labels or []) + if str(name).strip() + } + if not drop or not expr: + return expr + + def rewrite_selector(selector_text): + kept = [] + changed = False + for part in _split_top_level_csv(selector_text): + matcher = _NATIVE_PROMQL_LABEL_MATCHER_RE.match(part) + if matcher and matcher.group("label").strip() in drop: + changed = True + continue + kept.append(part) + if not changed: + return selector_text + return ", ".join(kept) + + rewritten = _map_promql_brace_selectors(expr, rewrite_selector) + return re.sub(r"([A-Za-z_:][A-Za-z0-9_:]*)\{\s*\}", r"\1", rewritten) + + def _trim_wrapping_parens(expr): text = str(expr or "").strip() while text.startswith("(") and text.endswith(")"): @@ -2325,6 +2374,17 @@ def _kibana_binds_promql_control_params(runtime_features=None) -> bool: ) +def _is_bare_instant_selector(promql_expr) -> bool: + """True when *promql_expr* is a bare instant-vector selector (gauge or counter).""" + if not promql_expr: + return False + try: + frag = _parse_fragment(promql_expr) + except Exception: + return False + return bool(frag and getattr(frag, "family", None) == "simple_metric") + + def _is_bare_counter_reference(promql_expr, resolver, rule_pack=None): """Return True if *promql_expr* is a bare counter instant-vector selector. @@ -2521,6 +2581,27 @@ def _translate_panel_native_promql( target = targets_with_expr[0][0] expr = target.get("expr", "") + collapsed_expr = collapse_or_for_native_promql( + expr, resolver=resolver, rule_pack=rule_pack + ) + if collapsed_expr != expr: + _append_unique( + panel_notes, + "PromQL same-metric 'or': preferred left range-window operand and " + "dropped the alternate-window fallback; Grafana uses the right " + "side only when the left lacks samples", + ) + expr = collapsed_expr + expr = _strip_ignored_promql_label_matchers( + expr, getattr(rule_pack, "ignored_labels", None) + ) + # Native PROMQL is attempted before the ES|QL live-missing loop. An + # absent instant gauge would otherwise stay native, score Green, and + # either smoke empty or 400 with ``value_$1``/``value_$2`` (issue #158 + # keeps native on field gaps *after* emit; this gate refuses emit when + # field-caps already proved the source metrics are gone). + if _live_missing_metrics_for_expr(expr, resolver): + return None runtime_features = getattr(rule_pack, "runtime_features", {}) _record_passthrough_native_labels(expr, resolver) if ( @@ -2652,16 +2733,18 @@ def _translate_panel_native_promql( # is a separate case: ``build_native_promql_query`` rejects it, so those # degrade to ES|QL regardless of this gate.) # - # A bare counter reference (``http_requests_total{job="api"}`` with no - # rate()) is likewise kept native even though it is a single metric: - # Kibana's PROMQL preview serves the raw cumulative value directly, so - # preserve the original expression instead of the ES|QL - # ``LAST_OVER_TIME`` fallback and its misleading "Counter referenced - # without rate()" warning (issue #139). Bare gauges and rate()/range - # functions are not bare counters and still degrade here. + # Parse the *source* expression for the bare-selector check: native + # cleaning rewrites ``{instance="$host"}`` to ``{instance=?host}``, + # which the PromQL AST parser classifies as ``unknown`` and would + # wrongly degrade Grafana 5 singlestat gauges (Uptime, buffer pool). + is_bare_selector = ( + getattr(native_fragment, "family", None) == "simple_metric" + or _is_bare_instant_selector(expr) + or _is_bare_counter_reference(expr, resolver, rule_pack) + ) if "_timeseries" in group_cols and ( len(_collect_source_metrics(native_fragment, dedup=False)) < 2 - and not _is_bare_counter_reference(expr, resolver, rule_pack) + and not is_bare_selector ): return None # Dashboard metric/gauge tiles collapse a *range* query via @@ -3806,13 +3889,23 @@ def translate_panel(panel, datasource_index="metrics-*", esql_index=None, rule_p ) if _native_panel: yaml_panel["esql"] = _native_panel + # Curated overrides skip PANEL_TRANSLATORS; honour + # pack-level timeFrom drops before enrich applies + # Grafana panel time_range. + if _override.get("drop_time_from"): + panel.pop("timeFrom", None) + panel.pop("timeShift", None) + # Apply display units first, then seriesOverrides so a + # right-axis ``format: none`` (Load 1m) can clear the + # inherited left-axis % / bytes format. Matches the + # generic PANEL_TRANSLATORS path. + enrich_yaml_panel_display(yaml_panel, panel) # Wide multi-metric curated queries (e.g. Memory Basic) # need Grafana field overrides like RAM Total # ``stack: false`` applied the same as the generic path. _apply_series_override_axes( yaml_panel, panel, _override_warnings ) - enrich_yaml_panel_display(yaml_panel, panel) _label_placeholder_value_metric( yaml_panel, title=title, @@ -3867,11 +3960,19 @@ def translate_panel(panel, datasource_index="metrics-*", esql_index=None, rule_p _optional_omitted = set( _optional_metric_result.omitted_metrics or [] ) - if _optional_omitted: + _optional_declared = { + str(name).strip() + for name in (rule_pack.live_optional_metrics or []) + if str(name).strip() + } + if _optional_omitted or _optional_declared: _dropped_curated_metrics = [ metric for metric in _dropped_curated_metrics if metric not in _optional_omitted + and not _live_optional_source_metric_absent( + metric, resolver, _optional_declared + ) ] if _dropped_curated_metrics: _append_unique( @@ -5874,6 +5975,19 @@ def _live_missing_metrics_for_expr(expr, resolver): return missing +def _live_optional_source_metric_absent(metric, resolver, optional_names): + """True when *metric* is pack-optional and live field-caps proved it absent. + + Curated overrides sometimes replace a source metric the target never + ingested (postgres_exporter dropping ``pg_postmaster_start_time_seconds``) + rather than listing it in the override text for later stripping. Those + substitutions must not yellow the panel as a pack omission. + """ + if metric not in optional_names or not metric: + return False + return metric in _live_missing_metrics_for_expr(metric, resolver) + + def _source_metrics_absent_from_query(source_exprs, query_text, resolver): """Prometheus metrics referenced by *source_exprs* that never appear in the final emitted *query_text*. @@ -7326,6 +7440,11 @@ def _apply_series_override_axes(yaml_panel: dict, grafana_panel: dict, warnings: metric["axis"] = "right" if right_format: metric["format"] = dict(right_format) + elif _grafana_right_axis_present(grafana_panel): + # Grafana ``yaxes[1].format: none`` is an explicit + # "no unit" on the overlay axis. Do not keep the + # left-axis format (Load 1m inheriting CPU %). + metric.pop("format", None) if stack_override is False: metric["stack"] = False if alias and not matched: @@ -7353,6 +7472,11 @@ def _grafana_yaxis_metric_format(grafana_panel: dict, axis: str) -> dict | None: return grafana_unit_to_yaml_format(unit) +def _grafana_right_axis_present(grafana_panel: dict) -> bool: + yaxes = grafana_panel.get("yaxes") + return isinstance(yaxes, list) and len(yaxes) > 1 and isinstance(yaxes[1], dict) + + def _field_override_targets_metric( override: dict[str, Any], metric: dict[str, Any] ) -> bool: @@ -8973,7 +9097,12 @@ def _ensure_param_controls( and control.get("type") == "esql" and control.get("variable_name") } - missing = sorted(name for name in emitted_params if name not in bound) + missing = sorted( + name + for name in emitted_params + if name not in bound + and name not in (getattr(rule_pack, "ignored_labels", None) or []) + ) # Inert controls (the reverse of ``missing``): a control whose ``?var`` is # bound by neither a migrated panel query nor another control's populate # query. Grafana cascade parents (e.g. ``$namespace`` narrowing the @@ -10499,6 +10628,9 @@ def _apply_panel_layout_overrides_recursively(panels: list[dict], overrides: lis if value is not None: size[key] = int(value) panel["size"] = size + new_title = override.get("title") + if isinstance(new_title, str) and new_title.strip(): + panel["title"] = new_title.strip() if "collapsed" in override and isinstance(panel.get("section"), dict): panel["section"]["collapsed"] = bool(override.get("collapsed")) section = panel.get("section") diff --git a/observability_migration/adapters/source/grafana/promql.py b/observability_migration/adapters/source/grafana/promql.py index 67b4fcd6..61000094 100644 --- a/observability_migration/adapters/source/grafana/promql.py +++ b/observability_migration/adapters/source/grafana/promql.py @@ -5542,6 +5542,15 @@ def _range_fallback_identity(frag) -> tuple | None: if inner is None: return None return ("binop", op, ("scalar", left_scalar), inner) + # ``rate(A)/rate(B) or irate(A)/irate(B)`` — both sides are range + # calls, not scalars. Ignore rate-vs-irate so the pair collapses to + # one operand before a later cloud-metric ``or`` is considered. + if left is not None and right is not None: + left_id = _range_fallback_identity(left) + right_id = _range_fallback_identity(right) + if left_id is None or right_id is None: + return None + return ("binop", op, left_id, right_id) return None if frag.family == "topk": if not frag.metric: @@ -5585,7 +5594,9 @@ def _operands_are_same_metric_range_fallback(operands: list) -> bool: # (``A{f1} or A{f2}``) — those differ in matchers and already fail the # identity check above, but also refuse when every operand is a bare # metric with no range shape (cross-metric COALESCE should own that). - if not any(getattr(op, "range_func", None) or getattr(op, "range_window", None) for op in operands): + if not any( + getattr(op, "range_func", None) or getattr(op, "range_window", None) for op in operands + ): # topk / scaled wrappers store range on the fragment itself for # range_agg children; walk one level. def _has_range(op) -> bool: @@ -5652,6 +5663,175 @@ def _same_metric_range_fallback_warning(frag) -> str: ) +_ABSENT_OR_OPERAND_WARNING = ( + "PromQL 'or': dropped operands whose metrics are absent from the live target" +) + + +def _collapse_same_metric_range_fallback_groups(operands: list) -> list: + """Keep the leftmost operand of each adjacent same-metric range-fallback run. + + Percona/MySQL Overview writes + ``sum(rate(node_…)) or sum(irate(node_…)) or sum(rdsosmetrics_…)``. + The first pair shares identity (rate vs irate) and should collapse to + the left before the cross-metric COALESCE rewrite sees a 4-way chain. + """ + if not operands: + return operands + out = [] + i = 0 + while i < len(operands): + ident = _range_fallback_identity(operands[i]) + j = i + 1 + if ident is not None: + while j < len(operands) and _range_fallback_identity(operands[j]) == ident: + j += 1 + if j > i + 1: + out.append(operands[i]) + i = j + continue + out.append(operands[i]) + i += 1 + return out + + +def _fragment_metric_names(frag, seen=None) -> list[str]: + """Source metric names reachable from *frag*, including nested arithmetic.""" + seen = seen if seen is not None else set() + names: list[str] = [] + if frag is None or id(frag) in seen: + return names + seen.add(id(frag)) + metric = str(getattr(frag, "metric", "") or "") + if metric and not metric.startswith("label_"): + names.append(metric) + extra = getattr(frag, "extra", None) or {} + for key in ("left_frag", "right_frag", "inner_frag"): + names.extend(_fragment_metric_names(extra.get(key), seen)) + names.extend(_fragment_metric_names(getattr(frag, "binary_rhs", None), seen)) + return list(dict.fromkeys(names)) + + +def _metric_known_absent(resolver, metric_name: str) -> bool: + """True only when live field-caps prove *metric_name* is missing. + + Offline / empty caches return False so we do not drop a cloud fallback + that we cannot actually disprove (issue #167: keep both metrics). + """ + if resolver is None or not metric_name: + return False + field_exists = getattr(resolver, "field_exists", None) + if not callable(field_exists): + return False + candidates: list[str] = [] + resolve = getattr(resolver, "resolve_metric_field", None) + if callable(resolve): + for prefer in (None, "gauge", "counter"): + try: + resolved = ( + resolve(metric_name) + if prefer is None + else resolve(metric_name, prefer=prefer) + ) + except TypeError: + try: + resolved = resolve(metric_name) + except Exception: + resolved = None + except Exception: + resolved = None + if resolved: + candidates.append(str(resolved)) + candidates.extend([metric_name, f"metrics.{metric_name}"]) + unique: list[str] = [] + seen: set[str] = set() + for cand in candidates: + if cand and cand not in seen: + seen.add(cand) + unique.append(cand) + statuses = [] + for cand in unique: + try: + statuses.append(field_exists(cand)) + except Exception: + statuses.append(None) + if any(status is True for status in statuses): + return False + if any(status is None for status in statuses): + return False + return bool(statuses) + + +def _operand_known_absent(operand, resolver) -> bool: + names = _fragment_metric_names(operand) + if not names: + return False + return all(_metric_known_absent(resolver, name) for name in names) + + +def _reduce_or_operands(frag, resolver) -> tuple[list, list]: + """Collapse range-window fallbacks, then drop live-absent OR operands. + + Returns ``(kept, dropped)``. ``kept`` is empty when the chain is malformed. + """ + operands = _flatten_or_operands(frag) + if not operands: + return [], [] + collapsed = _collapse_same_metric_range_fallback_groups(operands) + kept = [] + dropped = [] + for operand in collapsed: + if _operand_known_absent(operand, resolver): + dropped.append(operand) + else: + kept.append(operand) + return kept, dropped + + +def _expr_for_or_collapse_parse(expr: str, rule_pack=None) -> str: + """Make Grafana range macros parseable without rewriting label matchers. + + ``preprocess_grafana_macros`` also turns ``$host`` into ``label_host``, which + would break native PROMQL ``?host`` binding if we emitted that form. Only + the ``[$interval]`` / ``$__rate_interval`` tokens need a concrete window + so the AST parser can see the ``or``. + """ + default_window = (getattr(rule_pack, "default_rate_window", None) if rule_pack else None) or "5m" + result = substitute_grafana_range_macros(expr) + for pattern in (r"\$__rate_interval", r"\$__interval", r"\$interval"): + result = re.sub(pattern, default_window, result) + result = re.sub( + r"\[\s*\$(?!__)([A-Za-z_][A-Za-z0-9_]*)\s*\]", + f"[{default_window}]", + result, + ) + return result + + +def collapse_or_for_native_promql(expr, resolver=None, rule_pack=None) -> str: + """Drop Grafana same-metric range-fallback ``or`` so native PROMQL can run. + + Elasticsearch's PROMQL command rejects set operators (``or`` / ``and`` / + ``unless``). The ES|QL translator already prefers the left + ``rate(M[$interval])`` operand of ``rate(...) or irate(...)`` and drops + live-absent cloud fallbacks. Apply the same rewrite before the native + path so those single-target panels stay PROMQL instead of degrading. + """ + if not expr or not re.search(r"\bor\b", expr, re.IGNORECASE): + return expr + try: + parsed = _parse_fragment(_expr_for_or_collapse_parse(expr, rule_pack)) + except Exception: + return expr + left = _left_operand_of_same_metric_range_fallback(parsed) + if left is not None and (left.raw_expr or "").strip(): + return left.raw_expr.strip() + kept, _dropped = _reduce_or_operands(parsed, resolver) + if len(kept) == 1 and (kept[0].raw_expr or "").strip(): + return kept[0].raw_expr.strip() + return expr + + def _is_zero_scaled_operand(frag) -> bool: """True for ``0 * X`` / ``X * 0`` (including ``scaled_agg`` with scalar 0).""" if frag is None: @@ -5873,6 +6053,8 @@ def _try_rewrite_set_or_cross_metric( summary_mode=False, preferred_group_labels=None, preferred_group_labels_origin=None, + allow_direct_ts_gauge=False, + allow_tsds_gauge_promotion=False, ): """Rewrite a cross-metric ``A or B`` as a ``COALESCE(A, B)`` union. @@ -5885,19 +6067,20 @@ def _try_rewrite_set_or_cross_metric( the right value fills the groups the left never produced. Longer chains ``A or B or C`` collapse to ``COALESCE(A, B, C)`` in source order. - This keeps **both** metrics instead of silently dropping the right operand - (issue #167). When the operands cannot be aligned safely — different - grouping dimensions, divergent source commands, or an operand that itself - has no honest translation — we return ``None`` so the caller marks the - panel for manual review rather than emitting half the data. + Same-metric ``rate(M) or irate(M)`` pairs inside a longer chain (Linux + node_exporter followed by an RDS/OSMetrics fallback) collapse to the + left operand first. Operands whose metrics field-caps prove absent are + dropped so a missing cloud series cannot 400 the whole panel. + + This keeps **both** remaining metrics instead of silently dropping the + right operand (issue #167). When the operands cannot be aligned safely — + different grouping dimensions, divergent source commands, or an operand + that itself has no honest translation — we return ``None`` so the caller + marks the panel for manual review rather than emitting half the data. """ if (frag.binary_op or "").lower() != "or": return None - operand_frags = _flatten_or_operands(frag) - if not operand_frags or len(operand_frags) < 2: - return None - # An ``on(...)``/``ignoring(...)`` modifier changes which right-operand # series fill a missing left-operand series — PromQL matches by the # modifier's key, not the full label set. The COALESCE union groups by the @@ -5906,6 +6089,37 @@ def _try_rewrite_set_or_cross_metric( if _or_chain_has_vector_matching(frag): return None + kept, dropped = _reduce_or_operands(frag, resolver) + if not kept: + return None + + drop_warnings = [_ABSENT_OR_OPERAND_WARNING] if dropped else [] + + # After collapsing rate/irate pairs and dropping absent cloud fallbacks, + # a single leftover operand is the honest Linux-side translation. + if len(kept) == 1: + plan = _build_formula_plan( + kept[0], + resolver, + rule_pack, + alias_hint=alias_hint, + summary_mode=summary_mode, + preferred_group_labels=preferred_group_labels, + allow_direct_ts_gauge=allow_direct_ts_gauge, + preferred_group_labels_origin=preferred_group_labels_origin, + allow_tsds_gauge_promotion=allow_tsds_gauge_promotion, + ) + if plan is None: + return None + for warning in drop_warnings: + if warning not in plan.warnings: + plan.warnings.append(warning) + return plan + + operand_frags = kept + if len(operand_frags) < 2: + return None + # Every operand must be translatable on its own. A nested set operator or a # carried not-feasible reason means we cannot faithfully include that side, # so refuse the whole union (→ manual review) instead of dropping it. @@ -5951,7 +6165,7 @@ def _try_rewrite_set_or_cross_metric( coalesce_args = ", ".join(_esql_identifier(spec.final_alias) for spec in specs) expr = f"COALESCE({coalesce_args})" - warnings = [] + warnings = list(drop_warnings) for spec in specs: for w in spec.warnings: if w not in warnings: @@ -6129,6 +6343,8 @@ def _build_formula_plan( summary_mode=summary_mode, preferred_group_labels=preferred_group_labels, preferred_group_labels_origin=preferred_group_labels_origin, + allow_direct_ts_gauge=allow_direct_ts_gauge, + allow_tsds_gauge_promotion=allow_tsds_gauge_promotion, ) if cross is not None: return cross @@ -6635,8 +6851,29 @@ def colocated_binary_agg_plan(frag, resolver, rule_pack): """``(value_expr, leaf)`` for a renderable ``agg(A op B)``, else None.""" if not frag or not frag.outer_agg: return None - inner = (getattr(frag, "extra", {}) or {}).get("inner_frag") - if inner is None or getattr(inner, "family", "") != "binary_expr": + extra = getattr(frag, "extra", None) + if not isinstance(extra, dict): + return None + inner = extra.get("inner_frag") + if inner is None: + return None + # ``sum((rate(A)/rate(B)) or (irate(A)/irate(B)) or cloud_metric)`` lands + # as unknown+inner OR. Collapse the range-window pair and drop live-absent + # cloud fallbacks so the remaining ratio can render as co-located arithmetic. + if getattr(inner, "family", "") == "binary_expr" and (inner.binary_op or "").lower() == "or": + kept, dropped = _reduce_or_operands(inner, resolver) + preferred = None + if len(kept) == 1 or (kept and _operands_are_same_metric_range_fallback(kept)): + preferred = kept[0] + if preferred is None: + return None + extra["inner_frag"] = preferred + inner = preferred + if dropped: + extra["or_chain_dropped_absent"] = True + if getattr(inner, "family", "") != "binary_expr": + return None + if (inner.binary_op or "").lower() in _SET_OPERATORS: return None rendered = _render_colocated_arithmetic(inner, resolver, rule_pack) if rendered is None: diff --git a/observability_migration/adapters/source/grafana/rules.py b/observability_migration/adapters/source/grafana/rules.py index e6406387..8bbdbdbc 100644 --- a/observability_migration/adapters/source/grafana/rules.py +++ b/observability_migration/adapters/source/grafana/rules.py @@ -293,9 +293,11 @@ def load_rule_pack_files(paths: Sequence[str] | None) -> RulePackConfig: } if override.kibana_type_override: entry["kibana_type_override"] = override.kibana_type_override + if override.drop_time_from: + entry["drop_time_from"] = True pack.panel_query_overrides.append(entry) for override in panel_cfg.layout_overrides: - pack.panel_layout_overrides.append({ + entry = { "title_match": override.title_match, "position": { key: value @@ -314,7 +316,10 @@ def load_rule_pack_files(paths: Sequence[str] | None) -> RulePackConfig: if value is not None }, "collapsed": override.collapsed, - }) + } + if override.title: + entry["title"] = override.title + pack.panel_layout_overrides.append(entry) for field_name in ( "default_rate_window", diff --git a/observability_migration/targets/kibana/emit/display.py b/observability_migration/targets/kibana/emit/display.py index 41d69909..835725f4 100644 --- a/observability_migration/targets/kibana/emit/display.py +++ b/observability_migration/targets/kibana/emit/display.py @@ -11,6 +11,10 @@ import re from typing import Any +# Grafana 5 singlestat stores the unit on the panel (``format: bytes``). +# Target query formats are not units and must not be treated as such. +_NON_UNIT_PANEL_FORMATS = frozenset({"", "time_series", "table", "heatmap"}) + _OPAQUE_AXIS_TITLE_ALIASES = { "aqu-sz", # Community dashboards often copy Grafana's percent unit id into axisLabel. @@ -122,6 +126,11 @@ def extract_grafana_unit(panel: dict) -> str: for axis in (panel.get("yaxes") or []): if isinstance(axis, dict) and axis.get("format"): return str(axis["format"]) + # Grafana 5 singlestat / stat panels: ``"format": "bytes"`` / ``"s"`` / + # ``"percent"`` live on the panel root, not in fieldConfig. + legacy = str((panel or {}).get("format") or "").strip() + if legacy and legacy not in _NON_UNIT_PANEL_FORMATS: + return legacy return "" @@ -413,9 +422,12 @@ def enrich_yaml_panel_display( unit = extract_grafana_unit(grafana_panel) fmt = grafana_unit_to_yaml_format(unit) - # Carry fieldConfig.defaults.decimals into the format so the Kibana panel - # respects the same precision the operator set in Grafana. + # Carry fieldConfig.defaults.decimals (or Grafana 5 singlestat + # ``decimals``) into the format so the Kibana panel respects the same + # precision the operator set in Grafana. panel_decimals = _field_defaults(grafana_panel).get("decimals") + if panel_decimals is None: + panel_decimals = grafana_panel.get("decimals") if isinstance(panel_decimals, (int, float)) and panel_decimals >= 0: decimals_int = int(panel_decimals) if fmt is not None: diff --git a/observability_migration/targets/kibana/emit/esql_utils.py b/observability_migration/targets/kibana/emit/esql_utils.py index 4899109d..dc0a1c77 100644 --- a/observability_migration/targets/kibana/emit/esql_utils.py +++ b/observability_migration/targets/kibana/emit/esql_utils.py @@ -215,6 +215,17 @@ def _metric_fields_from_projection(projected_fields, group_fields): ] +def _eval_alias_is_series_identity(expr): + """True for string-identity EVAL expressions used as a series key. + + ``EVAL series_group = CONCAT(...)`` / ``TO_STRING(...)`` names the + breakdown, not a Y metric. Numeric derived columns (``ratio = a / b``) + must keep flowing into ``metric_fields``. + """ + text = str(expr or "").strip().upper() + return text.startswith("CONCAT(") or text.startswith("TO_STRING(") + + def extract_esql_shape(esql): commands = split_esql_pipeline(esql) shape = ESQLShape() @@ -251,8 +262,19 @@ def extract_esql_shape(esql): for assignment in _split_top_level_csv(command[5:].strip()): alias, _expr = split_top_level_assignment(assignment) field_name = _output_field_name(alias) - if field_name and field_name not in shape.projected_fields: + if not field_name: + continue + if field_name not in shape.projected_fields: shape.projected_fields.append(field_name) + if _eval_alias_is_series_identity(_expr): + if field_name not in shape.group_fields: + shape.group_fields.append(field_name) + continue + # Derived columns are first-class series (e.g. ``STATS a, b | + # EVAL ratio = a / b``). Without this, Lens Y accessors stay on + # the STATS aliases and the EVAL column never renders. + if field_name not in shape.metric_fields and field_name not in shape.group_fields: + shape.metric_fields.append(field_name) continue if lower_command.startswith("keep "): @@ -262,9 +284,10 @@ def extract_esql_shape(esql): if part.strip() ] group_fields = [field for field in shape.group_fields if field in projected_fields] - metric_fields = [field for field in shape.metric_fields if field in projected_fields] - if not metric_fields: - metric_fields = _metric_fields_from_projection(projected_fields, group_fields) + # KEEP is the operator-visible output: include EVAL aliases that + # survive the projection, in KEEP order — not only STATS names + # that happen to still be listed. + metric_fields = _metric_fields_from_projection(projected_fields, group_fields) time_fields = [ field for field in projected_fields diff --git a/parity-rig/curated/grafana_763_redis_exporter/docker-compose.yml b/parity-rig/curated/grafana_763_redis_exporter/docker-compose.yml index 76e42919..90daef24 100644 --- a/parity-rig/curated/grafana_763_redis_exporter/docker-compose.yml +++ b/parity-rig/curated/grafana_763_redis_exporter/docker-compose.yml @@ -238,6 +238,90 @@ services: mysql: condition: service_healthy + # Drive real mysqld_exporter series (commands, questions, handlers, InnoDB, + # processlist) so Grafana 7362 is not an idle SHOW STATUS scrape. + mysql-load: + image: mysql:8.0 + container_name: redis-rig-mysql-load + networks: + - redis-rig + depends_on: + mysql: + condition: service_healthy + restart: unless-stopped + entrypoint: ["/bin/sh", "-c"] + command: + - | + mysql_cmd="mysql -h mysql -uroot -prigpass -Drigdb --connect-timeout=5" + until $$mysql_cmd -e "SELECT 1" >/dev/null 2>&1; do sleep 2; done + $$mysql_cmd -e " + CREATE TABLE IF NOT EXISTS rig_load ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + k INT NOT NULL, + v VARCHAR(64), + KEY k (k) + ) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS rig_scan ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + payload VARCHAR(128) + ) ENGINE=InnoDB; + " + i=0 + while true; do + i=$$((i + 1)) + $$mysql_cmd -e " + START TRANSACTION; + INSERT INTO rig_load (k, v) VALUES ($$i, CONCAT('val-', $$i)); + UPDATE rig_load SET v = CONCAT('upd-', $$i) WHERE k = $$i; + SELECT v FROM rig_load WHERE k = $$i; + INSERT INTO rig_scan (payload) VALUES (REPEAT('x', 32)); + SELECT COUNT(*) FROM rig_scan; + SELECT payload FROM rig_scan ORDER BY payload LIMIT 20; + DELETE FROM rig_load WHERE id < $$((i - 400)); + COMMIT; + " >/dev/null 2>&1 + $$mysql_cmd -e "SHOW PROCESSLIST; SHOW STATUS LIKE 'Threads%';" >/dev/null 2>&1 + sleep 0.25 + done + + # Drive real postgres_exporter series (tup_*, xact_*) so Grafana 9628 + # insert/update/delete graphs are not an idle scrape of zeros. + postgres-load: + image: postgres:16-alpine + container_name: redis-rig-postgres-load + networks: + - redis-rig + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped + environment: + - PGPASSWORD=rigpass + entrypoint: ["/bin/sh", "-c"] + command: + - | + until pg_isready -h postgres -U postgres >/dev/null 2>&1; do sleep 2; done + psql -h postgres -U postgres -d rigdb -c " + CREATE TABLE IF NOT EXISTS rig_load ( + id SERIAL PRIMARY KEY, + k INT NOT NULL, + v VARCHAR(64) + ); + " + i=0 + while true; do + i=$$((i + 1)) + psql -h postgres -U postgres -d rigdb -c " + BEGIN; + INSERT INTO rig_load (k, v) VALUES ($$i, CONCAT('val-', $$i)); + UPDATE rig_load SET v = CONCAT('upd-', $$i) WHERE k = $$i; + SELECT v FROM rig_load WHERE k = $$i; + DELETE FROM rig_load WHERE id < $$((i - 10)); + COMMIT; + " >/dev/null 2>&1 + sleep 0.25 + done + postgres: image: postgres:16-alpine container_name: redis-rig-postgres diff --git a/parity-rig/curated/grafana_763_redis_exporter/redis_scraper.py b/parity-rig/curated/grafana_763_redis_exporter/redis_scraper.py index 74dc769a..53f9516a 100644 --- a/parity-rig/curated/grafana_763_redis_exporter/redis_scraper.py +++ b/parity-rig/curated/grafana_763_redis_exporter/redis_scraper.py @@ -100,6 +100,46 @@ "node_vmstat_pswpin": "counter", "node_vmstat_pswpout": "counter", }, + "mysql.prometheus": { + # mysqld_exporter # TYPE untyped — Elasticsearch infers gauge without + # _total. Dashboards that rate() these need counter mapping. + "mysql_global_status_queries": "counter", + "mysql_global_status_questions": "counter", + "mysql_global_status_threads_created": "counter", + "mysql_global_status_created_tmp_tables": "counter", + "mysql_global_status_created_tmp_disk_tables": "counter", + "mysql_global_status_created_tmp_files": "counter", + "mysql_global_status_select_full_join": "counter", + "mysql_global_status_select_full_range_join": "counter", + "mysql_global_status_select_range": "counter", + "mysql_global_status_select_range_check": "counter", + "mysql_global_status_select_scan": "counter", + "mysql_global_status_sort_rows": "counter", + "mysql_global_status_sort_range": "counter", + "mysql_global_status_sort_merge_passes": "counter", + "mysql_global_status_sort_scan": "counter", + "mysql_global_status_slow_queries": "counter", + "mysql_global_status_aborted_connects": "counter", + "mysql_global_status_aborted_clients": "counter", + "mysql_global_status_table_locks_immediate": "counter", + "mysql_global_status_table_locks_waited": "counter", + "mysql_global_status_bytes_received": "counter", + "mysql_global_status_bytes_sent": "counter", + "mysql_global_status_opened_files": "counter", + "mysql_global_status_opened_tables": "counter", + "mysql_global_status_table_open_cache_hits": "counter", + "mysql_global_status_table_open_cache_misses": "counter", + "mysql_global_status_table_open_cache_overflows": "counter", + "mysql_global_status_opened_table_definitions": "counter", + "mysql_global_status_innodb_buffer_pool_read_requests": "counter", + "mysql_global_status_innodb_buffer_pool_reads": "counter", + "mysql_global_status_innodb_data_reads": "counter", + "mysql_global_status_innodb_data_writes": "counter", + "mysql_global_status_qcache_hits": "counter", + "mysql_global_status_qcache_inserts": "counter", + "mysql_global_status_qcache_not_cached": "counter", + "mysql_global_status_qcache_lowmem_prunes": "counter", + }, } diff --git a/tests/snapshots/datadog_yaml/integrations__consul.txt b/tests/snapshots/datadog_yaml/integrations__consul.txt index 51cfbbc3..64e7b7fc 100644 --- a/tests/snapshots/datadog_yaml/integrations__consul.txt +++ b/tests/snapshots/datadog_yaml/integrations__consul.txt @@ -19,7 +19,7 @@ statuses: {'ok': 16, 'requires_manual': 4, 'skipped': 5, 'warning': 2} - title='Services Warning'; kind=esql; type=pie; dimension=; metrics=['query1']; breakdowns=['consul_node_id', 'consul_datacenter', 'host.name']; primary=; metric= - title='Services Critical'; kind=esql; type=datatable; dimension=; metrics=['query1']; breakdowns=['consul_node_id', 'consul_datacenter', 'host.name']; primary=; metric= - title='Datadog note 1689828293212788'; kind=markdown -- title='Leader Last Contact with Followers (in ms)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1_3']; breakdowns=[]; primary=; metric= +- title='Leader Last Contact with Followers (in ms)'; kind=esql; type=line; dimension=time_bucket; metrics=['max_time', 'average_time', 'query1_3']; breakdowns=[]; primary=; metric= - title='Latency of Leader Commit to Disk'; kind=esql; type=line; dimension=time_bucket; metrics=['max_time', 'average_time']; breakdowns=[]; primary=; metric= - title='New Leader Events'; kind=markdown - title='Consul Raft Commit Time'; kind=esql; type=line; dimension=time_bucket; metrics=['average_time', 'median_time']; breakdowns=[]; primary=; metric= diff --git a/tests/snapshots/datadog_yaml/integrations__mysql.txt b/tests/snapshots/datadog_yaml/integrations__mysql.txt index a9aff569..61b02e9c 100644 --- a/tests/snapshots/datadog_yaml/integrations__mysql.txt +++ b/tests/snapshots/datadog_yaml/integrations__mysql.txt @@ -14,5 +14,5 @@ statuses: {'ok': 10, 'warning': 1} - title='System load'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query1_2', 'query1_3']; breakdowns=[]; primary=; metric= - title='CPU usage (%)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query2', 'query3', 'query4', 'query5', 'query6']; breakdowns=[]; primary=; metric= - title='I/O wait (%)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1']; breakdowns=[]; primary=; metric= -- title='System memory'; kind=esql; type=line; dimension=time_bucket; metrics=['query1']; breakdowns=[]; primary=; metric= +- title='System memory'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query2_query1']; breakdowns=[]; primary=; metric= - title='Network traffic (per sec)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query1_2']; breakdowns=[]; primary=; metric= diff --git a/tests/snapshots/datadog_yaml/integrations__postgres.txt b/tests/snapshots/datadog_yaml/integrations__postgres.txt index 271d755c..d7baac3f 100644 --- a/tests/snapshots/datadog_yaml/integrations__postgres.txt +++ b/tests/snapshots/datadog_yaml/integrations__postgres.txt @@ -12,5 +12,5 @@ statuses: {'ok': 9} - title='System load'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query1_2', 'query1_3']; breakdowns=[]; primary=; metric= - title='CPU usage (%)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query2', 'query3', 'query4', 'query5', 'query6']; breakdowns=[]; primary=; metric= - title='I/O wait (%)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1']; breakdowns=[]; primary=; metric= -- title='System memory'; kind=esql; type=line; dimension=time_bucket; metrics=['query1']; breakdowns=[]; primary=; metric= +- title='System memory'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query2_query1']; breakdowns=[]; primary=; metric= - title='Network traffic (per sec)'; kind=esql; type=line; dimension=time_bucket; metrics=['query1', 'query1_2']; breakdowns=[]; primary=; metric= diff --git a/tests/targets/kibana/test_esql_utils.py b/tests/targets/kibana/test_esql_utils.py index 4807c875..3cb4d427 100644 --- a/tests/targets/kibana/test_esql_utils.py +++ b/tests/targets/kibana/test_esql_utils.py @@ -68,3 +68,65 @@ def test_extract_esql_shape_drop_removes_group_field(): assert shape.metric_fields == ["value"] assert shape.group_fields == [] assert shape.projected_fields == ["value"] + + +def test_extract_esql_shape_keep_includes_eval_aliases_alongside_stats(): + """STATS intermediates + EVAL derived column + KEEP of both. + + Curated MySQL CPU / Memory / Query Cache overrides emit this shape. + Lens Y accessors must follow KEEP, not only the surviving STATS names. + """ + query = ( + "TS metrics-* " + "| STATS Load = MAX(LAST_OVER_TIME(metrics.node_load1)), " + "non_idle = SUM(RATE(metrics.node_cpu_seconds_total)), " + "cpu_cores = COUNT_DISTINCT(labels.cpu) " + "BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) " + "| EVAL CPU_busy_pct = CASE(cpu_cores > 0, ((non_idle * 100) / cpu_cores), NULL) " + "| KEEP time_bucket, CPU_busy_pct, Load " + "| SORT time_bucket ASC" + ) + + shape = extract_esql_shape(query) + + assert shape.metric_fields == ["CPU_busy_pct", "Load"] + assert shape.time_fields == ["time_bucket"] + assert "non_idle" not in shape.metric_fields + assert "cpu_cores" not in shape.metric_fields + + +def test_extract_esql_shape_eval_is_a_metric_when_stats_intermediates_are_dropped(): + query = ( + "TS metrics-* " + "| STATS lhs = AVG(metrics.page_size), rhs = AVG(metrics.pages), " + "log_buffer = AVG(metrics.log_buffer) " + "BY time_bucket = TBUCKET(20, ?_tstart, ?_tend) " + "| EVAL pool_data = lhs * rhs " + "| DROP lhs, rhs" + ) + + shape = extract_esql_shape(query) + + assert shape.metric_fields == ["log_buffer", "pool_data"] + assert shape.time_fields == ["time_bucket"] + + +def test_extract_esql_shape_concat_eval_is_group_not_metric(): + """Identity EVAL aliases (``series_group = CONCAT(...)``) are the series + key, not a Y accessor. Treating them as metrics adds a phantom + ``Series Group`` axis on Lens XY panels.""" + query = ( + "TS metrics-* " + "| STATS process_resident_memory_bytes = MAX(LAST_OVER_TIME(" + "metrics.process_resident_memory_bytes)) " + "BY time_bucket = TBUCKET(20, ?_tstart, ?_tend), labels.instance " + '| EVAL series_group = CONCAT(COALESCE(TO_STRING(labels.instance), "")) ' + "| SORT time_bucket ASC" + ) + + shape = extract_esql_shape(query) + + assert shape.metric_fields == ["process_resident_memory_bytes"] + assert "series_group" in shape.group_fields + assert "series_group" not in shape.metric_fields + diff --git a/tests/test_curated_packs.py b/tests/test_curated_packs.py index 2ee9b27a..bdffca4d 100644 --- a/tests/test_curated_packs.py +++ b/tests/test_curated_packs.py @@ -126,6 +126,57 @@ def test_find_1860_by_title_fallback(): assert entry["gnet_id"] == 1860 +def test_find_7362_by_gnet_id(): + entry = find_curated_pack(gnet_id=7362, title="", tags=[]) + assert entry is not None + assert entry["gnet_id"] == 7362 + assert entry["name"] == "grafana_7362_mysql_overview" + + +def test_find_7362_by_title_fallback(): + entry = find_curated_pack( + gnet_id=None, + title="MySQL Overview", + tags=["Percona", "MySQL"], + ) + assert entry is not None + assert entry["gnet_id"] == 7362 + + +def test_find_9628_by_gnet_id(): + entry = find_curated_pack(gnet_id=9628, title="", tags=[]) + assert entry is not None + assert entry["gnet_id"] == 9628 + assert entry["name"] == "grafana_9628_postgresql_database" + + +def test_find_9628_by_title_fallback(): + entry = find_curated_pack( + gnet_id=None, + title="PostgreSQL Database", + tags=["postgres", "db", "stats"], + ) + assert entry is not None + assert entry["gnet_id"] == 9628 + + +def test_find_14114_by_gnet_id(): + entry = find_curated_pack(gnet_id=14114, title="", tags=[]) + assert entry is not None + assert entry["gnet_id"] == 14114 + assert entry["name"] == "grafana_14114_postgres_exporter_quickstart" + + +def test_find_14114_by_title_fallback(): + entry = find_curated_pack( + gnet_id=None, + title="PostgreSQL Exporter Quickstart and Dashboard", + tags=["postgres"], + ) + assert entry is not None + assert entry["gnet_id"] == 14114 + + def test_find_18406_by_title_fallback(): entry = find_curated_pack( gnet_id=None, @@ -939,6 +990,490 @@ def test_resolve_pack_14091_maps_renamed_fragmentation_metric(): assert target == "metrics.redis_mem_fragmentation_ratio" +def test_resolve_pack_7362_pins_untyped_status_counters_and_processlist_map(): + """mysqld_exporter publishes suffix-less status counters as untyped. + + Without metric_kinds, Elasticsearch stores them as gauges and RATE() 400s. + The processlist metric was also renamed after this dashboard's revision 5. + """ + dashboard = {"gnetId": 7362, "title": "MySQL Overview", "tags": ["Percona", "MySQL"]} + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + assert resolved.metric_kinds.get("mysql_global_status_queries") == "counter" + assert resolved.metric_kinds.get("mysql_global_status_questions") == "counter" + assert resolved.metric_kinds.get("mysql_global_status_bytes_received") == "counter" + assert resolved.metric_kinds.get("mysql_info_schema_processlist_threads") == "gauge" + entry = (resolved.metric_map or {}).get("mysql_info_schema_threads") + target = getattr(entry, "target", entry) + assert target == "metrics.mysql_info_schema_processlist_threads" + assert resolved.control_field_overrides.get("host") == "labels.instance" + titles = {o.get("title_match") for o in resolved.panel_query_overrides} + assert "Process States" in titles + assert "MySQL Query Cache Activity" in titles + assert "CPU Usage / Load" in titles + assert "mysql_global_variables_query_cache_size" in resolved.live_optional_metrics + assert "aws_rds_read_latency_average" in resolved.live_optional_metrics + cpu_override = next( + o for o in resolved.panel_query_overrides if o.get("title_match") == "CPU Usage / Load" + ) + assert cpu_override.get("kibana_type_override") == "line" + titles = {o.get("title_match") for o in resolved.panel_layout_overrides} + assert "Section 1" in titles + overview = next( + o for o in resolved.panel_layout_overrides if o.get("title_match") == "Section 1" + ) + assert overview.get("title") == "Overview" + + +def test_7362_hourly_panels_follow_dashboard_time_picker(): + """Grafana pins timeFrom=24h on the hourly charts. + + Mixed ``metrics-*`` Lens 24h windows render ``No results found`` even when + ``_query`` returns one or two sparse buckets. The pack drops timeFrom so + these panels follow the dashboard picker like the working sibling MySQL + rate charts. + """ + dashboard = {"gnetId": 7362, "title": "MySQL Overview", "tags": ["Percona", "MySQL"]} + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolver = SchemaResolver(resolved) + hourly = next( + o for o in resolved.panel_query_overrides + if o.get("title_match") == "MySQL Network Usage Hourly" + ) + assert hourly.get("drop_time_from") is True + panel = { + "id": 1, + "type": "graph", + "title": "MySQL Network Usage Hourly", + "timeFrom": "24h", + "targets": [ + { + "expr": "increase(mysql_global_status_bytes_received[1h])", + "refId": "A", + "legendFormat": "Received", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, + "fieldConfig": {"defaults": {}, "overrides": []}, + } + yaml_panel, result = translate_panel( + panel, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + resolver=resolver, + ) + assert result.status in {"migrated", "migrated_with_warnings"}, result.reasons + esql = yaml_panel.get("esql") or {} + assert "time_range" not in esql + assert "TBUCKET(20" in (esql.get("query") or "") + assert "mysql_global_status_bytes_received" in (esql.get("query") or "") + + +def test_7362_cpu_override_binds_busy_pct_and_load(): + dashboard = {"gnetId": 7362, "title": "MySQL Overview", "tags": ["Percona"]} + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolver = SchemaResolver(resolved) + panel = { + "id": 2, + "type": "graph", + "title": "CPU Usage / Load", + "stack": True, + "targets": [ + {"expr": 'node_load1{instance="$host"}', "refId": "C", "legendFormat": "Load 1m"} + ], + "seriesOverrides": [{"alias": "Load 1m", "yaxis": 2, "stack": False}], + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, + "yaxes": [ + {"format": "percent", "max": 100, "min": 0}, + {"format": "none", "min": 0}, + ], + } + yaml_panel, result = translate_panel( + panel, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + resolver=resolver, + ) + assert result.status in {"migrated", "migrated_with_warnings"}, result.reasons + esql = yaml_panel.get("esql") or {} + query = esql.get("query") or "" + assert "CPU_busy_pct" in query + y_cols = [item.get("field") for item in (esql.get("metrics") or [])] + assert "CPU_busy_pct" in y_cols + assert "Load 1m" in y_cols + assert esql.get("type") == "line" + load = next(item for item in (esql.get("metrics") or []) if item.get("field") == "Load 1m") + assert load.get("axis") == "right" + assert "suffix" not in (load.get("format") or {}) + + +def test_resolve_pack_9628_ignores_helm_release_and_pins_memory_gauges(): + """Revision 1 filters on Helm ``release``; typical scrapes do not store it. + + Grafana also ``rate()``s process RSS/VMS gauges, which Elasticsearch + rejects as RATE() on double. The pack pins those as gauges and overrides + Average Memory Usage to LAST_OVER_TIME. + """ + dashboard = { + "gnetId": 9628, + "title": "PostgreSQL Database", + "tags": ["postgres", "db", "stats"], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + assert "release" in resolved.ignored_labels + assert resolved.metric_kinds.get("process_resident_memory_bytes") == "gauge" + assert resolved.metric_kinds.get("process_virtual_memory_bytes") == "gauge" + assert resolved.metric_kinds.get("pg_stat_database_xact_commit") == "counter" + assert resolved.control_field_overrides.get("instance") == "labels.instance" + assert resolved.control_field_overrides.get("datname") == "labels.datname" + titles = {o.get("title_match") for o in resolved.panel_query_overrides} + assert "Average Memory Usage" in titles + assert "Start Time" in titles + assert "Version" in titles + assert "pg_postmaster_start_time_seconds" in resolved.live_optional_metrics + + +def test_9628_memory_override_uses_last_over_time(): + dashboard = {"gnetId": 9628, "title": "PostgreSQL Database", "tags": ["postgres"]} + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolver = SchemaResolver(resolved) + panel = { + "id": 24, + "type": "graph", + "title": "Average Memory Usage", + "targets": [ + { + "expr": 'avg(rate(process_resident_memory_bytes{instance="$instance"}[5m]))', + "refId": "A", + "legendFormat": "Resident Mem", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, + "yaxes": [{"format": "decbytes"}, {"format": "short"}], + } + yaml_panel, result = translate_panel( + panel, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + resolver=resolver, + ) + assert result.status in {"migrated", "migrated_with_warnings"}, result.reasons + query = (yaml_panel.get("esql") or {}).get("query") or "" + assert "LAST_OVER_TIME" in query + assert "RATE(" not in query + assert "process_resident_memory_bytes" in query + + +def test_9628_start_time_override_does_not_yellow_absent_postmaster_metric(): + """postgres_exporter v0.15 dropped pg_postmaster_start_time_seconds. + + The override substitutes process_start_time_seconds. That source metric is + live_optional, so an absent field-caps hit must not yellow the panel as a + pack omission. + """ + dashboard = {"gnetId": 9628, "title": "PostgreSQL Database", "tags": ["postgres"]} + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolver = SchemaResolver(resolved) + resolver._field_cache = { + "metrics.process_start_time_seconds": {"double": {"type": "double"}}, + "metrics.pg_static": {"double": {"type": "double"}}, + "labels.instance": {"keyword": {"type": "keyword"}}, + "labels.short_version": {"keyword": {"type": "keyword"}}, + } + resolver._discovery_attempted = True + resolver._discovery_status = "ok" + panel = { + "id": 28, + "type": "singlestat", + "title": "Start Time", + "format": "dateTimeFromNow", + "targets": [ + { + "expr": 'pg_postmaster_start_time_seconds{instance="$instance"} * 1000', + "refId": "A", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 2}, + } + yaml_panel, result = translate_panel( + panel, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + resolver=resolver, + ) + assert result.status == "migrated", result.reasons + assert not any("curated override" in reason for reason in result.reasons) + query = (yaml_panel.get("esql") or {}).get("query") or "" + assert "process_start_time_seconds" in query + assert "DATE_DIFF" in query + assert "pg_postmaster_start_time_seconds" not in query + + +def test_9628_instance_query_result_becomes_label_values_control(): + """Helm query_result(pg_up{release=...}) has no Kibana populate query. + + The pack plugin rewrites Instance to label_values(pg_up, instance) and + drops the unused namespace/release cascade so the dashboard still gets + an Instance control. + """ + dashboard = { + "gnetId": 9628, + "title": "PostgreSQL Database", + "tags": ["postgres"], + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "label": "Namespace", + "query": "query_result(pg_exporter_last_scrape_duration_seconds)", + }, + { + "name": "release", + "type": "query", + "label": "Release", + "query": 'query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace="$namespace"})', + }, + { + "name": "instance", + "type": "query", + "label": "Instance", + "query": 'query_result(pg_up{release="$release"})', + }, + { + "name": "datname", + "type": "query", + "label": "Database", + "query": "label_values(datname)", + "includeAll": True, + "multi": True, + }, + ] + }, + "panels": [ + { + "id": 38, + "type": "singlestat", + "title": "Max Connections", + "targets": [ + { + "expr": 'pg_settings_max_connections{instance="$instance"}', + "refId": "A", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 2}, + "format": "none", + } + ], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + result = translate_dashboard( + dashboard, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + ) + payload = result.dashboard_ir.to_yaml_dict() + controls = payload.get("controls") or [] + names = {control.get("variable_name") for control in controls} + assert "instance" in names, controls + assert "namespace" not in names + assert "release" not in names + assert not any("query_result" in warning for warning in (result.control_warnings or [])), ( + result.control_warnings + ) + instance = next(c for c in controls if c.get("variable_name") == "instance") + query = str(instance.get("query") or "") + assert "labels.instance" in query or "instance" in query + + +def test_9628_dashboard_does_not_emit_release_control(): + """Native PROMQL must not resurrect Helm $release as a Kibana control. + + Mixed metrics-* has a kernel ``release`` field; binding it filters Postgres + series to nothing (Max Connections / CPU / Open FDs empty in view mode). + """ + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + dashboard = { + "gnetId": 9628, + "title": "PostgreSQL Database", + "tags": ["postgres"], + "templating": { + "list": [ + { + "name": "release", + "type": "query", + "label": "Release", + "query": 'query_result(pg_up{release="x"})', + }, + { + "name": "instance", + "type": "query", + "label": "Instance", + "query": 'query_result(pg_up{release="$release"})', + }, + ] + }, + "panels": [ + { + "id": 38, + "type": "singlestat", + "title": "Max Connections", + "targets": [ + { + "expr": 'pg_settings_max_connections{release="$release", instance="$instance"}', + "refId": "A", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 2}, + "format": "none", + } + ], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolved.native_promql = True + set_runtime_feature( + resolved, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test" + ) + set_runtime_feature( + resolved, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test" + ) + result = translate_dashboard( + dashboard, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + ) + payload = result.dashboard_ir.to_yaml_dict() + names = {c.get("variable_name") for c in (payload.get("controls") or [])} + assert "release" not in names, names + max_conn = next(p for p in result.panel_results if p.title == "Max Connections") + query = max_conn.esql_query or "" + assert "release" not in query + assert "?instance" in query or "instance" in query + + +def test_resolve_pack_14114_pins_counters_and_bgwriter_map(): + """Mixin Buffers names lack OpenMetrics _total; v0.15 exporters add it.""" + dashboard = { + "gnetId": 14114, + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "tags": ["postgres"], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + assert resolved.metric_kinds.get("pg_stat_database_xact_commit") == "counter" + assert resolved.metric_kinds.get("pg_stat_database_numbackends") == "gauge" + entry = (resolved.metric_map or {}).get("pg_stat_bgwriter_buffers_alloc") + target = getattr(entry, "target", entry) + assert target == "metrics.pg_stat_bgwriter_buffers_alloc_total" + assert resolved.control_field_overrides.get("instance") == "labels.instance" + assert resolved.control_field_overrides.get("db") == "labels.datname" + + +def test_14114_buffers_override_uses_total_suffix_offline(): + dashboard = { + "gnetId": 14114, + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "tags": ["postgres"], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + resolver = SchemaResolver(resolved) + panel = { + "id": 2, + "type": "graph", + "title": "Buffers", + "targets": [ + { + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", + "refId": "A", + "legendFormat": "buffers_alloc", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 7}, + } + yaml_panel, result = translate_panel( + panel, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + resolver=resolver, + ) + assert result.status in {"migrated", "migrated_with_warnings"}, result.reasons + query = (yaml_panel.get("esql") or {}).get("query") or "" + assert "IRATE(metrics.pg_stat_bgwriter_buffers_alloc_total)" in query + assert "IRATE(metrics.pg_stat_bgwriter_buffers_alloc)" not in query + + +def test_14114_instance_up_becomes_pg_up_control(): + """Mixin Instance lists Prometheus ``up``; Elastic stores postgres as ``pg_up``.""" + dashboard = { + "gnetId": 14114, + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "tags": ["postgres"], + "templating": { + "list": [ + { + "name": "instance", + "type": "query", + "label": "instance", + "query": 'label_values(up{job=~"postgres.*"},instance)', + "includeAll": True, + "current": {"selected": False, "text": "All", "value": "$__all"}, + }, + { + "name": "job", + "type": "query", + "label": "job", + "query": "label_values(pg_up, job)", + "includeAll": False, + "current": {"selected": True, "text": "postgres", "value": "postgres"}, + }, + ] + }, + "panels": [ + { + "id": 11, + "type": "singlestat", + "title": "QPS", + "targets": [ + { + "expr": ( + 'sum(irate(pg_stat_database_xact_commit{instance=~"$instance"}[5m]))' + ), + "refId": "A", + } + ], + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 3}, + "format": "none", + } + ], + } + resolved = resolve_pack_for_dashboard(dashboard, RulePackConfig()) + result = translate_dashboard( + dashboard, + datasource_index="metrics-*", + esql_index="metrics-*", + rule_pack=resolved, + ) + payload = result.dashboard_ir.to_yaml_dict() + controls = payload.get("controls") or [] + names = {control.get("variable_name") for control in controls} + assert "instance" in names, controls + assert "job" not in names, controls + instance = next(c for c in controls if c.get("variable_name") == "instance") + query = str(instance.get("query") or "") + assert "pg_up" in query or "metrics.pg_up" in query + assert "metrics.up" not in query + + def test_prometheus_native_label_candidates_come_first_in_redis_packs(): """Offline runs take the FIRST candidate without probing the target. @@ -952,6 +1487,9 @@ def test_prometheus_native_label_candidates_come_first_in_redis_packs(): 18406: [("cluster", "labels.cluster"), ("bdb", "labels.bdb")], 14091: [("instance", "labels.instance"), ("job", "labels.job")], 11835: [("instance", "labels.instance"), ("job", "labels.job")], + 7362: [("instance", "labels.instance"), ("job", "labels.job")], + 9628: [("instance", "labels.instance"), ("job", "labels.job")], + 14114: [("instance", "labels.instance"), ("job", "labels.job")], } for gnet_id, pairs in expected_first.items(): resolved = resolve_pack_for_dashboard( @@ -1344,6 +1882,29 @@ def test_panel_layout_overrides_can_flip_section_collapsed_state(): assert panels[0]["section"]["collapsed"] is False +def test_panel_layout_overrides_can_rename_section_title(): + panels = [ + { + "title": "Section 1", + "section": { + "collapsed": False, + "panels": [ + { + "title": "MySQL Uptime", + "position": {"x": 0, "y": 0}, + "size": {"w": 6, "h": 6}, + } + ], + }, + } + ] + overrides = [{"title_match": "Section 1", "title": "Overview"}] + + _apply_panel_layout_overrides_recursively(panels, overrides) + + assert panels[0]["title"] == "Overview" + + def test_curated_query_override_materializes_control_and_metric_placeholders(): class _FakeResolver: def resolve_control_field(self, name, metric_field=None): @@ -2036,6 +2597,23 @@ def test_schema_validates_pack_with_collapsed_layout_override(): assert payload.panel.layout_overrides[0].collapsed is False +def test_schema_validates_pack_with_title_layout_override(): + from observability_migration.adapters.source.grafana.extension_schema import validate_rule_pack_payload + + raw = { + "panel": { + "layout_overrides": [ + { + "title_match": "Section 1", + "title": "Overview", + } + ] + } + } + payload = validate_rule_pack_payload(raw) + assert payload.panel.layout_overrides[0].title == "Overview" + + def test_panel_query_override_loaded_from_pack_yaml_round_trip(): """load_rule_pack_files parses query_overrides into RulePackConfig.panel_query_overrides.""" import os diff --git a/tests/test_grafana_passthrough_profile.py b/tests/test_grafana_passthrough_profile.py index 1fbb87bb..52fd51bd 100644 --- a/tests/test_grafana_passthrough_profile.py +++ b/tests/test_grafana_passthrough_profile.py @@ -423,7 +423,7 @@ def test_native_promql_label_rewrite_falls_back_to_esql(self): self.assertIn("host_name", result.esql_query) self.assertNotIn("instance ==", result.esql_query) - def test_native_promql_ignored_label_falls_back_to_esql(self): + def test_native_promql_strips_ignored_label_matchers(self): rule_pack = RulePackConfig( native_promql=True, ignored_labels=["metrics_path"], @@ -453,8 +453,9 @@ def test_native_promql_ignored_label_falls_back_to_esql(self): resolver=resolver, ) - self.assertNotIn("PROMQL ", result.esql_query) + self.assertIn("PROMQL ", result.esql_query) self.assertNotIn("metrics_path", result.esql_query) + self.assertIn("rate(http_requests_total[5m])", result.esql_query) def test_alert_label_rewrite_falls_back_to_esql(self): rule_pack = RulePackConfig(label_rewrites={"mode": "cpu_mode"}) diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 6f780d0d..aa658b78 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -3130,6 +3130,95 @@ def test_set_or_same_metric_topk_rate_or_irate_prefers_left(self): self.assertIn("RATE(", esql) self.assertNotIn("IRATE(", esql) + def test_mysql_network_or_chain_drops_absent_cloud_fallback(self): + """Percona MySQL Overview: ``sum(rate(node)) or sum(irate(node)) or sum(rds)``. + + When live field-caps prove the RDS/OSMetrics series absent, keep the + Linux node_exporter operand instead of emitting COALESCE over an + unknown column (which 400s the whole panel into a markdown placeholder). + """ + resolver = self._native_profile_resolver( + { + "metrics.node_network_receive_bytes_total": { + "double": { + "type": "double", + "time_series_metric": "counter", + "searchable": True, + "aggregatable": True, + } + }, + "labels.device": {"keyword": {"searchable": True, "aggregatable": True}}, + "labels.instance": {"keyword": {"searchable": True, "aggregatable": True}}, + } + ) + expr = ( + 'sum(rate(node_network_receive_bytes_total{instance="$host", device!="lo"}[5m])) ' + 'or sum(irate(node_network_receive_bytes_total{instance="$host", device!="lo"}[5m])) ' + 'or sum(max_over_time(rdsosmetrics_network_rx{instance="$host"}[5m])) ' + 'or sum(max_over_time(rdsosmetrics_network_rx{instance="$host"}[5m]))' + ) + translated = self.translate(expr, resolver=resolver) + self.assertNotEqual(translated.feasibility, "not_feasible", translated.warnings) + esql = translated.esql_query or "" + self.assertIn("node_network_receive_bytes_total", esql) + self.assertIn("RATE(", esql) + self.assertNotIn("rdsosmetrics_network_rx", esql) + self.assertNotIn("IRATE(", esql) + + def test_mysql_network_or_chain_keeps_cloud_fallback_without_field_caps(self): + """Offline / no caps: still COALESCE both sides (issue #167).""" + expr = ( + 'sum(rate(node_network_receive_bytes_total{device!="lo"}[5m])) ' + 'or sum(max_over_time(rdsosmetrics_network_rx[5m]))' + ) + translated = self.translate(expr) + self.assertNotEqual(translated.feasibility, "not_feasible", translated.warnings) + esql = translated.esql_query or "" + self.assertIn("node_network_receive_bytes_total", esql) + self.assertIn("rdsosmetrics_network_rx", esql) + + def test_mysql_disk_latency_or_chain_drops_absent_aws_fallback(self): + """Percona Disk Latency: ``sum((rate/rate) or (irate/irate) or aws_rds)``. + + Nested set-or inside sum was not_feasible. With live field-caps that + prove the AWS RDS series absent, prefer the Linux disk ratio. + """ + counter = { + "double": { + "type": "double", + "time_series_metric": "counter", + "searchable": True, + "aggregatable": True, + } + } + resolver = self._native_profile_resolver( + { + "metrics.node_disk_read_time_seconds_total": counter, + "metrics.node_disk_reads_completed_total": counter, + "labels.device": {"keyword": {"searchable": True, "aggregatable": True}}, + "labels.instance": {"keyword": {"searchable": True, "aggregatable": True}}, + } + ) + expr = ( + "sum((rate(node_disk_read_time_seconds_total{device!~\"dm-.+\"}[5m]) " + "/ rate(node_disk_reads_completed_total{device!~\"dm-.+\"}[5m])) " + "or (irate(node_disk_read_time_seconds_total{device!~\"dm-.+\"}[5m]) " + "/ irate(node_disk_reads_completed_total{device!~\"dm-.+\"}[5m])) " + "or avg_over_time(aws_rds_read_latency_average[5m]))" + ) + translated = self.translate(expr, resolver=resolver) + self.assertNotEqual( + translated.feasibility, + "not_feasible", + msg=f"warnings={translated.warnings} query={translated.esql_query}", + ) + esql = translated.esql_query or "" + self.assertIn("node_disk_read_time_seconds_total", esql) + self.assertIn("node_disk_reads_completed_total", esql) + self.assertIn("RATE(", esql) + self.assertNotIn("aws_rds_read_latency_average", esql) + self.assertNotIn("IRATE(", esql) + def test_load_over_nested_cpu_count_aligns_groupings(self): """Docker/node Load panel: ``load / count by(job, instance)(count by(..., cpu)(...))``. @@ -4868,6 +4957,226 @@ def test_native_promql_distinct_metric_difference_stays_native_on_stat(self): joined, ) + def test_native_promql_rate_or_irate_graph_collapses_to_left(self): + """Grafana MySQL/Percona ``rate(M[$interval]) or irate(M[5m])`` is a + same-metric window fallback. Native PROMQL rejects ``or``; collapse to + the left operand so the graph stays on the PROMQL path.""" + expr = ( + "rate(mysql_global_status_questions{instance=\"$host\"}[$interval]) " + "or irate(mysql_global_status_questions{instance=\"$host\"}[5m])" + ) + panel = { + "title": "MySQL Questions", + "type": "graph", + "targets": [{"refId": "A", "expr": expr, "legendFormat": "Questions"}], + } + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + rule_pack = rules.RulePackConfig(native_promql=True) + set_runtime_feature(rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test") + set_runtime_feature(rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test") + + yaml_panel, result = panels.translate_panel( + panel, + esql_index="metrics-*", + datasource_index="metrics-*", + rule_pack=rule_pack, + resolver=self.resolver, + ) + + query = yaml_panel["esql"]["query"] + self.assertIn("PROMQL", query) + self.assertIn("mysql_global_status_questions", query) + self.assertNotRegex(query, r"\bor\b") + self.assertNotIn("irate(", query.lower()) + joined = " ".join(result.notes) + " ".join(result.reasons) + self.assertRegex(joined, r"(?i)same-metric.*or|preferred left") + + def test_native_promql_bare_gauge_singlestat_keeps_bytes_format(self): + """Grafana 5 singlestat gauges (Uptime, buffer-pool size) are bare + selectors. Keep them native and map panel ``format: bytes`` into Lens.""" + panel = { + "title": "InnoDB Buffer Pool Size", + "type": "singlestat", + "format": "bytes", + "decimals": 0, + "targets": [ + { + "refId": "A", + "expr": 'mysql_global_variables_innodb_buffer_pool_size{instance="$host"}', + } + ], + } + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + rule_pack = rules.RulePackConfig(native_promql=True) + set_runtime_feature(rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test") + set_runtime_feature(rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test") + + yaml_panel, _result = panels.translate_panel( + panel, + esql_index="metrics-*", + datasource_index="metrics-*", + rule_pack=rule_pack, + resolver=self.resolver, + ) + + query = yaml_panel["esql"]["query"] + self.assertIn("PROMQL", query) + self.assertIn("mysql_global_variables_innodb_buffer_pool_size", query) + self.assertIn("| STATS value = LAST(value, step)", query) + primary = yaml_panel["esql"].get("primary") or {} + fmt = primary.get("format") or {} + self.assertEqual(fmt.get("type"), "bytes") + self.assertEqual(fmt.get("decimals"), 0) + + def test_native_promql_absent_gauge_is_telemetry_missing_markdown(self): + """Bare instant gauges stay native only when field-caps can see the + metric. An absent selector must keep the telemetry-missing markdown + instead of a green empty PROMQL tile.""" + panel = { + "title": "Version", + "type": "singlestat", + "targets": [ + {"refId": "A", "expr": "prometheus_build_info"}, + ], + } + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + rule_pack = rules.RulePackConfig(native_promql=True) + set_runtime_feature(rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test") + set_runtime_feature(rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test") + resolver = self._native_profile_resolver( + { + "metrics.up": {"double": {"type": "double", "aggregatable": True, "searchable": True}}, + } + ) + resolver._discovery_status = "ok" + + yaml_panel, result = panels.translate_panel( + panel, + esql_index="metrics-*", + datasource_index="metrics-*", + rule_pack=rule_pack, + resolver=resolver, + ) + + self.assertIn("markdown", yaml_panel) + self.assertNotIn("esql", yaml_panel) + self.assertEqual(result.kibana_type, "markdown") + self.assertEqual(result.status, "migrated_with_warnings") + self.assertTrue( + any("Target telemetry missing" in reason and "prometheus_build_info" in reason + for reason in result.reasons), + result.reasons, + ) + + def test_native_promql_present_gauge_stays_native_when_discovery_ok(self): + panel = { + "title": "InnoDB Buffer Pool Size", + "type": "singlestat", + "format": "bytes", + "targets": [ + { + "refId": "A", + "expr": 'mysql_global_variables_innodb_buffer_pool_size{instance="$host"}', + } + ], + } + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + rule_pack = rules.RulePackConfig(native_promql=True) + set_runtime_feature(rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test") + set_runtime_feature(rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test") + resolver = self._native_profile_resolver( + { + "metrics.mysql_global_variables_innodb_buffer_pool_size": { + "double": {"type": "double", "aggregatable": True, "searchable": True, "time_series_metric": "gauge"}, + }, + } + ) + resolver._discovery_status = "ok" + + yaml_panel, _result = panels.translate_panel( + panel, + esql_index="metrics-*", + datasource_index="metrics-*", + rule_pack=rule_pack, + resolver=resolver, + ) + + query = yaml_panel["esql"]["query"] + self.assertIn("PROMQL", query) + self.assertIn("mysql_global_variables_innodb_buffer_pool_size", query) + + def test_native_promql_absent_binary_agg_ratio_is_telemetry_missing_markdown(self): + """``sum((delta(A)))/sum((delta(B)))`` on missing metrics must not stay + native: Elasticsearch then emits ``value_$1``/``value_$2`` and + ``STATS LAST(value)`` 400s.""" + panel = { + "title": "Average query time", + "type": "gauge", + "targets": [ + { + "refId": "A", + "expr": ( + "sum((delta(pg_stat_statements_total_time_seconds" + '{instance=~"$Instance"}[5m])))' + "/sum((delta(pg_stat_statements_calls" + '{instance=~"$Instance"}[5m])))' + ), + } + ], + } + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + rule_pack = rules.RulePackConfig(native_promql=True) + set_runtime_feature(rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test") + set_runtime_feature(rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test") + resolver = self._native_profile_resolver( + { + "metrics.pg_up": {"double": {"type": "double", "aggregatable": True, "searchable": True}}, + } + ) + resolver._discovery_status = "ok" + + yaml_panel, result = panels.translate_panel( + panel, + esql_index="metrics-*", + datasource_index="metrics-*", + rule_pack=rule_pack, + resolver=resolver, + ) + + self.assertIn("markdown", yaml_panel) + query = (yaml_panel.get("esql") or {}).get("query") or "" + self.assertNotIn("PROMQL", query) + self.assertNotIn("STATS value = LAST(value, step)", query) + self.assertTrue( + any("Target telemetry missing" in reason for reason in result.reasons), + result.reasons, + ) + def test_native_promql_distinct_metric_ratio_with_macro_stays_native_on_gauge(self): """The single-value distinct-metric gate must count metrics from the *cleaned* expression, not the raw one. A common Grafana ratio uses a @@ -5634,6 +5943,45 @@ def test_series_override_yaxis_preserved_on_merged_metric(self): self.assertEqual(metrics_by_field["transmit"].get("format"), {"type": "bytes", "suffix": "/s"}) self.assertNotIn("Merged compatible panel targets into a single ES|QL query", result.reasons) + def test_series_override_right_axis_none_clears_inherited_format(self): + """Grafana CPU Usage / Load: left axis is percent, Load 1m is yaxis 2 + with format none. The overlay must not keep the % suffix.""" + panel = { + "id": 910, + "type": "graph", + "title": "CPU Usage / Load", + "datasource": {"type": "prometheus", "uid": "prom"}, + "yaxes": [ + {"format": "percent", "show": True}, + {"format": "none", "show": True}, + ], + "seriesOverrides": [{"alias": "Load 1m", "yaxis": 2}], + "targets": [ + { + "expr": "avg(rate(node_cpu_seconds_total{mode!=\"idle\"}[1m])) * 100", + "refId": "A", + "legendFormat": "busy", + }, + { + "expr": "node_load1", + "refId": "C", + "legendFormat": "Load 1m", + }, + ], + } + + yaml_panel, result = self.translate_panel(panel) + + self.assertIn(result.status, {"migrated", "migrated_with_warnings"}) + metrics_by_field = { + metric.get("label") or metric["field"]: metric + for metric in yaml_panel["esql"]["metrics"] + } + load = metrics_by_field["Load 1m"] + self.assertEqual(load.get("axis"), "right") + self.assertNotEqual((load.get("format") or {}).get("suffix"), "%") + self.assertNotIn("format", load) + def test_series_override_stack_false_marks_overlay_metrics(self): """kubernetes-mixin CPU/Memory Usage: stack containers but not requests/limits.""" panel = { @@ -12981,6 +13329,21 @@ def test_extract_unit_empty_panel(self): from observability_migration.targets.kibana.emit.display import extract_grafana_unit self.assertEqual(extract_grafana_unit({}), "") + def test_extract_unit_from_legacy_singlestat_format(self): + from observability_migration.targets.kibana.emit.display import extract_grafana_unit + self.assertEqual(extract_grafana_unit({"format": "bytes"}), "bytes") + self.assertEqual(extract_grafana_unit({"format": "s"}), "s") + self.assertEqual(extract_grafana_unit({"format": "percent"}), "percent") + self.assertEqual(extract_grafana_unit({"format": "time_series"}), "") + + def test_extract_unit_prefers_field_config_over_legacy_format(self): + from observability_migration.targets.kibana.emit.display import extract_grafana_unit + panel = { + "format": "bytes", + "fieldConfig": {"defaults": {"unit": "percent"}}, + } + self.assertEqual(extract_grafana_unit(panel), "percent") + def test_unit_to_yaml_format_bytes(self): from observability_migration.targets.kibana.emit.display import grafana_unit_to_yaml_format fmt = grafana_unit_to_yaml_format("bytes") @@ -17430,10 +17793,18 @@ def test_rejects_unless(self): def test_rejects_or_binary_op(self): from observability_migration.adapters.source.grafana.panels import can_use_native_promql + from observability_migration.adapters.source.grafana.promql import collapse_or_for_native_promql self.assertFalse(can_use_native_promql("foo or bar")) self.assertFalse(can_use_native_promql( "rate(http_requests_total[5m]) or vector(0)" )) + collapsed = collapse_or_for_native_promql( + "rate(mysql_global_status_queries[$interval]) " + "or irate(mysql_global_status_queries[5m])" + ) + self.assertNotIn(" or ", collapsed) + self.assertIn("rate(mysql_global_status_queries", collapsed) + self.assertTrue(can_use_native_promql(collapsed)) def test_rejects_and_binary_op(self): from observability_migration.adapters.source.grafana.panels import can_use_native_promql @@ -17649,9 +18020,17 @@ def test_gauge_panel_produces_gauge_type(self): yaml_panel, _result = self.translate_panel(panel) self.assertEqual(yaml_panel["esql"]["type"], "gauge") - def test_stat_panel_with_multi_series_skips_native_promql(self): + def test_stat_panel_bare_selector_stays_native_promql(self): + """Grafana 5 singlestat gauges (``up``, MySQL uptime) are bare + selectors. Keep them on native PROMQL and collapse with LAST.""" panel = self._make_panel("up", panel_type="stat") _, result = self.translate_panel(panel) + self.assertEqual(result.query_ir.get("family"), "native_promql") + self.assertIn("PROMQL index=", result.esql_query or "") + + def test_stat_panel_with_grouped_series_skips_native_promql(self): + panel = self._make_panel("sum by (job) (up)", panel_type="stat") + _, result = self.translate_panel(panel) self.assertNotEqual(result.query_ir.get("family"), "native_promql") self.assertNotIn("PROMQL index=", result.esql_query or "") @@ -17688,6 +18067,36 @@ def test_query_ir_clean_expression_uses_cleaned_native_promql(self): 'node_filesystem_avail_bytes{instance="node-1"}', ) + def test_ignored_labels_are_stripped_from_native_promql_matchers(self): + """Helm ``release`` filters must not become ``?release`` on native PROMQL. + + Mixed ``metrics-*`` often has a kernel ``release`` field. A synthesized + control bound to that field empties every panel that still filters on + the Prometheus ``release`` label. + """ + from observability_migration.adapters.source.grafana.runtime_features import ( + KIBANA_PROMQL_CONTROL_PARAMS, + PROMQL_LABEL_MATCHER_PARAMS, + set_runtime_feature, + ) + + set_runtime_feature( + self.rule_pack, PROMQL_LABEL_MATCHER_PARAMS, supported=True, source="test" + ) + set_runtime_feature( + self.rule_pack, KIBANA_PROMQL_CONTROL_PARAMS, supported=True, source="test" + ) + self.rule_pack.ignored_labels = list(self.rule_pack.ignored_labels) + ["release"] + panel = self._make_panel( + 'pg_settings_max_connections{release="$release", instance="$instance"}', + panel_type="stat", + ) + yaml_panel, result = self.translate_panel(panel) + query = result.esql_query or (yaml_panel.get("esql") or {}).get("query") or "" + self.assertIn("PROMQL", query) + self.assertNotIn("release", query) + self.assertIn("?instance", query) + def test_query_ir_target_query_is_promql_command(self): panel = self._make_panel("rate(foo[5m])") _, result = self.translate_panel(panel)