diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0b697..40d89a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * None ### Bug fixes - * [[`PR-12`](https://github.com/thiagoesteves/observer_web/pull/12)] Fixing bug fix reporting that graphics were displaying information in the reverse order + * [[`PR-12`](https://github.com/thiagoesteves/observer_web/pull/12)] Fixing bug reported that graphics were displaying information in the reverse order ### Enhancements * [[`PR-10`](https://github.com/thiagoesteves/observer_web/pull/10)] Adding iframe configuration to allow Observer Web to run with embedded pages diff --git a/README.md b/README.md index bde0c7f..1c1d12f 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,65 @@ mix assets.build Since Observer Web uses the OTP distribution, it is required to have a name when running. ``` -elixir --sname hello -S mix run --no-halt dev.exs +elixir --sname observer -S mix run --no-halt dev.exs ``` Now you can visit [`localhost:4000/observer`](http://localhost:4000/observer) from your browser. +### Run multiple nodes with Metric Hub configuration + +1. Start the nodes + +Open a new terminal (Terminal 1) and run the application in `observer` mode: +``` +export PORT=4000 +export OBSERVER_WEB_TELEMETRY_MODE=observer +elixir --sname observer --cookie cookie -S mix run --no-halt dev.exs +``` + +Open a new terminal (Terminal 2) and run the application in `broadcast` mode: +``` +export PORT=4001 +export OBSERVER_WEB_TELEMETRY_MODE=broadcast +elixir --sname broadcast --cookie cookie -S mix run --no-halt dev.exs +``` + +2. Connect the nodes + +Open a new terminal (Terminal 3) and run: +``` +iex --sname hub --cookie cookie -S mix phx.server +``` + +After the Phoenix server starts, you'll see the Elixir interactive shell prompt. Connect +the nodes by executing these commands: + +```elixir +# Connect to the observer node +{:ok, hostname} = :inet.gethostname() +Node.connect(:"observer@#{hostname}") +# Should return: true + +# Connect to the broadcast node +{:ok, hostname} = :inet.gethostname() +Node.connect(:"broadcast@#{hostname}") +# Should return: true +``` + +you can close the Terminal 3 if you want, this terminal is only for supporting the node connection. + +To verify everything is working properly: + + * Visit [`localhost:4000/observer/metrics`](http://localhost:4000/observer/metrics) in your browser to confirm + the application is running in `observer` mode. + * Visit [`localhost:4001/observer/metrics`](http://localhost:4001/observer/metrics) to confirm the application + is running in `broadcast` mode. + +You can now explore the `observer` mode, checking that the data is persisted even if the other app in +broadcast mode restarts. + + [dye]: https://github.com/thiagoesteves/deployex [edb]: https://www.erlang.org/doc/apps/runtime_tools/dbg.html [liv]: https://github.com/phoenixframework/phoenix_live_view diff --git a/config/config.exs b/config/config.exs index b3442f7..f5fa97b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -30,10 +30,6 @@ if config_env() == :dev do ), cd: Path.expand("../assets", __DIR__) ] - - config :observer_web, ObserverWeb.Telemetry, - adapter: ObserverWeb.Telemetry.Storage, - data_retention_period: :timer.minutes(30) end # Configures Elixir's Logger diff --git a/dev.exs b/dev.exs index 011c739..186fb4a 100644 --- a/dev.exs +++ b/dev.exs @@ -75,6 +75,22 @@ Application.put_env(:phoenix, :serve_endpoints, true) Application.put_env(:phoenix, :persistent, true) Task.async(fn -> + # Stop the default Telemetry server to start a new one with new defaults + mode = "OBSERVER_WEB_TELEMETRY_MODE" |> System.get_env("local") |> String.to_atom() + + retention_period = + "OBSERVER_WEB_TELEMETRY_RETENTION_PERIOD" |> System.get_env("1800000") |> String.to_integer() + + telemetry_module = ObserverWeb.Telemetry.Storage + :ok = Supervisor.terminate_child(ObserverWeb.Application, telemetry_module) + :ok = Supervisor.delete_child(ObserverWeb.Application, telemetry_module) + + {:ok, _} = + Supervisor.start_child( + ObserverWeb.Application, + {telemetry_module, [mode: mode, data_retention_period: retention_period]} + ) + {:ok, _} = Supervisor.start_child(ObserverWeb.Application, WebDev.Endpoint) Process.sleep(:infinity) diff --git a/guides/installation.md b/guides/installation.md index ef4cf19..635796f 100644 --- a/guides/installation.md +++ b/guides/installation.md @@ -54,23 +54,6 @@ After you've verified that the dashboard is loading you'll probably want to rest dashboard via authentication, either with a [custom resolver's][ac] access controls or [Basic Auth][ba]. -### Retention period for metrics - -The Observer Web can monitor Beam VM metrics by default, using ETS tables to store the data. -However, this means that the data is not persisted across restarts. The retention period -for this data can be configured. - -By default, without a retention time set, the metrics will only show data received during the -current session. If you'd like to persist this data for a longer period, you can configure -a retention time. - -To configure the retention period, use the following optional setting: - -```elixir -config :observer_web, ObserverWeb.Telemetry, - data_retention_period: :timer.minutes(5) -``` - ### Embedding Observer Web in your app page In some cases, you may prefer to run the Observer in the same page as your app rather than in @@ -139,6 +122,81 @@ path `/observer"`. However, using the iframe approach allows you to display your application's information alongside the Observer in your main page, providing a more integrated monitoring experience. +### Metrics + +#### Retention period for metrics + +Observer Web can monitor Beam VM metrics (along with many others) and uses ETS +tables to store the data and there is a possibility of configuration for the retention +period while the application is running. + +By default, without a retention time set, the metrics will only show data received during the +current session. If you'd like to persist this data for a longer period, you can configure +a retention time. + +To configure the retention period, use the following optional setting: + +```elixir +config :observer_web, ObserverWeb.Telemetry, + data_retention_period: :timer.minutes(30) +``` + +> #### Persistence Across Restarts {: .attention} +> +> Please note that data is not persisted across application restarts. For persistent +> storage, refer to the Configuration section to set up a Central Hub application, +> which can aggregate and retain metrics. + +#### Configuration + +Observer Web can operate in two distinct metrics configurations: `Standalone` and `Metric Hub`. +These configurations determine how metrics are collected, stored, and managed. + +#### Standalone Configuration (default) + +In this mode, all applications with Observer Web installed operate independently. Each +application receives and stores its own metrics within its ETS tables. The image below +illustrates this configuration: + +![Standalone Mode](./static/standalone.png) + +__**NOTE: No additional configuration is required for this mode**__ + +#### Metric Hub Configuration + +In this mode, one application is designated as the central hub to store all metrics, +while the remaining applications broadcast their data to this designated hub. This +configuration is ideal for scenarios where you have a dedicated application for monitoring +or deployment, such as [DeployEx][dye]. Additionally, this setup ensures that metrics +are retained even if any of the monitored applications restart. + +![Metric Hub Mode](./static/metric_hub.png) + +To configure applications to broadcast their metrics, use the following setting: + +```elixir +config :observer_web, ObserverWeb.Telemetry, + mode: :broadcast +``` + +> #### Disable Endpoint for Broadcast applications {: .attention} +> +> In this mode, since there is a centralized application dedicated to capturing metrics, +> we recommend disabling the `/observer` endpoint on all applications configured in +> **broadcast** mode. Only the **central observer (hub)** should expose the `/observer` endpoint +> to avoid redundancy and ensure efficient metric collection. + +To designate an application as the **central observer (hub)**, use the following setting: + +```elixir +config :observer_web, ObserverWeb.Telemetry, + mode: :observer, + data_retention_period: :timer.minutes(30) +``` + +The application in `observer mode` will also retain its own metrics in addition to +aggregating metrics from other applications. + ### Usage with Web and Clustering The Observer Web provides observer ability for the local application as well as any other that is @@ -158,3 +216,4 @@ via OTP distribution! [ac]: Observer.Web.Resolver.html#c:resolve_access/1 [ba]: https://hexdocs.pm/basic_auth/readme.html [oi]: installation.html +[dye]: https://github.com/thiagoesteves/deployex \ No newline at end of file diff --git a/guides/overview.md b/guides/overview.md index 3c6987c..a6384a0 100644 --- a/guides/overview.md +++ b/guides/overview.md @@ -8,6 +8,8 @@ and Beam VM statistics. Powered by [Phoenix LiveView][liv], it is distributed, lightweight, and fully real-time. This library is part of the [DeployEx][dye] project. +![Observer Dashboard](./static/dashboard.png) + [dye]: https://github.com/thiagoesteves/deployex [edb]: https://www.erlang.org/doc/apps/runtime_tools/dbg.html [liv]: https://github.com/phoenixframework/phoenix_live_view diff --git a/guides/static/metric_hub.png b/guides/static/metric_hub.png new file mode 100644 index 0000000..cf94c26 Binary files /dev/null and b/guides/static/metric_hub.png differ diff --git a/guides/static/standalone.png b/guides/static/standalone.png new file mode 100644 index 0000000..a66703e Binary files /dev/null and b/guides/static/standalone.png differ diff --git a/guides/templates/observer_metrics.excalidraw b/guides/templates/observer_metrics.excalidraw new file mode 100644 index 0000000..3ee5054 --- /dev/null +++ b/guides/templates/observer_metrics.excalidraw @@ -0,0 +1,2801 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "YDq1-o-iQzWpFVm4eJwhV", + "type": "rectangle", + "x": 555.6762931570853, + "y": 300.06038695014774, + "width": 790.5234375, + "height": 271.10887822403583, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a0", + "roundness": { + "type": 3 + }, + "seed": 1626826632, + "version": 530, + "versionNonce": 1409245861, + "isDeleted": false, + "boundElements": [], + "updated": 1741897719401, + "link": null, + "locked": false + }, + { + "id": "klmLu0034WAuLwH9yewl3", + "type": "text", + "x": 585.0741662212131, + "y": 303.5725318863018, + "width": 170.8798828125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a1", + "roundness": null, + "seed": 1165854856, + "version": 80, + "versionNonce": 1569912456, + "isDeleted": false, + "boundElements": [], + "updated": 1741890945723, + "link": null, + "locked": false, + "text": "OTP Distribution", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "OTP Distribution", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "-vqLc5NyhYIN9Gp4Ig4Qg", + "type": "rectangle", + "x": 605.8515625, + "y": 343.26239573407776, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a2", + "roundness": { + "type": 3 + }, + "seed": 95626120, + "version": 77, + "versionNonce": 65930232, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "3XyfV8FHEUC09a7sKhWdS" + }, + { + "id": "c8_9yiiyx4vCxQUAX5G9q", + "type": "arrow" + }, + { + "id": "06TZcuvI0VvPGbdn2kbOZ", + "type": "arrow" + }, + { + "id": "5s5IHtKs35Sy7lluNsCsn", + "type": "arrow" + } + ], + "updated": 1741891809561, + "link": null, + "locked": false + }, + { + "id": "3XyfV8FHEUC09a7sKhWdS", + "type": "text", + "x": 626.2071533203125, + "y": 352.17255198407776, + "width": 142.659912109375, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a3", + "roundness": null, + "seed": 503631240, + "version": 81, + "versionNonce": 189079176, + "isDeleted": false, + "boundElements": [], + "updated": 1741891811338, + "link": null, + "locked": false, + "text": "Elixir App #1 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "-vqLc5NyhYIN9Gp4Ig4Qg", + "originalText": "Elixir App #1 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "_3qyVfvIiGY72oPX6oQKV", + "type": "rectangle", + "x": 612.4179117794974, + "y": 500.2651659995528, + "width": 176.12109375, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a8", + "roundness": { + "type": 3 + }, + "seed": 1640343432, + "version": 242, + "versionNonce": 1900471032, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "xRmFhzVWxnBc6jukcqxz_" + }, + { + "id": "c8_9yiiyx4vCxQUAX5G9q", + "type": "arrow" + } + ], + "updated": 1741891090971, + "link": null, + "locked": false + }, + { + "id": "xRmFhzVWxnBc6jukcqxz_", + "type": "text", + "x": 644.4685022946342, + "y": 505.2651659995528, + "width": 112.01991271972656, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a9", + "roundness": null, + "seed": 1443839736, + "version": 181, + "versionNonce": 895468536, + "isDeleted": false, + "boundElements": [], + "updated": 1741891090971, + "link": null, + "locked": false, + "text": "node metric\n(ets)", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "_3qyVfvIiGY72oPX6oQKV", + "originalText": "node metric (ets)", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "c8_9yiiyx4vCxQUAX5G9q", + "type": "arrow", + "x": 646.3249620829991, + "y": 411.43518632592475, + "width": 1.0314844636499174, + "height": 88.43079911561892, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aE", + "roundness": { + "type": 2 + }, + "seed": 488756872, + "version": 648, + "versionNonce": 1070032376, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "IgIP3N5uWILgSmyb2Nay9" + } + ], + "updated": 1741891809561, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0314844636499174, + 41.45060761384781 + ], + [ + 1.0242615693019843, + 88.43079911561892 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "-vqLc5NyhYIN9Gp4Ig4Qg", + "focus": 0.5626835062972096, + "gap": 1 + }, + "endBinding": { + "elementId": "_3qyVfvIiGY72oPX6oQKV", + "focus": -0.6033476708850669, + "gap": 1.209561399714005 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "IgIP3N5uWILgSmyb2Nay9", + "type": "text", + "x": 675.6522140786608, + "y": 432.91015625, + "width": 51.503936767578125, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aF", + "roundness": null, + "seed": 2143957384, + "version": 12, + "versionNonce": 446900104, + "isDeleted": false, + "boundElements": [], + "updated": 1741890657180, + "link": null, + "locked": false, + "text": "publish", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "c8_9yiiyx4vCxQUAX5G9q", + "originalText": "publish", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "qnl-BC16hLOqCbl0s9Dhj", + "type": "rectangle", + "x": 877.0927955009174, + "y": 339.8199442879109, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aG", + "roundness": { + "type": 3 + }, + "seed": 1725966216, + "version": 301, + "versionNonce": 107251080, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "2h6UzW248cj_zreeg3dEL" + }, + { + "id": "f2YcQJ69i8c31Fhb4WSTz", + "type": "arrow" + } + ], + "updated": 1741890910189, + "link": null, + "locked": false + }, + { + "id": "2h6UzW248cj_zreeg3dEL", + "type": "text", + "x": 894.7183829642963, + "y": 348.7301005379109, + "width": 148.1199188232422, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aH", + "roundness": null, + "seed": 1317140104, + "version": 287, + "versionNonce": 1255053448, + "isDeleted": false, + "boundElements": [], + "updated": 1741891814893, + "link": null, + "locked": false, + "text": "Elixir App #2 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "qnl-BC16hLOqCbl0s9Dhj", + "originalText": "Elixir App #2 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "cupL5fxKqJSq_9CIrD1fr", + "type": "rectangle", + "x": 883.6591447804149, + "y": 496.9904483391426, + "width": 176.12109375, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aI", + "roundness": { + "type": 3 + }, + "seed": 521266568, + "version": 470, + "versionNonce": 1424641416, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "w81BR7XxdZTYcJJHV9P4F" + }, + { + "id": "f2YcQJ69i8c31Fhb4WSTz", + "type": "arrow" + }, + { + "id": "06TZcuvI0VvPGbdn2kbOZ", + "type": "arrow" + } + ], + "updated": 1741891092900, + "link": null, + "locked": false + }, + { + "id": "w81BR7XxdZTYcJJHV9P4F", + "type": "text", + "x": 915.7097352955516, + "y": 501.9904483391426, + "width": 112.01991271972656, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aJ", + "roundness": null, + "seed": 1439429768, + "version": 403, + "versionNonce": 826760840, + "isDeleted": false, + "boundElements": [], + "updated": 1741891092899, + "link": null, + "locked": false, + "text": "node metric\n(ets)", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cupL5fxKqJSq_9CIrD1fr", + "originalText": "node metric (ets)", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "f2YcQJ69i8c31Fhb4WSTz", + "type": "arrow", + "x": 947.3537855182229, + "y": 407.93326803483785, + "width": 2.240193984845291, + "height": 88.79433115847803, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aK", + "roundness": { + "type": 2 + }, + "seed": 1751917448, + "version": 1236, + "versionNonce": 1316558181, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "K8m96N_MlAeKkElLeGZ5Q" + } + ], + "updated": 1741897734288, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.1713367029260553, + 40.995186946776016 + ], + [ + 2.240193984845291, + 88.79433115847803 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "qnl-BC16hLOqCbl0s9Dhj", + "focus": 0.24177823308287294, + "gap": 1 + }, + "endBinding": { + "elementId": "cupL5fxKqJSq_9CIrD1fr", + "focus": -0.2417294665764151, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "K8m96N_MlAeKkElLeGZ5Q", + "type": "text", + "x": 646.7409019270488, + "y": 391.89027958105015, + "width": 51.503936767578125, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aL", + "roundness": null, + "seed": 892236424, + "version": 16, + "versionNonce": 95755768, + "isDeleted": false, + "boundElements": [], + "updated": 1741890753636, + "link": null, + "locked": false, + "text": "publish", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "f2YcQJ69i8c31Fhb4WSTz", + "originalText": "publish", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "D3ySxfsq_7BvpbqFIUB6s", + "type": "rectangle", + "x": 1115.9895904805162, + "y": 339.7274761889527, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aM", + "roundness": { + "type": 3 + }, + "seed": 1487395064, + "version": 150, + "versionNonce": 1919850120, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "i2bzP2HpbBwqzJBjPmfRa" + }, + { + "id": "x6cxffzO-mm248oHLGaBz", + "type": "arrow" + } + ], + "updated": 1741890860193, + "link": null, + "locked": false + }, + { + "id": "i2bzP2HpbBwqzJBjPmfRa", + "type": "text", + "x": 1134.535183742235, + "y": 348.6376324389527, + "width": 146.2799072265625, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aN", + "roundness": null, + "seed": 825358840, + "version": 136, + "versionNonce": 930386568, + "isDeleted": false, + "boundElements": [], + "updated": 1741891817962, + "link": null, + "locked": false, + "text": "Elixir App #3 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "D3ySxfsq_7BvpbqFIUB6s", + "originalText": "Elixir App #3 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "yLOUgGF2FPruM3YBx3oGM", + "type": "rectangle", + "x": 1122.5559397600136, + "y": 497.0588137740604, + "width": 176.12109375, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aO", + "roundness": { + "type": 3 + }, + "seed": 365951736, + "version": 322, + "versionNonce": 1222234248, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "sDXXxMdTo9mluq2ttbBqx" + }, + { + "id": "x6cxffzO-mm248oHLGaBz", + "type": "arrow" + }, + { + "id": "5s5IHtKs35Sy7lluNsCsn", + "type": "arrow" + } + ], + "updated": 1741891094171, + "link": null, + "locked": false + }, + { + "id": "sDXXxMdTo9mluq2ttbBqx", + "type": "text", + "x": 1154.6065302751504, + "y": 502.0588137740604, + "width": 112.01991271972656, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aP", + "roundness": null, + "seed": 419814392, + "version": 255, + "versionNonce": 1974157704, + "isDeleted": false, + "boundElements": [], + "updated": 1741891094170, + "link": null, + "locked": false, + "text": "node metric\n(ets)", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "yLOUgGF2FPruM3YBx3oGM", + "originalText": "node metric (ets)", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "x6cxffzO-mm248oHLGaBz", + "type": "arrow", + "x": 1194.8603069666424, + "y": 405.8081150549725, + "width": 2.242331550699646, + "height": 88.955164692354, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aQ", + "roundness": { + "type": 2 + }, + "seed": 1557634296, + "version": 799, + "versionNonce": 840480613, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "Rs5DppiuvXj4vz-du-a9o" + } + ], + "updated": 1741897755030, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.1734905939063083, + 41.15879347261546 + ], + [ + 2.242331550699646, + 88.955164692354 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "D3ySxfsq_7BvpbqFIUB6s", + "focus": 0.14821048470332127, + "gap": 1.7396736339802032 + }, + "endBinding": { + "elementId": "yLOUgGF2FPruM3YBx3oGM", + "focus": -0.1441613703568487, + "gap": 2.2955340267338897 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "Rs5DppiuvXj4vz-du-a9o", + "type": "text", + "x": 954.9923956966193, + "y": 394.47352711277046, + "width": 51.503936767578125, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aR", + "roundness": null, + "seed": 583613944, + "version": 16, + "versionNonce": 1140636040, + "isDeleted": false, + "boundElements": [], + "updated": 1741890758750, + "link": null, + "locked": false, + "text": "publish", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "x6cxffzO-mm248oHLGaBz", + "originalText": "publish", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "06TZcuvI0VvPGbdn2kbOZ", + "type": "arrow", + "x": 790.1584763153801, + "y": 378.55666192722384, + "width": 130.94581876032055, + "height": 118.10505372082827, + "angle": 0, + "strokeColor": "#f08c00", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aS", + "roundness": { + "type": 2 + }, + "seed": 451194104, + "version": 553, + "versionNonce": 114812664, + "isDeleted": false, + "boundElements": [], + "updated": 1741891809561, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 122.66860589620194, + 11.159380462508125 + ], + [ + 130.94581876032055, + 118.10505372082827 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "-vqLc5NyhYIN9Gp4Ig4Qg", + "focus": -0.16666653545462018, + "gap": 1 + }, + "endBinding": { + "elementId": "cupL5fxKqJSq_9CIrD1fr", + "focus": -0.5340424673000342, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "5s5IHtKs35Sy7lluNsCsn", + "type": "arrow", + "x": 781.0587112304619, + "y": 344.122278855864, + "width": 382.19229996278466, + "height": 177.87732460715768, + "angle": 0, + "strokeColor": "#f08c00", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aU", + "roundness": { + "type": 2 + }, + "seed": 723644808, + "version": 1572, + "versionNonce": 2049984504, + "isDeleted": false, + "boundElements": [], + "updated": 1741891809561, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 344.4240351593643, + -25.7760611329806 + ], + [ + 382.19229996278466, + 152.10126347417707 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "-vqLc5NyhYIN9Gp4Ig4Qg", + "focus": -0.6573141231689354, + "gap": 1 + }, + "endBinding": { + "elementId": "yLOUgGF2FPruM3YBx3oGM", + "focus": -0.43225812106126915, + "gap": 2.473934561494275 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "cy9wkFoIvjLqAaMGXJQVa", + "type": "arrow", + "x": 736.6809533767753, + "y": 414.04392011783386, + "width": 1.0231939376614037, + "height": 87.66229163304496, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aV", + "roundness": { + "type": 2 + }, + "seed": 155860984, + "version": 808, + "versionNonce": 2057705208, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "hftJDLFvPixBylm4A9t_B" + } + ], + "updated": 1741890832752, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0231939376614037, + 41.0016613954603 + ], + [ + 1.0154354873143348, + 87.66229163304496 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "hftJDLFvPixBylm4A9t_B", + "type": "text", + "x": 716.4401626342609, + "y": 445.04558151329417, + "width": 42.52796936035156, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aW", + "roundness": null, + "seed": 250577144, + "version": 30, + "versionNonce": 1721431688, + "isDeleted": false, + "boundElements": [], + "updated": 1741890831444, + "link": null, + "locked": false, + "text": "fetch", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cy9wkFoIvjLqAaMGXJQVa", + "originalText": "fetch", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "a3TdIhwJFjnm09k5q6jh0", + "type": "text", + "x": 795.5780499090774, + "y": 386.9656126814658, + "width": 69.24795532226562, + "height": 40, + "angle": 0, + "strokeColor": "#f08c00", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aX", + "roundness": null, + "seed": 1162192376, + "version": 362, + "versionNonce": 677443464, + "isDeleted": false, + "boundElements": [ + { + "id": "5s5IHtKs35Sy7lluNsCsn", + "type": "arrow" + }, + { + "id": "06TZcuvI0VvPGbdn2kbOZ", + "type": "arrow" + } + ], + "updated": 1741891036053, + "link": null, + "locked": false, + "text": "fetch\nby GRPC", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "fetch\nby GRPC", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "2nyDVnaJbJJcAE9iN14vK", + "type": "rectangle", + "x": 561.2119506473259, + "y": 647.6515262623763, + "width": 790.5234375, + "height": 351.6096209760424, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aY", + "roundness": { + "type": 3 + }, + "seed": 905227512, + "version": 1007, + "versionNonce": 888312824, + "isDeleted": false, + "boundElements": [], + "updated": 1741891544193, + "link": null, + "locked": false + }, + { + "id": "pooBz-7g8yotlsw6_hXCx", + "type": "text", + "x": 630.81792850059, + "y": 652.8796896461592, + "width": 170.8798828125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aZ", + "roundness": null, + "seed": 1975799288, + "version": 294, + "versionNonce": 1248679160, + "isDeleted": false, + "boundElements": [], + "updated": 1741891544193, + "link": null, + "locked": false, + "text": "OTP Distribution", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "OTP Distribution", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "v6OxnNxAVWws3CBc2cNOC", + "type": "rectangle", + "x": 660.2052033032335, + "y": 766.6169409681737, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aa", + "roundness": { + "type": 3 + }, + "seed": 1022873336, + "version": 242, + "versionNonce": 170317960, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "fb3L35otfM67MjWBopmjZ" + }, + { + "id": "SirlCPSzyyYfo9JOlsueo", + "type": "arrow" + } + ], + "updated": 1741891786377, + "link": null, + "locked": false + }, + { + "id": "fb3L35otfM67MjWBopmjZ", + "type": "text", + "x": 684.8807861889757, + "y": 775.5270972181737, + "width": 134.01992797851562, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ab", + "roundness": null, + "seed": 873672696, + "version": 245, + "versionNonce": 1187628792, + "isDeleted": false, + "boundElements": [], + "updated": 1741891796786, + "link": null, + "locked": false, + "text": "Monitor app +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "v6OxnNxAVWws3CBc2cNOC", + "originalText": "Monitor app +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "CUVKw5aZb3vxjD8EpawhK", + "type": "rectangle", + "x": 666.7715525827309, + "y": 923.6197112336488, + "width": 176.12109375, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ac", + "roundness": { + "type": 3 + }, + "seed": 1394241784, + "version": 420, + "versionNonce": 1541895759, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "zSss0uN5Hh9ATip7UIWTH" + }, + { + "id": "SirlCPSzyyYfo9JOlsueo", + "type": "arrow" + }, + { + "id": "wyBcIXJW0CiH9R7-gjbVk", + "type": "arrow" + }, + { + "id": "jaBfydEfrg7mQqfdWq4UY", + "type": "arrow" + }, + { + "id": "NqNW3Zz6DYDdIYcbEaMtO", + "type": "arrow" + } + ], + "updated": 1741953348792, + "link": null, + "locked": false + }, + { + "id": "zSss0uN5Hh9ATip7UIWTH", + "type": "text", + "x": 691.3121562204262, + "y": 941.1197112336488, + "width": 127.03988647460938, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ad", + "roundness": null, + "seed": 1905365496, + "version": 362, + "versionNonce": 1570847224, + "isDeleted": false, + "boundElements": [], + "updated": 1741891544193, + "link": null, + "locked": false, + "text": "metrics (ets)", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "CUVKw5aZb3vxjD8EpawhK", + "originalText": "metrics (ets)", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "SirlCPSzyyYfo9JOlsueo", + "type": "arrow", + "x": 700.6770141292467, + "y": 834.7897315600206, + "width": 1.0330732206358562, + "height": 88.50124698253762, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ae", + "roundness": { + "type": 2 + }, + "seed": 1451004664, + "version": 1140, + "versionNonce": 857757320, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "TwbW4myuGEF0CRV1gfct_" + } + ], + "updated": 1741891786378, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0330732206358562, + 41.689770906177216 + ], + [ + 1.025847111139342, + 88.50124698253762 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "v6OxnNxAVWws3CBc2cNOC", + "focus": 0.5626835062972094, + "gap": 1 + }, + "endBinding": { + "elementId": "CUVKw5aZb3vxjD8EpawhK", + "focus": -0.6033476708850694, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "TwbW4myuGEF0CRV1gfct_", + "type": "text", + "x": 1058.6283977401656, + "y": 607.7958017279275, + "width": 51.503936767578125, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "af", + "roundness": null, + "seed": 1214215160, + "version": 16, + "versionNonce": 2086318840, + "isDeleted": false, + "boundElements": [], + "updated": 1741891248242, + "link": null, + "locked": false, + "text": "publish", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "SirlCPSzyyYfo9JOlsueo", + "originalText": "publish", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "uGZBc-X3JUt2VIETsdxFp", + "type": "rectangle", + "x": 1032.7924561019963, + "y": 684.2533748902545, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ag", + "roundness": { + "type": 3 + }, + "seed": 490616056, + "version": 942, + "versionNonce": 1985354561, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "aS4VkuCgreOr1aX1n4inr" + }, + { + "id": "wyBcIXJW0CiH9R7-gjbVk", + "type": "arrow" + } + ], + "updated": 1741953344698, + "link": null, + "locked": false + }, + { + "id": "aS4VkuCgreOr1aX1n4inr", + "type": "text", + "x": 1053.1480469223088, + "y": 693.1635311402545, + "width": 142.659912109375, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ah", + "roundness": null, + "seed": 1251792376, + "version": 925, + "versionNonce": 1328950520, + "isDeleted": false, + "boundElements": [], + "updated": 1741891800433, + "link": null, + "locked": false, + "text": "Elixir App #1 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "uGZBc-X3JUt2VIETsdxFp", + "originalText": "Elixir App #1 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "NbkVN8knlSelFb3S088Gv", + "type": "rectangle", + "x": 1032.2584201540553, + "y": 782.2327563444948, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "am", + "roundness": { + "type": 3 + }, + "seed": 1795169016, + "version": 678, + "versionNonce": 1541534401, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "CItSvROuEGMRsggOvPd-y" + }, + { + "id": "jaBfydEfrg7mQqfdWq4UY", + "type": "arrow" + } + ], + "updated": 1741953347248, + "link": null, + "locked": false + }, + { + "id": "CItSvROuEGMRsggOvPd-y", + "type": "text", + "x": 1049.8840076174342, + "y": 791.1429125944948, + "width": 148.1199188232422, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "an", + "roundness": null, + "seed": 322042872, + "version": 658, + "versionNonce": 158741640, + "isDeleted": false, + "boundElements": [], + "updated": 1741891803915, + "link": null, + "locked": false, + "text": "Elixir App #2 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "NbkVN8knlSelFb3S088Gv", + "originalText": "Elixir App #2 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "AB0sIvk8JiJtaif7E7zxs", + "type": "arrow", + "x": 791.0345941800085, + "y": 837.6376286442592, + "width": 1.0231939376614037, + "height": 87.66229163304496, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "au", + "roundness": { + "type": 2 + }, + "seed": 494641912, + "version": 972, + "versionNonce": 49425400, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "vR7NG_IrJjLlo3H0ly5E0" + } + ], + "updated": 1741891544193, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0231939376614037, + 41.0016613954603 + ], + [ + 1.0154354873143348, + 87.66229163304496 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "vR7NG_IrJjLlo3H0ly5E0", + "type": "text", + "x": 1153.4640822115662, + "y": 609.9555893014492, + "width": 42.52796936035156, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "av", + "roundness": null, + "seed": 2096923640, + "version": 34, + "versionNonce": 665777656, + "isDeleted": false, + "boundElements": [], + "updated": 1741891248242, + "link": null, + "locked": false, + "text": "fetch", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "AB0sIvk8JiJtaif7E7zxs", + "originalText": "fetch", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "Hy63cJgmDB7Ig8R0Vd6ps", + "type": "rectangle", + "x": 1036.167358917286, + "y": 883.7870789060302, + "width": 183.37109375, + "height": 67.8203125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ax", + "roundness": { + "type": 3 + }, + "seed": 1143067128, + "version": 719, + "versionNonce": 569627695, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "jX-HVD1AscnOWGw8EX13I" + }, + { + "id": "NqNW3Zz6DYDdIYcbEaMtO", + "type": "arrow" + } + ], + "updated": 1741953348792, + "link": null, + "locked": false + }, + { + "id": "jX-HVD1AscnOWGw8EX13I", + "type": "text", + "x": 1054.7129521790048, + "y": 892.6972351560302, + "width": 146.2799072265625, + "height": 50, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ay", + "roundness": null, + "seed": 1829032696, + "version": 697, + "versionNonce": 2089034232, + "isDeleted": false, + "boundElements": [], + "updated": 1741891807289, + "link": null, + "locked": false, + "text": "Elixir App #3 +\nObserver Web", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Hy63cJgmDB7Ig8R0Vd6ps", + "originalText": "Elixir App #3 +\nObserver Web", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "wyBcIXJW0CiH9R7-gjbVk", + "type": "arrow", + "x": 1028.968913860861, + "y": 717.8643259362889, + "width": 347.6882004750488, + "height": 205.4266526062695, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b00", + "roundness": { + "type": 2 + }, + "seed": 732423304, + "version": 2698, + "versionNonce": 625670561, + "isDeleted": false, + "boundElements": [], + "updated": 1741953356520, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -311.35491006854227, + 34.80468873621314 + ], + [ + -347.6882004750488, + 205.4266526062695 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "uGZBc-X3JUt2VIETsdxFp", + "focus": 0.24854740324962374, + "gap": 3.8235422411353284 + }, + "endBinding": { + "elementId": "CUVKw5aZb3vxjD8EpawhK", + "focus": -0.847122056023372, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "vJ14Rl8qmMFPSFpiX9TGI", + "type": "text", + "x": 675.8541516898042, + "y": 696.439986811912, + "width": 80.17593383789062, + "height": 40, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b01", + "roundness": null, + "seed": 1814286984, + "version": 670, + "versionNonce": 2007173512, + "isDeleted": false, + "boundElements": [], + "updated": 1741891557626, + "link": null, + "locked": false, + "text": "publish\nby PubSub", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "publish\nby PubSub", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "jaBfydEfrg7mQqfdWq4UY", + "type": "arrow", + "x": 1030.0265378520892, + "y": 815.4656723801778, + "width": 349.4908898604999, + "height": 162.39550089183376, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b02", + "roundness": { + "type": 2 + }, + "seed": 17503480, + "version": 2915, + "versionNonce": 533821967, + "isDeleted": false, + "boundElements": [], + "updated": 1741953359169, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -313.50025181518174, + -54.56876477665389 + ], + [ + -349.4908898604999, + 107.82673611517987 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "NbkVN8knlSelFb3S088Gv", + "focus": -0.3142279722454315, + "gap": 2.2318823019661522 + }, + "endBinding": { + "elementId": "CUVKw5aZb3vxjD8EpawhK", + "focus": -0.8554358970257638, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "NqNW3Zz6DYDdIYcbEaMtO", + "type": "arrow", + "x": 1033.8767547120083, + "y": 917.8951704560008, + "width": 352.5661853471868, + "height": 162.50411320622652, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b03", + "roundness": { + "type": 2 + }, + "seed": 2009640184, + "version": 3076, + "versionNonce": 1359445647, + "isDeleted": false, + "boundElements": [], + "updated": 1741953362272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -316.5513978796472, + -157.1083051196689 + ], + [ + -352.5661853471868, + 5.395808086557622 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "Hy63cJgmDB7Ig8R0Vd6ps", + "focus": -0.5898074000049356, + "gap": 2.2906042052777593 + }, + "endBinding": { + "elementId": "CUVKw5aZb3vxjD8EpawhK", + "focus": -0.8472571593642386, + "gap": 1 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "S2pEXBq8ULew7NrHemHI0", + "type": "rectangle", + "x": 597.3926928048896, + "y": 400.2798223595205, + "width": 55.48526934891117, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b06", + "roundness": { + "type": 3 + }, + "seed": 132425720, + "version": 207, + "versionNonce": 56668293, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "9UUcsi5UP-C-JiizsItBx" + } + ], + "updated": 1741897718519, + "link": null, + "locked": false + }, + { + "id": "9UUcsi5UP-C-JiizsItBx", + "type": "text", + "x": 608.0953494520014, + "y": 405.2798223595205, + "width": 34.0799560546875, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b07", + "roundness": null, + "seed": 1439288200, + "version": 146, + "versionNonce": 1533171685, + "isDeleted": false, + "boundElements": [], + "updated": 1741897718519, + "link": null, + "locked": false, + "text": "local", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "S2pEXBq8ULew7NrHemHI0", + "originalText": "local", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "ofdIXs6mkiZdQct1N8G_g", + "type": "rectangle", + "x": 1021.7007949830277, + "y": 398.61661385206827, + "width": 55.56365746266988, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b08", + "roundness": { + "type": 3 + }, + "seed": 2038392056, + "version": 533, + "versionNonce": 1981231397, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "9BxDzAOjO5UW-C4_tB6yJ" + } + ], + "updated": 1741897759736, + "link": null, + "locked": false + }, + { + "id": "9BxDzAOjO5UW-C4_tB6yJ", + "type": "text", + "x": 1032.4426456870187, + "y": 403.61661385206827, + "width": 34.0799560546875, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b09", + "roundness": null, + "seed": 1731724792, + "version": 472, + "versionNonce": 12567685, + "isDeleted": false, + "boundElements": [], + "updated": 1741897759736, + "link": null, + "locked": false, + "text": "local", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "ofdIXs6mkiZdQct1N8G_g", + "originalText": "local", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "uoeta_pjDk8GeCGHNuyrl", + "type": "rectangle", + "x": 1264.047109301503, + "y": 399.4031980280578, + "width": 57.158450121891974, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0A", + "roundness": { + "type": 3 + }, + "seed": 873062792, + "version": 617, + "versionNonce": 1488077227, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "9TJmGgFCdyh5kN4l_hbCx" + } + ], + "updated": 1741897757487, + "link": null, + "locked": false + }, + { + "id": "9TJmGgFCdyh5kN4l_hbCx", + "type": "text", + "x": 1275.5863563351052, + "y": 404.4031980280578, + "width": 34.0799560546875, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0B", + "roundness": null, + "seed": 40136840, + "version": 556, + "versionNonce": 804443211, + "isDeleted": false, + "boundElements": [], + "updated": 1741897757487, + "link": null, + "locked": false, + "text": "local", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "uoeta_pjDk8GeCGHNuyrl", + "originalText": "local", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "rnzCYR4pWsEQmRlwEjkXG", + "type": "rectangle", + "x": 1174.103506359767, + "y": 734.4801168241995, + "width": 110.77862297103457, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0C", + "roundness": { + "type": 3 + }, + "seed": 353027576, + "version": 554, + "versionNonce": 986176136, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "EOAQ6dKeg_35Qhm8iOGQy" + } + ], + "updated": 1741891764312, + "link": null, + "locked": false + }, + { + "id": "EOAQ6dKeg_35Qhm8iOGQy", + "type": "text", + "x": 1190.1008577012412, + "y": 739.4801168241995, + "width": 78.78392028808594, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0D", + "roundness": null, + "seed": 481249016, + "version": 505, + "versionNonce": 690976760, + "isDeleted": false, + "boundElements": [], + "updated": 1741892052705, + "link": null, + "locked": false, + "text": "broadcast", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "rnzCYR4pWsEQmRlwEjkXG", + "originalText": "broadcast", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "h9UvhsjhLwDhPBdqMZavt", + "type": "rectangle", + "x": 1177.6796261701927, + "y": 834.8736662414503, + "width": 110.77862297103457, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0E", + "roundness": { + "type": 3 + }, + "seed": 2094022392, + "version": 614, + "versionNonce": 618033912, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "PZBXPMm78Y7pKIhJxuzMK" + } + ], + "updated": 1741891771496, + "link": null, + "locked": false + }, + { + "id": "PZBXPMm78Y7pKIhJxuzMK", + "type": "text", + "x": 1193.676977511667, + "y": 839.8736662414503, + "width": 78.78392028808594, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0F", + "roundness": null, + "seed": 1063100408, + "version": 557, + "versionNonce": 1520294648, + "isDeleted": false, + "boundElements": [], + "updated": 1741892056347, + "link": null, + "locked": false, + "text": "broadcast", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "h9UvhsjhLwDhPBdqMZavt", + "originalText": "broadcast", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "tPmIqPBinTASgcfYDwqxw", + "type": "rectangle", + "x": 1182.7045745659457, + "y": 934.9752875109116, + "width": 110.77862297103457, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0G", + "roundness": { + "type": 3 + }, + "seed": 2120598920, + "version": 638, + "versionNonce": 1210196360, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "9N2TnrwQdUPQfjYc5EefU" + } + ], + "updated": 1741891778162, + "link": null, + "locked": false + }, + { + "id": "9N2TnrwQdUPQfjYc5EefU", + "type": "text", + "x": 1198.70192590742, + "y": 939.9752875109116, + "width": 78.78392028808594, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0H", + "roundness": null, + "seed": 1604572296, + "version": 581, + "versionNonce": 534978040, + "isDeleted": false, + "boundElements": [], + "updated": 1741892057563, + "link": null, + "locked": false, + "text": "broadcast", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "tPmIqPBinTASgcfYDwqxw", + "originalText": "broadcast", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "Gqe3GirohO6TE54XPLmKj", + "type": "rectangle", + "x": 571.428657401228, + "y": 747.108712254516, + "width": 110.77862297103457, + "height": 30, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0I", + "roundness": { + "type": 3 + }, + "seed": 601955720, + "version": 663, + "versionNonce": 876299000, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "peKx3C-ShotQonakUixT4" + } + ], + "updated": 1741892046006, + "link": null, + "locked": false + }, + { + "id": "peKx3C-ShotQonakUixT4", + "type": "text", + "x": 593.8499970849874, + "y": 752.108712254516, + "width": 65.93594360351562, + "height": 20, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0J", + "roundness": null, + "seed": 1811164296, + "version": 613, + "versionNonce": 480406776, + "isDeleted": false, + "boundElements": [], + "updated": 1741892047920, + "link": null, + "locked": false, + "text": "observer", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Gqe3GirohO6TE54XPLmKj", + "originalText": "observer", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "44h8NHPgzeYBCvl3ftUHi", + "type": "rectangle", + "x": 848.2697963310011, + "y": 621.6152502124413, + "width": 215.74571337032648, + "height": 40.62126115727756, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0K", + "roundness": { + "type": 3 + }, + "seed": 1887795051, + "version": 118, + "versionNonce": 1560494155, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "b8WDWEAFMs7KWtj0BG063" + } + ], + "updated": 1741897567797, + "link": null, + "locked": false + }, + { + "id": "b8WDWEAFMs7KWtj0BG063", + "type": "text", + "x": 874.9027085581565, + "y": 631.9258807910801, + "width": 162.47988891601562, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#3bc9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0L", + "roundness": null, + "seed": 238719627, + "version": 74, + "versionNonce": 29837867, + "isDeleted": false, + "boundElements": [], + "updated": 1741897587850, + "link": null, + "locked": false, + "text": "METRIC HUB Config", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "44h8NHPgzeYBCvl3ftUHi", + "originalText": "METRIC HUB Config", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "9NX196l-2AwCzRsPfQDze", + "type": "rectangle", + "x": 832.3056515083448, + "y": 262.3041561221829, + "width": 215.74571337032648, + "height": 40.62126115727756, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0M", + "roundness": { + "type": 3 + }, + "seed": 1364276971, + "version": 205, + "versionNonce": 1662726795, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "8wLPVMCDHNUe7Mzl0WVAj" + } + ], + "updated": 1741897537736, + "link": null, + "locked": false + }, + { + "id": "8wLPVMCDHNUe7Mzl0WVAj", + "type": "text", + "x": 859.2745622706565, + "y": 272.6147867008217, + "width": 161.80789184570312, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#3bc9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0N", + "roundness": null, + "seed": 998318475, + "version": 142, + "versionNonce": 1004869381, + "isDeleted": false, + "boundElements": [], + "updated": 1741897585421, + "link": null, + "locked": false, + "text": "STANDALONE Config", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "9NX196l-2AwCzRsPfQDze", + "originalText": "STANDALONE Config", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "finwU6DtjU5lygJ-GwcIM", + "type": "arrow", + "x": 1014.2295029291383, + "y": 407.59834041877417, + "width": 1.0231939376614037, + "height": 87.66229163304496, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0O", + "roundness": { + "type": 2 + }, + "seed": 505577643, + "version": 948, + "versionNonce": 2014792395, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "IczIhEiGF3ihmFUWpMNCu" + } + ], + "updated": 1741897741371, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0231939376614037, + 41.0016613954603 + ], + [ + 1.0154354873143348, + 87.66229163304496 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "IczIhEiGF3ihmFUWpMNCu", + "type": "text", + "x": 717.8300904542298, + "y": 411.2344409973408, + "width": 42.52796936035156, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0P", + "roundness": null, + "seed": 2063807307, + "version": 34, + "versionNonce": 1008010661, + "isDeleted": false, + "boundElements": [], + "updated": 1741897736168, + "link": null, + "locked": false, + "text": "fetch", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "finwU6DtjU5lygJ-GwcIM", + "originalText": "fetch", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "2CC7uxF1Yo8GW3hF3mybM", + "type": "arrow", + "x": 1252.3347499894433, + "y": 407.7875531071565, + "width": 1.0231939376614037, + "height": 87.66229163304496, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0Q", + "roundness": { + "type": 2 + }, + "seed": 111281701, + "version": 882, + "versionNonce": 1852710725, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "BZycArq7LOTqbJb9PYuQK" + } + ], + "updated": 1741897748186, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 1.0231939376614037, + 41.0016613954603 + ], + [ + 1.0154354873143348, + 87.66229163304496 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "BZycArq7LOTqbJb9PYuQK", + "type": "text", + "x": 1308.471012431653, + "y": 446.4847648429661, + "width": 42.52796936035156, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0R", + "roundness": null, + "seed": 1126693253, + "version": 34, + "versionNonce": 1252095627, + "isDeleted": false, + "boundElements": [], + "updated": 1741897744322, + "link": null, + "locked": false, + "text": "fetch", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "2CC7uxF1Yo8GW3hF3mybM", + "originalText": "fetch", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/lib/observer_web/application.ex b/lib/observer_web/application.ex index d5f6e25..c2ef04c 100644 --- a/lib/observer_web/application.ex +++ b/lib/observer_web/application.ex @@ -10,12 +10,11 @@ defmodule ObserverWeb.Application do @impl true def start(_type, _args) do children = - telemetry_servers() ++ - [ - Observer.Web.Telemetry, - {Phoenix.PubSub, [name: ObserverWeb.PubSub]}, - ObserverWeb.Tracer.Server - ] + [ + Observer.Web.Telemetry, + ObserverWeb.Tracer.Server, + {Phoenix.PubSub, [name: ObserverWeb.PubSub]} + ] ++ telemetry_servers() # # See https://hexdocs.pm/elixir/Supervisor.html # # for other strategies and supported options @@ -24,10 +23,17 @@ defmodule ObserverWeb.Application do # NOTE: DO NOT start these servers when running tests. if_not_test do - defp telemetry_servers, - do: [ - ObserverWeb.Telemetry.Storage + defp telemetry_servers do + [{ObserverWeb.Telemetry.Storage, telemetry_server_config()}] + end + + defp telemetry_server_config do + [ + mode: Application.get_env(:observer_web, ObserverWeb.Telemetry)[:mode] || :local, + data_retention_period: + Application.get_env(:observer_web, ObserverWeb.Telemetry)[:data_retention_period] ] + end else defp telemetry_servers, do: [] end diff --git a/lib/observer_web/apps/process.ex b/lib/observer_web/apps/process.ex index ce9f05d..4bf20aa 100644 --- a/lib/observer_web/apps/process.ex +++ b/lib/observer_web/apps/process.ex @@ -46,8 +46,9 @@ defmodule ObserverWeb.Apps.Process do state = :sys.get_state(pid, 100) {:ok, state} catch - _, _ -> - {:error, "Could not retrieve the state for pid: #{inspect(pid)}"} + _, reason -> + {:error, + "Could not retrieve the state for pid: #{inspect(pid)} reason: #{inspect(reason)}"} end else {:error, diff --git a/lib/observer_web/telemetry.ex b/lib/observer_web/telemetry.ex index 83bd5a6..664c794 100644 --- a/lib/observer_web/telemetry.ex +++ b/lib/observer_web/telemetry.ex @@ -11,15 +11,15 @@ defmodule ObserverWeb.Telemetry do """ @type t :: %__MODULE__{ timestamp: non_neg_integer(), - value: integer() | float(), - unit: String.t(), + value: nil | integer() | float(), + unit: nil | String.t(), tags: map(), measurements: map() } defstruct timestamp: nil, - value: "", - unit: "", + value: nil, + unit: nil, tags: %{}, measurements: %{} end @@ -67,6 +67,18 @@ defmodule ObserverWeb.Telemetry do @spec get_keys_by_node(atom()) :: list() def get_keys_by_node(node), do: default().get_keys_by_node(node) + @doc """ + List all available nodes considering the current metric configured mode + """ + @spec list_active_nodes() :: list() + def list_active_nodes, do: default().list_active_nodes() + + @doc """ + Retrieve the configured mode (from ets table) + """ + @spec cached_mode() :: :local | :broadcast | :observer + def cached_mode, do: default().cached_mode() + ### ========================================================================== ### Private functions ### ========================================================================== diff --git a/lib/observer_web/telemetry/adapter.ex b/lib/observer_web/telemetry/adapter.ex index f5cd4dc..2fe79a5 100644 --- a/lib/observer_web/telemetry/adapter.ex +++ b/lib/observer_web/telemetry/adapter.ex @@ -9,4 +9,6 @@ defmodule ObserverWeb.Telemetry.Adapter do @callback unsubscribe_for_new_data(String.t(), String.t()) :: :ok @callback list_data_by_node_key(atom() | String.t(), String.t(), Keyword.t()) :: list() @callback get_keys_by_node(atom()) :: list() + @callback list_active_nodes() :: list() + @callback cached_mode() :: nil | :local | :broadcast | :observer end diff --git a/lib/observer_web/telemetry/storage.ex b/lib/observer_web/telemetry/storage.ex index 46cb106..8ff6077 100644 --- a/lib/observer_web/telemetry/storage.ex +++ b/lib/observer_web/telemetry/storage.ex @@ -4,115 +4,219 @@ defmodule ObserverWeb.Telemetry.Storage do """ use GenServer - import ObserverWeb.Macros - alias ObserverWeb.Rpc @behaviour ObserverWeb.Telemetry.Adapter + @storage_table :storage_table + @mode_key "mode" @metric_keys "metric-keys" - @metric_table :observer_web_metrics + @registry_key "registry-nodes" @one_minute_in_milliseconds 60_000 @retention_data_delete_interval :timer.minutes(1) + @type t :: %__MODULE__{ + nodes: [atom()], + node_metric_tables: map(), + persist_data?: boolean(), + mode: :local | :broadcast | :observer, + data_retention_period: nil | non_neg_integer() + } + + defstruct nodes: [], + node_metric_tables: %{}, + persist_data?: false, + mode: :local, + data_retention_period: nil + ### ========================================================================== ### Callback functions ### ========================================================================== @spec start_link(any()) :: :ignore | {:error, any()} | {:ok, pid()} def start_link(args) do - GenServer.start_link(__MODULE__, args, name: __MODULE__) + name = Keyword.get(args, :name, __MODULE__) + + GenServer.start_link(__MODULE__, args, name: name) end @impl true - def init(_args) do - node = Node.self() + def init(args) do + # Create a general table to store information + :ets.new(@storage_table, [:set, :protected, :named_table]) - # Create metric tables for the node - :ets.new(@metric_table, [:set, :protected, :named_table]) - :ets.insert(@metric_table, {@metric_keys, []}) + node_self = Node.self() - persist_data? = - if data_retention_period() do + mode = Keyword.fetch!(args, :mode) + data_retention_period = Keyword.fetch!(args, :data_retention_period) + + :ets.insert(@storage_table, {@mode_key, mode}) + + persist_data? = fn -> + if data_retention_period do :timer.send_interval(@retention_data_delete_interval, :prune_expired_entries) true else false end + end - {:ok, %{node: node, persist_data?: persist_data?}} + case mode do + :local -> + {:ok, + %__MODULE__{ + nodes: [node_self], + persist_data?: persist_data?.(), + node_metric_tables: create_update_metric_table(node_self, %{}), + mode: mode, + data_retention_period: data_retention_period + }} + + :observer -> + # List all nodes including self() + nodes = [node_self] ++ Node.list() + + # Subscribe to receive metrics data via PubSub + Phoenix.PubSub.subscribe(ObserverWeb.PubSub, broadcast_topic()) + + # Subscribe to receive notifications if any node is UP or Down + :net_kernel.monitor_nodes(true) + + {:ok, + %{ + nodes: nodes, + persist_data?: persist_data?.(), + node_metric_tables: Enum.reduce(nodes, %{}, &create_update_metric_table(&1, &2)), + mode: mode, + data_retention_period: data_retention_period + }} + + :broadcast -> + # NOTE: In Broadcast mode, data is not stored. + {:ok, %__MODULE__{nodes: [node_self], mode: mode}} + end end @impl true - def handle_cast( - {:observer_web_telemetry, - %{metrics: metrics, reporter: reporter, measurements: measurements}}, - %{node: node, persist_data?: persist_data?} = state - ) - when reporter in [node] do - now = System.os_time(:millisecond) - minute = unix_to_minutes(now) + def handle_cast({:observer_web_telemetry, event}, state) do + do_handle_metrics(event, state) + end - keys = get_keys_by_node(reporter) + @impl true + def handle_info({:observer_web_telemetry, event}, state) do + do_handle_metrics(event, state) + end - new_keys = - Enum.reduce(metrics, [], fn metric, acc -> - {key, timed_key, data} = build_telemetry_data(metric, measurements, now, minute) + def handle_info({:nodeup, node}, state) do + nodes = state.nodes ++ [node] - # credo:disable-for-lines:3 - if persist_data? do - current_data = - case :ets.lookup(@metric_table, timed_key) do - [{_, current_list_data}] -> [data | current_list_data] - _ -> [data] - end + if node |> metric_table() |> ets_table_exists?() do + {:noreply, %{state | nodes: nodes}} + else + node_metric_tables = create_update_metric_table(node, state.node_metric_tables) - :ets.insert(@metric_table, {timed_key, current_data}) - end + {:noreply, %{state | nodes: nodes, node_metric_tables: node_metric_tables}} + end + end - Phoenix.PubSub.broadcast( - ObserverWeb.PubSub, - metrics_topic(reporter, key), - {:metrics_new_data, reporter, key, data} - ) + def handle_info( + {:nodedown, node}, + %{nodes: nodes, persist_data?: persist_data?, node_metric_tables: node_metric_tables} = + state + ) do + metric_table = Map.get(node_metric_tables, node) + now = System.os_time(:millisecond) + minute = unix_to_minutes(now) - if key in keys do - acc - else - [key | acc] - end - end) + node + |> get_keys_by_node() + |> Enum.each(fn key -> + if persist_data? do + metric_key = metric_key(key, minute) - if new_keys != [] do - :ets.insert(@metric_table, {@metric_keys, new_keys ++ keys}) + data = %ObserverWeb.Telemetry.Data{timestamp: now} - Phoenix.PubSub.broadcast( - ObserverWeb.PubSub, - keys_topic(), - {:metrics_new_keys, reporter, new_keys} - ) - end + # credo:disable-for-lines:2 + current_data = + case :ets.lookup(metric_table, metric_key) do + [{_, current_list_data}] -> [data | current_list_data] + _ -> [data] + end - {:noreply, state} + :ets.insert(metric_table, {metric_key, current_data}) + + notify_new_metric_data(node, key, data) + end + end) + + nodes = nodes -- [node] + + {:noreply, %{state | nodes: nodes}} end - @impl true - def handle_info(:prune_expired_entries, state) do + def handle_info( + :prune_expired_entries, + %{node_metric_tables: tables, data_retention_period: data_retention_period} = state + ) do now_minutes = unix_to_minutes() - retention_period = trunc(data_retention_period() / @one_minute_in_milliseconds) + retention_period = trunc(data_retention_period / @one_minute_in_milliseconds) deletion_period_to = now_minutes - retention_period - 1 deletion_period_from = deletion_period_to - 2 - prune_keys = fn key -> + prune_keys = fn key, table -> Enum.each(deletion_period_from..deletion_period_to, fn timestamp -> - :ets.delete(@metric_table, metric_key(key, timestamp)) + :ets.delete(table, metric_key(key, timestamp)) end) end - Node.self() - |> get_keys_by_node() - |> Enum.each(&prune_keys.(&1)) + Enum.each(tables, fn {node, table} -> + node + |> get_keys_by_node() + |> Enum.each(&prune_keys.(&1, table)) + end) + + {:noreply, state} + end + + defp do_handle_metrics( + %{metrics: metrics, reporter: reporter, measurements: measurements}, + %{nodes: nodes, persist_data?: persist_data?, node_metric_tables: node_metric_tables} = + state + ) do + if reporter in nodes do + metric_table = Map.get(node_metric_tables, reporter) + now = System.os_time(:millisecond) + minute = unix_to_minutes(now) + + keys = get_keys_by_node(reporter) + + new_keys = + Enum.reduce(metrics, [], fn metric, acc -> + {key, timed_key, data} = build_telemetry_data(metric, measurements, now, minute) + + if persist_data?, do: ets_append_to_list(metric_table, timed_key, data) + + notify_new_metric_data(reporter, key, data) + + # credo:disable-for-lines:3 + if key in keys do + acc + else + [key | acc] + end + end) + + if new_keys != [] do + :ets.insert(metric_table, {@metric_keys, new_keys ++ keys}) + + Phoenix.PubSub.broadcast( + ObserverWeb.PubSub, + keys_topic(), + {:metrics_new_keys, reporter, new_keys} + ) + end + end {:noreply, state} end @@ -122,7 +226,15 @@ defmodule ObserverWeb.Telemetry.Storage do ### ========================================================================== @impl true def push_data(event) do - GenServer.cast(__MODULE__, {:observer_web_telemetry, event}) + msg = {:observer_web_telemetry, event} + + case cached_mode() do + :broadcast -> + Phoenix.PubSub.broadcast(ObserverWeb.PubSub, broadcast_topic(), msg) + + mode when mode in [:local, :observer] -> + GenServer.cast(__MODULE__, msg) + end end @impl true @@ -150,64 +262,160 @@ defmodule ObserverWeb.Telemetry.Storage do end def list_data_by_node_key(node, key, options) when is_atom(node) do - from = Keyword.get(options, :from, 15) - order = Keyword.get(options, :order, :asc) + case cached_mode() do + :broadcast -> + [] - now_minutes = unix_to_minutes() - from_minutes = now_minutes - from - - result = - Enum.reduce(from_minutes..now_minutes, [], fn minute, acc -> - case Rpc.call( - node, - :ets, - :lookup, - [@metric_table, metric_key(key, minute)], - :infinity - ) do - [{_, value}] -> - value ++ acc + mode when mode in [:local, :observer] -> + from = Keyword.get(options, :from, 15) + order = Keyword.get(options, :order, :asc) - _ -> - acc + now_minutes = unix_to_minutes() + from_minutes = now_minutes - from + + metric_table = metric_table(node) + + fetch_data = fn + :local, node, minute -> + Rpc.call(node, :ets, :lookup, [metric_table, metric_key(key, minute)], :infinity) + + :observer, _node, minute -> + if ets_table_exists?(metric_table) do + :ets.lookup(metric_table, metric_key(key, minute)) + else + [] + end end - end) - if order == :asc, do: Enum.reverse(result), else: result + # credo:disable-for-lines:3 + result = + Enum.reduce(from_minutes..now_minutes, [], fn minute, acc -> + case fetch_data.(mode, node, minute) do + [{_, value}] -> + value ++ acc + + _ -> + acc + end + end) + + if order == :asc, do: Enum.reverse(result), else: result + end end @impl true def get_keys_by_node(nil), do: [] def get_keys_by_node(node) do - case Rpc.call(node, :ets, :lookup, [@metric_table, @metric_keys], :infinity) do - [{_, value}] -> - value + case cached_mode() do + :broadcast -> + [] - # coveralls-ignore-start - _ -> + :local -> + case Rpc.call(node, :ets, :lookup, [metric_table(node), @metric_keys], :infinity) do + [{_, value}] -> + value + + # coveralls-ignore-start + _ -> + [] + # coveralls-ignore-stop + end + + :observer -> + node + |> metric_table() + |> ets_lookup_if_exist(@metric_keys, []) + end + end + + @impl true + def list_active_nodes do + case cached_mode() do + :broadcast -> [] - # coveralls-ignore-stop + + :local -> + [Node.self()] ++ Node.list() + + :observer -> + ets_lookup_if_exist(@storage_table, @registry_key, []) end end + @impl true + def cached_mode do + ets_lookup_if_exist(@storage_table, @mode_key, nil) + end + ### ========================================================================== ### Private functions ### ========================================================================== - if_not_test do - defp data_retention_period, - do: Application.get_env(:observer_web, ObserverWeb.Telemetry)[:data_retention_period] - else - defp data_retention_period, do: :timer.minutes(1) - end + defp metric_table(node), do: String.to_atom("#{node}::observer-web-metrics") defp metric_key(metric, timestamp), do: "#{metric}|#{timestamp}" defp unix_to_minutes(time \\ System.os_time(:millisecond)), do: trunc(time / @one_minute_in_milliseconds) + defp ets_table_exists?(table_name) do + case :ets.info(table_name) do + :undefined -> false + _info -> true + end + end + + defp ets_lookup_if_exist(table, key, default_return) do + with true <- ets_table_exists?(table), + [{_, value}] <- :ets.lookup(table, key) do + value + else + # coveralls-ignore-start + _ -> + default_return + # coveralls-ignore-stop + end + end + + defp ets_append_to_list(table, key, new_item) do + case :ets.lookup(table, key) do + [{^key, current_list_data}] -> + updated_list = [new_item | current_list_data] + :ets.insert(table, {key, updated_list}) + updated_list + + [] -> + # Key doesn't exist yet, create new list with just this item + :ets.insert(table, {key, [new_item]}) + [new_item] + end + end + + # NOTE: PubSub topics defp keys_topic, do: "metrics::keys" defp metrics_topic(node, key), do: "metrics::#{node}::#{key}" + defp broadcast_topic, do: "metrics::broadcast" + + defp notify_new_metric_data(reporter, key, data) do + Phoenix.PubSub.broadcast( + ObserverWeb.PubSub, + metrics_topic(reporter, key), + {:metrics_new_data, reporter, key, data} + ) + end + + defp create_update_metric_table(node, current_map) do + table = metric_table(node) + + # Create Metric table + :ets.new(table, [:set, :protected, :named_table]) + :ets.insert(table, {@metric_keys, []}) + + # Add node the to registry + ets_append_to_list(@storage_table, @registry_key, node) + + Map.put(current_map, node, table) + end defp build_telemetry_data(%{name: name} = metric, measurements, now, minute) do {name, metric_key(name, minute), diff --git a/lib/web/components/metrics/common.ex b/lib/web/components/metrics/common.ex new file mode 100644 index 0000000..75760e4 --- /dev/null +++ b/lib/web/components/metrics/common.ex @@ -0,0 +1,10 @@ +defmodule Observer.Web.Components.Metrics.Common do + @moduledoc false + + def timestamp_to_string(timestamp) do + timestamp + |> trunc() + |> DateTime.from_unix!(:millisecond) + |> DateTime.to_string() + end +end diff --git a/lib/web/components/metrics/phx_lv_socket.ex b/lib/web/components/metrics/phx_lv_socket.ex index 1676bdc..361bf93 100644 --- a/lib/web/components/metrics/phx_lv_socket.ex +++ b/lib/web/components/metrics/phx_lv_socket.ex @@ -4,6 +4,8 @@ defmodule Observer.Web.Components.Metrics.PhxLvSocket do use Phoenix.Component + alias Observer.Web.Components.Metrics.Common + attr :title, :string, required: true attr :service, :string, required: true attr :metric, :string, required: true @@ -56,18 +58,24 @@ defmodule Observer.Web.Components.Metrics.PhxLvSocket do } {series_data, categories_data} = - Enum.reduce(metrics, {empty_series_data, []}, fn metric, {series_data, categories_data} -> - timestamp = - metric.timestamp - |> trunc() - |> DateTime.from_unix!(:millisecond) - |> DateTime.to_string() + Enum.reduce(metrics, {empty_series_data, []}, fn + %ObserverWeb.Telemetry.Data{value: nil} = metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) + + {%{ + total: [nil] ++ series_data.total, + supervisors: [nil] ++ series_data.supervisors, + connected: [nil] ++ series_data.connected + }, [timestamp] ++ categories_data} + + metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) - {%{ - total: [metric.measurements.total] ++ series_data.total, - supervisors: [metric.measurements.supervisors] ++ series_data.supervisors, - connected: [metric.measurements.connected] ++ series_data.connected - }, [timestamp] ++ categories_data} + {%{ + total: [metric.measurements.total] ++ series_data.total, + supervisors: [metric.measurements.supervisors] ++ series_data.supervisors, + connected: [metric.measurements.connected] ++ series_data.connected + }, [timestamp] ++ categories_data} end) datasets = diff --git a/lib/web/components/metrics/vm_memory.ex b/lib/web/components/metrics/vm_memory.ex index 02e015a..a6ac6d2 100644 --- a/lib/web/components/metrics/vm_memory.ex +++ b/lib/web/components/metrics/vm_memory.ex @@ -4,6 +4,8 @@ defmodule Observer.Web.Components.Metrics.VmMemory do use Phoenix.Component + alias Observer.Web.Components.Metrics.Common + attr :title, :string, required: true attr :service, :string, required: true attr :metric, :string, required: true @@ -59,24 +61,36 @@ defmodule Observer.Web.Components.Metrics.VmMemory do # NOTE: Streams are retrieved in the reverse order {series_data, categories_data} = - Enum.reduce(metrics, {empty_series_data, []}, fn metric, {series_data, categories_data} -> - timestamp = - metric.timestamp - |> trunc() - |> DateTime.from_unix!(:millisecond) - |> DateTime.to_string() + Enum.reduce(metrics, {empty_series_data, []}, fn + %ObserverWeb.Telemetry.Data{value: nil} = metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) + + {%{ + atom: [nil] ++ series_data.atom, + atom_used: [nil] ++ series_data.atom_used, + binary: [nil] ++ series_data.binary, + code: [nil] ++ series_data.code, + ets: [nil] ++ series_data.ets, + processes: [nil] ++ series_data.processes, + processes_used: [nil] ++ series_data.processes_used, + system: [nil] ++ series_data.system, + total: [nil] ++ series_data.total + }, [timestamp] ++ categories_data} + + metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) - {%{ - atom: [metric.measurements.atom] ++ series_data.atom, - atom_used: [metric.measurements.atom_used] ++ series_data.atom_used, - binary: [metric.measurements.binary] ++ series_data.binary, - code: [metric.measurements.code] ++ series_data.code, - ets: [metric.measurements.ets] ++ series_data.ets, - processes: [metric.measurements.processes] ++ series_data.processes, - processes_used: [metric.measurements.processes_used] ++ series_data.processes_used, - system: [metric.measurements.system] ++ series_data.system, - total: [metric.measurements.total] ++ series_data.total - }, [timestamp] ++ categories_data} + {%{ + atom: [metric.measurements.atom] ++ series_data.atom, + atom_used: [metric.measurements.atom_used] ++ series_data.atom_used, + binary: [metric.measurements.binary] ++ series_data.binary, + code: [metric.measurements.code] ++ series_data.code, + ets: [metric.measurements.ets] ++ series_data.ets, + processes: [metric.measurements.processes] ++ series_data.processes, + processes_used: [metric.measurements.processes_used] ++ series_data.processes_used, + system: [metric.measurements.system] ++ series_data.system, + total: [metric.measurements.total] ++ series_data.total + }, [timestamp] ++ categories_data} end) datasets = diff --git a/lib/web/components/metrics/vm_run_queue.ex b/lib/web/components/metrics/vm_run_queue.ex index 725a04f..f41b532 100644 --- a/lib/web/components/metrics/vm_run_queue.ex +++ b/lib/web/components/metrics/vm_run_queue.ex @@ -4,6 +4,8 @@ defmodule Observer.Web.Components.Metrics.VmRunQueue do use Phoenix.Component + alias Observer.Web.Components.Metrics.Common + attr :title, :string, required: true attr :service, :string, required: true attr :metric, :string, required: true @@ -54,14 +56,16 @@ defmodule Observer.Web.Components.Metrics.VmRunQueue do # NOTE: Streams are retrieved in the reverse order defp normalize(metrics) do {series_data, categories_data} = - Enum.reduce(metrics, {[], []}, fn metric, {series_data, categories_data} -> - timestamp = - metric.timestamp - |> trunc() - |> DateTime.from_unix!(:millisecond) - |> DateTime.to_string() + Enum.reduce(metrics, {[], []}, fn + %ObserverWeb.Telemetry.Data{value: nil} = metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) + + {[nil] ++ series_data, [timestamp] ++ categories_data} + + metric, {series_data, categories_data} -> + timestamp = Common.timestamp_to_string(metric.timestamp) - {[metric.value] ++ series_data, [timestamp] ++ categories_data} + {[metric.value] ++ series_data, [timestamp] ++ categories_data} end) datasets = diff --git a/lib/web/helpers.ex b/lib/web/helpers.ex index ebd846a..ec09b18 100644 --- a/lib/web/helpers.ex +++ b/lib/web/helpers.ex @@ -42,6 +42,7 @@ defmodule Observer.Web.Helpers do ## Examples iex> alias Observer.Web.Helpers + ...> assert "/" = Helpers.observer_path(:root) ...> assert "/" = Helpers.observer_path(:root, :any) ...> assert_raise RuntimeError, ~r/nothing stored in the :routing key/, fn -> Helpers.observer_path(nil, ["path", "to", "resource"]) end ...> Process.put(:routing, :nowhere) diff --git a/lib/web/pages/metrics/page.ex b/lib/web/pages/metrics/page.ex index 445c98d..14f84a5 100644 --- a/lib/web/pages/metrics/page.ex +++ b/lib/web/pages/metrics/page.ex @@ -27,11 +27,24 @@ defmodule Observer.Web.Metrics.Page do attention_msg = "" + mode_color = + case assigns.mode do + :observer -> + "text-white bg-gradient-to-r from-teal-400 via-teal-500 to-teal-600 hover:bg-gradient-to-br focus:ring-4 focus:outline-none focus:ring-teal-300 dark:focus:ring-teal-800 shadow-lg shadow-teal-500/50 dark:shadow-lg dark:shadow-teal-800/80" + + :broadcast -> + "text-white bg-gradient-to-r from-pink-400 via-pink-500 to-pink-600 hover:bg-gradient-to-br focus:ring-4 focus:outline-none focus:ring-pink-300 dark:focus:ring-pink-800 shadow-lg shadow-pink-500/50 dark:shadow-lg dark:shadow-pink-800/80" + + _local_or_nil -> + "text-white bg-gradient-to-r from-cyan-400 via-cyan-500 to-cyan-600 hover:bg-gradient-to-br focus:ring-4 focus:outline-none focus:ring-cyan-300 dark:focus:ring-cyan-800 shadow-lg shadow-cyan-500/50 dark:shadow-lg dark:shadow-cyan-800/80" + end + assigns = assigns |> assign(unselected_services_keys: unselected_services_keys) |> assign(unselected_metrics_keys: unselected_metrics_keys) |> assign(attention_msg: attention_msg) + |> assign(mode_color: mode_color) ~H"""
@@ -60,6 +73,16 @@ defmodule Observer.Web.Metrics.Page do label="Start Time" options={["1m", "5m", "15m", "30m", "1h"]} /> + +
+ Mode + +
@@ -77,7 +100,6 @@ defmodule Observer.Web.Metrics.Page do ]} show_options={@show_metric_options} /> -
<%= for service <- @node_info.selected_services_keys do %> @@ -138,6 +160,7 @@ defmodule Observer.Web.Metrics.Page do |> assign(:metric_config, %{}) |> assign(form: to_form(default_form_options())) |> assign(:show_metric_options, false) + |> assign(:mode, Telemetry.cached_mode()) end def handle_mount(socket) do @@ -148,6 +171,7 @@ defmodule Observer.Web.Metrics.Page do |> assign(:metric_config, %{}) |> assign(form: to_form(default_form_options())) |> assign(:show_metric_options, false) + |> assign(:mode, nil) end @impl Page @@ -342,7 +366,7 @@ defmodule Observer.Web.Metrics.Page do {:noreply, assign(socket, :node_info, node_info)} end - def handle_info({:nodedown, node}, %{assigns: %{node_info: node_info}} = socket) do + def handle_info({:nodedown, node}, %{assigns: %{node_info: node_info, mode: :local}} = socket) do service_key = node |> to_string node_info = @@ -354,6 +378,12 @@ defmodule Observer.Web.Metrics.Page do {:noreply, assign(socket, :node_info, node_info)} end + def handle_info({:nodedown, _node}, socket) do + # NOTE: Do nothing, nodedown MUST NOT change the current + # socket information + {:noreply, socket} + end + defp data_key(service, metric), do: "#{service}::#{metric}" defp assign_metric_config( @@ -395,13 +425,12 @@ defmodule Observer.Web.Metrics.Page do selected_metrics_keys: selected_metrics_keys } - ([Node.self()] ++ Node.list()) - |> Enum.reduce(initial_map, fn target_node, - %{ - services_keys: services_keys, - metrics_keys: metrics_keys, - node: node - } = acc -> + Enum.reduce(Telemetry.list_active_nodes(), initial_map, fn target_node, + %{ + services_keys: services_keys, + metrics_keys: metrics_keys, + node: node + } = acc -> node_metrics_keys = Telemetry.get_keys_by_node(target_node) service = target_node |> to_string [name, _hostname] = String.split(service, "@") diff --git a/mix.exs b/mix.exs index 937f7a2..4211e4b 100644 --- a/mix.exs +++ b/mix.exs @@ -87,6 +87,15 @@ defmodule ObserverWeb.MixProject do ] end + defp copy_ex_doc(_) do + static_destination_path = "./doc/static" + File.mkdir_p!(static_destination_path) + + File.cp_r("./guides/static", static_destination_path, fn _source, _destination -> + true + end) + end + # Specifies your project dependencies. # # Type `mix help deps` for examples and options. @@ -133,6 +142,7 @@ defmodule ObserverWeb.MixProject do # See the documentation for `Mix` for more info on aliases. defp aliases do [ + docs: ["docs", ©_ex_doc/1], "assets.build": ["tailwind default", "esbuild default"], release: [ "assets.build", diff --git a/test/observer_web/apps/port_test.exs b/test/observer_web/apps/port_test.exs index cf7157d..10ab99a 100644 --- a/test/observer_web/apps/port_test.exs +++ b/test/observer_web/apps/port_test.exs @@ -3,15 +3,13 @@ defmodule ObserverWeb.Apps.PortTest do import Mox + alias Observer.Web.Mocks.RpcStubber alias ObserverWeb.Apps.Port, as: AppsPort setup :verify_on_exit! test "info/2" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() invalid_port = "#Port<0.1000>" diff --git a/test/observer_web/apps/process_test.exs b/test/observer_web/apps/process_test.exs index 0703e3a..5d2e327 100644 --- a/test/observer_web/apps/process_test.exs +++ b/test/observer_web/apps/process_test.exs @@ -3,13 +3,13 @@ defmodule ObserverWeb.Apps.ProcessTest do import Mox + alias Observer.Web.Mocks.RpcStubber alias ObserverWeb.Apps.Process, as: AppsPort setup :verify_on_exit! test "info/1" do - ObserverWeb.RpcMock - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) + RpcStubber.defaults() kernel_pid = :application_controller.get_master(:kernel) diff --git a/test/observer_web/apps_test.exs b/test/observer_web/apps_test.exs index da0f14e..8d9a596 100644 --- a/test/observer_web/apps_test.exs +++ b/test/observer_web/apps_test.exs @@ -2,26 +2,20 @@ defmodule ObserverWeb.AppsTest do use ExUnit.Case, async: true import Mox - alias ObserverWeb.Apps + alias Observer.Web.Mocks.RpcStubber + setup :verify_on_exit! test "list/0" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() assert Enum.find(Apps.list(), &(&1.name == :kernel)) end test "info/0" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) + RpcStubber.defaults() assert %Apps{id: _, name: _, children: _, symbol: _, lineStyle: _, itemStyle: _} = Apps.info() diff --git a/test/observer_web/phx_lv_socket_test.exs b/test/observer_web/phx_lv_socket_test.exs index 74465ef..c08921b 100644 --- a/test/observer_web/phx_lv_socket_test.exs +++ b/test/observer_web/phx_lv_socket_test.exs @@ -4,15 +4,13 @@ defmodule ObserverWeb.PhxLvSocket do import Mock import Mox - setup :verify_on_exit! - + alias Observer.Web.Mocks.RpcStubber alias ObserverWeb.Telemetry.Producer.PhxLvSocket + setup :verify_on_exit! + test "Check the phoenix liveview socket info is being published" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() with_mock :telemetry, execute: fn [:phoenix, :liveview, :socket, :"observer.web"], diff --git a/test/observer_web/telemetry/broadcast_storage_test.exs b/test/observer_web/telemetry/broadcast_storage_test.exs new file mode 100644 index 0000000..65809e2 --- /dev/null +++ b/test/observer_web/telemetry/broadcast_storage_test.exs @@ -0,0 +1,47 @@ +defmodule ObserverWeb.BroadcastStorageTest do + use ExUnit.Case, async: false + + alias ObserverWeb.Telemetry.Storage + alias ObserverWeb.TelemetryFixtures + + setup [:create_consumer] + + test "push_data/1", %{node: node} do + Phoenix.PubSub.subscribe(ObserverWeb.PubSub, "metrics::broadcast") + + node + |> TelemetryFixtures.build_reporter_vm_memory_total() + |> Storage.push_data() + + assert_receive {:observer_web_telemetry, %{reporter: ^node, metrics: _}}, 1_000 + end + + test "list_active_nodes/0" do + assert [] == Storage.list_active_nodes() + end + + test "Check broadcast doesn't store any data", %{node: node} do + key_name = "vm.memory.total" + + Phoenix.PubSub.subscribe(ObserverWeb.PubSub, "metrics::broadcast") + + node + |> TelemetryFixtures.build_reporter_vm_memory_total() + |> Storage.push_data() + + assert_receive {:observer_web_telemetry, _}, 1_000 + + assert [] == Storage.list_data_by_node_key(node |> to_string(), key_name) + assert [] == Storage.get_keys_by_node(node) + assert [] == Storage.list_active_nodes() + end + + defp create_consumer(context) do + node = Node.self() + {:ok, pid} = Storage.start_link(mode: :broadcast, data_retention_period: :timer.minutes(1)) + + context + |> Map.put(:node, node) + |> Map.put(:pid, pid) + end +end diff --git a/test/observer_web/telemetry_storage_test.exs b/test/observer_web/telemetry/local_storage_test.exs similarity index 82% rename from test/observer_web/telemetry_storage_test.exs rename to test/observer_web/telemetry/local_storage_test.exs index 15d90c2..116623b 100644 --- a/test/observer_web/telemetry_storage_test.exs +++ b/test/observer_web/telemetry/local_storage_test.exs @@ -1,4 +1,4 @@ -defmodule ObserverWeb.TelemetryStorageTest do +defmodule ObserverWeb.LocalStorageTest do use ExUnit.Case, async: false import Mock @@ -9,15 +9,15 @@ defmodule ObserverWeb.TelemetryStorageTest do setup [ :set_mox_global, - :verify_on_exit!, - :create_consumer + :create_consumer, + :metric_table ] - test "[un]subscribe_for_new_keys/0", %{node: node} do + test "[un]subscribe_for_new_keys/0", %{node: node, metric_table: metric_table} do Storage.subscribe_for_new_keys() ObserverWeb.RpcMock - |> stub(:call, fn ^node, :ets, :lookup, [:observer_web_metrics, "metric-keys"], :infinity -> + |> stub(:call, fn ^node, :ets, :lookup, [^metric_table, "metric-keys"], :infinity -> [{"metric-keys", []}] end) @@ -28,11 +28,11 @@ defmodule ObserverWeb.TelemetryStorageTest do assert_receive {:metrics_new_keys, ^node, ["vm.memory.total"]}, 1_000 end - test "[un]subscribe_for_new_data/0", %{node: node} do + test "[un]subscribe_for_new_data/0", %{node: node, metric_table: metric_table} do Storage.subscribe_for_new_data(node, "vm.memory.total") ObserverWeb.RpcMock - |> stub(:call, fn ^node, :ets, :lookup, [:observer_web_metrics, "metric-keys"], :infinity -> + |> stub(:call, fn ^node, :ets, :lookup, [^metric_table, "metric-keys"], :infinity -> [{"metric-keys", []}] end) @@ -48,12 +48,12 @@ defmodule ObserverWeb.TelemetryStorageTest do Storage.unsubscribe_for_new_data(node, "vm.memory.total") end - test "get_keys_by_node/1 valid node", %{node: node} do + test "get_keys_by_node/1 valid node", %{node: node, metric_table: metric_table} do Storage.subscribe_for_new_data(node, "vm.memory.total") test_pid = self() ObserverWeb.RpcMock - |> stub(:call, fn ^node, :ets, :lookup, [:observer_web_metrics, "metric-keys"], :infinity -> + |> stub(:call, fn ^node, :ets, :lookup, [^metric_table, "metric-keys"], :infinity -> if test_pid != self() do # GenServer Cast [{"metric-keys", []}] @@ -78,7 +78,11 @@ defmodule ObserverWeb.TelemetryStorageTest do assert [] == Storage.get_keys_by_node(nil) end - test "list_data_by_node_key/3", %{node: node} do + test "list_active_nodes/0", %{node: node} do + assert [^node] = Storage.list_active_nodes() + end + + test "list_data_by_node_key/3", %{node: node, metric_table: metric_table} do key_name = "test.phoenix" Storage.subscribe_for_new_data(node, key_name) @@ -87,7 +91,7 @@ defmodule ObserverWeb.TelemetryStorageTest do |> stub( :call, fn - ^node, :ets, :lookup, [:observer_web_metrics, "metric-keys"], :infinity -> + ^node, :ets, :lookup, [^metric_table, "metric-keys"], :infinity -> # First time: Empty keys # Second time: Added key called = Process.get("ets_lookup", 0) @@ -134,7 +138,7 @@ defmodule ObserverWeb.TelemetryStorageTest do ] = Storage.list_data_by_node_key(node |> to_string(), key_name) end - test "Pruning expiring entries", %{node: node, pid: pid} do + test "Pruning expiring entries", %{node: node, pid: pid, metric_table: metric_table} do key_name = "test.phoenix" now = System.os_time(:millisecond) @@ -145,7 +149,7 @@ defmodule ObserverWeb.TelemetryStorageTest do |> stub( :call, fn - ^node, :ets, :lookup, [:observer_web_metrics, "metric-keys"], :infinity -> + ^node, :ets, :lookup, [^metric_table, "metric-keys"], :infinity -> # First time: Empty keys # Second time: Added key called = Process.get("ets_lookup", 0) @@ -202,10 +206,15 @@ defmodule ObserverWeb.TelemetryStorageTest do defp create_consumer(context) do node = Node.self() - {:ok, pid} = Storage.start_link([]) + {:ok, pid} = Storage.start_link(mode: :local, data_retention_period: :timer.minutes(1)) context |> Map.put(:node, node) |> Map.put(:pid, pid) end + + defp metric_table(context) do + node = Node.self() + Map.put(context, :metric_table, String.to_atom("#{node}::observer-web-metrics")) + end end diff --git a/test/observer_web/telemetry/observer_storage_test.exs b/test/observer_web/telemetry/observer_storage_test.exs new file mode 100644 index 0000000..95c6883 --- /dev/null +++ b/test/observer_web/telemetry/observer_storage_test.exs @@ -0,0 +1,297 @@ +defmodule ObserverWeb.ObserverStorageTest do + use ExUnit.Case, async: false + + import Mock + import Mox + + alias ObserverWeb.Telemetry.Storage + alias ObserverWeb.TelemetryFixtures + + setup [ + :set_mox_global, + :create_consumer + ] + + test "[un]subscribe_for_new_keys/0", %{node: node} do + Storage.subscribe_for_new_keys() + + node + |> TelemetryFixtures.build_reporter_vm_memory_total() + |> Storage.push_data() + + assert_receive {:metrics_new_keys, ^node, ["vm.memory.total"]}, 1_000 + end + + test "[un]subscribe_for_new_data/0", %{node: node} do + Storage.subscribe_for_new_data(node, "vm.memory.total") + + node + |> TelemetryFixtures.build_reporter_vm_memory_total() + |> Storage.push_data() + + assert_receive {:metrics_new_data, ^node, "vm.memory.total", + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: _, measurements: _}}, + 1_000 + + # Validate by inspection + Storage.unsubscribe_for_new_data(node, "vm.memory.total") + end + + test "get_keys_by_node/1 valid node", %{node: node} do + Storage.subscribe_for_new_data(node, "vm.memory.total") + + node + |> TelemetryFixtures.build_reporter_vm_memory_total() + |> Storage.push_data() + + assert_receive {:metrics_new_data, ^node, "vm.memory.total", + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: _, measurements: _}}, + 1_000 + + assert ["vm.memory.total"] == Storage.get_keys_by_node(node) + end + + test "node/1 invalid node" do + assert [] == Storage.get_keys_by_node(nil) + end + + test "list_data_by_node_key/3 when data is received by observer - handle_cast", %{node: node} do + key_name = "test.phoenix" + + Storage.subscribe_for_new_data(node, key_name) + + Enum.each(1..5, &Storage.push_data(build_metric(node, key_name, &1))) + + assert_receive {:metrics_new_data, ^node, ^key_name, %{timestamp: _, unit: _, value: 5}}, + 1_000 + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :desc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name) + end + + test "list_data_by_node_key/3 when data is received by broadcast - handle_info" do + key_name = "test.phoenix" + node = :fake@nohost + + Storage.subscribe_for_new_data(node, key_name) + + send(ObserverWeb.Telemetry.Storage, {:nodeup, node}) + + Enum.each( + 1..5, + &Phoenix.PubSub.broadcast( + ObserverWeb.PubSub, + "metrics::broadcast", + {:observer_web_telemetry, build_metric(node, key_name, &1)} + ) + ) + + assert_receive {:metrics_new_data, ^node, ^key_name, %{timestamp: _, unit: _, value: 5}}, + 1_000 + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :desc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name) + end + + test "list_data_by_node_key/3 ignore when node is not previously up - handle_info" do + key_name = "test.phoenix" + node = :fake@nohost + + Storage.subscribe_for_new_data(node, key_name) + + Phoenix.PubSub.broadcast( + ObserverWeb.PubSub, + "metrics::broadcast", + {:observer_web_telemetry, build_metric(node, key_name, 1)} + ) + + assert [] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + end + + test "Add new node, check if data is stored, remove node and check node is available and data contains nil" do + key_name = "test.phoenix" + node = :fake@nohost + + Storage.subscribe_for_new_data(node, key_name) + + send(ObserverWeb.Telemetry.Storage, {:nodeup, node}) + + Enum.each(1..5, &Storage.push_data(build_metric(node, key_name, &1))) + + assert_receive {:metrics_new_data, ^node, ^key_name, %{timestamp: _, unit: _, value: 5}}, + 1_000 + + assert node in Storage.list_active_nodes() + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :desc) + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name) + + send(ObserverWeb.Telemetry.Storage, {:nodedown, node}) + + :timer.sleep(100) + + assert node in Storage.list_active_nodes() + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: nil, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name) + end + + test "Testing Node down within a minute that doesn't contain any data" do + key_name = "test.phoenix" + node = :fake@nohost + + now = System.os_time(:millisecond) + + Storage.subscribe_for_new_data(node, key_name) + + send(ObserverWeb.Telemetry.Storage, {:nodeup, node}) + + with_mock System, os_time: fn _ -> now - 120_000 end do + Storage.push_data(build_metric(node, key_name, 999)) + + assert_receive {:metrics_new_data, ^node, ^key_name, %{timestamp: _, unit: _, value: 999}}, + 1_000 + end + + assert node in Storage.list_active_nodes() + + assert [%ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 999, tags: _}] = + Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + + send(ObserverWeb.Telemetry.Storage, {:nodedown, node}) + + :timer.sleep(100) + + assert node in Storage.list_active_nodes() + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 999, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: nil, tags: _} + ] = + Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + end + + test "Pruning expiring entries", %{node: node, pid: pid} do + key_name = "test.phoenix" + + now = System.os_time(:millisecond) + + Storage.subscribe_for_new_data(node, key_name) + + with_mock System, os_time: fn _ -> now - 120_000 end do + Enum.each(1..5, &Storage.push_data(build_metric(node, key_name, &1))) + + assert_receive {:metrics_new_data, ^node, ^key_name, %{timestamp: _, unit: _, value: 5}}, + 1_000 + end + + assert [ + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 1, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 2, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 3, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 4, tags: _}, + %ObserverWeb.Telemetry.Data{timestamp: _, unit: _, value: 5, tags: _} + ] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + + send(pid, :prune_expired_entries) + + :timer.sleep(100) + + assert [] = Storage.list_data_by_node_key(node |> to_string(), key_name, order: :asc) + end + + defp build_metric(node, name, value) do + %{ + metrics: [ + %{ + name: name, + value: value, + unit: " millisecond", + info: "", + tags: %{status: 200, method: "GET"}, + type: "summary" + } + ], + reporter: node, + measurements: %{duration: 1_311_711} + } + end + + defp create_consumer(context) do + node = Node.self() + {:ok, pid} = Storage.start_link(mode: :observer, data_retention_period: :timer.minutes(1)) + + context + |> Map.put(:node, node) + |> Map.put(:pid, pid) + end +end diff --git a/test/observer_web/tracer_test.exs b/test/observer_web/tracer_test.exs index c9515ad..e31fbf7 100644 --- a/test/observer_web/tracer_test.exs +++ b/test/observer_web/tracer_test.exs @@ -3,26 +3,21 @@ defmodule ObserverWeb.TracerTest do import Mox + alias Observer.Web.Mocks.RpcStubber alias ObserverWeb.Tracer alias ObserverWeb.TracerFixtures setup :verify_on_exit! test "get_modules/1" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() assert list = Tracer.get_modules(Node.self()) assert Enum.member?(list, :kernel) end test "get_module_functions_info/2" do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() assert %{ functions: %{"config_change/3" => %{arity: 3, name: :config_change}}, diff --git a/test/observer_web/web/live/apps/page_test.exs b/test/observer_web/web/live/apps/page_test.exs index a8aaf7d..d6a1e5e 100644 --- a/test/observer_web/web/live/apps/page_test.exs +++ b/test/observer_web/web/live/apps/page_test.exs @@ -5,20 +5,17 @@ defmodule Observer.Web.Apps.PageLiveTest do import Mox import Mock + alias Observer.Web.Mocks.RpcStubber + alias Observer.Web.Mocks.TelemetryStubber + setup [ :set_mox_global, :verify_on_exit! ] test "GET /observer/applications", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, _index_live, html} = live(conn, "/observer/applications") @@ -26,14 +23,8 @@ defmodule Observer.Web.Apps.PageLiveTest do end test "Adjust Initial Tree Depth", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -56,14 +47,8 @@ defmodule Observer.Web.Apps.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -104,14 +89,8 @@ defmodule Observer.Web.Apps.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -162,8 +141,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -216,8 +194,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -314,8 +291,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -360,8 +336,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -412,8 +387,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -458,8 +432,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -501,14 +474,8 @@ defmodule Observer.Web.Apps.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") @@ -548,8 +515,7 @@ defmodule Observer.Web.Apps.PageLiveTest do :rpc.pinfo(pid, information) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/applications") diff --git a/test/observer_web/web/live/index.test.exs b/test/observer_web/web/live/index.test.exs index 07a0902..53ba753 100644 --- a/test/observer_web/web/live/index.test.exs +++ b/test/observer_web/web/live/index.test.exs @@ -3,24 +3,22 @@ defmodule Observer.Web.IndexLiveTest do import Mox + alias Observer.Web.Mocks.RpcStubber + alias Observer.Web.Mocks.TelemetryStubber + setup :verify_on_exit! test "forbidding mount using a resolver callback", %{conn: conn} do - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() assert {:error, {:redirect, redirect}} = live(conn, "/observer-limited") assert %{to: "/", flash: %{"error" => "Access forbidden"}} = redirect end test "Check iframe OFF allows root buttom", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, _index_live, html} = live(conn, "/observer/tracing") @@ -29,13 +27,9 @@ defmodule Observer.Web.IndexLiveTest do end test "Check iframe ON doesn't allow root buttom", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) + RpcStubber.defaults() - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, _index_live, html} = live(conn, "/observer/tracing?iframe=true") diff --git a/test/observer_web/web/live/metrics/page_test.exs b/test/observer_web/web/live/metrics/page_test.exs index 7261ea9..f5bc0d8 100644 --- a/test/observer_web/web/live/metrics/page_test.exs +++ b/test/observer_web/web/live/metrics/page_test.exs @@ -4,6 +4,7 @@ defmodule Observer.Web.Metrics.PageLiveTest do import Phoenix.LiveViewTest import Mox + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.TelemetryFixtures setup [ @@ -11,15 +12,43 @@ defmodule Observer.Web.Metrics.PageLiveTest do :verify_on_exit! ] - test "GET /metrics", %{conn: conn} do + test "GET /metrics - default :local mode", %{conn: conn} do + TelemetryStubber.defaults() + |> expect(:subscribe_for_new_keys, fn -> :ok end) + |> expect(:get_keys_by_node, fn _node -> [] end) + + {:ok, _index_live, html} = live(conn, "/observer/metrics") + + assert html =~ "Live Metrics" + assert html =~ "local" + end + + test "GET /metrics - :broadcast mode", %{conn: conn} do ObserverWeb.TelemetryMock + |> stub(:push_data, fn _event -> :ok end) + |> stub(:list_active_nodes, fn -> [Node.self()] ++ Node.list() end) + |> stub(:cached_mode, fn -> :broadcast end) |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:get_keys_by_node, fn _node -> [] end) + + {:ok, _index_live, html} = live(conn, "/observer/metrics") + + assert html =~ "Live Metrics" + assert html =~ "broadcast" + end + + test "GET /metrics :observer mode", %{conn: conn} do + ObserverWeb.TelemetryMock |> stub(:push_data, fn _event -> :ok end) + |> stub(:list_active_nodes, fn -> [Node.self()] ++ Node.list() end) + |> stub(:cached_mode, fn -> :observer end) + |> expect(:subscribe_for_new_keys, fn -> :ok end) + |> expect(:get_keys_by_node, fn _node -> [] end) {:ok, _index_live, html} = live(conn, "/observer/metrics") assert html =~ "Live Metrics" + assert html =~ "observer" end test "GET /metrics + new key", %{conn: conn} do @@ -27,7 +56,7 @@ defmodule Observer.Web.Metrics.PageLiveTest do metric = "fake.phoenix.metric" - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> send(test_pid_process, {:liveview_pid, self()}) :ok @@ -44,7 +73,6 @@ defmodule Observer.Web.Metrics.PageLiveTest do [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, html} = live(conn, "/observer/metrics") @@ -72,7 +100,7 @@ defmodule Observer.Web.Metrics.PageLiveTest do metric_id = String.replace(metric, ".", "-") test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> send(test_pid_process, {:liveview_pid, self()}) :ok @@ -86,7 +114,6 @@ defmodule Observer.Web.Metrics.PageLiveTest do ] end) |> stub(:get_keys_by_node, fn _node -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -140,13 +167,12 @@ defmodule Observer.Web.Metrics.PageLiveTest do service_id = String.replace(node, "@", "-") test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> send(test_pid_process, {:liveview_pid, self()}) :ok end) |> stub(:get_keys_by_node, fn _node -> [] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -168,18 +194,52 @@ defmodule Observer.Web.Metrics.PageLiveTest do refute render(liveview) =~ "services:#{node}" end - test "Testing NodeUp, no previous service is removed", %{conn: conn} do + test "Testing NodeDown in oberver mode MUST NOT affect selected services", %{conn: conn} do node = Node.self() |> to_string service_id = String.replace(node, "@", "-") test_pid_process = self() ObserverWeb.TelemetryMock + |> stub(:push_data, fn _event -> :ok end) + |> stub(:list_active_nodes, fn -> [Node.self()] ++ Node.list() end) + |> stub(:cached_mode, fn -> :observer end) + |> expect(:subscribe_for_new_keys, fn -> + send(test_pid_process, {:liveview_pid, self()}) + :ok + end) + |> stub(:get_keys_by_node, fn _node -> [] end) + + {:ok, liveview, _html} = live(conn, "/observer/metrics") + + liveview + |> element("#metrics-multi-select-toggle-options") + |> render_click() + + assert_receive {:liveview_pid, liveview_pid}, 1_000 + + html = + liveview + |> element("#metrics-multi-select-services-#{service_id}-add-item") + |> render_click() + + assert html =~ "services:#{node}" + + send(liveview_pid, {:nodedown, Node.self()}) + + assert render(liveview) =~ "services:#{node}" + end + + test "Testing NodeUp, no previous service is removed", %{conn: conn} do + node = Node.self() |> to_string + service_id = String.replace(node, "@", "-") + test_pid_process = self() + + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> send(test_pid_process, {:liveview_pid, self()}) :ok end) |> stub(:get_keys_by_node, fn _node -> [] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") diff --git a/test/observer_web/web/live/metrics/phoenix_test.exs b/test/observer_web/web/live/metrics/phoenix_test.exs index 1b99ef2..6123d99 100644 --- a/test/observer_web/web/live/metrics/phoenix_test.exs +++ b/test/observer_web/web/live/metrics/phoenix_test.exs @@ -4,6 +4,7 @@ defmodule Observer.Web.Metrics.PhoenixTest do import Phoenix.LiveViewTest import Mox + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.TelemetryFixtures setup [ @@ -28,13 +29,12 @@ defmodule Observer.Web.Metrics.PhoenixTest do metric = unquote(metric) metric_id = String.replace(metric, ".", "-") - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [] end) |> stub(:get_keys_by_node, fn _node -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -88,13 +88,12 @@ defmodule Observer.Web.Metrics.PhoenixTest do metric = unquote(metric) metric_id = String.replace(metric, ".", "-") - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [] end) |> stub(:get_keys_by_node, fn _node -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -158,7 +157,7 @@ defmodule Observer.Web.Metrics.PhoenixTest do test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> send(test_pid_process, {:liveview_pid, self()}) @@ -170,7 +169,6 @@ defmodule Observer.Web.Metrics.PhoenixTest do ] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -228,7 +226,7 @@ defmodule Observer.Web.Metrics.PhoenixTest do test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> send(test_pid_process, {:liveview_pid, self()}) @@ -240,7 +238,6 @@ defmodule Observer.Web.Metrics.PhoenixTest do ] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") diff --git a/test/observer_web/web/live/metrics/phx_lv_socket_test.exs b/test/observer_web/web/live/metrics/phx_lv_socket_test.exs index ce65a2e..eae02eb 100644 --- a/test/observer_web/web/live/metrics/phx_lv_socket_test.exs +++ b/test/observer_web/web/live/metrics/phx_lv_socket_test.exs @@ -4,6 +4,7 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do import Phoenix.LiveViewTest import Mox + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.TelemetryFixtures setup [ @@ -18,13 +19,12 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do metric_id = String.replace(metric, ".", "-") telemetry_data = TelemetryFixtures.build_telemetry_data_phx_lv_socket_total() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [telemetry_data] end) |> stub(:get_keys_by_node, fn _node -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -68,13 +68,12 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do metric_id = String.replace(metric, ".", "-") telemetry_data = TelemetryFixtures.build_telemetry_data_phx_lv_socket_total() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [telemetry_data] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -119,7 +118,7 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> send(test_pid_process, {:liveview_pid, self()}) @@ -131,7 +130,6 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do ] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -153,14 +151,24 @@ defmodule Observer.Web.Metrics.PhxLvSocketTest do # assert initial data assert html =~ "2025-01-27 12:52:59.123Z" + # assert live updated data send( liveview_pid, {:metrics_new_data, node, metric, TelemetryFixtures.build_telemetry_data_phx_lv_socket_total(1_737_982_379_456)} ) - # assert live updated data html = render(liveview) assert html =~ "2025-01-27 12:52:59.456Z" + + # Assert nil data received, this will indicate the application has restarted + send( + liveview_pid, + {:metrics_new_data, node, metric, + TelemetryFixtures.build_telemetry_data(1_737_982_379_789, nil)} + ) + + html = render(liveview) + assert html =~ "2025-01-27 12:52:59.789Z" end end diff --git a/test/observer_web/web/live/metrics/vm_memory_test.exs b/test/observer_web/web/live/metrics/vm_memory_test.exs index af028f5..a947274 100644 --- a/test/observer_web/web/live/metrics/vm_memory_test.exs +++ b/test/observer_web/web/live/metrics/vm_memory_test.exs @@ -4,6 +4,7 @@ defmodule Observer.Web.Metrics.VmMemoryTest do import Phoenix.LiveViewTest import Mox + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.TelemetryFixtures setup [ @@ -18,13 +19,12 @@ defmodule Observer.Web.Metrics.VmMemoryTest do metric_id = String.replace(metric, ".", "-") telemetry_data = TelemetryFixtures.build_telemetry_data_vm_total_memory() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [telemetry_data] end) |> stub(:get_keys_by_node, fn _node -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -68,13 +68,12 @@ defmodule Observer.Web.Metrics.VmMemoryTest do metric_id = String.replace(metric, ".", "-") telemetry_data = TelemetryFixtures.build_telemetry_data_vm_total_memory() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [telemetry_data] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -119,7 +118,7 @@ defmodule Observer.Web.Metrics.VmMemoryTest do test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> send(test_pid_process, {:liveview_pid, self()}) @@ -131,7 +130,6 @@ defmodule Observer.Web.Metrics.VmMemoryTest do ] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -153,14 +151,24 @@ defmodule Observer.Web.Metrics.VmMemoryTest do # assert initial data assert html =~ "2025-01-27 12:52:59.123Z" + # assert live updated data send( liveview_pid, {:metrics_new_data, node, metric, TelemetryFixtures.build_telemetry_data_vm_total_memory(1_737_982_379_456)} ) - # assert live updated data html = render(liveview) assert html =~ "2025-01-27 12:52:59.456Z" + + # Assert nil data received, this will indicate the application has restarted + send( + liveview_pid, + {:metrics_new_data, node, metric, + TelemetryFixtures.build_telemetry_data(1_737_982_379_789, nil)} + ) + + html = render(liveview) + assert html =~ "2025-01-27 12:52:59.789Z" end end diff --git a/test/observer_web/web/live/metrics/vm_run_queue_test.exs b/test/observer_web/web/live/metrics/vm_run_queue_test.exs index 5652f2e..8bd126d 100644 --- a/test/observer_web/web/live/metrics/vm_run_queue_test.exs +++ b/test/observer_web/web/live/metrics/vm_run_queue_test.exs @@ -4,6 +4,7 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do import Phoenix.LiveViewTest import Mox + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.TelemetryFixtures setup [ @@ -23,13 +24,12 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do metric = unquote(metric) metric_id = String.replace(metric, ".", "-") - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -79,13 +79,12 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do metric = unquote(metric) metric_id = String.replace(metric, ".", "-") - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:unsubscribe_for_new_data, fn ^node, ^metric -> :ok end) |> expect(:list_data_by_node_key, fn ^node, ^metric, _ -> [] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -140,7 +139,7 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do test_pid_process = self() - ObserverWeb.TelemetryMock + TelemetryStubber.defaults() |> expect(:subscribe_for_new_keys, fn -> :ok end) |> expect(:subscribe_for_new_data, fn ^node, ^metric -> send(test_pid_process, {:liveview_pid, self()}) @@ -152,7 +151,6 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do ] end) |> stub(:get_keys_by_node, fn _ -> [metric] end) - |> stub(:push_data, fn _event -> :ok end) {:ok, liveview, _html} = live(conn, "/observer/metrics") @@ -175,16 +173,26 @@ defmodule Observer.Web.Metrics.VmRunQueueTest do assert html =~ "2025-01-27 12:53:20.666" assert html =~ "#{init}" + # assert live updated data send( liveview_pid, {:metrics_new_data, node, metric, TelemetryFixtures.build_telemetry_data(1_737_982_379_777, update)} ) - # assert live updated data html = render(liveview) assert html =~ "2025-01-27 12:52:59.777Z" assert html =~ "#{update}" + + # Assert nil data received, this will indicate the application has restarted + send( + liveview_pid, + {:metrics_new_data, node, metric, + TelemetryFixtures.build_telemetry_data(1_737_982_379_999, nil)} + ) + + html = render(liveview) + assert html =~ "2025-01-27 12:52:59.999Z" end end) end diff --git a/test/observer_web/web/live/tracing/page_test.exs b/test/observer_web/web/live/tracing/page_test.exs index 44018f9..0e18c26 100644 --- a/test/observer_web/web/live/tracing/page_test.exs +++ b/test/observer_web/web/live/tracing/page_test.exs @@ -2,6 +2,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do use Observer.Web.ConnCase, async: false import Mox + alias Observer.Web.Mocks.RpcStubber + alias Observer.Web.Mocks.TelemetryStubber alias ObserverWeb.Tracer setup [ @@ -17,13 +19,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do end test "GET /tracing - fallback on the default", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, _index_live, html} = live(conn, "/observer") @@ -31,13 +28,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do end test "GET /tracing", %{conn: conn} do - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, _index_live, html} = live(conn, "/observer/tracing") @@ -48,13 +40,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -129,13 +116,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -166,13 +148,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -207,13 +184,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -258,13 +230,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -308,13 +275,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -355,13 +317,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -404,13 +361,8 @@ defmodule Observer.Web.Tracing.PageLiveTest do node = Node.self() |> to_string service = String.replace(node, "@", "-") - ObserverWeb.RpcMock - |> stub(:call, fn node, module, function, args, timeout -> - :rpc.call(node, module, function, args, timeout) - end) - - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + RpcStubber.defaults() + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") @@ -466,8 +418,7 @@ defmodule Observer.Web.Tracing.PageLiveTest do :rpc.call(node, module, function, args, timeout) end) - ObserverWeb.TelemetryMock - |> stub(:push_data, fn _event -> :ok end) + TelemetryStubber.defaults() {:ok, index_live, _html} = live(conn, "/observer/tracing") diff --git a/test/support/mocks/rpc_stubber.ex b/test/support/mocks/rpc_stubber.ex new file mode 100644 index 0000000..e1dde17 --- /dev/null +++ b/test/support/mocks/rpc_stubber.ex @@ -0,0 +1,13 @@ +defmodule Observer.Web.Mocks.RpcStubber do + @moduledoc false + + import Mox + + def defaults do + ObserverWeb.RpcMock + |> stub(:call, fn node, module, function, args, timeout -> + :rpc.call(node, module, function, args, timeout) + end) + |> stub(:pinfo, fn pid, information -> :rpc.pinfo(pid, information) end) + end +end diff --git a/test/support/mocks/telemetry_stubber.ex b/test/support/mocks/telemetry_stubber.ex new file mode 100644 index 0000000..0c6afc6 --- /dev/null +++ b/test/support/mocks/telemetry_stubber.ex @@ -0,0 +1,12 @@ +defmodule Observer.Web.Mocks.TelemetryStubber do + @moduledoc false + + import Mox + + def defaults do + ObserverWeb.TelemetryMock + |> stub(:push_data, fn _event -> :ok end) + |> stub(:list_active_nodes, fn -> [Node.self()] ++ Node.list() end) + |> stub(:cached_mode, fn -> :local end) + end +end