Add Machine Learning section to docs & example page to demo app (#787)

sd2k · web-flow · commit af902ec63a7c · 2024-07-11T14:46:42.000+01:00
diff --git a/docusaurus/docs/scenes-ml/advanced-callbacks.md b/docusaurus/docs/scenes-ml/advanced-callbacks.md
@@ -0,0 +1,16 @@
+---
+id: advanced-callbacks
+title: Using callbacks
+---
+
+Many of the components in `scenes-ml` allow a callback to be passed in the constructor. For example, in `SceneOutlierDetector`:
+
+```ts
+const outlierDetector = new SceneOutlierDetector({
+  onOutlierDetected: (outlier: Outlier) => {},
+});
+```
+
+This callback can be used to create more customised experiences specific to your Scene.
+
+For example, you may have a custom scene showing all pods for a given Kubernetes deployment. By enabling the outlier detector, you could use the callback to store all pods and timestamps which appear to be behaving differently, and render a second panel to show the logs for those pods and timestamps.
diff --git a/docusaurus/docs/scenes-ml/baselines-and-forecasts.md b/docusaurus/docs/scenes-ml/baselines-and-forecasts.md
@@ -0,0 +1,55 @@
+---
+id: baselines-and-forecasts
+title: Baselines and forecasts
+---
+
+import DiscoverSeasonalitiesSvg from '/static/img/discover-seasonalities.svg';
+import PinnedSvg from '/static/img/pinned.svg';
+
+**Baselines** provide smoothed estimates of time series data along with lower and upper bounds for the data over time. They can also be used as **forecasts**, using historical data to predict the behaviour of future data.
+
+This functionality can also be used for **anomaly detection** by highlighting timestamps where the true value strayed outside of the lower and upper predicted bounds.
+
+Baselines can be added to panels by using the `SceneBaseliner` component of `scenes-ml`, which will add a control to enable/disable the calculation, adjust the prediction intervals, discover seasonalities, and pin the results.
+
+![Panel with baselines added](/img/baseliner.png)
+
+## Usage
+
+The code example below demonstrates how to add baselines to a time series panel.
+
+```ts
+import { SceneBaseliner } from '@grafana/scenes-ml';
+
+// Default values are shown here, all are optional.
+const baseliner = new SceneBaseliner({
+  interval: 0.95,
+  discoverSeasonalities: false,
+  pinned: false,
+});
+const panel = PanelBuilders.timeseries()
+  .setHeaderActions([baseliner])
+  .build();
+```
+
+:::note
+Make sure you only add baselines to **time series** panels, as they rarely make sense for other panel types.
+:::
+
+### Pinning results
+
+By default, baselines are recalculated on every state change, i.e. whenever the time range, query or interval changes. This isn't always desirable: for example, the user may want to zoom out and view the current forecasts in a future time range.
+
+Enabling the **pinned <PinnedSvg className="ml-icon" />** setting will freeze the current results, so they won't be recalculated as the time range or other settings are changed.
+
+## Technical details
+
+`scenes-ml` uses the [MSTL][mstl] algorithm to produce in-sample and out-of-sample forecasts. This algorithm decomposes the data into **trend**, **seasonality** and **residuals**, then uses an an [ETS][ets] algorithm to model the trend series.
+
+By default, the algorithm assumes **hourly**, **daily**, **weekly** and **yearly** seasonality (if the data spans at least two of the given season length, i.e. at least two hours for hourly or at least two days for daily).
+
+If the **discover seasonalities <DiscoverSeasonalitiesSvg className="ml-icon"/>** setting is enabled, the baseliner will first attempt to detect any non-standard seasonality in the data using a [periodogram] and account for these seasonalities when modeling the data.
+
+[mstl]: https://arxiv.org/abs/2107.13462
+[ets]: https://otexts.com/fpp3/ets-forecasting.html
+[periodogram]: https://www.sktime.net/en/latest/api_reference/auto_generated/sktime.param_est.seasonality.SeasonalityPeriodogram.html
diff --git a/docusaurus/docs/scenes-ml/changepoint-detection.md b/docusaurus/docs/scenes-ml/changepoint-detection.md
@@ -0,0 +1,50 @@
+---
+id: changepoint-detection
+title: Changepoint detection
+---
+
+import PinnedSvg from '/static/img/pinned.svg';
+
+**Changepoint detection** attempts to identify timestamps where a time series has changed behaviour. For example, it could be used to identify sudden changes in the **magnitude** or the **variance** of a time series.
+
+The `SceneChangepointDetector` component from `scenes-ml` can be used to add this functionality to all series in a panel. This component will add an annotation at every detected changepoint.
+
+![Panel with changepoints added](/img/changepoints.png)
+
+:::warning
+Changepoint detection is currently a beta feature. The underlying algorithm may perform slowly for certain panels, so be sure to test it thoroughly before using it.
+:::
+
+## Usage
+
+The code example below demonstrates how to add changepoint detection to a time series panel.
+
+```ts
+import { SceneChangepointDetector } from '@grafana/scenes-ml';
+
+// Default values are shown here, all are optional.
+const changepointDetector = new SceneChangepointDetector({
+  enabled: false,
+  pinned: false,
+  onChangepointDetected: (changepoint: Changepoint) => {},
+});
+const panel = PanelBuilders.timeseries()
+  .setHeaderActions([outlierDetector])
+  .build();
+```
+
+:::note
+Make sure you only add changepoint detection to **time series** panels, as it rarely makes sense for other panel types.
+:::
+
+### Pinning results
+
+By default, baselines are recalculated on every state change, i.e. whenever the time range, query or interval changes. This isn't always desirable: for example, the user may want to zoom out and view the current forecasts in a future time range.
+
+Enabling the **pinned <PinnedSvg className="ml-icon" />** setting will freeze the current results, so they won't be recalculated as the time range or other settings are changed.
+
+## Technical details
+
+`scenes-ml` currently uses the [AutoRegressive Gaussian Process Change Point detection][argpcp] (ARGPCP) algorithm, which can be slow in some cases. Alternative algorithms may be added in future.
+
+[argpcp]: https://redpoll.ai/blog/changepoint/#autoregressive-gaussian-process-change-point-detector-argpcp
diff --git a/docusaurus/docs/scenes-ml/getting-started.md b/docusaurus/docs/scenes-ml/getting-started.md
@@ -0,0 +1,85 @@
+---
+id: getting-started
+title: Getting started
+---
+
+`@grafana/scenes-ml` is a separate npm package which lets you add Machine Learning powered functionality to your scenes.
+
+This topic explains hows to install Scenes ML and use it within a Grafana App plugin.
+
+## Installation
+
+If you're adding Scenes ML to an existing Scenes app plugin, first make sure your plugin config is up-to-date by running:
+
+```bash
+npx @grafana/create-plugin@latest --update
+```
+
+Then add `@grafana/scenes-ml` to your plugin by running the following commands in your project:
+
+```bash
+yarn add @grafana/scenes-ml
+```
+
+## Add ML features to a scene
+
+### 1. Create a scene
+
+Create a scene using the snippet below. This will add a time series panel to the scene with built-in controls to add trend, lower and upper bounds to all series in the panel.
+
+```ts
+// helloMLScene.ts
+
+import { EmbeddedScene, SceneFlexLayout, SceneFlexItem, SceneQueryRunner, PanelBuilders, sceneUtils } from '@grafana/scenes';
+import { SceneBaseliner, MLDemoDS } from '@grafana/scenes-ml';
+
+// Register the demo datasource from `scenes-ml`.
+// This isn't required for normal usage, it just gives us some sensible demo data.
+sceneUtils.registerRuntimeDataSource({ dataSource: new MLDemoDS('ml-test', 'ml-test') })
+
+function getForecastQueryRunner() {
+  return new SceneQueryRunner({
+    queries: [
+      { refId: 'A', datasource: { uid: 'ml-test', type: 'ml-test', }, type: 'forecasts' },
+    ],
+  });
+}
+
+export function getScene() {
+  return new EmbeddedScene({
+    body: new SceneFlexLayout({
+      children: [
+        new SceneFlexItem({
+          width: '50%',
+          height: 300,
+          body: PanelBuilders.timeseries()
+            .setTitle('Forecast demo')
+            .setData(getForecastQueryRunner())
+            // Add the `SceneBaseliner` to the panel.
+            .setHeaderActions([new SceneBaseliner({ interval: 0.95 })])
+            .build()
+        }),
+      ],
+    }),
+  });
+}
+```
+
+### 2. Render the scene
+
+Use the following code in your Grafana app plugin page to render the "Hello ML" scene:
+
+```tsx
+import React from 'react';
+import { getScene } from './helloMLScene';
+
+export const HelloMLPluginPage = () => {
+  const scene = getScene();
+
+  return <scene.Component model={scene} />;
+};
+```
+
+## Source code
+
+[View the example source code](https://github.com/grafana/scenes/tree/main/docusaurus/docs/scenes-ml/getting-started.tsx)
diff --git a/docusaurus/docs/scenes-ml/getting-started.tsx b/docusaurus/docs/scenes-ml/getting-started.tsx
@@ -0,0 +1,34 @@
+import { EmbeddedScene, SceneFlexLayout, SceneFlexItem, SceneQueryRunner, PanelBuilders, sceneUtils } from '@grafana/scenes';
+import { SceneBaseliner, MLDemoDS } from '@grafana/scenes-ml';
+
+// Register the demo datasource from `scenes-ml`.
+// This isn't required for normal usage, it just gives us some sensible demo data.
+sceneUtils.registerRuntimeDataSource({ dataSource: new MLDemoDS('ml-test', 'ml-test') })
+
+function getForecastQueryRunner() {
+  return new SceneQueryRunner({
+    queries: [
+      { refId: 'A', datasource: { uid: 'ml-test', type: 'ml-test', }, type: 'forecasts' },
+    ],
+  });
+}
+
+export function getScene() {
+  return new EmbeddedScene({
+    body: new SceneFlexLayout({
+      children: [
+        new SceneFlexItem({
+          width: '50%',
+          height: 300,
+          body: PanelBuilders.timeseries()
+            .setTitle('Forecast demo')
+            .setData(getForecastQueryRunner())
+            // Add the `SceneBaseliner` to the panel.
+            .setHeaderActions([new SceneBaseliner({ interval: 0.95 })])
+            .build()
+        }),
+      ],
+    }),
+  });
+}
+
diff --git a/docusaurus/docs/scenes-ml/outlier-detection.md b/docusaurus/docs/scenes-ml/outlier-detection.md
@@ -0,0 +1,47 @@
+---
+id: outlier-detection
+title: Outlier detection
+---
+
+import PinnedSvg from '/static/img/pinned.svg';
+
+**Outlier detection** is the problem of identifying when one or more series within a group is behaving differently to the rest.
+
+`scenes-ml` provides a `SceneOutlierDetector` component which will perform outlier detection and highlight any misbehaving series. It will also add a grey band indicating the 'cluster range' (the range of data that can be considered non-outlying), and (optionally) add annotations at time ranges where an outlier was detected.
+
+![Panel with outliers added](/img/outliers.png)
+
+## Usage
+
+The code example below demonstrates how to add outlier detection to a time series panel.
+
+```ts
+import { SceneOutlierDetector } from '@grafana/scenes-ml';
+
+// Default values are shown here, all are optional.
+const outlierDetector = new SceneOutlierDetector({
+  sensitivity: 0.5,
+  addAnnotations: false,
+  pinned: false,
+  onOutlierDetected: (outlier: Outlier) => {},
+});
+const panel = PanelBuilders.timeseries()
+  .setHeaderActions([outlierDetector])
+  .build();
+```
+
+:::note
+Make sure you only add outlier detection to **time series** panels, as it rarely makes sense for other panel types.
+:::
+
+### Pinning results
+
+By default, baselines are recalculated on every state change, i.e. whenever the time range, query or interval changes. This isn't always desirable: for example, the user may want to zoom out and view the current forecasts in a future time range.
+
+Enabling the **pinned <PinnedSvg className="ml-icon" />** setting will freeze the current results, so they won't be recalculated as the time range or other settings are changed.
+
+## Technical details
+
+`scenes-ml` currently uses a variant of the [DBSCAN][dbscan] algorithm to detect outliers. Additional algorithms may be added in future.
+
+[dbscan]: https://en.wikipedia.org/wiki/DBSCAN
diff --git a/docusaurus/website/sidebars.js b/docusaurus/website/sidebars.js
@@ -60,6 +60,18 @@ const sidebars = {
         'advanced-time-range-comparison',
       ],
     },
+    {
+      type: 'category',
+      label: '@grafana/scenes-ml',
+      collapsible: true,
+      collapsed: false,
+      items: [
+        'getting-started',
+        'baselines-and-forecasts',
+        'outlier-detection',
+        'changepoint-detection',
+      ].map(id => `scenes-ml/${id}`),
+    }
   ],
 };
 module.exports = sidebars;
diff --git a/docusaurus/website/src/css/custom.css b/docusaurus/website/src/css/custom.css
@@ -219,9 +219,7 @@ html[data-theme='dark'] .algolia-autocomplete .ds-with-1.ds-dropdown-menu:before
   color: #63676d;
 }
 
-.algolia-autocomplete.algolia-autocomplete
-  .algolia-docsearch-suggestion--text
-  .algolia-docsearch-suggestion--highlight {
+.algolia-autocomplete.algolia-autocomplete .algolia-docsearch-suggestion--text .algolia-docsearch-suggestion--highlight {
   box-shadow: inset 0 -2px 0 0 var(--primary-border);
 }
 
@@ -237,3 +235,9 @@ html[data-theme='dark'] .algolia-autocomplete .ds-with-1.ds-dropdown-menu:before
 .algolia-autocomplete.algolia-autocomplete .algolia-docsearch-suggestion--content:before {
   background: var(--ifm-toc-border-color);
 }
+
+/* icons */
+.ml-icon {
+  height: 16px;
+  fill: #FFFFFF;
+}
diff --git a/docusaurus/website/static/img/baseliner.png b/docusaurus/website/static/img/baseliner.png
diff --git a/docusaurus/website/static/img/changepoints.png b/docusaurus/website/static/img/changepoints.png
diff --git a/docusaurus/website/static/img/discover-seasonalities.svg b/docusaurus/website/static/img/discover-seasonalities.svg
@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 28.34 20">
+    <circle cx="14.17" cy="2.67" r="2.67" />
+    <circle cx="2.67" cy="17.33" r="2.67" />
+    <path d="M3.42,17.33H1.92c0-6.46,4.39-15.41,12.64-15.41v1.5C7.29,3.42,3.42,11.5,3.42,17.33Z" />
+    <circle cx="25.67" cy="17.33" r="2.67" />
+    <path d="M26.42,17.33h-1.5c0-5.83-3.87-13.91-11.14-13.91V1.92C22,1.92,26.42,10.87,26.42,17.33Z" />
+</svg>
diff --git a/docusaurus/website/static/img/outliers.png b/docusaurus/website/static/img/outliers.png
diff --git a/docusaurus/website/static/img/pinned.svg b/docusaurus/website/static/img/pinned.svg
@@ -0,0 +1,4 @@
+<svg viewBox="0 0 27 37"
+    xmlns="http://www.w3.org/2000/svg">
+    <path fill-rule="evenodd" clip-rule="evenodd" d="M23.7532 6.88866L21.4172 9.24594L15.7269 6.28202L16.3818 3.04909L23.7532 6.88866ZM14.9358 8.61917L9.97459 14.8326L17.9308 18.9768L20.2659 11.3955L14.9358 8.61917ZM22.8561 11.2556L20.1258 20.1201L20.9501 20.5495C22.7392 21.4814 23.4359 23.6906 22.5062 25.484L20.5985 29.164C20.4125 29.5226 19.9717 29.6622 19.6139 29.4758L11.0445 25.0122L5.07869 36.3536C4.76675 36.9466 4.03497 37.1725 3.44425 36.8583C2.85351 36.5441 2.62748 35.8086 2.93942 35.2155L8.89542 23.8929L0.393614 19.4645C0.035803 19.2781 -0.103532 18.8363 0.0824013 18.4776L1.99015 14.7976C2.91983 13.0043 5.12378 12.3059 6.91284 13.2378L7.77297 13.6858L13.1353 6.97009C13.1331 6.88005 13.1409 6.78876 13.1593 6.69769L14.3179 0.977899C14.4789 0.183404 15.3539 -0.23557 16.0716 0.138248L26.344 5.4889C27.0725 5.86832 27.2248 6.84706 26.6463 7.43087L22.8561 11.2556ZM5.79063 15.4025L6.2226 15.6275L8.4242 16.7743L17.2011 21.346L19.396 22.4892L19.8279 22.7142C20.4243 23.0248 20.6565 23.7612 20.3466 24.3591L19.2244 26.5237L3.02752 18.0872L4.14975 15.9225C4.45964 15.3247 5.19429 15.0919 5.79063 15.4025Z" />
+</svg>
diff --git a/packages/scenes-app/package.json b/packages/scenes-app/package.json
@@ -6,8 +6,8 @@
   "license": "AGPL-3.0-only",
   "description": "A basic grafana app plugin",
   "scripts": {
-    "build": "TS_NODE_PROJECT=\"./.config/webpack/tsconfig.webpack.json\" webpack -c ./.config/webpack/webpack.config.ts --env production",
-    "dev": "TS_NODE_PROJECT=\"./.config/webpack/tsconfig.webpack.json\" webpack -w -c ./.config/webpack/webpack.config.ts --env development",
+    "build": "TS_NODE_PROJECT=\"./.config/webpack/tsconfig.webpack.json\" webpack -c ./webpack.config.ts --env production",
+    "dev": "TS_NODE_PROJECT=\"./.config/webpack/tsconfig.webpack.json\" webpack -w -c ./webpack.config.ts --env development",
     "e2e": "yarn cypress install && TZ=UTC yarn grafana-e2e run",
     "test": "jest --passWithNoTests",
     "test:ci": "jest --maxWorkers 4",
@@ -63,6 +63,7 @@
     "@grafana/data": "^11.0.0",
     "@grafana/runtime": "^11.0.0",
     "@grafana/scenes": "workspace:*",
+    "@grafana/scenes-ml": "^0.2.0",
     "@grafana/scenes-react": "workspace:*",
     "@grafana/schema": "^11.0.0",
     "@grafana/ui": "^11.0.0",
diff --git a/packages/scenes-app/src/demos/index.ts b/packages/scenes-app/src/demos/index.ts
@@ -37,6 +37,7 @@ import { getQueryControllerDemo } from './queryController';
 import { getDynamicDataLayersDemo } from './dynamicDataLayers';
 import { getInteropDemo } from './interopDemo';
 import { getUrlSyncTest } from './urlSyncTest';
+import { getMlDemo } from './ml';
 
 export interface DemoDescriptor {
   title: string;
@@ -83,5 +84,6 @@ export function getDemos(): DemoDescriptor[] {
     { title: 'Query controller demo', getPage: getQueryControllerDemo },
     { title: 'Interop with hooks and context', getPage: getInteropDemo },
     { title: 'Url sync test', getPage: getUrlSyncTest },
+    { title: 'Machine Learning', getPage: getMlDemo },
   ].sort((a, b) => a.title.localeCompare(b.title));
 }
diff --git a/packages/scenes-app/src/demos/ml.tsx b/packages/scenes-app/src/demos/ml.tsx
diff --git a/packages/scenes-app/webpack.config.ts b/packages/scenes-app/webpack.config.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -219,9 +219,7 @@ html[data-theme='dark'] .algolia-autocomplete .ds-with-1.ds-dropdown-menu:before`
`219`	`219`	`color: #63676d;`
`220`	`220`	`}`
`221`	`221`
`222`		`-.algolia-autocomplete.algolia-autocomplete`
`223`		`- .algolia-docsearch-suggestion--text`
`224`		`- .algolia-docsearch-suggestion--highlight {`
	`222`	`+.algolia-autocomplete.algolia-autocomplete .algolia-docsearch-suggestion--text .algolia-docsearch-suggestion--highlight {`
`225`	`223`	`box-shadow: inset 0 -2px 0 0 var(--primary-border);`
`226`	`224`	`}`
`227`	`225`
`@@ -237,3 +235,9 @@ html[data-theme='dark'] .algolia-autocomplete .ds-with-1.ds-dropdown-menu:before`
`237`	`235`	`.algolia-autocomplete.algolia-autocomplete .algolia-docsearch-suggestion--content:before {`
`238`	`236`	`background: var(--ifm-toc-border-color);`
`239`	`237`	`}`
	`238`	`+`
	`239`	`+/* icons */`
	`240`	`+.ml-icon {`
	`241`	`+ height: 16px;`
	`242`	`+ fill: #FFFFFF;`
	`243`	`+}`