Variables: Notify scene after each variable completion or value change (

#525)

docusaurus/docs/advanced-variables.md

@@ -209,6 +209,23 @@ function registerMacro() {
 }
 ```
 
+### Waiting for variables
+
+When you have state logic that depends on variables you can check if all your variable dependencies are ready (non loading state) with `sceneGraph.hasVariableDependencyInLoadingState`. This will return true if any dependency is in a loading state, includes checks of the complete dependency chain.
+
+For objects that subscribe to both time & variable we recommend using `VariableDependencyConfig` and it's `onVariableUpdateCompleted` callback and `hasDependencyInLoadingState` function. Since variables can also react and change based on time and to avoid double reactions the `VariableDependencyConfig` has internal state to remember that a scene object is waiting for variables. To leverage this specify the `onVariableUpdateCompleted` callback. This callback is called whenever a dependency changes value or if the scene object is waiting for variables, when a variable update process is completed.
+
+Example setup:
+
+Variables: A, B, C (B depends on A, C depends on B). A depends on time range so when ever time range change it will load new values which could result in a new value (which would then cause B and C to also update).
+
+SceneQueryRunner with a query that depends on variable C
+
+* 1. Time range changes value
+* 2. Variable A starts loading
+* 3. SceneQueryRunner responds to time range change tries to start new query, but before new query is issued calls `variableDependency.hasDependencyInLoadingState`. This checks if variable C is loading wich it is not, so then checks if variable B is loading (since it's a dependency of C), which it is not so then checks A, A is loading so it returns true and SceneQueryRunner will skip issuing a new query. When this happens the VariableDependencyConfig will set an internal flag that it is waiting for a variable dependency, this makes sure that the moment a next variable completes onVariableUpdateCompleted is called (no matter if the variable that was completed is a direct dependency or if it has changed value or not, we just care that it completed loading).
+* 4. Variable A completes loading. The options (possible values) are the same so no change value.
+* 5. SceneQueryRunner's VariableDependencyConfig receives the notification that variable A has completed it's loading phase, since it is in a waiting for variables state it will call the onVariableUpdateCompleted callback even though A is not a direct dependency and it has not changed value.
 
 ## Source code
 

packages/scenes-app/src/demos/nestedVariables.tsx

@@ -26,7 +26,15 @@ export function getNestedScenesAndVariablesDemo(defaults: SceneAppPageState) {
           query: 'A.*',
           value: 'server',
           text: '',
-          delayMs: 1000,
+          delayMs: 2000,
+          options: [],
+        }),
+        new TestVariable({
+          name: 'notUsed',
+          query: 'A.$A.*',
+          value: 'server',
+          text: '',
+          delayMs: 5000,
           options: [],
         }),
       ],
@@ -45,7 +53,7 @@ export function getNestedScenesAndVariablesDemo(defaults: SceneAppPageState) {
                   name: 'pod',
                   query: 'A.$server.*',
                   value: 'pod',
-                  delayMs: 1000,
+                  delayMs: 2000,
                   isMulti: true,
                   text: '',
                   options: [],
@@ -98,7 +106,7 @@ export function getInnerScene(title: string) {
           name: 'handler',
           query: 'A.$server.$pod.*',
           value: 'pod',
-          delayMs: 1000,
+          delayMs: 2000,
           isMulti: true,
           text: '',
           options: [],

packages/scenes/src/behaviors/ActWhenVariableChanged.ts

@@ -26,24 +26,20 @@ export class ActWhenVariableChanged extends SceneObjectBase<ActWhenVariableChang
 
   protected _variableDependency = new VariableDependencyConfig(this, {
     variableNames: [this.state.variableName],
-    onVariableUpdatesCompleted: this._onVariableChanged.bind(this),
+    onReferencedVariableValueChanged: this._onVariableChanged.bind(this),
   });
 
-  private _onVariableChanged(changedVariables: Set<SceneVariable>): void {
+  private _onVariableChanged(variable: SceneVariable): void {
     const effect = this.state.onChange;
 
-    for (const variable of changedVariables) {
-      if (this.state.variableName === variable.state.name) {
-        if (this._runningEffect) {
-          this._runningEffect();
-          this._runningEffect = null;
-        }
+    if (this._runningEffect) {
+      this._runningEffect();
+      this._runningEffect = null;
+    }
 
-        const cancellation = effect(variable, this);
-        if (cancellation) {
-          this._runningEffect = cancellation;
-        }
-      }
+    const cancellation = effect(variable, this);
+    if (cancellation) {
+      this._runningEffect = cancellation;
     }
   }
 }

packages/scenes/src/core/types.ts

@@ -216,9 +216,6 @@ export interface SceneDataLayerProviderState extends SceneObjectState {
   data?: PanelData;
   isEnabled?: boolean;
   isHidden?: boolean;
-
-  // Private runtime state
-  _isWaitingForVariables?: boolean;
 }
 
 export interface SceneDataLayerProvider extends SceneObject<SceneDataLayerProviderState> {

packages/scenes/src/querying/SceneQueryRunner.test.ts

@@ -8,6 +8,7 @@ import {
   LoadingState,
   PanelData,
   toDataFrame,
+  VariableRefresh,
 } from '@grafana/data';
 
 import { SceneTimeRange } from '../core/SceneTimeRange';
@@ -645,6 +646,94 @@ describe('SceneQueryRunner', () => {
       expect(runRequestMock.mock.calls.length).toBe(2);
     });
 
+    it('When depending on a variable that depends on a variable that depends on time range', async () => {
+      const varA = new TestVariable({
+        name: 'A',
+        value: 'AA',
+        query: 'A.*',
+        refresh: VariableRefresh.onTimeRangeChanged,
+      });
+
+      const varB = new TestVariable({ name: 'B', value: 'AA', query: 'A.$A.*' });
+      const queryRunner = new SceneQueryRunner({ queries: [{ refId: 'A', query: '$B' }] });
+      const timeRange = new SceneTimeRange();
+
+      const scene = new TestScene({
+        $variables: new SceneVariableSet({ variables: [varA, varB] }),
+        $timeRange: timeRange,
+        $data: queryRunner,
+      });
+
+      scene.activate();
+
+      varA.signalUpdateCompleted();
+      varB.signalUpdateCompleted();
+
+      await new Promise((r) => setTimeout(r, 1));
+
+      // Should run query
+      expect(runRequestMock.mock.calls.length).toBe(1);
+
+      // Now change time range
+      timeRange.onRefresh();
+
+      // Allow rxjs logic time run
+      await new Promise((r) => setTimeout(r, 1));
+
+      expect(varA.state.loading).toBe(true);
+
+      varA.signalUpdateCompleted();
+
+      await new Promise((r) => setTimeout(r, 1));
+
+      // Since varA did not change here varB should not be loading
+      expect(varB.state.loading).toBe(false);
+
+      // should execute new query
+      expect(runRequestMock.mock.calls.length).toBe(2);
+    });
+
+    it('Should not issue query when unrealted variable completes and _isWaitingForVariables is false', async () => {
+      const varA = new TestVariable({ name: 'A', value: 'AA', query: 'A.*' });
+      const varB = new TestVariable({ name: 'B', value: 'AA', query: 'A.$A.*' });
+
+      // Query only depends on A
+      const queryRunner = new SceneQueryRunner({
+        queries: [{ refId: 'A', query: '$A' }],
+        maxDataPointsFromWidth: true,
+      });
+
+      const scene = new TestScene({
+        $variables: new SceneVariableSet({ variables: [varA, varB] }),
+        $timeRange: new SceneTimeRange(),
+        $data: queryRunner,
+      });
+
+      scene.activate();
+
+      // should execute query when variable completes update
+      varA.signalUpdateCompleted();
+
+      await new Promise((r) => setTimeout(r, 1));
+
+      // still waiting for containerWidth
+      expect(runRequestMock.mock.calls.length).toBe(0);
+
+      queryRunner.setContainerWidth(1000);
+
+      await new Promise((r) => setTimeout(r, 10));
+
+      expect(runRequestMock.mock.calls.length).toBe(1);
+
+      // Variable that is not a dependency completes
+      varB.signalUpdateCompleted();
+
+      await new Promise((r) => setTimeout(r, 1));
+
+      // should not result in a new query
+      expect(runRequestMock.mock.calls.length).toBe(1);
+    });
+
     it('Should set data and loadingState to Done when there are no queries', async () => {
       const queryRunner = new SceneQueryRunner({
         queries: [],

packages/scenes/src/querying/SceneQueryRunner.ts

@@ -28,7 +28,6 @@ import {
 } from '../core/types';
 import { getDataSource } from '../utils/getDataSource';
 import { VariableDependencyConfig } from '../variables/VariableDependencyConfig';
-import { SceneVariable } from '../variables/types';
 import { writeSceneLog } from '../utils/writeSceneLog';
 import { VariableValueRecorder } from '../variables/VariableValueRecorder';
 import { emptyPanelData } from '../core/SceneDataNode';
@@ -57,7 +56,6 @@ export interface QueryRunnerState extends SceneObjectState {
   // Filters to be applied to data layer results before combining them with SQR results
   dataLayerFilter?: DataLayerFilter;
   // Private runtime state
-  _isWaitingForVariables?: boolean;
   _hasFetchedData?: boolean;
 }
 
@@ -85,8 +83,7 @@ export class SceneQueryRunner extends SceneObjectBase<QueryRunnerState> implemen
 
   protected _variableDependency: VariableDependencyConfig<QueryRunnerState> = new VariableDependencyConfig(this, {
     statePaths: ['queries', 'datasource'],
-    onVariableUpdatesCompleted: (variables, dependencyChanged) =>
-      this.onVariableUpdatesCompleted(variables, dependencyChanged),
+    onVariableUpdateCompleted: this.onVariableUpdatesCompleted.bind(this),
   });
 
   public constructor(initialState: QueryRunnerState) {
@@ -209,31 +206,23 @@ export class SceneQueryRunner extends SceneObjectBase<QueryRunnerState> implemen
   }
 
   /**
-   * Handles some tricky cases where we need to run queries even when they have not changed in case
-   * the query execution on activate was stopped due to VariableSet still not having processed all variables.
+   * This tries to start a new query whenever a variable completes or is changed.
+   *
+   * We care about variable update completions even when the variable has not changed and even when it is not a direct dependency.
+   * Example: Variables A and B (B depends on A). A update depends on time range. So when time change query runner will
+   * find that variable A is loading which is a dependency on of variable B so will set _isWaitingForVariables to true and
+   * not issue any query.
+   *
+   * When A completes it's loading (with no value change, so B never updates) it will cause a call of this function letting
+   * the query runner know that A has completed, and in case _isWaitingForVariables we try to run the query. The query will
+   * only run if all variables are in a non loading state so in other scenarios where a query depends on many variables this will
+   * be called many times until all dependencies are in a non loading state.   *
    */
-  private onVariableUpdatesCompleted(_variablesThatHaveChanged: Set<SceneVariable>, dependencyChanged: boolean) {
-    // If no maxDataPoints specified we might need to wait for container width to be set from the outside
-    if (!this.state.maxDataPoints && this.state.maxDataPointsFromWidth && !this._containerWidth) {
-      return;
-    }
-
-    if (this.state._isWaitingForVariables) {
-      this.runQueries();
-      return;
-    }
-
-    if (dependencyChanged) {
-      this.runQueries();
-    }
+  private onVariableUpdatesCompleted() {
+    this.runQueries();
   }
 
   private shouldRunQueriesOnActivate() {
-    // If no maxDataPoints specified we might need to wait for container width to be set from the outside
-    if (!this.state.maxDataPoints && this.state.maxDataPointsFromWidth && !this._containerWidth) {
-      return false;
-    }
-
     if (this._variableValueRecorder.hasDependenciesChanged(this)) {
       writeSceneLog(
         'SceneQueryRunner',
@@ -342,28 +331,25 @@ export class SceneQueryRunner extends SceneObjectBase<QueryRunnerState> implemen
   }
 
   private async runWithTimeRange(timeRange: SceneTimeRangeLike) {
+    // If no maxDataPoints specified we might need to wait for container width to be set from the outside
+    if (!this.state.maxDataPoints && this.state.maxDataPointsFromWidth && !this._containerWidth) {
+      return;
+    }
+
     // If data layers subscription doesn't exist, create one
-    //
     if (!this._dataLayersSub) {
       this._handleDataLayers();
     }
 
-    const runRequest = getRunRequest();
     // Cancel any running queries
     this._querySub?.unsubscribe();
 
     // Skip executing queries if variable dependency is in loading state
-    if (sceneGraph.hasVariableDependencyInLoadingState(this)) {
+    if (this._variableDependency.hasDependencyInLoadingState()) {
       writeSceneLog('SceneQueryRunner', 'Variable dependency is in loading state, skipping query execution');
-      this.setState({ _isWaitingForVariables: true });
       return;
     }
 
-    // If we were waiting for variables, clear that flag
-    if (this.state._isWaitingForVariables) {
-      this.setState({ _isWaitingForVariables: false });
-    }
-
     const { queries } = this.state;
 
     // Simple path when no queries exist
@@ -380,6 +366,7 @@ export class SceneQueryRunner extends SceneObjectBase<QueryRunnerState> implemen
         this.findAndSubscribeToAdhocFilters(datasource?.uid);
       }
 
+      const runRequest = getRunRequest();
       const [request, secondaryRequest] = this.prepareRequests(timeRange, ds);
 
       writeSceneLog('SceneQueryRunner', 'Starting runRequest', this.state.key);

packages/scenes/src/querying/layers/SceneDataLayerBase.ts

@@ -10,7 +10,6 @@ import {
 } from '../../core/types';
 import { setBaseClassState } from '../../utils/utils';
 import { writeSceneLog } from '../../utils/writeSceneLog';
-import { SceneVariable } from '../../variables/types';
 import { VariableDependencyConfig } from '../../variables/VariableDependencyConfig';
 import { VariableValueRecorder } from '../../variables/VariableValueRecorder';
 
@@ -56,8 +55,7 @@ export abstract class SceneDataLayerBase<T extends SceneDataLayerProviderState>
   private _variableValueRecorder = new VariableValueRecorder();
 
   protected _variableDependency: VariableDependencyConfig<T> = new VariableDependencyConfig(this, {
-    onVariableUpdatesCompleted: (variables, dependencyChanged) =>
-      this.onVariableUpdatesCompleted(variables, dependencyChanged),
+    onVariableUpdateCompleted: this.onVariableUpdateCompleted.bind(this),
   });
 
   /**
@@ -124,16 +122,8 @@ export abstract class SceneDataLayerBase<T extends SceneDataLayerProviderState>
     this._variableValueRecorder.recordCurrentDependencyValuesForSceneObject(this);
   }
 
-  protected onVariableUpdatesCompleted(variables: Set<SceneVariable>, dependencyChanged: boolean): void {
-    writeSceneLog('SceneDataLayerBase', 'onVariableUpdatesCompleted');
-    if (this.state._isWaitingForVariables && this.shouldRunLayerOnActivate()) {
-      this.runLayer();
-      return;
-    }
-
-    if (dependencyChanged) {
-      this.runLayer();
-    }
+  protected onVariableUpdateCompleted(): void {
+    this.runLayer();
   }
 
   public cancelQuery() {

packages/scenes/src/querying/layers/annotations/AnnotationsDataLayer.ts

@@ -57,9 +57,8 @@ export class AnnotationsDataLayer
       this.querySub.unsubscribe();
     }
 
-    if (sceneGraph.hasVariableDependencyInLoadingState(this)) {
+    if (this._variableDependency.hasDependencyInLoadingState()) {
       writeSceneLog('AnnotationsDataLayer', 'Variable dependency is in loading state, skipping query execution');
-      this.setState({ _isWaitingForVariables: true });
       return;
     }