Implement filtering for imported data (#4118)

* move imported.ex to imported subfolder * move constructing base imported query into a separate module * Implement imported table deciding and filtering + tests for pages, entry_pages, exit_pages and common filter types * add top stats test with country filter * add timeseries test * Drop bounce_rate and time_on_page from imported & page-filtered Top Stats * rename field returned by top stats * turn pages into a fn comp * Move dashboard API results under a results key ...and also return the skip_imported_reason to the frontend to be used for displaying warnings. * extend ListReport component with an optional afterFetchData prop * turn Devices into a fn comp * add not_requested as a skip_imported_reason * display warning icons in the dashboard * Implement filtering suggestions and translate filter fields for imported * WIP * Improve and cover filtering suggestions with tests * Rename imported suggestions query helpers * fix screen size breakdown with screen size filter * support filtering by the same suggestion property * support location filters when fetching location suggestions * support filtering by multiple props from the same table * Implement filtering by goals * Make views per visit metric work for import entry and exit pages * Get rid of circular dependencies between Stats.Imported and Stats.Imported.Base * Clean up Query struct manipulation in Breakdown * Rename helper function for clarity * Automatically refresh query struct state after modifications * Shutup credo * display imported warning bubble in prop breakdown section * Render warning bubble for funnels whenever imported data is in the view * Transform any operator on respective goal filters * Fix percentage and conversion_rate calculation in presence of custom props * add tests for for combining page and pageview goal filters * add skip_refresh option to query tweaking functions * add imported CR support for timeseries * still show url breakdown when special goal + url in filter * rename Query.refresh * use flat_map instead of map and concat * fix darkmode color * Handle invalid imported region codes in suggestions gracefully * Add an entry to CHANGELOG.md --------- Co-authored-by: Adrian Gruntkowski <[email protected]>
plausible · Jun 3, 2024 · 1d3b068 · 1d3b068
1 parent 7cd9bea
commit 1d3b068
Show file tree

Hide file tree

Showing 54 changed files with 2,915 additions and 602 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Snippet integration verification
+- Limited filtering support for imported data in the dashboard and via Stats API
 
 ### Removed
 

diff --git a/assets/js/dashboard/historical.js b/assets/js/dashboard/historical.js
@@ -31,7 +31,7 @@ function Historical(props) {
           <ComparisonInput site={props.site} query={props.query} />
         </div>
       </div>
-      <VisitorGraph site={props.site} query={props.query} />
+      <VisitorGraph site={props.site} query={props.query} updateImportedDataInView={props.updateImportedDataInView}/>
 
       <div className="w-full md:flex">
         <div className={ statsBoxClass }>
@@ -51,7 +51,7 @@ function Historical(props) {
         </div>
       </div>
 
-      <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} />
+      <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} importedDataInView={props.importedDataInView}/>
     </div>
   )
 }

diff --git a/assets/js/dashboard/index.js b/assets/js/dashboard/index.js
@@ -13,8 +13,10 @@ class Dashboard extends React.Component {
   constructor(props) {
     super(props)
     this.updateLastLoadTimestamp = this.updateLastLoadTimestamp.bind(this)
+    this.updateImportedDataInView = this.updateImportedDataInView.bind(this)
     this.state = {
       query: parseQuery(props.location.search, this.props.site),
+      importedDataInView: false,
       lastLoadTimestamp: new Date()
     }
   }
@@ -35,14 +37,18 @@ class Dashboard extends React.Component {
     this.setState({lastLoadTimestamp: new Date()})
   }
 
+  updateImportedDataInView(newBoolean) {
+    this.setState({importedDataInView: newBoolean})
+  }
+
   render() {
     const { site, loggedIn, currentUserRole } = this.props
     const { query, lastLoadTimestamp } = this.state
 
     if (this.state.query.period === 'realtime') {
       return <Realtime site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>
     } else {
-      return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>
+      return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp} importedDataInView={this.state.importedDataInView} updateImportedDataInView={this.updateImportedDataInView}/>
     }
   }
 }

diff --git a/assets/js/dashboard/stats/behaviours/conversions.js b/assets/js/dashboard/stats/behaviours/conversions.js
@@ -6,7 +6,7 @@ import { CR_METRIC } from '../reports/metrics';
 import ListReport from '../reports/list';
 
 export default function Conversions(props) {
-  const { site, query } = props
+  const { site, query, afterFetchData } = props
 
   function fetchConversions() {
     return api.get(url.apiPath(site, '/conversions'), query, { limit: 9 })
@@ -23,6 +23,7 @@ export default function Conversions(props) {
   return (
     <ListReport
       fetchData={fetchConversions}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Goal"
       onClick={props.onGoalFilterClick}

diff --git a/assets/js/dashboard/stats/behaviours/goal-conversions.js b/assets/js/dashboard/stats/behaviours/goal-conversions.js
@@ -31,7 +31,7 @@ export function specialTitleWhenGoalFilter(query, defaultTitle) {
 }
 
 function SpecialPropBreakdown(props) {
-  const { site, query, prop } = props
+  const { site, query, prop, afterFetchData } = props
 
   function fetchData() {
     return api.get(url.apiPath(site, `/custom-prop-values/${prop}`), query)
@@ -55,6 +55,7 @@ function SpecialPropBreakdown(props) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel={prop}
       metrics={[
@@ -73,12 +74,12 @@ function SpecialPropBreakdown(props) {
 }
 
 export default function GoalConversions(props) {
-  const {site, query} = props
+  const {site, query, afterFetchData} = props
 
   const specialGoal = getSpecialGoal(query)
   if (specialGoal) {
-    return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} />
+    return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} afterFetchData={afterFetchData} />
   } else {
-    return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick}/>
+    return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick} afterFetchData={afterFetchData} />
   }
 }
diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js
@@ -3,7 +3,7 @@ import { Menu, Transition } from '@headlessui/react'
 import { ChevronDownIcon } from '@heroicons/react/20/solid'
 import classNames from 'classnames'
 import * as storage from '../../util/storage'
-
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning'
 import GoalConversions, { specialTitleWhenGoalFilter } from './goal-conversions'
 import Properties from './props'
 import { FeatureSetupNotice } from '../../components/notice'
@@ -48,6 +48,8 @@ export default function Behaviours(props) {
 
   const [showingPropsForGoalFilter, setShowingPropsForGoalFilter] = useState(false)
 
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
+
   const onGoalFilterClick = useCallback((e) => {
     const goalName = e.target.innerHTML
     const isSpecialGoal = Object.keys(SPECIAL_GOALS).includes(goalName)
@@ -170,9 +172,14 @@ export default function Behaviours(props) {
     )
   }
 
+  function afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime())
+  }
+
   function renderConversions() {
     if (site.hasGoals) {
-      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} />
+      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} afterFetchData={afterFetchData}/>
     }
     else if (adminAccess) {
       return (
@@ -224,7 +231,7 @@ export default function Behaviours(props) {
 
   function renderProps() {
     if (site.hasProps && site.propsAvailable) {
-      return <Properties site={site} query={query} />
+      return <Properties site={site} query={query} afterFetchData={afterFetchData}/>
     } else if (adminAccess) {
       let callToAction
 
@@ -330,9 +337,14 @@ export default function Behaviours(props) {
       <div className="items-start justify-between block w-full mt-6 md:flex">
         <div className="w-full p-4 bg-white rounded shadow-xl dark:bg-gray-825">
           <div className="flex justify-between w-full">
-            <h3 className="font-bold dark:text-gray-100">
-              {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}
-            </h3>
+            <div className="flex gap-x-1">
+              <h3 className="font-bold dark:text-gray-100">
+                {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}
+              </h3>
+              <ImportedQueryUnsupportedWarning condition={mode === CONVERSIONS  && importedQueryUnsupported}/>
+              <ImportedQueryUnsupportedWarning condition={mode === PROPS && importedQueryUnsupported} message="Imported data is unavailable in this view"/>
+              <ImportedQueryUnsupportedWarning condition={mode === FUNNELS && props.importedDataInView} message="Imported data is unavailable in this view"/>
+            </div>
             {tabs()}
           </div>
           {renderContent()}

diff --git a/assets/js/dashboard/stats/behaviours/props.js b/assets/js/dashboard/stats/behaviours/props.js
@@ -75,6 +75,7 @@ export default function Properties(props) {
     return (
       <ListReport
         fetchData={fetchProps}
+        afterFetchData={props.afterFetchData}
         getFilterFor={getFilterFor}
         keyLabel={propKey}
         metrics={[

diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js
@@ -1,13 +1,13 @@
-import React from 'react';
-
+import React, {useState} from 'react';
 import * as storage from '../../util/storage'
 import { getFiltersByKeyPrefix, isFilteringOnFixedValue } from '../../util/filters'
 import ListReport from '../reports/list'
 import * as api from '../../api'
 import * as url from '../../util/url'
 import { VISITORS_METRIC, PERCENTAGE_METRIC, maybeWithCR } from '../reports/metrics';
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';
 
-function Browsers({ query, site }) {
+function Browsers({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/browsers'), query)
   }
@@ -22,6 +22,7 @@ function Browsers({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Browser"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -30,7 +31,7 @@ function Browsers({ query, site }) {
   )
 }
 
-function BrowserVersions({ query, site }) {
+function BrowserVersions({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/browser-versions'), query)
   }
@@ -48,6 +49,7 @@ function BrowserVersions({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Browser version"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -57,7 +59,7 @@ function BrowserVersions({ query, site }) {
 
 }
 
-function OperatingSystems({ query, site }) {
+function OperatingSystems({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/operating-systems'), query)
   }
@@ -72,6 +74,7 @@ function OperatingSystems({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Operating system"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -80,7 +83,7 @@ function OperatingSystems({ query, site }) {
   )
 }
 
-function OperatingSystemVersions({ query, site }) {
+function OperatingSystemVersions({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/operating-system-versions'), query)
   }
@@ -98,6 +101,7 @@ function OperatingSystemVersions({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Operating System Version"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -107,7 +111,7 @@ function OperatingSystemVersions({ query, site }) {
 
 }
 
-function ScreenSizes({ query, site }) {
+function ScreenSizes({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/screen-sizes'), query)
   }
@@ -126,6 +130,7 @@ function ScreenSizes({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Screen size"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -157,45 +162,44 @@ function iconFor(screenSize) {
   }
 }
 
-export default class Devices extends React.Component {
-  constructor(props) {
-    super(props)
-    this.tabKey = `deviceTab__${props.site.domain}`
-    const storedTab = storage.getItem(this.tabKey)
-    this.state = {
-      mode: storedTab || 'browser'
-    }
+export default function Devices(props) {
+  const {site, query} = props
+  const tabKey = `deviceTab__${site.domain}`
+  const storedTab = storage.getItem(tabKey)
+  const [mode, setMode] = useState(storedTab || 'browser')
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
+
+  function switchTab(mode) {
+    storage.setItem(tabKey, mode)
+    setMode(mode)
   }
 
-  setMode(mode) {
-    return () => {
-      storage.setItem(this.tabKey, mode)
-      this.setState({ mode })
-    }
+  function afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = query.period === 'realtime'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime)
   }
 
-  renderContent() {
-    switch (this.state.mode) {
+  function renderContent() {
+    switch (mode) {
       case 'browser':
-        if (isFilteringOnFixedValue(this.props.query, 'browser')) {
-          return <BrowserVersions site={this.props.site} query={this.props.query} />
+        if (isFilteringOnFixedValue(query, 'browser')) {
+          return <BrowserVersions site={site} query={query} afterFetchData={afterFetchData} />
         }
-        return <Browsers site={this.props.site} query={this.props.query} />
+        return <Browsers site={site} query={query} afterFetchData={afterFetchData} />
       case 'os':
-        if (isFilteringOnFixedValue(this.props.query, 'os')) {
-          return <OperatingSystemVersions site={this.props.site} query={this.props.query} />
+        if (isFilteringOnFixedValue(query, 'os')) {
+          return <OperatingSystemVersions site={site} query={query} afterFetchData={afterFetchData} />
         }
-        return <OperatingSystems site={this.props.site} query={this.props.query} />
+        return <OperatingSystems site={site} query={query} afterFetchData={afterFetchData} />
       case 'size':
       default:
-        return (
-          <ScreenSizes site={this.props.site} query={this.props.query} />
-        )
+        return <ScreenSizes site={site} query={query} afterFetchData={afterFetchData} />
     }
   }
 
-  renderPill(name, mode) {
-    const isActive = this.state.mode === mode
+  function renderPill(name, pill) {
+    const isActive = mode === pill
 
     if (isActive) {
       return (
@@ -210,28 +214,29 @@ export default class Devices extends React.Component {
     return (
       <button
         className="cursor-pointer hover:text-indigo-600"
-        onClick={this.setMode(mode)}
+        onClick={() => switchTab(pill)}
       >
         {name}
       </button>
     )
   }
 
-  render() {
-    return (
-      <div>
-        <div className="flex justify-between w-full">
+  return (
+    <div>
+      <div className="flex justify-between w-full">
+        <div className="flex gap-x-1">
           <h3 className="font-bold dark:text-gray-100">Devices</h3>
-          <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
-            {this.renderPill('Browser', 'browser')}
-            {this.renderPill('OS', 'os')}
-            {this.renderPill('Size', 'size')}
-          </div>
+          <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>
+        </div>
+        <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
+          {renderPill('Browser', 'browser')}
+          {renderPill('OS', 'os')}
+          {renderPill('Size', 'size')}
         </div>
-        {this.renderContent()}
       </div>
-    )
-  }
+      {renderContent()}
+    </div>
+  )
 }
 
 function getSingleFilter(query, filterKey) {

diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.js
@@ -81,6 +81,9 @@ export default function VisitorGraph(props) {
   function fetchTopStatsAndGraphData() {
     fetchTopStats(site, query)
       .then((res) => {
+        if (props.updateImportedDataInView) {
+          props.updateImportedDataInView(res.includes_imported)
+        }
         setTopStatData(res)
         setTopStatsLoading(false)
       })