APIv2: TimeSeries using QueryBuilder, release `experimental_session_c…

…ount` (#4305)

* Move fragments module under Plausible.Stats.SQL

* Introduce select_merge_as macro

This simplifies some select_merge calls

* Simplify select_join_fields

* Remove a needless dynamic

* wrap_select_columns macro

* Move metrics from base.ex to expression.ex

* Move WhereBuilder under Plausible.Stats.SQL

* Moduledoc

* Improved macros

* Wrap more code

* select_merge_as more

* Move defp to the end

* include.time_labels parsing

* include.time_labels in result

Note that the previous implementation of the labels from TimeSeries.ex was broken

* Apply consistent function in imports and timeseries.ex

* Remove boilerplate

* WIP: Limited support for timeseries-with-querybuilder

* time:week dimension

* cleanup: property -> dimension

* Make querying with time series work

* Refactor: Move special metrics (percentage, conversion rate) to own module

* Explicitly format datetimes

* Consistent include_imported in special metrics

* Solve week-related crash

* conversion_rate hacking

* Keep include_imported consistent after splitting the query

* Simplify do_decide_tables

* Handle time dimensions in imports cleaner

* Allow time dimensions in custom property queries

* time:week handling continued

* cast_revenue_metrics_to_money

* fix `full_intervals` support

* Handle minute/realtime graphs

* experimental_session_count? with timeseries

This becomes required as we try to include visits from sessions by default

* Support hourly data in imports

* Update bounce_rate in more csv tests

* Update some time-series query tests

* Fix for meta.warning being included incorrectly

* Simplify imported.ex

* experimental_session_count flag removal

* moduledoc

* Split interval and time modules

CHANGELOG.md

@@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+- Realtime and hourly graphs now show visits lasting their whole duration instead when specific events occur
 - Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200
 - Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245
 - `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.

assets/js/dashboard/api.js

@@ -38,7 +38,6 @@ export function serializeQuery(query, extraQuery = []) {
   if (query.from) { queryObj.from = formatISO(query.from) }
   if (query.to) { queryObj.to = formatISO(query.to) }
   if (query.filters) { queryObj.filters = serializeApiFilters(query.filters) }
-  if (query.experimental_session_count) { queryObj.experimental_session_count = query.experimental_session_count }
   if (query.with_imported) { queryObj.with_imported = query.with_imported }
   if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH }
 

assets/js/dashboard/query.js

@@ -42,7 +42,6 @@ export function parseQuery(querystring, site) {
     to: q.get('to') ? dayjs.utc(q.get('to')) : undefined,
     match_day_of_week: matchDayOfWeek == 'true',
     with_imported: q.get('with_imported') ? q.get('with_imported') === 'true' : true,
-    experimental_session_count: q.get('experimental_session_count'),
     filters: parseJsonUrl(q.get('filters'), []),
     labels: parseJsonUrl(q.get('labels'), {})
   }

lib/plausible/stats/base.ex

@@ -139,84 +139,4 @@ defmodule Plausible.Stats.Base do
 
     "^#{escaped}$"
   end
-
-  defp total_visitors(site, query) do
-    base_event_query(site, query)
-    |> select([e],
-      total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
-    )
-  end
-
-  # `total_visitors_subquery` returns a subquery which selects `total_visitors` -
-  # the number used as the denominator in the calculation of `conversion_rate` and
-  # `percentage` metrics.
-
-  # Usually, when calculating the totals, a new query is passed into this function,
-  # where certain filters (e.g. goal, props) are removed. That might make the query
-  # able to include imported data. However, we always want to include imported data
-  # only if it's included in the base query - otherwise the total will be based on
-  # a different data set, making the metric inaccurate. This is why we're using an
-  # explicit `include_imported` argument here.
-  def total_visitors_subquery(site, query, include_imported)
-
-  def total_visitors_subquery(site, query, true = _include_imported) do
-    wrap_alias([], %{
-      total_visitors:
-        subquery(total_visitors(site, query)) +
-          subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))
-    })
-  end
-
-  def total_visitors_subquery(site, query, false = _include_imported) do
-    wrap_alias([], %{
-      total_visitors: subquery(total_visitors(site, query))
-    })
-  end
-
-  def add_percentage_metric(q, site, query, metrics) do
-    if :percentage in metrics do
-      total_query = Query.set_dimensions(query, [])
-
-      q
-      |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
-      |> select_merge_as([], %{
-        percentage:
-          fragment(
-            "if(? > 0, round(? / ? * 100, 1), null)",
-            selected_as(:total_visitors),
-            selected_as(:visitors),
-            selected_as(:total_visitors)
-          )
-      })
-    else
-      q
-    end
-  end
-
-  # Adds conversion_rate metric to query, calculated as
-  # X / Y where Y is the same breakdown value without goal or props
-  # filters.
-  def maybe_add_conversion_rate(q, site, query, metrics) do
-    if :conversion_rate in metrics do
-      total_query =
-        query
-        |> Query.remove_filters(["event:goal", "event:props"])
-        |> Query.set_dimensions([])
-
-      # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
-      subquery(q)
-      |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
-      |> select_merge_as([e], %{
-        conversion_rate:
-          fragment(
-            "if(? > 0, round(? / ? * 100, 1), 0)",
-            selected_as(:total_visitors),
-            e.visitors,
-            selected_as(:total_visitors)
-          )
-      })
-    else
-      q
-    end
-  end
 end

lib/plausible/stats/comparisons.ex

@@ -163,21 +163,21 @@ defmodule Plausible.Stats.Comparisons do
   end
 
   defp maybe_include_imported(query, source_query) do
-    requested? = source_query.imported_data_requested
+    requested? = source_query.include.imports
 
     case Query.ensure_include_imported(query, requested?) do
       :ok ->
         struct!(query,
-          imported_data_requested: true,
           include_imported: true,
-          skip_imported_reason: nil
+          skip_imported_reason: nil,
+          include: Map.put(query.include, :imports, true)
         )
 
       {:error, reason} ->
         struct!(query,
-          imported_data_requested: requested?,
           include_imported: false,
-          skip_imported_reason: reason
+          skip_imported_reason: reason,
+          include: Map.put(query.include, :imports, requested?)
         )
     end
   end

lib/plausible/stats/filters/filters.ex

@@ -83,8 +83,8 @@ defmodule Plausible.Stats.Filters do
 
   def parse(_), do: []
 
-  def without_prefix(property) do
-    property
+  def without_prefix(dimension) do
+    dimension
     |> String.split(":")
     |> List.last()
     |> String.to_existing_atom()

lib/plausible/stats/filters/query_parser.ex

@@ -6,6 +6,11 @@ defmodule Plausible.Stats.Filters.QueryParser do
   alias Plausible.Stats.Query
   alias Plausible.Stats.Metrics
 
+  @default_include %{
+    imports: false,
+    time_labels: false
+  }
+
   def parse(site, params, now \\ nil) when is_map(params) do
     with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
          {:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
@@ -22,13 +27,14 @@ defmodule Plausible.Stats.Filters.QueryParser do
            dimensions: dimensions,
            order_by: order_by,
            timezone: site.timezone,
-           imported_data_requested: Map.get(include, :imports, false),
-           preloaded_goals: preloaded_goals
+           preloaded_goals: preloaded_goals,
+           include: include
          },
          :ok <- validate_order_by(query),
          :ok <- validate_goal_filters(query),
          :ok <- validate_custom_props_access(site, query),
-         :ok <- validate_metrics(query) do
+         :ok <- validate_metrics(query),
+         :ok <- validate_include(query) do
       {:ok, query}
     end
   end
@@ -219,16 +225,32 @@ defmodule Plausible.Stats.Filters.QueryParser do
   defp parse_time("time"), do: {:ok, "time"}
   defp parse_time("time:hour"), do: {:ok, "time:hour"}
   defp parse_time("time:day"), do: {:ok, "time:day"}
+  defp parse_time("time:week"), do: {:ok, "time:week"}
   defp parse_time("time:month"), do: {:ok, "time:month"}
   defp parse_time(_), do: :error
 
   defp parse_order_direction([_, "asc"]), do: {:ok, :asc}
   defp parse_order_direction([_, "desc"]), do: {:ok, :desc}
   defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}
 
-  defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}}
-  defp parse_include(%{}), do: {:ok, %{}}
-  defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"}
+  defp parse_include(include) when is_map(include) do
+    with {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do
+      include = Map.merge(@default_include, Enum.into(parsed_include_list, %{}))
+
+      {:ok, include}
+    end
+  end
+
+  defp parse_include(entry), do: {:error, "Invalid include passed '#{inspect(entry)}'"}
+
+  defp parse_include_value({"imports", value}) when is_boolean(value),
+    do: {:ok, {:imports, value}}
+
+  defp parse_include_value({"time_labels", value}) when is_boolean(value),
+    do: {:ok, {:time_labels, value}}
+
+  defp parse_include_value({key, value}),
+    do: {:error, "Invalid include entry '#{inspect(%{key => value})}'"}
 
   defp parse_filter_key_string(filter_key, error_message \\ "") do
     case filter_key do
@@ -386,14 +408,24 @@ defmodule Plausible.Stats.Filters.QueryParser do
     end
   end
 
-  def event_dimensions_not_allowing_session_metrics?(dimensions) do
+  defp event_dimensions_not_allowing_session_metrics?(dimensions) do
     Enum.any?(dimensions, fn
       "event:page" -> false
       "event:" <> _ -> true
       _ -> false
     end)
   end
 
+  defp validate_include(query) do
+    time_dimension? = Enum.any?(query.dimensions, &String.starts_with?(&1, "time"))
+
+    if query.include.time_labels and not time_dimension? do
+      {:error, "Invalid include.time_labels: requires a time dimension"}
+    else
+      :ok
+    end
+  end
+
   defp parse_list(list, parser_function) do
     Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->
       case parser_function.(value) do

lib/plausible/stats/imported/base.ex

@@ -29,11 +29,19 @@ defmodule Plausible.Stats.Imported.Base do
     "event:page" => "imported_pages",
     "event:name" => "imported_custom_events",
 
-    # NOTE: these properties can be only filtered by
+    # NOTE: these dimensions can be only filtered by
     "visit:screen" => "imported_devices",
-    "event:hostname" => "imported_pages"
+    "event:hostname" => "imported_pages",
+
+    # NOTE: These dimensions are only used in group by
+    "time:month" => "imported_visitors",
+    "time:week" => "imported_visitors",
+    "time:day" => "imported_visitors",
+    "time:hour" => "imported_visitors"
   }
 
+  @queriable_time_dimensions ["time:month", "time:week", "time:day", "time:hour"]
+
   @imported_custom_props Imported.imported_custom_props()
 
   @db_field_mappings %{
@@ -121,9 +129,10 @@ defmodule Plausible.Stats.Imported.Base do
     do_decide_custom_prop_table(query, dimension)
   end
 
+  @queriable_custom_prop_dimensions ["event:goal", "event:name"] ++ @queriable_time_dimensions
   defp do_decide_custom_prop_table(%{dimensions: dimensions} = query) do
     if dimensions == [] or
-         (length(dimensions) == 1 and hd(dimensions) in ["event:goal", "event:name"]) do
+         (length(dimensions) == 1 and hd(dimensions) in @queriable_custom_prop_dimensions) do
       custom_prop_filters =
         query.filters
         |> Enum.map(&Enum.at(&1, 1))
@@ -169,14 +178,6 @@ defmodule Plausible.Stats.Imported.Base do
     ["imported_pages", "imported_custom_events"]
   end
 
-  defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do
-    if Map.has_key?(@property_to_table_mappings, dimension) do
-      [@property_to_table_mappings[dimension]]
-    else
-      []
-    end
-  end
-
   defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do
     filter_props = Enum.map(filters, &Enum.at(&1, 1))
 
@@ -197,13 +198,15 @@ defmodule Plausible.Stats.Imported.Base do
       filters
       |> Enum.map(fn [_, filter_key | _] -> filter_key end)
       |> Enum.concat(dimensions)
-      |> Enum.map(fn
-        "visit:screen" -> "visit:device"
-        dimension -> dimension
+      |> Enum.reject(&(&1 in @queriable_time_dimensions))
+      |> Enum.flat_map(fn
+        "visit:screen" -> ["visit:device"]
+        dimension -> [dimension]
       end)
       |> Enum.map(&@property_to_table_mappings[&1])
 
     case Enum.uniq(table_candidates) do
+      [] -> ["imported_visitors"]
       [nil] -> []
       [candidate] -> [candidate]
       _ -> []