From 19e9736086ef2834c52ff57903506e2818b09d93 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 13 May 2024 19:41:51 +0100
Subject: [PATCH 01/41] move imported.ex to imported subfolder

---
 lib/plausible/stats/{ => imported}/imported.ex | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/plausible/stats/{ => imported}/imported.ex (100%)

diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported/imported.ex
similarity index 100%
rename from lib/plausible/stats/imported.ex
rename to lib/plausible/stats/imported/imported.ex

From 41df563ebb5604a0d248dc1925dca59c3ecaf885 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 13 May 2024 21:00:23 +0100
Subject: [PATCH 02/41] move constructing base imported query into a separate
 module

---
 lib/plausible/stats/imported/base.ex     | 20 +++++++
 lib/plausible/stats/imported/imported.ex | 70 +++++++-----------------
 2 files changed, 41 insertions(+), 49 deletions(-)
 create mode 100644 lib/plausible/stats/imported/base.ex

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
new file mode 100644
index 000000000000..799031088b81
--- /dev/null
+++ b/lib/plausible/stats/imported/base.ex
@@ -0,0 +1,20 @@
+defmodule Plausible.Stats.Imported.Base do
+  @moduledoc """
+  A module for building the base of an imported stats query
+  """
+
+  import Ecto.Query
+
+  def query_imported(table, site, query) do
+    import_ids = site.complete_import_ids
+    %{first: date_from, last: date_to} = query.date_range
+
+    from(i in table,
+      where: i.site_id == ^site.id,
+      where: i.import_id in ^import_ids,
+      where: i.date >= ^date_from,
+      where: i.date <= ^date_to,
+      select: %{}
+    )
+  end
+end
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index b6623587aaeb..2543f703af8a 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -1,6 +1,6 @@
 defmodule Plausible.Stats.Imported do
   use Plausible.ClickhouseRepo
-  alias Plausible.Stats.{Query, Base}
+  alias Plausible.Stats.{Query, Base, Imported}
 
   import Ecto.Query
   import Plausible.Stats.Fragments
@@ -69,15 +69,9 @@ defmodule Plausible.Stats.Imported do
         query,
         metrics
       ) do
-    import_ids = site.complete_import_ids
-
     imported_q =
-      from(v in "imported_visitors",
-        where: v.site_id == ^site.id,
-        where: v.import_id in ^import_ids,
-        where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,
-        select: %{}
-      )
+      "imported_visitors"
+      |> Imported.Base.query_imported(site, query)
       |> select_imported_metrics(metrics)
       |> apply_interval(query, site)
 
@@ -113,17 +107,11 @@ defmodule Plausible.Stats.Imported do
       when property in @imported_properties do
     table = Map.fetch!(@property_to_table_mappings, property)
     dim = Plausible.Stats.Filters.without_prefix(property)
-    import_ids = site.complete_import_ids
 
     imported_q =
-      from(
-        i in table,
-        where: i.site_id == ^site.id,
-        where: i.import_id in ^import_ids,
-        where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
-        where: i.visitors > 0,
-        select: %{}
-      )
+      table
+      |> Imported.Base.query_imported(site, query)
+      |> where([i], i.visitors > 0)
       |> maybe_apply_filter(query, property, dim)
       |> group_imported_by(dim)
       |> select_imported_metrics(metrics)
@@ -155,7 +143,8 @@ defmodule Plausible.Stats.Imported do
 
   def merge_imported(q, site, %Query{property: nil} = query, metrics) do
     imported_q =
-      imported_visitors(site, query)
+      "imported_visitors"
+      |> Imported.Base.query_imported(site, query)
       |> select_imported_metrics(metrics)
 
     from(
@@ -174,24 +163,18 @@ defmodule Plausible.Stats.Imported do
     page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
 
     imported_q =
-      from(
-        i in "imported_pages",
-        where: i.site_id == ^site.id,
-        where: i.import_id in ^site.complete_import_ids,
-        where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
-        where: i.visitors > 0,
-        where:
-          fragment(
-            "notEmpty(multiMatchAllIndices(?, ?) as indices)",
-            i.page,
-            ^page_regexes
-          ),
-        array_join: index in fragment("indices"),
-        group_by: index,
-        select: %{
-          name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
-        }
+      "imported_pages"
+      |> Imported.Base.query_imported(site, query)
+      |> where([i], i.visitors > 0)
+      |> where(
+        [i],
+        fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
       )
+      |> join(:array, index in fragment("indices"))
+      |> group_by([_i, index], index)
+      |> select_merge([_i, index], %{
+        name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
+      })
       |> select_imported_metrics(metrics)
 
     from(s in Ecto.Query.subquery(q),
@@ -204,22 +187,11 @@ defmodule Plausible.Stats.Imported do
   end
 
   def total_imported_visitors(site, query) do
-    imported_visitors(site, query)
+    "imported_visitors"
+    |> Imported.Base.query_imported(site, query)
     |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
   end
 
-  defp imported_visitors(site, query) do
-    import_ids = site.complete_import_ids
-
-    from(
-      i in "imported_visitors",
-      where: i.site_id == ^site.id,
-      where: i.import_id in ^import_ids,
-      where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
-      select: %{}
-    )
-  end
-
   defp maybe_apply_filter(q, query, "event:props:url", _) do
     if name = find_special_goal_filter(query, @goals_with_url) do
       where(q, [i], i.name == ^name)

From 2491c7f955c16e1fa358b8deed8de165e3290053 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 14 May 2024 15:22:16 +0100
Subject: [PATCH 03/41] Implement imported table deciding and filtering

+ tests for pages, entry_pages, exit_pages and common filter types
---
 lib/plausible/stats/base.ex                   |   7 +-
 lib/plausible/stats/breakdown.ex              |   1 +
 lib/plausible/stats/imported/base.ex          |  75 ++++++
 lib/plausible/stats/imported/imported.ex      |  85 +++---
 .../controllers/api/stats_controller.ex       |   5 +-
 .../api/stats_controller/pages_test.exs       | 252 ++++++++++++++++++
 6 files changed, 375 insertions(+), 50 deletions(-)

diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 07d14a7871b1..8b30b2910adf 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -308,6 +308,8 @@ defmodule Plausible.Stats.Base do
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
+      query = struct!(query, property: nil)
+
       q
       |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
       |> select_merge(%{
@@ -329,7 +331,10 @@ defmodule Plausible.Stats.Base do
   # filters.
   def maybe_add_conversion_rate(q, site, query, metrics) do
     if :conversion_rate in metrics do
-      total_query = query |> Query.remove_filters(["event:goal", "event:props"])
+      total_query =
+        query
+        |> Query.remove_filters(["event:goal", "event:props"])
+        |> struct!(property: nil)
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 305d80838b3d..cfc2929cad30 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -180,6 +180,7 @@ defmodule Plausible.Stats.Breakdown do
 
         pages ->
           query
+          |> Query.remove_filters(["event:page"])
           |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])
           |> struct!(property: "visit:entry_page")
       end
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 799031088b81..657c7aebf8c6 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -2,10 +2,24 @@ defmodule Plausible.Stats.Imported.Base do
   @moduledoc """
   A module for building the base of an imported stats query
   """
+  alias Plausible.Stats.{Query, Imported, Filters}
 
   import Ecto.Query
 
+  @goals_with_url Imported.goals_with_url()
+  @goals_with_path Imported.goals_with_path()
+  @special_goals @goals_with_path ++ @goals_with_url
+
+  def query_imported(site, query) do
+    query = Imported.drop_redundant_filters(query)
+
+    query
+    |> decide_table()
+    |> query_imported(site, query)
+  end
+
   def query_imported(table, site, query) do
+    query = Imported.drop_redundant_filters(query)
     import_ids = site.complete_import_ids
     %{first: date_from, last: date_to} = query.date_range
 
@@ -16,5 +30,66 @@ defmodule Plausible.Stats.Imported.Base do
       where: i.date <= ^date_to,
       select: %{}
     )
+    |> apply_filter(query)
+  end
+
+  def decide_table(%Query{filters: [], property: nil}), do: "imported_visitors"
+  def decide_table(%Query{filters: [], property: "event:props:url"}), do: nil
+  def decide_table(%Query{filters: [], property: "event:props:path"}), do: nil
+
+  def decide_table(%Query{filters: filters, property: "event:props:url"}) do
+    case filters do
+      [[:is, "event:goal", {:event, name}]] when name in @goals_with_url ->
+        "imported_custom_events"
+
+      _ ->
+        nil
+    end
+  end
+
+  def decide_table(%Query{filters: filters, property: "event:props:path"}) do
+    case filters do
+      [[:is, "event:goal", {:event, name}]] when name in @goals_with_path ->
+        "imported_custom_events"
+
+      _ ->
+        nil
+    end
+  end
+
+  def decide_table(%Query{filters: [], property: property}) do
+    Imported.property_to_table_mappings()[property]
   end
+
+  def decide_table(%Query{filters: [filter], property: property}) do
+    [_op, filtered_prop | _] = filter
+    table_candidate = Imported.property_to_table_mappings()[filtered_prop]
+
+    cond do
+      is_nil(property) -> table_candidate
+      property == filtered_prop -> table_candidate
+      true -> nil
+    end
+  end
+
+  def decide_table(_query_with_more_than_one_filter), do: nil
+
+  defp apply_filter(q, %Query{filters: [[:is, "event:goal", {:event, name}]]})
+       when name in @special_goals do
+    where(q, [i], i.name == ^name)
+  end
+
+  defp apply_filter(_q, %Query{filters: [[_, "event:goal" | _]]}) do
+    # TODO: implement and test.
+    raise "Unimplemented"
+  end
+
+  defp apply_filter(q, %Query{filters: [[_, filtered_prop | _] = filter]}) do
+    db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
+    condition = Filters.WhereBuilder.build_condition(db_field, filter)
+
+    where(q, ^condition)
+  end
+
+  defp apply_filter(q, _), do: q
 end
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 2543f703af8a..55e0e8ff6a95 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -33,11 +33,18 @@ defmodule Plausible.Stats.Imported do
     "event:props:path" => "imported_custom_events"
   }
 
+  def property_to_table_mappings(), do: @property_to_table_mappings
+
   @imported_properties Map.keys(@property_to_table_mappings)
 
   @goals_with_url Plausible.Imported.goals_with_url()
+
+  def goals_with_url(), do: @goals_with_url
+
   @goals_with_path Plausible.Imported.goals_with_path()
 
+  def goals_with_path(), do: @goals_with_path
+
   @doc """
   Returns a boolean indicating whether the combination of filters and
   breakdown property is possible to query from the imported tables.
@@ -49,17 +56,32 @@ defmodule Plausible.Stats.Imported do
   (see `@goals_with_url` and `@goals_with_path`).
   """
   def schema_supports_query?(query) do
-    filter_count = length(query.filters)
-
-    case {filter_count, query.property} do
-      {0, "event:props:" <> _} -> false
-      {0, _} -> true
-      {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
-      {1, "event:props:path"} -> has_special_goal_filter?(query, @goals_with_path)
-      {_, _} -> false
+    query = drop_redundant_filters(query)
+
+    # TODO: We want to be able to filter by a goal and still
+    # show a goal breakdown
+    case query.property do
+      "event:goal" -> query.filters == []
+      _ -> not is_nil(Imported.Base.decide_table(query))
     end
   end
 
+  @doc """
+  This function gets rid of filters that do not make sense in the context
+  of imported data, for example `event:name == "pageview"`. They will be
+  ignored when deciding whether to include imported data in the query or
+  not, as well as when actually applying those filters to the query.
+  """
+  def drop_redundant_filters(query) do
+    struct!(query,
+      filters:
+        Enum.reject(query.filters, fn
+          [:is, "event:name", "pageview"] -> true
+          _ -> false
+        end)
+    )
+  end
+
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 
@@ -70,8 +92,8 @@ defmodule Plausible.Stats.Imported do
         metrics
       ) do
     imported_q =
-      "imported_visitors"
-      |> Imported.Base.query_imported(site, query)
+      site
+      |> Imported.Base.query_imported(query)
       |> select_imported_metrics(metrics)
       |> apply_interval(query, site)
 
@@ -105,12 +127,11 @@ defmodule Plausible.Stats.Imported do
 
   def merge_imported(q, site, %Query{property: property} = query, metrics)
       when property in @imported_properties do
-    table = Map.fetch!(@property_to_table_mappings, property)
     dim = Plausible.Stats.Filters.without_prefix(property)
 
     imported_q =
-      table
-      |> Imported.Base.query_imported(site, query)
+      site
+      |> Imported.Base.query_imported(query)
       |> where([i], i.visitors > 0)
       |> maybe_apply_filter(query, property, dim)
       |> group_imported_by(dim)
@@ -143,8 +164,8 @@ defmodule Plausible.Stats.Imported do
 
   def merge_imported(q, site, %Query{property: nil} = query, metrics) do
     imported_q =
-      "imported_visitors"
-      |> Imported.Base.query_imported(site, query)
+      site
+      |> Imported.Base.query_imported(query)
       |> select_imported_metrics(metrics)
 
     from(
@@ -187,27 +208,11 @@ defmodule Plausible.Stats.Imported do
   end
 
   def total_imported_visitors(site, query) do
-    "imported_visitors"
-    |> Imported.Base.query_imported(site, query)
+    site
+    |> Imported.Base.query_imported(query)
     |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
   end
 
-  defp maybe_apply_filter(q, query, "event:props:url", _) do
-    if name = find_special_goal_filter(query, @goals_with_url) do
-      where(q, [i], i.name == ^name)
-    else
-      q
-    end
-  end
-
-  defp maybe_apply_filter(q, query, "event:props:path", _) do
-    if name = find_special_goal_filter(query, @goals_with_path) do
-      where(q, [i], i.name == ^name)
-    else
-      q
-    end
-  end
-
   defp maybe_apply_filter(q, query, property, dim) do
     case Query.get_filter(query, property) do
       [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
@@ -215,20 +220,6 @@ defmodule Plausible.Stats.Imported do
     end
   end
 
-  defp has_special_goal_filter?(query, event_names) do
-    not is_nil(find_special_goal_filter(query, event_names))
-  end
-
-  defp find_special_goal_filter(query, event_names) do
-    case Query.get_filter(query, "event:goal") do
-      [:is, "event:goal", {:event, name}] ->
-        if name in event_names, do: name, else: nil
-
-      _ ->
-        nil
-    end
-  end
-
   defp select_imported_metrics(q, []), do: q
 
   defp select_imported_metrics(q, [:visitors | rest]) do
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index bc9086ca848c..a970c461354c 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -845,14 +845,15 @@ defmodule PlausibleWeb.Api.StatsController do
     else
       pages = Enum.map(breakdown_results, & &1[:exit_page])
 
-      total_visits_query =
+      total_pageviews_query =
         query
+        |> Query.remove_filters(["visit:exit_page"])
         |> Query.put_filter([:member, "event:page", pages])
         |> Query.put_filter([:is, "event:name", "pageview"])
         |> struct!(property: "event:page")
 
       total_pageviews =
-        Stats.breakdown(site, total_visits_query, [:pageviews], {limit, 1})
+        Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})
 
       Enum.map(breakdown_results, fn result ->
         exit_rate =
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index ea9ccfe6ae3a..ab7b2a230b1c 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -1199,6 +1199,156 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
                %{"total_visitors" => 3, "visitors" => 1, "name" => "/", "conversion_rate" => 33.3}
              ]
     end
+
+    test "filter by :is page with imported data", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, user_id: 1, pathname: "/", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),
+        build(:imported_entry_pages,
+          entry_page: "/",
+          visitors: 1,
+          bounces: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/",
+          visitors: 3,
+          pageviews: 3,
+          time_on_page: 300,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{"page" => "/"})
+      q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"
+
+      conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
+
+      assert json_response(conn, 200) == [
+               %{
+                 "bounce_rate" => 50,
+                 "name" => "/",
+                 "pageviews" => 4,
+                 "time_on_page" => 90.0,
+                 "visitors" => 4
+               }
+             ]
+    end
+
+    test "filter by :member page with imported data", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, user_id: 1, pathname: "/", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),
+        build(:imported_entry_pages,
+          entry_page: "/",
+          visitors: 1,
+          bounces: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_entry_pages,
+          entry_page: "/a",
+          visitors: 1,
+          bounces: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/",
+          visitors: 3,
+          pageviews: 3,
+          time_on_page: 300,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/a",
+          visitors: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{"page" => "/|/a"})
+      q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"
+
+      conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
+
+      assert json_response(conn, 200) == [
+               %{
+                 "bounce_rate" => 50,
+                 "name" => "/",
+                 "pageviews" => 4,
+                 "time_on_page" => 90.0,
+                 "visitors" => 4
+               },
+               %{
+                 "bounce_rate" => 100,
+                 "name" => "/a",
+                 "pageviews" => 1,
+                 "time_on_page" => 10.0,
+                 "visitors" => 1
+               }
+             ]
+    end
+
+    test "filter by :matches page with imported data", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, user_id: 1, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),
+        build(:imported_entry_pages,
+          entry_page: "/aaa",
+          visitors: 1,
+          bounces: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_entry_pages,
+          entry_page: "/a",
+          visitors: 1,
+          bounces: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/aaa",
+          visitors: 3,
+          pageviews: 3,
+          time_on_page: 300,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/a",
+          visitors: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{"page" => "/a**"})
+      q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"
+
+      conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
+
+      assert json_response(conn, 200) == [
+               %{
+                 "bounce_rate" => 50,
+                 "name" => "/aaa",
+                 "pageviews" => 4,
+                 "time_on_page" => 90.0,
+                 "visitors" => 4
+               },
+               %{
+                 "bounce_rate" => 100,
+                 "name" => "/a",
+                 "pageviews" => 1,
+                 "time_on_page" => 10.0,
+                 "visitors" => 1
+               }
+             ]
+    end
   end
 
   describe "GET /api/stats/:domain/entry-pages" do
@@ -1552,6 +1702,56 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       assert json_response(conn, 200) == []
     end
+
+    test "filter by :matches_member entry_page with imported data", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),
+        build(:imported_entry_pages,
+          entry_page: "/a",
+          visitors: 5,
+          entrances: 9,
+          visit_duration: 1000,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_entry_pages,
+          entry_page: "/bbb",
+          visitors: 2,
+          entrances: 2,
+          visit_duration: 100,
+          date: ~D[2021-01-01]
+        )
+      ])
+
+      filters = Jason.encode!(%{"entry_page" => "/a**|/b**"})
+      q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"
+
+      conn = get(conn, "/api/stats/#{site.domain}/entry-pages#{q}")
+
+      assert json_response(conn, 200) == [
+               %{
+                 "visit_duration" => 100.0,
+                 "name" => "/a",
+                 "visits" => 10,
+                 "visitors" => 6
+               },
+               %{
+                 "visit_duration" => 50.0,
+                 "name" => "/bbb",
+                 "visits" => 2,
+                 "visitors" => 2
+               },
+               %{
+                 "visit_duration" => 0,
+                 "name" => "/aaa",
+                 "visits" => 1,
+                 "visitors" => 1
+               }
+             ]
+    end
   end
 
   describe "GET /api/stats/:domain/exit-pages" do
@@ -1849,5 +2049,57 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       assert json_response(conn, 200) == []
     end
+
+    test "filter by :is_not exit_page with imported data", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 12:00:00]),
+        build(:pageview, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),
+        build(:imported_exit_pages,
+          exit_page: "/a",
+          visitors: 5,
+          exits: 9,
+          visit_duration: 1000,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_exit_pages,
+          exit_page: "/bbb",
+          visitors: 2,
+          exits: 2,
+          visit_duration: 100,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages, page: "/a", pageviews: 19, date: ~D[2021-01-01]),
+        build(:imported_pages, page: "/bbb", pageviews: 2, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{"exit_page" => "!/ignored"})
+      q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"
+
+      conn = get(conn, "/api/stats/#{site.domain}/exit-pages#{q}")
+
+      assert json_response(conn, 200) == [
+               %{
+                 "exit_rate" => 50.0,
+                 "name" => "/a",
+                 "visits" => 10,
+                 "visitors" => 6
+               },
+               %{
+                 "exit_rate" => 100.0,
+                 "name" => "/bbb",
+                 "visits" => 2,
+                 "visitors" => 2
+               },
+               %{
+                 "exit_rate" => 100.0,
+                 "name" => "/aaa",
+                 "visits" => 1,
+                 "visitors" => 1
+               }
+             ]
+    end
   end
 end

From 02001aaa5d95640516966af2480289a7b61f253c Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 14 May 2024 16:34:35 +0100
Subject: [PATCH 04/41] add top stats test with country filter

---
 .../api/stats_controller/top_stats_test.exs   | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index 4ec122086c76..c8285d4c6e18 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -530,6 +530,63 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
                %{"name" => "Visit duration", "value" => 303, "graph_metric" => "visit_duration"}
              ]
     end
+
+    test ":member filter on country", %{
+      conn: conn,
+      site: site
+    } do
+      populate_stats(site, [
+        build(:pageview,
+          country_code: "EE",
+          user_id: @user_id,
+          timestamp: ~N[2021-01-01 00:00:00]
+        ),
+        build(:pageview,
+          country_code: "EE",
+          user_id: @user_id,
+          timestamp: ~N[2021-01-01 00:01:00]
+        ),
+        build(:imported_locations,
+          country: "EE",
+          date: ~D[2021-01-01],
+          visitors: 1,
+          visits: 3,
+          pageviews: 34,
+          bounces: 0,
+          visit_duration: 420
+        ),
+        build(:imported_locations,
+          country: "FR",
+          date: ~D[2021-01-01],
+          visitors: 3,
+          visits: 7,
+          pageviews: 65,
+          bounces: 1,
+          visit_duration: 300
+        ),
+        build(:imported_locations, country: "US", date: ~D[2021-01-01], visitors: 999)
+      ])
+
+      filters = Jason.encode!(%{country: "EE|FR"})
+      q = "?period=day&date=2021-01-01&with_imported=true&filters=#{filters}"
+
+      conn = get(conn, "/api/stats/#{site.domain}/top-stats#{q}")
+
+      res = json_response(conn, 200)
+
+      assert res["top_stats"] == [
+               %{"name" => "Unique visitors", "value" => 5, "graph_metric" => "visitors"},
+               %{"name" => "Total visits", "value" => 11, "graph_metric" => "visits"},
+               %{"name" => "Total pageviews", "value" => 101, "graph_metric" => "pageviews"},
+               %{
+                 "name" => "Views per visit",
+                 "value" => 9.18,
+                 "graph_metric" => "views_per_visit"
+               },
+               %{"name" => "Bounce rate", "value" => 9, "graph_metric" => "bounce_rate"},
+               %{"name" => "Visit duration", "value" => 71, "graph_metric" => "visit_duration"}
+             ]
+    end
   end
 
   describe "GET /api/stats/top-stats - realtime" do

From c0c1d3635116c73742ec384ddd1e7b16e8720a87 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Wed, 15 May 2024 11:08:48 +0100
Subject: [PATCH 05/41] add timeseries test

---
 .../timeseries_test.exs                       | 96 +++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
index d1ecccfa326e..f4cc1081ad85 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
@@ -1647,5 +1647,101 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
 
       refute json_response(conn, 200)["warning"]
     end
+
+    test "returns all metrics based on imported/native data when filtering by browser", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview, browser: "Chrome", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, browser: "Chrome", user_id: 1, timestamp: ~N[2021-01-01 00:03:00]),
+        build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+        build(:imported_browsers, browser: "Firefox", date: ~D[2021-01-02]),
+        build(:imported_browsers,
+          browser: "Chrome",
+          visitors: 1,
+          pageviews: 1,
+          bounces: 1,
+          visit_duration: 3,
+          visits: 1,
+          date: ~D[2021-01-03]
+        ),
+        build(:pageview, browser: "Chrome", user_id: 2, timestamp: ~N[2021-01-04 00:00:00]),
+        build(:event,
+          name: "Signup",
+          browser: "Chrome",
+          user_id: 2,
+          timestamp: ~N[2021-01-04 00:10:00]
+        ),
+        build(:imported_browsers,
+          browser: "Chrome",
+          visitors: 4,
+          pageviews: 6,
+          bounces: 1,
+          visit_duration: 300,
+          visits: 5,
+          date: ~D[2021-01-04]
+        )
+      ])
+
+      results =
+        conn
+        |> get("/api/v1/stats/timeseries", %{
+          "site_id" => site.domain,
+          "period" => "custom",
+          "date" => "2021-01-01,2021-01-04",
+          "metrics" =>
+            "visitors,pageviews,events,visits,views_per_visit,bounce_rate,visit_duration",
+          "filters" => "visit:browser==Chrome",
+          "with_imported" => "true"
+        })
+        |> json_response(200)
+        |> Map.get("results")
+
+      assert results == [
+               %{
+                 "bounce_rate" => 0.0,
+                 "date" => "2021-01-01",
+                 "events" => 2,
+                 "pageviews" => 2,
+                 "views_per_visit" => 2.0,
+                 "visit_duration" => 180.0,
+                 "visitors" => 1,
+                 "visits" => 1
+               },
+               %{
+                 "bounce_rate" => nil,
+                 "date" => "2021-01-02",
+                 "events" => 0,
+                 "pageviews" => 0,
+                 "views_per_visit" => 0.0,
+                 "visit_duration" => nil,
+                 "visitors" => 0,
+                 "visits" => 0
+               },
+               %{
+                 "bounce_rate" => 100,
+                 "date" => "2021-01-03",
+                 "events" => 1,
+                 "pageviews" => 1,
+                 "views_per_visit" => 1.0,
+                 "visit_duration" => 3,
+                 "visitors" => 1,
+                 "visits" => 1
+               },
+               %{
+                 "bounce_rate" => 17.0,
+                 "date" => "2021-01-04",
+                 "events" => 8,
+                 "pageviews" => 7,
+                 "views_per_visit" => 1.17,
+                 "visit_duration" => 150,
+                 "visitors" => 5,
+                 "visits" => 6
+               }
+             ]
+    end
   end
 end

From c49f8f8bf1902a0061b137f8aaeaba7702dc8574 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Wed, 15 May 2024 12:39:28 +0100
Subject: [PATCH 06/41] Drop bounce_rate and time_on_page from imported &
 page-filtered Top Stats

---
 .../controllers/api/stats_controller.ex       | 27 ++++---------
 .../api/stats_controller/top_stats_test.exs   | 39 +++++++++++++++++++
 2 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index a970c461354c..a88122ac98e8 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -381,26 +381,15 @@ defmodule PlausibleWeb.Api.StatsController do
   end
 
   defp fetch_other_top_stats(site, query, comparison_query) do
+    page_filter? = Query.get_filter(query, "event:page")
+
+    metrics = [:visitors, :visits, :pageviews, :sample_percent]
+
     metrics =
-      if Query.get_filter(query, "event:page") do
-        [
-          :visitors,
-          :visits,
-          :pageviews,
-          :bounce_rate,
-          :time_on_page,
-          :sample_percent
-        ]
-      else
-        [
-          :visitors,
-          :visits,
-          :pageviews,
-          :views_per_visit,
-          :bounce_rate,
-          :visit_duration,
-          :sample_percent
-        ]
+      cond do
+        page_filter? && query.include_imported -> metrics
+        page_filter? -> metrics ++ [:bounce_rate, :time_on_page]
+        true -> metrics ++ [:views_per_visit, :bounce_rate, :visit_duration]
       end
 
     current_results = Stats.aggregate(site, query, metrics)
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index c8285d4c6e18..c2c33217167c 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -587,6 +587,45 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
                %{"name" => "Visit duration", "value" => 71, "graph_metric" => "visit_duration"}
              ]
     end
+
+    test ":is filter on page returns only visitors, visits and pageviews", %{
+      conn: conn,
+      site: site
+    } do
+      populate_stats(site, [
+        build(:pageview,
+          pathname: "/",
+          user_id: @user_id,
+          timestamp: ~N[2021-01-01 00:00:00]
+        ),
+        build(:pageview,
+          pathname: "/",
+          user_id: @user_id,
+          timestamp: ~N[2021-01-01 00:01:00]
+        ),
+        build(:imported_pages,
+          page: "/",
+          date: ~D[2021-01-01],
+          visitors: 1,
+          visits: 3,
+          pageviews: 34
+        ),
+        build(:imported_pages, page: "/ignored", date: ~D[2021-01-01], visitors: 999)
+      ])
+
+      filters = Jason.encode!(%{page: "/"})
+      q = "?period=day&date=2021-01-01&with_imported=true&filters=#{filters}"
+
+      conn = get(conn, "/api/stats/#{site.domain}/top-stats#{q}")
+
+      res = json_response(conn, 200)
+
+      assert res["top_stats"] == [
+               %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"},
+               %{"name" => "Total visits", "value" => 4, "graph_metric" => "visits"},
+               %{"name" => "Total pageviews", "value" => 36, "graph_metric" => "pageviews"}
+             ]
+    end
   end
 
   describe "GET /api/stats/top-stats - realtime" do

From f38cc04fce09912bccbd7ee75357da118cb71f52 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 20 May 2024 17:44:30 +0100
Subject: [PATCH 07/41] rename field returned by top stats

---
 .../dashboard/stats/graph/with-imported-switch.js  |  2 +-
 .../controllers/api/stats_controller.ex            | 10 +++++-----
 .../api/stats_controller/main_graph_test.exs       | 14 +++++++-------
 .../api/stats_controller/top_stats_test.exs        |  4 ++--
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/assets/js/dashboard/stats/graph/with-imported-switch.js b/assets/js/dashboard/stats/graph/with-imported-switch.js
index b376e1ee8716..b7c81befa373 100644
--- a/assets/js/dashboard/stats/graph/with-imported-switch.js
+++ b/assets/js/dashboard/stats/graph/with-imported-switch.js
@@ -22,7 +22,7 @@ export default function WithImportedSwitch({site, topStatData}) {
   const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
 
   if (isQueryingImportedPeriod || isComparingImportedPeriod) {
-    const withImported = topStatData.with_imported;
+    const withImported = topStatData.includes_imported;
     const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
     const target = url.setQuery('with_imported', (!withImported).toString())
     const tip = withImported ? "" : "do not ";
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index a88122ac98e8..e21aaa3db98d 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
     * `interval` - the interval used for querying.
 
-    * `with_imported` - boolean indicating whether the Google Analytics data
+    * `includes_imported` - boolean indicating whether imported data
       was queried or not.
 
     * `imports_exist` - boolean indicating whether there are any completed
@@ -92,7 +92,7 @@ defmodule PlausibleWeb.Api.StatsController do
     "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
     "plot" => [0, 0, 0, 0],
     "present_index" => nil,
-    "with_imported" => false
+    "includes_imported" => false
   }
   ```
 
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController do
         comparison_labels: comparison_result && label_timeseries(comparison_result, nil),
         present_index: present_index,
         interval: query.interval,
-        with_imported: with_imported?(query, comparison_query),
+        includes_imported: includes_imported?(query, comparison_query),
         imports_exist: site.complete_import_ids != [],
         full_intervals: full_intervals
       })
@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController do
       top_stats: top_stats,
       interval: query.interval,
       sample_percent: sample_percent,
-      with_imported: with_imported?(query, comparison_query),
+      includes_imported: includes_imported?(query, comparison_query),
       imports_exist: site.complete_import_ids != [],
       comparing_from: comparison_query && comparison_query.date_range.first,
       comparing_to: comparison_query && comparison_query.date_range.last,
@@ -1391,7 +1391,7 @@ defmodule PlausibleWeb.Api.StatsController do
     ]
   end
 
-  defp with_imported?(source_query, comparison_query) do
+  defp includes_imported?(source_query, comparison_query) do
     cond do
       source_query.include_imported -> true
       comparison_query && comparison_query.include_imported -> true
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index b88fae0fa582..773bc8cbf534 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -53,7 +53,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
         )
 
       zeroes = List.duplicate(0, 30)
-      assert %{"plot" => ^zeroes, "with_imported" => false} = json_response(conn, 200)
+      assert %{"plot" => ^zeroes, "includes_imported" => false} = json_response(conn, 200)
     end
 
     test "displays visitors for a day with imported data", %{conn: conn, site: site} do
@@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
           "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
+      assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =
                json_response(conn, 200)
 
       assert plot == [2] ++ List.duplicate(0, 23)
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
           "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
         )
 
-      assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
+      assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =
                json_response(conn, 200)
 
       assert Enum.count(plot) == 31
@@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
           "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
         )
 
-      assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
+      assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =
                json_response(conn, 200)
 
       assert Enum.count(plot) == 31
@@ -1157,7 +1157,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
                "plot" => plot,
                "comparison_plot" => comparison_plot,
                "imports_exist" => true,
-               "with_imported" => true
+               "includes_imported" => true
              } = json_response(conn, 200)
 
       assert 4 == Enum.sum(plot)
@@ -1203,7 +1203,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
                "plot" => plot,
                "comparison_plot" => comparison_plot,
                "imports_exist" => true,
-               "with_imported" => false
+               "includes_imported" => false
              } = json_response(conn, 200)
 
       assert 4 == Enum.sum(plot)
@@ -1233,7 +1233,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
                "plot" => this_week_plot,
                "comparison_plot" => last_week_plot,
                "imports_exist" => true,
-               "with_imported" => false
+               "includes_imported" => false
              } = json_response(conn, 200)
 
       assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index c2c33217167c..6483b6619e54 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -1454,7 +1454,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
           "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true&comparison=year_over_year"
         )
 
-      assert %{"top_stats" => top_stats, "with_imported" => true} = json_response(conn, 200)
+      assert %{"top_stats" => top_stats, "includes_imported" => true} = json_response(conn, 200)
 
       assert %{
                "change" => 100,
@@ -1484,7 +1484,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
           "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=false&comparison=year_over_year"
         )
 
-      assert %{"top_stats" => top_stats, "with_imported" => false} = json_response(conn, 200)
+      assert %{"top_stats" => top_stats, "includes_imported" => false} = json_response(conn, 200)
 
       assert %{
                "change" => 100,

From 961d0e864c7b6f64943f378a5f89fccfdad88059 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 07:43:51 +0100
Subject: [PATCH 08/41] turn pages into a fn comp

---
 assets/js/dashboard/stats/pages/index.js | 72 +++++++++++-------------
 1 file changed, 32 insertions(+), 40 deletions(-)

diff --git a/assets/js/dashboard/stats/pages/index.js b/assets/js/dashboard/stats/pages/index.js
index 8e730fc2d4e6..d82492a248dc 100644
--- a/assets/js/dashboard/stats/pages/index.js
+++ b/assets/js/dashboard/stats/pages/index.js
@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useState } from 'react';
 
 import * as storage from '../../util/storage'
 import * as url from '../../util/url'
@@ -102,38 +102,32 @@ const labelFor = {
   'exit-pages': 'Exit Pages',
 }
 
-export default class Pages extends React.Component {
-  constructor(props) {
-    super(props)
-    this.tabKey = `pageTab__${props.site.domain}`
-    const storedTab = storage.getItem(this.tabKey)
-    this.state = {
-      mode: storedTab || 'pages'
-    }
-  }
+export default function Pages(props) {
+  const {site, query} = props
+  const tabKey = `pageTab__${site.domain}`
+  const storedTab = storage.getItem(tabKey)
+  const [mode, setMode] = useState(storedTab || 'pages')
 
-  setMode(mode) {
-    return () => {
-      storage.setItem(this.tabKey, mode)
-      this.setState({ mode })
-    }
+  function switchTab(mode) {
+    storage.setItem(tabKey, mode)
+    setMode(mode)
   }
 
-  renderContent() {
-    switch (this.state.mode) {
+  function renderContent() {
+    switch (mode) {
       case "entry-pages":
-        return <EntryPages site={this.props.site} query={this.props.query} />
+        return <EntryPages site={site} query={query} />
       case "exit-pages":
-        return <ExitPages site={this.props.site} query={this.props.query} />
+        return <ExitPages site={site} query={query} />
       case "pages":
       default:
-        return <TopPages site={this.props.site} query={this.props.query} />
+        return <TopPages site={site} query={query} />
     }
   }
 
 
-  renderPill(name, mode) {
-    const isActive = this.state.mode === mode
+  function renderPill(name, pill) {
+    const isActive = mode === pill
 
     if (isActive) {
       return (
@@ -148,30 +142,28 @@ export default class Pages extends React.Component {
     return (
       <button
         className="hover:text-indigo-600 cursor-pointer"
-        onClick={this.setMode(mode)}
+        onClick={() => switchTab(pill)}
       >
         {name}
       </button>
     )
   }
 
-  render() {
-    return (
-      <div>
-        {/* Header Container */}
-        <div className="w-full flex justify-between">
-          <h3 className="font-bold dark:text-gray-100">
-            {labelFor[this.state.mode] || 'Page Visits'}
-          </h3>
-          <div className="flex font-medium text-xs text-gray-500 dark:text-gray-400 space-x-2">
-            {this.renderPill('Top Pages', 'pages')}
-            {this.renderPill('Entry Pages', 'entry-pages')}
-            {this.renderPill('Exit Pages', 'exit-pages')}
-          </div>
+  return (
+    <div>
+      {/* Header Container */}
+      <div className="w-full flex justify-between">
+        <h3 className="font-bold dark:text-gray-100">
+          {labelFor[mode] || 'Page Visits'}
+        </h3>
+        <div className="flex font-medium text-xs text-gray-500 dark:text-gray-400 space-x-2">
+          {renderPill('Top Pages', 'pages')}
+          {renderPill('Entry Pages', 'entry-pages')}
+          {renderPill('Exit Pages', 'exit-pages')}
         </div>
-        {/* Main Contents */}
-        {this.renderContent()}
       </div>
-    )
-  }
+      {/* Main Contents */}
+      {renderContent()}
+    </div>
+  )
 }

From a59cc5b956e2b61cfb3eb5e8229edc6db3723f60 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 10:09:37 +0100
Subject: [PATCH 09/41] Move dashboard API results under a results key

...and also return the skip_imported_reason to the frontend to be used
for displaying warnings.
---
 assets/js/dashboard/stats/locations/map.js    |   3 +-
 .../js/dashboard/stats/modals/conversions.js  |   5 +-
 .../js/dashboard/stats/modals/entry-pages.js  |  12 +--
 .../js/dashboard/stats/modals/exit-pages.js   |   5 +-
 assets/js/dashboard/stats/modals/pages.js     |   5 +-
 assets/js/dashboard/stats/modals/props.js     |   6 +-
 .../stats/modals/referrer-drilldown.js        |   5 +-
 assets/js/dashboard/stats/modals/sources.js   |   5 +-
 assets/js/dashboard/stats/modals/table.js     |   5 +-
 assets/js/dashboard/stats/reports/list.js     |   3 +-
 .../controllers/api/stats_controller.ex       | 102 ++++++++++++++----
 .../api/stats_controller/browsers_test.exs    |  25 ++---
 .../api/stats_controller/cities_test.exs      |   6 +-
 .../api/stats_controller/conversions_test.exs |  37 ++++---
 .../api/stats_controller/countries_test.exs   |  18 ++--
 .../custom_prop_breakdown_test.exs            |  59 +++++-----
 .../api/stats_controller/imported_test.exs    |  37 +++----
 .../operating_systems_test.exs                |  24 ++---
 .../api/stats_controller/pages_test.exs       | 100 ++++++++---------
 .../api/stats_controller/regions_test.exs     |   4 +-
 .../stats_controller/screen_sizes_test.exs    |  24 ++---
 .../api/stats_controller/sources_test.exs     |  90 ++++++++--------
 22 files changed, 315 insertions(+), 265 deletions(-)

diff --git a/assets/js/dashboard/stats/locations/map.js b/assets/js/dashboard/stats/locations/map.js
index fbc29b0f3433..5628f306f7f7 100644
--- a/assets/js/dashboard/stats/locations/map.js
+++ b/assets/js/dashboard/stats/locations/map.js
@@ -76,8 +76,7 @@ class Countries extends React.Component {
   fetchCountries() {
     return api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/countries`, this.props.query, {limit: 300})
       .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState({loading: false, countries: results})
+        this.setState({loading: false, countries: response.results})
       })
   }
 
diff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.js
index eb49612dcfe6..1a245918cc2f 100644
--- a/assets/js/dashboard/stats/modals/conversions.js
+++ b/assets/js/dashboard/stats/modals/conversions.js
@@ -37,11 +37,10 @@ function ConversionsModal(props) {
   function fetchData() {
     api.get(url.apiPath(site, `/conversions`), query, { limit: 100, page })
       .then((response) => {
-        const results = response.results ? response.results : response
         setLoading(false)
-        setList(list.concat(results))
+        setList(list.concat(response.results))
         setPage(page + 1)
-        setMoreResultsAvailable(results.length >= 100)
+        setMoreResultsAvailable(response.results.length >= 100)
       })
   }
 
diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.js
index f94ccdd91980..3c7004f42184 100644
--- a/assets/js/dashboard/stats/modals/entry-pages.js
+++ b/assets/js/dashboard/stats/modals/entry-pages.js
@@ -33,15 +33,13 @@ class EntryPagesModal extends React.Component {
       query,
       { limit: 100, page }
     )
-      .then((response) => {
-        const results = response.results ? response.results : response
-        
-        this.setState((state) => ({
+      .then(
+        (response) => this.setState((state) => ({
           loading: false,
-          pages: state.pages.concat(results),
-          moreResultsAvailable: results.length === 100
+          pages: state.pages.concat(response.results),
+          moreResultsAvailable: response.results.length === 100
         }))
-      })
+      )
   }
 
   loadMore() {
diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js
index e64bd7ebb0ee..8dc82bec08ff 100644
--- a/assets/js/dashboard/stats/modals/exit-pages.js
+++ b/assets/js/dashboard/stats/modals/exit-pages.js
@@ -28,10 +28,7 @@ class ExitPagesModal extends React.Component {
     const { query, page } = this.state;
 
     api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/exit-pages`, query, { limit: 100, page })
-      .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState((state) => ({ loading: false, pages: state.pages.concat(results), moreResultsAvailable: results.length === 100 }))
-      })
+      .then((response) => this.setState((state) => ({ loading: false, pages: state.pages.concat(response.results), moreResultsAvailable: response.results.length === 100 })))
   }
 
   loadMore() {
diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js
index cad2805d4b56..643938352fde 100644
--- a/assets/js/dashboard/stats/modals/pages.js
+++ b/assets/js/dashboard/stats/modals/pages.js
@@ -30,10 +30,7 @@ class PagesModal extends React.Component {
     const { query, page } = this.state;
 
     api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, query, { limit: 100, page, detailed })
-      .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState((state) => ({ loading: false, pages: state.pages.concat(results), moreResultsAvailable: results.length === 100 }))
-      })
+      .then((response) => this.setState((state) => ({ loading: false, pages: state.pages.concat(response.results), moreResultsAvailable: response.results.length === 100 })))
   }
 
   loadMore() {
diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.js
index 23876c333936..0fcd1c04f562 100644
--- a/assets/js/dashboard/stats/modals/props.js
+++ b/assets/js/dashboard/stats/modals/props.js
@@ -39,12 +39,10 @@ function PropsModal(props) {
   function fetchData() {
     api.get(url.apiPath(site, `/custom-prop-values/${propKey}`), query, { limit: 100, page })
       .then((response) => {
-        const results = response.results ? response.results : response
-
         setLoading(false)
-        setList(list.concat(results))
+        setList(list.concat(response.results))
         setPage(page + 1)
-        setMoreResultsAvailable(results.length >= 100)
+        setMoreResultsAvailable(response.results.length >= 100)
       })
   }
 
diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js
index a4045c233d28..49d33c547690 100644
--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js
+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js
@@ -21,10 +21,7 @@ class ReferrerDrilldownModal extends React.Component {
     const detailed = this.showExtra()
 
     api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/${this.props.match.params.referrer}`, this.state.query, {limit: 100, detailed})
-      .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState({loading: false, referrers: results})
-      })
+      .then((response) => this.setState({loading: false, referrers: response.results}))
   }
 
   showExtra() {
diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.js
index 55b9448d779e..2f272f19e017 100644
--- a/assets/js/dashboard/stats/modals/sources.js
+++ b/assets/js/dashboard/stats/modals/sources.js
@@ -35,10 +35,7 @@ class SourcesModal extends React.Component {
 
     const detailed = this.showExtra()
     api.get(`/api/stats/${encodeURIComponent(site.domain)}/${this.currentView()}`, query, { limit: 100, page, detailed })
-      .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState({ loading: false, sources: sources.concat(results), moreResultsAvailable: results.length === 100 })
-      })
+      .then((response) => this.setState({ loading: false, sources: sources.concat(response.results), moreResultsAvailable: response.results.length === 100 }))
   }
 
   componentDidMount() {
diff --git a/assets/js/dashboard/stats/modals/table.js b/assets/js/dashboard/stats/modals/table.js
index db02943cf6c2..20eea170c3bd 100644
--- a/assets/js/dashboard/stats/modals/table.js
+++ b/assets/js/dashboard/stats/modals/table.js
@@ -19,10 +19,7 @@ class ModalTable extends React.Component {
 
   componentDidMount() {
     api.get(this.props.endpoint, this.state.query, {limit: 100})
-      .then((response) => {
-        const results = response.results ? response.results : response
-        this.setState({loading: false, list: results})
-      })
+      .then((response) => this.setState({loading: false, list: response.results}))
   }
 
   showConversionRate() {
diff --git a/assets/js/dashboard/stats/reports/list.js b/assets/js/dashboard/stats/reports/list.js
index c81ae9a0fde7..ba81c908597c 100644
--- a/assets/js/dashboard/stats/reports/list.js
+++ b/assets/js/dashboard/stats/reports/list.js
@@ -125,8 +125,7 @@ export default function ListReport(props) {
     }
     props.fetchData()
       .then((response) => {
-        const results = response.results ? response.results : response
-        setState({ loading: false, list: results })
+        setState({ loading: false, list: response.results })
       })
   }, [props.keyLabel, props.query])
 
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index e21aaa3db98d..29aef361a853 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -481,7 +481,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -559,7 +562,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -583,7 +589,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -607,7 +616,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -631,7 +643,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -655,7 +670,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -679,7 +697,10 @@ defmodule PlausibleWeb.Api.StatsController do
         res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])
       end
     else
-      json(conn, res)
+      json(conn, %{
+        results: res,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -732,7 +753,10 @@ defmodule PlausibleWeb.Api.StatsController do
       Stats.breakdown(site, query, metrics, pagination)
       |> transform_keys(%{referrer: :name})
 
-    json(conn, referrers)
+      json(conn, %{
+        results: referrers,
+        skip_imported_reason: query.skip_imported_reason
+      })
   end
 
   def pages(conn, params) do
@@ -761,7 +785,10 @@ defmodule PlausibleWeb.Api.StatsController do
         pages |> to_csv([:name, :visitors, :pageviews, :bounce_rate, :time_on_page])
       end
     else
-      json(conn, pages)
+      json(conn, %{
+        results: pages,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -792,7 +819,10 @@ defmodule PlausibleWeb.Api.StatsController do
         ])
       end
     else
-      json(conn, entry_pages)
+      json(conn, %{
+        results: entry_pages,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -824,7 +854,10 @@ defmodule PlausibleWeb.Api.StatsController do
         ])
       end
     else
-      json(conn, exit_pages)
+      json(conn, %{
+        results: exit_pages,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -907,7 +940,10 @@ defmodule PlausibleWeb.Api.StatsController do
           end
         end)
 
-      json(conn, countries)
+        json(conn, %{
+          results: countries,
+          skip_imported_reason: query.skip_imported_reason
+        })
     end
   end
 
@@ -942,7 +978,10 @@ defmodule PlausibleWeb.Api.StatsController do
         regions |> to_csv([:name, :visitors])
       end
     else
-      json(conn, regions)
+      json(conn, %{
+        results: regions,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -982,7 +1021,10 @@ defmodule PlausibleWeb.Api.StatsController do
         cities |> to_csv([:name, :visitors])
       end
     else
-      json(conn, cities)
+      json(conn, %{
+        results: cities,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1006,7 +1048,10 @@ defmodule PlausibleWeb.Api.StatsController do
         browsers |> to_csv([:name, :visitors])
       end
     else
-      json(conn, browsers)
+      json(conn, %{
+        results: browsers,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1036,7 +1081,10 @@ defmodule PlausibleWeb.Api.StatsController do
         |> to_csv([:name, :version, :visitors])
       end
     else
-      json(conn, versions)
+      json(conn, %{
+        results: versions,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1060,7 +1108,10 @@ defmodule PlausibleWeb.Api.StatsController do
         systems |> to_csv([:name, :visitors])
       end
     else
-      json(conn, systems)
+      json(conn, %{
+        results: systems,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1086,7 +1137,10 @@ defmodule PlausibleWeb.Api.StatsController do
         |> to_csv([:name, :version, :visitors])
       end
     else
-      json(conn, versions)
+      json(conn, %{
+        results: versions,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1110,7 +1164,10 @@ defmodule PlausibleWeb.Api.StatsController do
         sizes |> to_csv([:name, :visitors])
       end
     else
-      json(conn, sizes)
+      json(conn, %{
+        results: sizes,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1146,7 +1203,10 @@ defmodule PlausibleWeb.Api.StatsController do
         :total_conversions
       ])
     else
-      json(conn, conversions)
+      json(conn, %{
+        results: conversions,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1157,7 +1217,7 @@ defmodule PlausibleWeb.Api.StatsController do
     case Plausible.Props.ensure_prop_key_accessible(prop_key, site.owner) do
       :ok ->
         props = breakdown_custom_prop_values(site, params)
-        json(conn, props)
+        json(conn, %{results: props})
 
       {:error, :upgrade_required} ->
         H.payment_required(
diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
index bd504c50bab0..1c2d8fc80637 100644
--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Chrome", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Firefox", "visitors" => 1, "percentage" => 33.3}
              ]
@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
       filters = Jason.encode!(%{props: %{"author" => "John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Chrome", "visitors" => 1, "percentage" => 100}
              ]
     end
@@ -82,7 +82,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
       filters = Jason.encode!(%{props: %{"author" => "!John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Firefox", "visitors" => 1, "percentage" => 50},
                %{"name" => "Safari", "visitors" => 1, "percentage" => 50}
              ]
@@ -99,7 +99,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Chrome",
                  "total_visitors" => 2,
@@ -123,13 +123,13 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Chrome", "visitors" => 1, "percentage" => 100}
              ]
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Chrome", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Firefox", "visitors" => 1, "percentage" => 33.3}
              ]
@@ -154,7 +154,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == []
+      assert json_response(conn, 200)["results"] == []
     end
 
     test "returns (not set) when appropriate", %{conn: conn, site: site} do
@@ -167,7 +167,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 1, "percentage" => 100.0}
              ]
     end
@@ -185,7 +185,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
              ]
     end
@@ -220,6 +220,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
           "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}"
         )
         |> json_response(200)
+        |> Map.get("results")
 
       assert %{
                "browser" => "Chrome",
@@ -254,7 +255,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
           "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "78.0", "visitors" => 2, "percentage" => 66.7, "browser" => "Chrome"},
                %{"name" => "77.0", "visitors" => 1, "percentage" => 33.3, "browser" => "Chrome"}
              ]
@@ -273,7 +274,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
           "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "(not set)",
                  "visitors" => 1,
@@ -317,7 +318,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
           "/api/stats/#{site.domain}/browser-versions?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "browser" => "(not set)",
                  "name" => "(not set)",
diff --git a/test/plausible_web/controllers/api/stats_controller/cities_test.exs b/test/plausible_web/controllers/api/stats_controller/cities_test.exs
index 903d6d625e60..0f97188f234f 100644
--- a/test/plausible_web/controllers/api/stats_controller/cities_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/cities_test.exs
@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do
     test "returns top cities by new visitors", %{conn: conn, site: site} do
       conn = get(conn, "/api/stats/#{site.domain}/cities?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"code" => 588_409, "country_flag" => "🇪🇪", "name" => "Tallinn", "visitors" => 3},
                %{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}
              ]
@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do
       filters = Jason.encode!(%{city: "591632"})
       conn = get(conn, "/api/stats/#{site.domain}/cities?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}
              ]
     end
@@ -61,7 +61,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/cities?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"code" => 588_409, "country_flag" => "🇪🇪", "name" => "Tallinn", "visitors" => 4},
                %{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}
              ]
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index f932456dfbd2..1f5125464a70 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Signup",
                  "visitors" => 2,
@@ -79,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       filters = Jason.encode!(%{props: %{"logged_in" => "true"}})
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 1,
@@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       filters = Jason.encode!(%{props: %{"logged_in" => "!true"}})
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 2,
@@ -157,7 +157,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       filters = Jason.encode!(%{props: %{"logged_in" => "(none)"}})
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 2,
@@ -197,7 +197,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       filters = Jason.encode!(%{props: %{"logged_in" => "!(none)"}})
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 2,
@@ -215,6 +215,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
         conn
         |> get("/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
         |> json_response(200)
+        |> Map.get("results")
 
       assert resp == []
     end
@@ -249,7 +250,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       filters = Jason.encode!(%{browser: "Firefox"})
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 1,
@@ -294,7 +295,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 5,
@@ -340,7 +341,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Payment",
                  "visitors" => 5,
@@ -372,7 +373,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       insert(:goal, %{site: site, event_name: "Payment", currency: :EUR})
 
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")
-      response = json_response(conn, 200)
+      response = json_response(conn, 200)["results"]
 
       assert [
                %{
@@ -414,7 +415,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Signup",
                  "visitors" => 1,
@@ -447,6 +448,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
         get(conn, path <> query)
         |> json_response(200)
+        |> Map.get("results")
       end
 
       expected = [
@@ -488,6 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
 
         get(conn, path <> query)
         |> json_response(200)
+        |> Map.get("results")
       end
 
       expected = [
@@ -539,7 +542,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Signup",
                  "visitors" => 2,
@@ -573,7 +576,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Visit /blog/**",
                  "visitors" => 2,
@@ -611,7 +614,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Visit /blog**",
                  "visitors" => 2,
@@ -649,7 +652,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Signup",
                  "visitors" => 1,
@@ -713,7 +716,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day&date=2019-07-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "conversion_rate" => 100.0,
                  "visitors" => 8,
@@ -801,7 +804,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
                  "events" => 3,
                  "conversion_rate" => 37.5
                }
-             ] = json_response(conn, 200)
+             ] = json_response(conn, 200)["results"]
     end
 
     test "calculates conversion_rate for goals with glob pattern with imported data", %{
@@ -832,7 +835,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
           "/api/stats/#{site.domain}/conversions?period=day"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Visit /blog**",
                  "visitors" => 2,
diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs
index de616c28d1c3..587548782087 100644
--- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs
@@ -16,7 +16,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "EE",
                  "alpha_3" => "EST",
@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "EE",
                  "alpha_3" => "EST",
@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == []
+      assert json_response(conn, 200)["results"] == []
     end
 
     test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do
@@ -90,7 +90,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "EE",
                  "alpha_3" => "EST",
@@ -140,7 +140,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
       filters = Jason.encode!(%{props: %{"author" => "John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "EE",
                  "alpha_3" => "EST",
@@ -182,7 +182,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
       filters = Jason.encode!(%{props: %{"author" => "!John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "GB",
                  "alpha_3" => "GBR",
@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
       filters = Jason.encode!(%{props: %{"author" => "(none)"}})
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "GB",
                  "alpha_3" => "GBR",
@@ -257,7 +257,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
       filters = Jason.encode!(%{props: %{"author" => "!(none)"}})
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "EE",
                  "alpha_3" => "EST",
@@ -279,7 +279,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
       filters = Jason.encode!(%{country: "GB"})
       conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "code" => "GB",
                  "alpha_3" => "GBR",
diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
index 9c1898a02c0f..8f972a139e71 100644
--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
@@ -21,7 +21,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "K2sna Kalle",
@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "K2sna Kalle",
@@ -82,7 +82,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "K2sna Kalle",
@@ -122,7 +122,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&limit=2&page=2"
         )
 
-      assert json_response(conn1, 200) == [
+      assert json_response(conn1, 200)["results"] == [
                %{
                  "visitors" => 3,
                  "name" => "Tiit",
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
                }
              ]
 
-      assert json_response(conn2, 200) == [
+      assert json_response(conn2, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "(none)",
@@ -171,7 +171,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "B",
@@ -207,7 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "(none)",
@@ -250,7 +250,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "0",
                  "visitors" => 1,
@@ -287,7 +287,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "(none)",
                  "visitors" => 1,
@@ -334,7 +334,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "20",
                  "visitors" => 2,
@@ -377,7 +377,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "0",
                  "visitors" => 1,
@@ -424,7 +424,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "1",
                  "visitors" => 2,
@@ -475,7 +475,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "1",
                  "visitors" => 2,
@@ -533,7 +533,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "20",
                  "visitors" => 2,
@@ -584,7 +584,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "20",
                  "visitors" => 2,
@@ -611,7 +611,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/variant?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "A",
@@ -645,7 +645,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "B",
@@ -690,7 +690,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "A",
                  "visitors" => 1,
@@ -735,7 +735,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "A",
                  "visitors" => 1,
@@ -783,7 +783,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "true",
@@ -844,7 +844,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "true",
@@ -889,6 +889,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
 
       returned_metrics =
         json_response(conn, 200)
+        |> Map.get("results")
         |> List.first()
         |> Map.keys()
 
@@ -916,7 +917,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "Sipsik",
@@ -946,7 +947,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "Sipsik",
@@ -973,7 +974,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "Sipsik",
@@ -1004,7 +1005,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "K2sna Kalle",
@@ -1040,7 +1041,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "name" => "bar",
@@ -1078,7 +1079,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
           "/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "name" => "bar",
@@ -1116,11 +1117,13 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
         conn
         |> get("/api/stats/#{site.domain}/custom-prop-values/url?period=day")
         |> json_response(200)
+        |> Map.get("results")
 
       [%{"visitors" => 1, "name" => "two"}] =
         conn
         |> get("/api/stats/#{site.domain}/custom-prop-values/path?period=day")
         |> json_response(200)
+        |> Map.get("results")
     end
 
     test "returns 402 'upgrade required' for any other prop key", %{conn: conn, site: site} do
@@ -1175,7 +1178,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
             "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "visitors" => 5,
                    "name" => "https://two.com",
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index 42eafec8694c..7380984d87cc 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -261,13 +261,14 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
           "imported_sources"
         )
 
-        conn =
-          get(
-            conn,
-            "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"
-          )
-
-        assert conn |> json_response(200) |> Enum.sort() == [
+        results =
+          conn
+          |> get("/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true")
+          |> json_response(200)
+          |> Map.get("results")
+          |> Enum.sort()
+
+        assert results == [
                  %{"name" => "A Nice Newsletter", "visitors" => 1},
                  %{"name" => "Direct / None", "visitors" => 1},
                  %{"name" => "DuckDuckGo", "visitors" => 2},
@@ -338,7 +339,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "bounce_rate" => 100.0,
                    "name" => "social",
@@ -420,7 +421,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "name" => "august",
                    "visitors" => 2,
@@ -509,7 +510,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "name" => "Sweden",
                    "visitors" => 3,
@@ -597,7 +598,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "name" => "blog",
                    "visitors" => 2,
@@ -703,7 +704,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "bounce_rate" => nil,
                    "time_on_page" => 60,
@@ -786,7 +787,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "name" => "/page2",
                    "visitors" => 3,
@@ -859,7 +860,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "🇪🇪"},
                  %{
                    "code" => 2_650_225,
@@ -933,7 +934,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{
                    "code" => "EE",
                    "alpha_3" => "EST",
@@ -997,7 +998,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},
                  %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},
                  %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}
@@ -1050,7 +1051,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert stats = json_response(conn, 200)
+        assert stats = json_response(conn, 200)["results"]
         assert length(stats) == 3
         assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in stats
         assert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats
@@ -1104,7 +1105,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
             "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true"
           )
 
-        assert json_response(conn, 200) == [
+        assert json_response(conn, 200)["results"] == [
                  %{"name" => "Mac", "visitors" => 3, "percentage" => 60},
                  %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}
                ]
diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
index d383fca24b0f..ea9f8990a282 100644
--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Mac", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Android", "visitors" => 1, "percentage" => 33.3}
              ]
@@ -31,7 +31,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 1, "percentage" => 50},
                %{"name" => "Linux", "visitors" => 1, "percentage" => 50}
              ]
@@ -41,7 +41,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 1, "percentage" => 100}
              ]
     end
@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
              ]
     end
@@ -74,7 +74,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Mac",
                  "total_visitors" => 2,
@@ -114,7 +114,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Mac", "visitors" => 1, "percentage" => 100}
              ]
     end
@@ -151,7 +151,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Android", "visitors" => 1, "percentage" => 50},
                %{"name" => "Mac", "visitors" => 1, "percentage" => 50}
              ]
@@ -172,7 +172,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Mac", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Android", "visitors" => 1, "percentage" => 33.3}
              ]
@@ -180,7 +180,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Mac", "visitors" => 3, "percentage" => 60},
                %{"name" => "Android", "visitors" => 2, "percentage" => 40}
              ]
@@ -199,7 +199,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Mac",
                  "total_visitors" => 2,
@@ -241,7 +241,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
           "/api/stats/#{site.domain}/operating-system-versions?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "10.16", "visitors" => 2, "percentage" => 66.7, "os" => "Mac"},
                %{"name" => "10.15", "visitors" => 1, "percentage" => 33.3, "os" => "Mac"}
              ]
@@ -281,7 +281,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
           "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "os" => "(not set)",
                  "name" => "(not set)",
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index ab7b2a230b1c..0df5dc2260b5 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -18,7 +18,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 3, "name" => "/"},
                %{"visitors" => 2, "name" => "/register"},
                %{"visitors" => 1, "name" => "/contact"}
@@ -40,7 +40,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{"hostname" => "*.example.com"})
       conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 3, "name" => "/"},
                %{"visitors" => 2, "name" => "/register"},
                %{"visitors" => 1, "name" => "/contact"},
@@ -50,7 +50,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{"hostname" => "d.example.com"})
       conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 2, "name" => "/register"},
                %{"visitors" => 1, "name" => "/"}
              ]
@@ -74,7 +74,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{props: %{"author" => "John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 1, "name" => "/blog/john-1"}
              ]
     end
@@ -100,7 +100,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{props: %{"author" => "!John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 1, "name" => "/"},
                %{"visitors" => 1, "name" => "/blog/other-post"}
              ]
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{props: %{"prop" => "~bar"}})
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 1, "name" => "/1"},
                %{"visitors" => 1, "name" => "/2"}
              ]
@@ -179,7 +179,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{props: %{"prop" => "~bar|nea"}})
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 1, "name" => "/1"},
                %{"visitors" => 1, "name" => "/2"},
                %{"visitors" => 1, "name" => "/6"}
@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       filters = Jason.encode!(%{props: %{"prop" => "bar", "number" => "1"}})
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 1, "name" => "/1"}
              ]
     end
@@ -266,7 +266,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/blog/john-2",
                  "visitors" => 2,
@@ -328,7 +328,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/blog",
                  "visitors" => 2,
@@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/blog",
                  "visitors" => 2,
@@ -436,7 +436,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/blog/other-post",
                  "visitors" => 2,
@@ -494,7 +494,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/firefox",
                  "visitors" => 2
@@ -534,7 +534,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       conn =
         get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/safari",
                  "visitors" => 1
@@ -578,7 +578,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/",
                  "visitors" => 2,
@@ -625,7 +625,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/",
                  "visitors" => 2,
@@ -679,7 +679,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/",
                  "visitors" => 2,
@@ -725,7 +725,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/articles/post-1",
                  "visitors" => 2,
@@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/blog/(/post-1",
                  "visitors" => 1,
@@ -830,7 +830,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/",
                  "visitors" => 2,
@@ -862,7 +862,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 3, "name" => "/"},
                %{"visitors" => 2, "name" => "/register"},
                %{"visitors" => 1, "name" => "/contact"}
@@ -870,7 +870,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 4, "name" => "/"},
                %{"visitors" => 3, "name" => "/register"},
                %{"visitors" => 1, "name" => "/contact"}
@@ -901,7 +901,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50.0,
                  "time_on_page" => 900.0,
@@ -948,7 +948,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50,
                  "name" => "/about",
@@ -1027,7 +1027,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50,
                  "name" => "/about-blog",
@@ -1055,6 +1055,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
                conn
                |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
                |> json_response(200)
+               |> Map.get("results")
     end
 
     test "ignores page refresh when calculating time on page", %{conn: conn, site: site} do
@@ -1072,6 +1073,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
                conn
                |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
                |> json_response(200)
+               |> Map.get("results")
     end
 
     test "calculates time on page per unique transition within session", %{conn: conn, site: site} do
@@ -1105,6 +1107,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
                conn
                |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
                |> json_response(200)
+               |> Map.get("results")
     end
 
     test "calculates bounce rate and time on page for pages with imported data", %{
@@ -1150,7 +1153,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 40.0,
                  "time_on_page" => 800.0,
@@ -1177,7 +1180,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"visitors" => 2, "name" => "/page1"},
                %{"visitors" => 1, "name" => "/page2"}
              ]
@@ -1195,7 +1198,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"total_visitors" => 3, "visitors" => 1, "name" => "/", "conversion_rate" => 33.3}
              ]
     end
@@ -1227,7 +1230,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50,
                  "name" => "/",
@@ -1276,7 +1279,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50,
                  "name" => "/",
@@ -1332,7 +1335,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "bounce_rate" => 50,
                  "name" => "/aaa",
@@ -1386,7 +1389,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "visits" => 2,
@@ -1442,7 +1445,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 1,
                  "visits" => 1,
@@ -1497,7 +1500,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 2,
                  "visits" => 2,
@@ -1518,7 +1521,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visitors" => 3,
                  "visits" => 5,
@@ -1581,7 +1584,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
         )
 
       # We're going to only join sessions where the exit hostname matches the filter
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "/page1", "visit_duration" => 0, "visitors" => 1, "visits" => 1},
                %{"name" => "/page2", "visit_duration" => 0, "visitors" => 1, "visits" => 1}
              ]
@@ -1610,6 +1613,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/pages?date=2021-01-01&period=day&filters=#{filters}&limit=#{limit}&page=#{page}"
         )
         |> json_response(200)
+        |> Map.get("results")
         |> Enum.map(fn %{"name" => "/signup/" <> seq} ->
           seq
         end)
@@ -1669,7 +1673,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "total_visitors" => 2,
                  "visitors" => 1,
@@ -1700,7 +1704,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == []
+      assert json_response(conn, 200)["results"] == []
     end
 
     test "filter by :matches_member entry_page with imported data", %{conn: conn, site: site} do
@@ -1731,7 +1735,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/entry-pages#{q}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "visit_duration" => 100.0,
                  "name" => "/a",
@@ -1781,7 +1785,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66},
                %{"name" => "/page2", "visitors" => 1, "visits" => 1, "exit_rate" => 100}
              ]
@@ -1829,7 +1833,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
         )
 
       # We're going to only join sessions where the entry hostname matches the filter
-      assert json_response(conn, 200) ==
+      assert json_response(conn, 200)["results"] ==
                [%{"name" => "/page1", "visitors" => 1, "visits" => 1}]
     end
 
@@ -1867,7 +1871,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "/", "visitors" => 1, "visits" => 1}
              ]
     end
@@ -1911,7 +1915,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66},
                %{"name" => "/page2", "visitors" => 1, "visits" => 1, "exit_rate" => 100}
              ]
@@ -1922,7 +1926,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/page2",
                  "visitors" => 3,
@@ -1973,7 +1977,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/exit1",
                  "visitors" => 1,
@@ -2026,7 +2030,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "/exit1", "visitors" => 1, "visits" => 1},
                %{"name" => "/exit2", "visitors" => 1, "visits" => 1}
              ]
@@ -2047,7 +2051,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == []
+      assert json_response(conn, 200)["results"] == []
     end
 
     test "filter by :is_not exit_page with imported data", %{conn: conn, site: site} do
@@ -2080,7 +2084,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/exit-pages#{q}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "exit_rate" => 50.0,
                  "name" => "/a",
diff --git a/test/plausible_web/controllers/api/stats_controller/regions_test.exs b/test/plausible_web/controllers/api/stats_controller/regions_test.exs
index 23ac063bab3d..711ae957f8eb 100644
--- a/test/plausible_web/controllers/api/stats_controller/regions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/regions_test.exs
@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do
     test "returns top cities by new visitors", %{conn: conn, site: site} do
       conn = get(conn, "/api/stats/#{site.domain}/regions?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"code" => "EE-37", "country_flag" => "🇪🇪", "name" => "Harjumaa", "visitors" => 3},
                %{"code" => "EE-39", "country_flag" => "🇪🇪", "name" => "Hiiumaa", "visitors" => 2}
              ]
@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do
       filters = Jason.encode!(%{region: "EE-39"})
       conn = get(conn, "/api/stats/#{site.domain}/regions?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"code" => "EE-39", "country_flag" => "🇪🇪", "name" => "Hiiumaa", "visitors" => 2}
              ]
     end
diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
index 128d1293df65..13f7736bc4c6 100644
--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Laptop", "visitors" => 1, "percentage" => 33.3}
              ]
@@ -39,7 +39,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
           "date" => "2021-01-01"
         })
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 1, "percentage" => 100},
                %{"name" => "Laptop", "visitors" => 1, "percentage" => 100}
              ]
@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 1, "percentage" => 50},
                %{"name" => "Desktop", "visitors" => 1, "percentage" => 50}
              ]
@@ -67,7 +67,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
       filters = Jason.encode!(%{screen: "(not set)"})
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 1, "percentage" => 100}
              ]
     end
@@ -84,7 +84,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
              ]
     end
@@ -117,7 +117,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
       filters = Jason.encode!(%{props: %{"author" => "John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 1, "percentage" => 100}
              ]
     end
@@ -152,7 +152,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
       filters = Jason.encode!(%{props: %{"author" => "!John Doe"}})
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Mobile", "visitors" => 1, "percentage" => 50},
                %{"name" => "Tablet", "visitors" => 1, "percentage" => 50}
              ]
@@ -173,14 +173,14 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Laptop", "visitors" => 1, "percentage" => 33.3}
              ]
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},
                %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},
                %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}
@@ -215,7 +215,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
           "with_imported" => "true"
         })
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 2, "percentage" => 100},
                %{"name" => "Laptop", "visitors" => 2, "percentage" => 100}
              ]
@@ -232,7 +232,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Desktop",
                  "total_visitors" => 2,
@@ -258,7 +258,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},
                %{"name" => "Mobile", "visitors" => 1, "percentage" => 33.3}
              ]
diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
index dfa9480c02a0..a76b1da27415 100644
--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
@@ -33,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/sources")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 3},
                %{"name" => "DuckDuckGo", "visitors" => 2},
                %{"name" => "Direct / None", "visitors" => 1}
@@ -83,7 +83,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -187,7 +187,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Facebook", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -241,7 +241,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -270,14 +270,14 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/sources")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
 
       conn = get(conn, "/api/stats/#{site.domain}/sources?with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 4},
                %{"name" => "DuckDuckGo", "visitors" => 2}
              ]
@@ -310,7 +310,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "DuckDuckGo",
                  "visitors" => 1,
@@ -375,7 +375,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "DuckDuckGo",
                  "visitors" => 1,
@@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Google",
                  "visitors" => 3,
@@ -433,7 +433,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/sources?period=realtime")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -460,13 +460,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
 
       conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "DuckDuckGo", "visitors" => 2}
              ]
     end
@@ -490,7 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
       filters = Jason.encode!(%{"page" => "/page1"})
       conn = get(conn, "/api/stats/#{site.domain}/sources?filters=#{filters}")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "Google", "visitors" => 2},
                %{"name" => "DuckDuckGo", "visitors" => 1}
              ]
@@ -538,7 +538,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           )
 
         # nobody landed on one.example.com from utm_param=ad
-        assert json_response(conn, 200) == []
+        assert json_response(conn, 200)["results"] == []
       end
     end
   end
@@ -589,7 +589,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "social",
                  "visitors" => 1,
@@ -610,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "social",
                  "visitors" => 2,
@@ -669,7 +669,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "social",
                  "visitors" => 1,
@@ -684,7 +684,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "social",
                  "visitors" => 2,
@@ -745,7 +745,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "august",
                  "visitors" => 2,
@@ -766,7 +766,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "august",
                  "visitors" => 3,
@@ -829,7 +829,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "profile",
                  "visitors" => 1,
@@ -844,7 +844,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "profile",
                  "visitors" => 2,
@@ -886,7 +886,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_sources?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "newsletter",
                  "visitors" => 2,
@@ -953,7 +953,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Sweden",
                  "visitors" => 2,
@@ -974,7 +974,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Sweden",
                  "visitors" => 3,
@@ -1037,7 +1037,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "oat milk",
                  "visitors" => 1,
@@ -1052,7 +1052,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "oat milk",
                  "visitors" => 2,
@@ -1113,7 +1113,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "blog",
                  "visitors" => 2,
@@ -1134,7 +1134,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "blog",
                  "visitors" => 3,
@@ -1197,7 +1197,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "ad",
                  "visitors" => 1,
@@ -1212,7 +1212,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "ad",
                  "visitors" => 2,
@@ -1257,7 +1257,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Twitter",
                  "total_visitors" => 2,
@@ -1299,7 +1299,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == []
+      assert json_response(conn, 200)["results"] == []
     end
 
     test "returns top referrers for a custom goal and filtered by hostname (2)",
@@ -1330,7 +1330,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "conversion_rate" => 100.0,
                  "name" => "Facebook",
@@ -1380,7 +1380,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "DuckDuckGo",
                  "visitors" => 1,
@@ -1431,7 +1431,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "DuckDuckGo",
                  "visitors" => 1,
@@ -1473,7 +1473,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "Twitter",
                  "total_visitors" => 2,
@@ -1513,7 +1513,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/referrers/10words?period=day"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "10words.com", "visitors" => 2},
                %{"name" => "10words.com/page1", "visitors" => 1}
              ]
@@ -1559,7 +1559,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/referrers/example?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "example.com/page1", "visitors" => 1}
              ]
     end
@@ -1596,7 +1596,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/referrers/10words?period=day&date=2021-01-01&detailed=true"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "10words.com",
                  "visitors" => 2,
@@ -1649,13 +1649,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
 
       conn = get(conn, "/api/stats/#{site.domain}/referrers/!Google?period=day")
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{"name" => "duckduckgo.com", "visitors" => 1}
              ]
 
       conn = get(conn, "/api/stats/#{site.domain}/referrers/Google|DuckDuckGo?period=day")
 
-      assert [entry1, entry2] = json_response(conn, 200)
+      assert [entry1, entry2] = json_response(conn, 200)["results"]
       assert %{"name" => "google.com", "visitors" => 2} in [entry1, entry2]
       assert %{"name" => "duckduckgo.com", "visitors" => 1} in [entry1, entry2]
     end
@@ -1688,7 +1688,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "10words.com",
                  "total_visitors" => 2,
@@ -1726,7 +1726,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
           "/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}"
         )
 
-      assert json_response(conn, 200) == [
+      assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "10words.com",
                  "total_visitors" => 2,

From 453cdab161af6cbc5072b5981ec4afd3e9c6d098 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 10:38:00 +0100
Subject: [PATCH 10/41] extend ListReport component with an optional
 afterFetchData prop

---
 assets/js/dashboard/stats/reports/list.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/assets/js/dashboard/stats/reports/list.js b/assets/js/dashboard/stats/reports/list.js
index ba81c908597c..52fe138db5b7 100644
--- a/assets/js/dashboard/stats/reports/list.js
+++ b/assets/js/dashboard/stats/reports/list.js
@@ -110,6 +110,12 @@ function ExternalLink({ item, externalLinkDest }) {
 
 //   * `color` - color of the comparison bars in light-mode
 
+//   * `afterFetchData` - a function to be called directly after `fetchData`. Receives the,
+//     raw API response as an argument. The return value is ignored by ListReport. Allows
+//     hooking into the request lifecycle and doing actions with returned metadata. For
+//     example, the parent component might want to control what happens when imported data
+//     is included or not.
+
 export default function ListReport(props) {
   const [state, setState] = useState({ loading: true, list: null })
   const [visible, setVisible] = useState(false)
@@ -125,6 +131,10 @@ export default function ListReport(props) {
     }
     props.fetchData()
       .then((response) => {
+        if (props.afterFetchData) {
+          props.afterFetchData(response)
+        }
+
         setState({ loading: false, list: response.results })
       })
   }, [props.keyLabel, props.query])

From 12384373ed2060a3c932d7b828a91746cea280e7 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 11:48:03 +0100
Subject: [PATCH 11/41] turn Devices into a fn comp

---
 assets/js/dashboard/stats/devices/index.js | 75 +++++++++-------------
 1 file changed, 32 insertions(+), 43 deletions(-)

diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js
index efc22fbc4e40..9c7668ded333 100644
--- a/assets/js/dashboard/stats/devices/index.js
+++ b/assets/js/dashboard/stats/devices/index.js
@@ -1,5 +1,4 @@
-import React from 'react';
-
+import React, {useState} from 'react';
 import * as storage from '../../util/storage'
 import { getFiltersByKeyPrefix, isFilteringOnFixedValue } from '../../util/filters'
 import ListReport from '../reports/list'
@@ -157,45 +156,37 @@ function iconFor(screenSize) {
   }
 }
 
-export default class Devices extends React.Component {
-  constructor(props) {
-    super(props)
-    this.tabKey = `deviceTab__${props.site.domain}`
-    const storedTab = storage.getItem(this.tabKey)
-    this.state = {
-      mode: storedTab || 'browser'
-    }
-  }
+export default function Devices(props) {
+  const {site, query} = props
+  const tabKey = `deviceTab__${site.domain}`
+  const storedTab = storage.getItem(tabKey)
+  const [mode, setMode] = useState(storedTab || 'browser')
 
-  setMode(mode) {
-    return () => {
-      storage.setItem(this.tabKey, mode)
-      this.setState({ mode })
-    }
+  function switchTab(mode) {
+    storage.setItem(tabKey, mode)
+    setMode(mode)
   }
 
-  renderContent() {
-    switch (this.state.mode) {
+  function renderContent() {
+    switch (mode) {
       case 'browser':
-        if (isFilteringOnFixedValue(this.props.query, 'browser')) {
-          return <BrowserVersions site={this.props.site} query={this.props.query} />
+        if (isFilteringOnFixedValue(query, 'browser')) {
+          return <BrowserVersions site={site} query={query} />
         }
-        return <Browsers site={this.props.site} query={this.props.query} />
+        return <Browsers site={site} query={query} />
       case 'os':
-        if (isFilteringOnFixedValue(this.props.query, 'os')) {
-          return <OperatingSystemVersions site={this.props.site} query={this.props.query} />
+        if (isFilteringOnFixedValue(query, 'os')) {
+          return <OperatingSystemVersions site={site} query={query} />
         }
-        return <OperatingSystems site={this.props.site} query={this.props.query} />
+        return <OperatingSystems site={site} query={query} />
       case 'size':
       default:
-        return (
-          <ScreenSizes site={this.props.site} query={this.props.query} />
-        )
+        return <ScreenSizes site={site} query={query} />
     }
   }
 
-  renderPill(name, mode) {
-    const isActive = this.state.mode === mode
+  function renderPill(name, pill) {
+    const isActive = mode === pill
 
     if (isActive) {
       return (
@@ -210,28 +201,26 @@ export default class Devices extends React.Component {
     return (
       <button
         className="cursor-pointer hover:text-indigo-600"
-        onClick={this.setMode(mode)}
+        onClick={() => switchTab(pill)}
       >
         {name}
       </button>
     )
   }
 
-  render() {
-    return (
-      <div>
-        <div className="flex justify-between w-full">
-          <h3 className="font-bold dark:text-gray-100">Devices</h3>
-          <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
-            {this.renderPill('Browser', 'browser')}
-            {this.renderPill('OS', 'os')}
-            {this.renderPill('Size', 'size')}
-          </div>
+  return (
+    <div>
+      <div className="flex justify-between w-full">
+        <h3 className="font-bold dark:text-gray-100">Devices</h3>
+        <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
+          {renderPill('Browser', 'browser')}
+          {renderPill('OS', 'os')}
+          {renderPill('Size', 'size')}
         </div>
-        {this.renderContent()}
       </div>
-    )
-  }
+      {renderContent()}
+    </div>
+  )
 }
 
 function getSingleFilter(query, filterKey) {

From 3a6c22936343e957824118bb982ff4dd706cf9e3 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 14:17:44 +0100
Subject: [PATCH 12/41] add not_requested as a skip_imported_reason

---
 lib/plausible/stats/comparisons.ex                  | 10 ++++------
 lib/plausible/stats/query.ex                        | 13 +++++++------
 .../controllers/api/external_stats_controller.ex    |  2 +-
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index 86ee0f3554c4..c7bd4bf85898 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -162,12 +162,10 @@ defmodule Plausible.Stats.Comparisons do
     Date.add(date, -days_to_subtract)
   end
 
-  defp maybe_include_imported(query, %Query{imported_data_requested: false}, _site) do
-    %Stats.Query{query | include_imported: false}
-  end
+  defp maybe_include_imported(query, source_query, site) do
+    requested? = source_query.imported_data_requested
 
-  defp maybe_include_imported(query, _source_query, site) do
-    case Query.ensure_include_imported(query, site) do
+    case Query.ensure_include_imported(query, site, requested?) do
       :ok ->
         struct!(query,
           imported_data_requested: true,
@@ -176,7 +174,7 @@ defmodule Plausible.Stats.Comparisons do
 
       {:error, reason} ->
         struct!(query,
-          imported_data_requested: true,
+          imported_data_requested: requested?,
           include_imported: false,
           skip_imported_reason: reason
         )
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 90f347d61a4b..3fe853e6a4f1 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -249,11 +249,11 @@ defmodule Plausible.Stats.Query do
   defp put_imported_opts(query, site, params) do
     requested? = params["with_imported"] == "true"
 
-    case ensure_include_imported(query, site) do
+    case ensure_include_imported(query, site, requested?) do
       :ok ->
         struct!(query,
-          imported_data_requested: requested?,
-          include_imported: requested?
+          imported_data_requested: true,
+          include_imported: true
         )
 
       {:error, reason} ->
@@ -265,10 +265,11 @@ defmodule Plausible.Stats.Query do
     end
   end
 
-  @spec ensure_include_imported(t(), Plausible.Site.t()) ::
-          :ok | {:error, :no_imported_data | :out_of_range | :unsupported_query}
-  def ensure_include_imported(query, site) do
+  @spec ensure_include_imported(t(), Plausible.Site.t(), boolean()) ::
+          :ok | {:error, :not_requested | :no_imported_data | :out_of_range | :unsupported_query}
+  def ensure_include_imported(query, site, requested?) do
     cond do
+      not requested? -> {:error, :not_requested}
       is_nil(site.latest_import_end_date) -> {:error, :no_imported_data}
       Date.after?(query.date_range.first, site.latest_import_end_date) -> {:error, :out_of_range}
       not Imported.schema_supports_query?(query) -> {:error, :unsupported_query}
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 77e3d51ca500..7fcb2fca3bc6 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
   end
 
   defp maybe_add_warning(payload, %{skip_imported_reason: reason})
-       when reason in [nil, :no_imported_data, :out_of_range] do
+       when reason in [nil, :not_requested, :no_imported_data, :out_of_range] do
     payload
   end
 

From ad5d16f8f5592945c9487255a80061309be09105 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 21 May 2024 14:25:05 +0100
Subject: [PATCH 13/41] display warning icons in the dashboard

---
 .../dashboard/stats/behaviours/conversions.js |  3 +-
 .../stats/behaviours/goal-conversions.js      |  9 +++--
 assets/js/dashboard/stats/behaviours/index.js | 20 +++++++---
 assets/js/dashboard/stats/devices/index.js    | 38 +++++++++++++------
 .../imported-query-unsupported-warning.js     | 14 +++++++
 assets/js/dashboard/stats/locations/index.js  | 37 ++++++++++++------
 assets/js/dashboard/stats/locations/map.js    |  4 ++
 assets/js/dashboard/stats/pages/index.js      | 32 +++++++++++-----
 .../dashboard/stats/sources/referrer-list.js  | 17 ++++++++-
 .../js/dashboard/stats/sources/source-list.js | 23 ++++++++---
 .../controllers/api/stats_controller.ex       | 36 ++++++++++--------
 .../api/stats_controller/imported_test.exs    |  4 +-
 12 files changed, 172 insertions(+), 65 deletions(-)
 create mode 100644 assets/js/dashboard/stats/imported-query-unsupported-warning.js

diff --git a/assets/js/dashboard/stats/behaviours/conversions.js b/assets/js/dashboard/stats/behaviours/conversions.js
index c318c11d3946..c474b5786714 100644
--- a/assets/js/dashboard/stats/behaviours/conversions.js
+++ b/assets/js/dashboard/stats/behaviours/conversions.js
@@ -6,7 +6,7 @@ import { CR_METRIC } from '../reports/metrics';
 import ListReport from '../reports/list';
 
 export default function Conversions(props) {
-  const { site, query } = props
+  const { site, query, afterFetchData } = props
 
   function fetchConversions() {
     return api.get(url.apiPath(site, '/conversions'), query, { limit: 9 })
@@ -23,6 +23,7 @@ export default function Conversions(props) {
   return (
     <ListReport
       fetchData={fetchConversions}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Goal"
       onClick={props.onGoalFilterClick}
diff --git a/assets/js/dashboard/stats/behaviours/goal-conversions.js b/assets/js/dashboard/stats/behaviours/goal-conversions.js
index bd3c867878db..2d7b5263cc9a 100644
--- a/assets/js/dashboard/stats/behaviours/goal-conversions.js
+++ b/assets/js/dashboard/stats/behaviours/goal-conversions.js
@@ -31,7 +31,7 @@ export function specialTitleWhenGoalFilter(query, defaultTitle) {
 }
 
 function SpecialPropBreakdown(props) {
-  const { site, query, prop } = props
+  const { site, query, prop, afterFetchData } = props
 
   function fetchData() {
     return api.get(url.apiPath(site, `/custom-prop-values/${prop}`), query)
@@ -55,6 +55,7 @@ function SpecialPropBreakdown(props) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel={prop}
       metrics={[
@@ -73,12 +74,12 @@ function SpecialPropBreakdown(props) {
 }
 
 export default function GoalConversions(props) {
-  const {site, query} = props
+  const {site, query, afterFetchData} = props
 
   const specialGoal = getSpecialGoal(query)
   if (specialGoal) {
-    return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} />
+    return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} afterFetchData={afterFetchData} />
   } else {
-    return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick}/>
+    return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick} afterFetchData={afterFetchData} />
   }
 }
diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js
index 55966a703c8a..85b8b48f90e9 100644
--- a/assets/js/dashboard/stats/behaviours/index.js
+++ b/assets/js/dashboard/stats/behaviours/index.js
@@ -3,7 +3,7 @@ import { Menu, Transition } from '@headlessui/react'
 import { ChevronDownIcon } from '@heroicons/react/20/solid'
 import classNames from 'classnames'
 import * as storage from '../../util/storage'
-
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning'
 import GoalConversions, { specialTitleWhenGoalFilter } from './goal-conversions'
 import Properties from './props'
 import { FeatureSetupNotice } from '../../components/notice'
@@ -48,6 +48,8 @@ export default function Behaviours(props) {
 
   const [showingPropsForGoalFilter, setShowingPropsForGoalFilter] = useState(false)
 
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
+
   const onGoalFilterClick = useCallback((e) => {
     const goalName = e.target.innerHTML
     const isSpecialGoal = Object.keys(SPECIAL_GOALS).includes(goalName)
@@ -170,9 +172,14 @@ export default function Behaviours(props) {
     )
   }
 
+  function afterFetchGoalData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime())
+  }
+
   function renderConversions() {
     if (site.hasGoals) {
-      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} />
+      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} afterFetchData={afterFetchGoalData}/>
     }
     else if (adminAccess) {
       return (
@@ -330,9 +337,12 @@ export default function Behaviours(props) {
       <div className="items-start justify-between block w-full mt-6 md:flex">
         <div className="w-full p-4 bg-white rounded shadow-xl dark:bg-gray-825">
           <div className="flex justify-between w-full">
-            <h3 className="font-bold dark:text-gray-100">
-              {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}
-            </h3>
+            <div className="flex gap-x-1">
+              <h3 className="font-bold dark:text-gray-100">
+                {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}
+              </h3>
+              <ImportedQueryUnsupportedWarning condition={mode === CONVERSIONS  && importedQueryUnsupported}/>
+            </div>
             {tabs()}
           </div>
           {renderContent()}
diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js
index 9c7668ded333..9abb656c7876 100644
--- a/assets/js/dashboard/stats/devices/index.js
+++ b/assets/js/dashboard/stats/devices/index.js
@@ -5,8 +5,9 @@ import ListReport from '../reports/list'
 import * as api from '../../api'
 import * as url from '../../util/url'
 import { VISITORS_METRIC, PERCENTAGE_METRIC, maybeWithCR } from '../reports/metrics';
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';
 
-function Browsers({ query, site }) {
+function Browsers({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/browsers'), query)
   }
@@ -21,6 +22,7 @@ function Browsers({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Browser"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -29,7 +31,7 @@ function Browsers({ query, site }) {
   )
 }
 
-function BrowserVersions({ query, site }) {
+function BrowserVersions({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/browser-versions'), query)
   }
@@ -47,6 +49,7 @@ function BrowserVersions({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Browser version"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -56,7 +59,7 @@ function BrowserVersions({ query, site }) {
 
 }
 
-function OperatingSystems({ query, site }) {
+function OperatingSystems({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/operating-systems'), query)
   }
@@ -71,6 +74,7 @@ function OperatingSystems({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Operating system"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -79,7 +83,7 @@ function OperatingSystems({ query, site }) {
   )
 }
 
-function OperatingSystemVersions({ query, site }) {
+function OperatingSystemVersions({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/operating-system-versions'), query)
   }
@@ -97,6 +101,7 @@ function OperatingSystemVersions({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Operating System Version"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -106,7 +111,7 @@ function OperatingSystemVersions({ query, site }) {
 
 }
 
-function ScreenSizes({ query, site }) {
+function ScreenSizes({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/screen-sizes'), query)
   }
@@ -125,6 +130,7 @@ function ScreenSizes({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Screen size"
       metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}
@@ -161,27 +167,34 @@ export default function Devices(props) {
   const tabKey = `deviceTab__${site.domain}`
   const storedTab = storage.getItem(tabKey)
   const [mode, setMode] = useState(storedTab || 'browser')
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
 
   function switchTab(mode) {
     storage.setItem(tabKey, mode)
     setMode(mode)
   }
 
+  function afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = query.period === 'realtime'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime)
+  }
+
   function renderContent() {
     switch (mode) {
       case 'browser':
         if (isFilteringOnFixedValue(query, 'browser')) {
-          return <BrowserVersions site={site} query={query} />
+          return <BrowserVersions site={site} query={query} afterFetchData={afterFetchData} />
         }
-        return <Browsers site={site} query={query} />
+        return <Browsers site={site} query={query} afterFetchData={afterFetchData} />
       case 'os':
         if (isFilteringOnFixedValue(query, 'os')) {
-          return <OperatingSystemVersions site={site} query={query} />
+          return <OperatingSystemVersions site={site} query={query} afterFetchData={afterFetchData} />
         }
-        return <OperatingSystems site={site} query={query} />
+        return <OperatingSystems site={site} query={query} afterFetchData={afterFetchData} />
       case 'size':
       default:
-        return <ScreenSizes site={site} query={query} />
+        return <ScreenSizes site={site} query={query} afterFetchData={afterFetchData} />
     }
   }
 
@@ -211,7 +224,10 @@ export default function Devices(props) {
   return (
     <div>
       <div className="flex justify-between w-full">
-        <h3 className="font-bold dark:text-gray-100">Devices</h3>
+        <div className="flex gap-x-1">
+          <h3 className="font-bold dark:text-gray-100">Devices</h3>
+          <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>
+        </div>
         <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
           {renderPill('Browser', 'browser')}
           {renderPill('OS', 'os')}
diff --git a/assets/js/dashboard/stats/imported-query-unsupported-warning.js b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
new file mode 100644
index 000000000000..5c6c64adb0de
--- /dev/null
+++ b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
@@ -0,0 +1,14 @@
+import React from "react";
+import { ExclamationCircleIcon } from '@heroicons/react/24/outline'
+
+export default function ImportedQueryUnsupportedWarning({condition}) {
+  if (condition) {
+    return (
+      <span tooltip="Imported data is excluded due to applied filters">
+        <ExclamationCircleIcon className="w-6 h-6" />
+      </span>
+    )
+  } else {
+    return null
+  }
+}
\ No newline at end of file
diff --git a/assets/js/dashboard/stats/locations/index.js b/assets/js/dashboard/stats/locations/index.js
index 60fad2d0e888..0a80592c0cbb 100644
--- a/assets/js/dashboard/stats/locations/index.js
+++ b/assets/js/dashboard/stats/locations/index.js
@@ -8,8 +8,9 @@ import {apiPath, sitePath} from '../../util/url'
 import ListReport from '../reports/list'
 import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';
 import { getFiltersByKeyPrefix } from '../../util/filters';
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';
 
-function Countries({query, site, onClick}) {
+function Countries({query, site, onClick, afterFetchData}) {
   function fetchData() {
     return api.get(apiPath(site, '/countries'), query, { limit: 9 })
   }
@@ -29,6 +30,7 @@ function Countries({query, site, onClick}) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       onClick={onClick}
       keyLabel="Country"
@@ -41,7 +43,7 @@ function Countries({query, site, onClick}) {
   )
 }
 
-function Regions({query, site, onClick}) {
+function Regions({query, site, onClick, afterFetchData}) {
   function fetchData() {
     return api.get(apiPath(site, '/regions'), query, {limit: 9})
   }
@@ -61,6 +63,7 @@ function Regions({query, site, onClick}) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       onClick={onClick}
       keyLabel="Region"
@@ -73,7 +76,7 @@ function Regions({query, site, onClick}) {
   )
 }
 
-function Cities({query, site}) {
+function Cities({query, site, afterFetchData}) {
   function fetchData() {
     return api.get(apiPath(site, '/cities'), query, {limit: 9})
   }
@@ -93,6 +96,7 @@ function Cities({query, site}) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="City"
       metrics={maybeWithCR([VISITORS_METRIC], query)}
@@ -116,10 +120,12 @@ export default class Locations extends React.Component {
     super(props)
     this.onCountryFilter = this.onCountryFilter.bind(this)
     this.onRegionFilter = this.onRegionFilter.bind(this)
+    this.afterFetchData = this.afterFetchData.bind(this)
     this.tabKey = `geoTab__${  props.site.domain}`
     const storedTab = storage.getItem(this.tabKey)
     this.state = {
-      mode: storedTab || 'map'
+      mode: storedTab || 'map',
+      importedQueryUnsupported: false
     }
   }
 
@@ -156,17 +162,23 @@ export default class Locations extends React.Component {
     this.setMode('cities')()
   }
 
+  afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = this.props.query.period === 'realtime'
+    this.setState({importedQueryUnsupported: unsupportedQuery && !isRealtime})
+  }
+
 	renderContent() {
     switch(this.state.mode) {
 		case "cities":
-      return <Cities site={this.props.site} query={this.props.query} />
+      return <Cities site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />
 		case "regions":
-      return <Regions onClick={this.onRegionFilter} site={this.props.site} query={this.props.query} />
+      return <Regions onClick={this.onRegionFilter} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />
 		case "countries":
-      return <Countries onClick={this.onCountryFilter('countries')} site={this.props.site} query={this.props.query} />
+      return <Countries onClick={this.onCountryFilter('countries')} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />
     case "map":
     default:
-      return <CountriesMap onClick={this.onCountryFilter('map')} site={this.props.site} query={this.props.query}/>
+      return <CountriesMap onClick={this.onCountryFilter('map')} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />
     }
   }
 
@@ -197,9 +209,12 @@ export default class Locations extends React.Component {
     return (
       <div>
         <div className="w-full flex justify-between">
-          <h3 className="font-bold dark:text-gray-100">
-            {labelFor[this.state.mode] || 'Locations'}
-          </h3>
+          <div className="flex gap-x-1">
+            <h3 className="font-bold dark:text-gray-100">
+              {labelFor[this.state.mode] || 'Locations'}
+            </h3>
+            <ImportedQueryUnsupportedWarning condition={this.state.importedQueryUnsupported} />
+          </div>
           <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">
             { this.renderPill('Map', 'map') }
             { this.renderPill('Countries', 'countries') }
diff --git a/assets/js/dashboard/stats/locations/map.js b/assets/js/dashboard/stats/locations/map.js
index 5628f306f7f7..4006607c80ea 100644
--- a/assets/js/dashboard/stats/locations/map.js
+++ b/assets/js/dashboard/stats/locations/map.js
@@ -76,6 +76,10 @@ class Countries extends React.Component {
   fetchCountries() {
     return api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/countries`, this.props.query, {limit: 300})
       .then((response) => {
+        if (this.props.afterFetchData) {
+          this.props.afterFetchData(response)
+        }
+
         this.setState({loading: false, countries: response.results})
       })
   }
diff --git a/assets/js/dashboard/stats/pages/index.js b/assets/js/dashboard/stats/pages/index.js
index d82492a248dc..ec023ee2c91d 100644
--- a/assets/js/dashboard/stats/pages/index.js
+++ b/assets/js/dashboard/stats/pages/index.js
@@ -5,8 +5,9 @@ import * as url from '../../util/url'
 import * as api from '../../api'
 import ListReport from './../reports/list'
 import { VISITORS_METRIC, maybeWithCR } from './../reports/metrics';
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';
 
-function EntryPages({ query, site }) {
+function EntryPages({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/entry-pages'), query, { limit: 9 })
   }
@@ -25,6 +26,7 @@ function EntryPages({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Entry page"
       metrics={maybeWithCR([{ ...VISITORS_METRIC, label: 'Unique Entrances' }], query)}
@@ -36,7 +38,7 @@ function EntryPages({ query, site }) {
   )
 }
 
-function ExitPages({ query, site }) {
+function ExitPages({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/exit-pages'), query, { limit: 9 })
   }
@@ -55,6 +57,7 @@ function ExitPages({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Exit page"
       metrics={maybeWithCR([{ ...VISITORS_METRIC, label: "Unique Exits" }], query)}
@@ -66,7 +69,7 @@ function ExitPages({ query, site }) {
   )
 }
 
-function TopPages({ query, site }) {
+function TopPages({ query, site, afterFetchData }) {
   function fetchData() {
     return api.get(url.apiPath(site, '/pages'), query, { limit: 9 })
   }
@@ -85,6 +88,7 @@ function TopPages({ query, site }) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Page"
       metrics={maybeWithCR([VISITORS_METRIC], query)}
@@ -107,21 +111,28 @@ export default function Pages(props) {
   const tabKey = `pageTab__${site.domain}`
   const storedTab = storage.getItem(tabKey)
   const [mode, setMode] = useState(storedTab || 'pages')
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
 
   function switchTab(mode) {
     storage.setItem(tabKey, mode)
     setMode(mode)
   }
 
+  function afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = query.period === 'realtime'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime)
+  }
+
   function renderContent() {
     switch (mode) {
       case "entry-pages":
-        return <EntryPages site={site} query={query} />
+        return <EntryPages site={site} query={query} afterFetchData={afterFetchData} />
       case "exit-pages":
-        return <ExitPages site={site} query={query} />
+        return <ExitPages site={site} query={query} afterFetchData={afterFetchData} />
       case "pages":
       default:
-        return <TopPages site={site} query={query} />
+        return <TopPages site={site} query={query} afterFetchData={afterFetchData} />
     }
   }
 
@@ -153,9 +164,12 @@ export default function Pages(props) {
     <div>
       {/* Header Container */}
       <div className="w-full flex justify-between">
-        <h3 className="font-bold dark:text-gray-100">
-          {labelFor[mode] || 'Page Visits'}
-        </h3>
+        <div className="flex gap-x-1">
+          <h3 className="font-bold dark:text-gray-100">
+            {labelFor[mode] || 'Page Visits'}
+          </h3>
+          <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>
+        </div>
         <div className="flex font-medium text-xs text-gray-500 dark:text-gray-400 space-x-2">
           {renderPill('Top Pages', 'pages')}
           {renderPill('Entry Pages', 'entry-pages')}
diff --git a/assets/js/dashboard/stats/sources/referrer-list.js b/assets/js/dashboard/stats/sources/referrer-list.js
index 62800335b456..daadbab21c72 100644
--- a/assets/js/dashboard/stats/sources/referrer-list.js
+++ b/assets/js/dashboard/stats/sources/referrer-list.js
@@ -1,14 +1,23 @@
-import React from 'react';
+import React, { useState } from 'react';
 import * as api from '../../api'
 import * as url from '../../util/url'
 import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics'
 import ListReport from '../reports/list'
+import ImportedQueryUnsupportedWarning from '../../stats/imported-query-unsupported-warning'
 
 export default function Referrers({source, site, query}) {
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
+
   function fetchReferrers() {
     return api.get(url.apiPath(site, `/referrers/${encodeURIComponent(source)}`), query, {limit: 9})
   }
 
+  function afterFetchReferrers(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = query.period === 'realtime'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime)
+  }
+
   function externalLinkDest(referrer) {
     if (referrer.name === 'Direct / None') { return null }
     return `https://${referrer.name}`
@@ -35,9 +44,13 @@ export default function Referrers({source, site, query}) {
 
   return (
     <div className="flex flex-col flex-grow">
-      <h3 className="font-bold dark:text-gray-100">Top Referrers</h3>
+      <div className="flex gap-x-1">
+        <h3 className="font-bold dark:text-gray-100">Top Referrers</h3>
+        <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>
+      </div>
       <ListReport
         fetchData={fetchReferrers}
+        afterFetchData={afterFetchReferrers}
         getFilterFor={getFilterFor}
         keyLabel="Referrer"
         metrics={maybeWithCR([VISITORS_METRIC], query)}
diff --git a/assets/js/dashboard/stats/sources/source-list.js b/assets/js/dashboard/stats/sources/source-list.js
index c9bd1a7f8e6d..07a36bfc3044 100644
--- a/assets/js/dashboard/stats/sources/source-list.js
+++ b/assets/js/dashboard/stats/sources/source-list.js
@@ -8,6 +8,7 @@ import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';
 import { Menu, Transition } from '@headlessui/react'
 import { ChevronDownIcon } from '@heroicons/react/20/solid'
 import classNames from 'classnames'
+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';
 
 const UTM_TAGS = {
   utm_medium: { label: 'UTM Medium', shortLabel: 'UTM Medium', endpoint: '/utm_mediums' },
@@ -43,6 +44,7 @@ function AllSources(props) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={props.afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel="Source"
       metrics={maybeWithCR([VISITORS_METRIC], query)}
@@ -72,6 +74,7 @@ function UTMSources(props) {
   return (
     <ListReport
       fetchData={fetchData}
+      afterFetchData={props.afterFetchData}
       getFilterFor={getFilterFor}
       keyLabel={utmTag.label}
       metrics={maybeWithCR([VISITORS_METRIC], query)}
@@ -87,6 +90,7 @@ export default function SourceList(props) {
   const tabKey = 'sourceTab__' + props.site.domain
   const storedTab = storage.getItem(tabKey)
   const [currentTab, setCurrentTab] = useState(storedTab || 'all')
+  const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)
 
   function setTab(tab) {
     return () => {
@@ -152,19 +156,28 @@ export default function SourceList(props) {
 
   function renderContent() {
     if (currentTab === 'all') {
-      return <AllSources site={site} query={query} />
+      return <AllSources site={site} query={query} afterFetchData={afterFetchData} />
     } else {
-      return <UTMSources tab={currentTab} site={site} query={query} />
+      return <UTMSources tab={currentTab} site={site} query={query} afterFetchData={afterFetchData} />
     }
   }
 
+  function afterFetchData(apiResponse) {
+    const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
+    const isRealtime = query.period === 'realtime'
+    setImportedQueryUnsupported(unsupportedQuery && !isRealtime)
+  }
+
   return (
     <div>
       {/* Header Container */}
       <div className="w-full flex justify-between">
-        <h3 className="font-bold dark:text-gray-100">
-          Top Sources
-        </h3>
+        <div className="flex gap-x-1">
+          <h3 className="font-bold dark:text-gray-100">
+            Top Sources
+          </h3>
+          <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>
+        </div>
         {renderTabs()}
       </div>
       {/* Main Contents */}
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 29aef361a853..1a2c579f7582 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -753,10 +753,10 @@ defmodule PlausibleWeb.Api.StatsController do
       Stats.breakdown(site, query, metrics, pagination)
       |> transform_keys(%{referrer: :name})
 
-      json(conn, %{
-        results: referrers,
-        skip_imported_reason: query.skip_imported_reason
-      })
+    json(conn, %{
+      results: referrers,
+      skip_imported_reason: query.skip_imported_reason
+    })
   end
 
   def pages(conn, params) do
@@ -940,10 +940,10 @@ defmodule PlausibleWeb.Api.StatsController do
           end
         end)
 
-        json(conn, %{
-          results: countries,
-          skip_imported_reason: query.skip_imported_reason
-        })
+      json(conn, %{
+        results: countries,
+        skip_imported_reason: query.skip_imported_reason
+      })
     end
   end
 
@@ -1216,8 +1216,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
     case Plausible.Props.ensure_prop_key_accessible(prop_key, site.owner) do
       :ok ->
-        props = breakdown_custom_prop_values(site, params)
-        json(conn, %{results: props})
+        json(conn, breakdown_custom_prop_values(site, params))
 
       {:error, :upgrade_required} ->
         H.payment_required(
@@ -1245,6 +1244,7 @@ defmodule PlausibleWeb.Api.StatsController do
         prop_names
         |> Enum.map(fn prop_key ->
           breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))
+          |> Map.get(:results)
           |> Enum.map(&Map.put(&1, :property, prop_key))
           |> transform_keys(%{:name => :value})
         end)
@@ -1274,12 +1274,16 @@ defmodule PlausibleWeb.Api.StatsController do
         [:visitors, :events, :percentage] ++ @revenue_metrics
       end
 
-    Stats.breakdown(site, query, metrics, pagination)
-    |> transform_keys(%{prop_key => :name})
-    |> Enum.map(fn entry ->
-      Enum.map(entry, &format_revenue_metric/1)
-      |> Map.new()
-    end)
+    props =
+      Stats.breakdown(site, query, metrics, pagination)
+      |> transform_keys(%{prop_key => :name})
+      |> Enum.map(fn entry ->
+        Enum.map(entry, &format_revenue_metric/1)
+        |> Map.new()
+      end)
+      |> IO.inspect()
+
+    %{results: props, skip_imported_reason: query.skip_imported_reason}
   end
 
   def current_visitors(conn, _) do
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index 7380984d87cc..370eb0ecf6b6 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -263,7 +263,9 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
 
         results =
           conn
-          |> get("/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true")
+          |> get(
+            "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"
+          )
           |> json_response(200)
           |> Map.get("results")
           |> Enum.sort()

From 92580e2f94e9a7d12107a0caf4055ccf4d117cd9 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Fri, 24 May 2024 14:29:58 +0200
Subject: [PATCH 14/41] Implement filtering suggestions and translate filter
 fields for imported

---
 lib/plausible/stats/filter_suggestions.ex     |  27 +++-
 lib/plausible/stats/imported/base.ex          |  18 ++-
 lib/plausible/stats/imported/imported.ex      | 147 +++++++++++++++++-
 .../controllers/api/stats_controller.ex       |   1 -
 4 files changed, 182 insertions(+), 11 deletions(-)

diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 6e38044f998c..636f309c9199 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -16,6 +16,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.country_code
       )
+      |> Plausible.Stats.Imported.merged_imported_countries(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end)
@@ -35,9 +36,10 @@ defmodule Plausible.Stats.FilterSuggestions do
       group_by: e.subdivision1_code,
       order_by: [desc: fragment("count(*)")],
       select: e.subdivision1_code,
-      where: e.subdivision1_code != "",
-      limit: 24
+      where: e.subdivision1_code != ""
     )
+    |> Plausible.Stats.Imported.merge_imported_regions(site, query)
+    |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
       subdiv = Location.get_subdivision(c)
@@ -59,6 +61,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.subdivision1_code
       )
+      |> Plausible.Stats.Imported.merge_imported_regions(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)
@@ -78,9 +81,10 @@ defmodule Plausible.Stats.FilterSuggestions do
       group_by: e.city_geoname_id,
       order_by: [desc: fragment("count(*)")],
       select: e.city_geoname_id,
-      where: e.city_geoname_id != 0,
-      limit: 24
+      where: e.city_geoname_id != 0
     )
+    |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+    |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
       city = Location.get_city(c)
@@ -101,9 +105,10 @@ defmodule Plausible.Stats.FilterSuggestions do
         group_by: e.city_geoname_id,
         order_by: [desc: fragment("count(*)")],
         select: e.city_geoname_id,
-        where: e.city_geoname_id != 0,
-        limit: 5000
+        where: e.city_geoname_id != 0
       )
+      |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+      |> limit(5000)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Location.get_city(c) end)
@@ -223,10 +228,16 @@ defmodule Plausible.Stats.FilterSuggestions do
       where: fragment("? ilike ?", field(e, ^filter_name), ^filter_query),
       select: field(e, ^filter_name),
       group_by: ^filter_name,
-      order_by: [desc: fragment("count(*)")],
-      limit: 25
+      order_by: [desc: fragment("count(*)")]
     )
     |> apply_additional_filters(filter_name, site)
+    |> Plausible.Stats.Imported.merge_imported_filter_suggestions(
+      site,
+      query,
+      filter_name,
+      filter_query
+    )
+    |> limit(25)
     |> ClickhouseRepo.all()
     |> Enum.filter(fn suggestion -> suggestion != "" end)
     |> wrap_suggestions()
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 657c7aebf8c6..c64f08e906e0 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -10,6 +10,21 @@ defmodule Plausible.Stats.Imported.Base do
   @goals_with_path Imported.goals_with_path()
   @special_goals @goals_with_path ++ @goals_with_url
 
+  @db_field_mappings %{
+    referrer_source: :source,
+    screen_size: :device,
+    screen: :device,
+    os: :operating_system,
+    os_version: :operating_system_version,
+    country_code: :country,
+    subdivision1_code: :region,
+    city_geoname_id: :city,
+    entry_page_hostname: :hostname,
+    pathname: :page
+  }
+
+  def db_field_mappings(), do: @db_field_mappings
+
   def query_imported(site, query) do
     query = Imported.drop_redundant_filters(query)
 
@@ -86,7 +101,8 @@ defmodule Plausible.Stats.Imported.Base do
 
   defp apply_filter(q, %Query{filters: [[_, filtered_prop | _] = filter]}) do
     db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
-    condition = Filters.WhereBuilder.build_condition(db_field, filter)
+    mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
+    condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)
 
     where(q, ^condition)
   end
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 55e0e8ff6a95..f5774a5e3792 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -30,7 +30,11 @@ defmodule Plausible.Stats.Imported do
     "event:page" => "imported_pages",
     "event:name" => "imported_custom_events",
     "event:props:url" => "imported_custom_events",
-    "event:props:path" => "imported_custom_events"
+    "event:props:path" => "imported_custom_events",
+
+    # NOTE: these properties can be only filtered by
+    "visit:screen" => "imported_devices",
+    "event:hostname" => "imported_pages"
   }
 
   def property_to_table_mappings(), do: @property_to_table_mappings
@@ -82,6 +86,123 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  def merged_imported_countries(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{country_code: e.country_code, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        group_by: i.country,
+        select_merge: %{country_code: i.country, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.country_code == i.country_code,
+      select: fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_regions(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        where: i.region != "",
+        group_by: i.region,
+        select_merge: %{region_code: i.region, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.region_code == i.region_code,
+      select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_cities(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        where: i.city != 0,
+        group_by: i.city,
+        select_merge: %{city_id: i.city, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.city_id == i.city_id,
+      select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_filter_suggestions(
+        native_q,
+        _site,
+        %Plausible.Stats.Query{include_imported: false},
+        _filter_name,
+        _filter_search
+      ) do
+    native_q
+  end
+
+  def merge_imported_filter_suggestions(
+        native_q,
+        site,
+        query,
+        filter_name,
+        filter_query
+      ) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})
+
+    db_field = Map.get(Imported.Base.db_field_mappings(), filter_name, filter_name)
+
+    property_field =
+      case db_field do
+        :operating_system -> :os
+        :operating_system_version -> :os_version
+        other -> other
+      end
+
+    table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property_field}")
+    table_by_event = Map.get(@property_to_table_mappings, "event:#{property_field}")
+    table = table_by_visit || table_by_event
+
+    if db_field && table do
+      imported_q =
+        from i in Imported.Base.query_imported(table, site, query),
+          where: fragment("? ilike ?", field(i, ^db_field), ^filter_query),
+          group_by: field(i, ^db_field),
+          select_merge: %{name: field(i, ^db_field), count: fragment("count(*)")}
+
+      from(s in subquery(native_q),
+        full_join: i in subquery(imported_q),
+        on: s.name == i.name,
+        select: fragment("if(not empty(?), ?, ?)", s.name, s.name, i.name),
+        order_by: [desc: fragment("? + ?", s.count, i.count)],
+        limit: 25
+      )
+    else
+      native_q
+    end
+  end
+
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 
@@ -283,6 +404,18 @@ defmodule Plausible.Stats.Imported do
     |> select_imported_metrics(rest)
   end
 
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
+         [:bounce_rate | rest]
+       ) do
+    q
+    |> select_merge([i], %{
+      bounces: 0,
+      __internal_visits: 0
+    })
+    |> select_imported_metrics(rest)
+  end
+
   defp select_imported_metrics(
          %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
          [:bounce_rate | rest]
@@ -316,6 +449,18 @@ defmodule Plausible.Stats.Imported do
     |> select_imported_metrics(rest)
   end
 
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
+         [:visit_duration | rest]
+       ) do
+    q
+    |> select_merge([i], %{
+      visit_duration: 0,
+      __internal_visits: 0
+    })
+    |> select_imported_metrics(rest)
+  end
+
   defp select_imported_metrics(
          %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
          [:visit_duration | rest]
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 1a2c579f7582..9c50482d8a34 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -1281,7 +1281,6 @@ defmodule PlausibleWeb.Api.StatsController do
         Enum.map(entry, &format_revenue_metric/1)
         |> Map.new()
       end)
-      |> IO.inspect()
 
     %{results: props, skip_imported_reason: query.skip_imported_reason}
   end

From cec119c000606d429cb64a937794240edddcdf31 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 27 May 2024 12:15:44 +0200
Subject: [PATCH 15/41] WIP

---
 lib/plausible/stats/filter_suggestions.ex     |  2 +-
 lib/plausible/stats/imported/imported.ex      |  3 ++-
 .../api/stats_controller/suggestions_test.exs | 26 +++++++++++++++++++
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 636f309c9199..31772493b641 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -16,7 +16,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.country_code
       )
-      |> Plausible.Stats.Imported.merged_imported_countries(site, query)
+      |> Plausible.Stats.Imported.merge_imported_countries(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end)
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index f5774a5e3792..af001a68fe54 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -33,6 +33,7 @@ defmodule Plausible.Stats.Imported do
     "event:props:path" => "imported_custom_events",
 
     # NOTE: these properties can be only filtered by
+    # TODO: add support for breaking down by translating to device breakdown
     "visit:screen" => "imported_devices",
     "event:hostname" => "imported_pages"
   }
@@ -86,7 +87,7 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  def merged_imported_countries(native_q, site, query) do
+  def merge_imported_countries(native_q, site, query) do
     native_q =
       native_q
       |> exclude(:order_by)
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index fd77e5279d5a..fc37312e8715 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -634,4 +634,30 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
              }
     end
   end
+
+  describe "imported data - country" do
+    setup [:create_user, :log_in, :create_site, :create_site_import]
+
+    test "it works", %{conn: conn, site: site, site_import: site_import} do
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 23:00:01],
+          pathname: "/",
+          country_code: "US"
+        ),
+        build(:imported_locations,
+          date: ~D[2019-01-01],
+          country: "PL"
+        )
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q=Unit"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "US", "label" => "United States"}]
+    end
+  end
 end

From cb40a286a70fc4d724564f07cf3c43bddddc15b5 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 27 May 2024 14:51:10 +0200
Subject: [PATCH 16/41] Improve and cover filtering suggestions with tests

---
 lib/plausible/stats/imported/base.ex          |   2 -
 lib/plausible/stats/imported/imported.ex      |  53 +-
 .../api/stats_controller/suggestions_test.exs | 475 +++++++++++++++++-
 3 files changed, 512 insertions(+), 18 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index c64f08e906e0..affb7f33331d 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -23,8 +23,6 @@ defmodule Plausible.Stats.Imported.Base do
     pathname: :page
   }
 
-  def db_field_mappings(), do: @db_field_mappings
-
   def query_imported(site, query) do
     query = Imported.drop_redundant_filters(query)
 
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index af001a68fe54..68c3257eab17 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -87,6 +87,14 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  def merge_imported_countries(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+    native_q
+  end
+
+  def merge_imported_countries(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+    native_q
+  end
+
   def merge_imported_countries(native_q, site, query) do
     native_q =
       native_q
@@ -97,7 +105,7 @@ defmodule Plausible.Stats.Imported do
     imported_q =
       from i in Imported.Base.query_imported("imported_locations", site, query),
         group_by: i.country,
-        select_merge: %{country_code: i.country, count: fragment("count(*)")}
+        select_merge: %{country_code: i.country, count: fragment("sum(?)", i.pageviews)}
 
     from(s in subquery(native_q),
       full_join: i in subquery(imported_q),
@@ -107,6 +115,14 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  def merge_imported_regions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+    native_q
+  end
+
+  def merge_imported_regions(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+    native_q
+  end
+
   def merge_imported_regions(native_q, site, query) do
     native_q =
       native_q
@@ -118,7 +134,7 @@ defmodule Plausible.Stats.Imported do
       from i in Imported.Base.query_imported("imported_locations", site, query),
         where: i.region != "",
         group_by: i.region,
-        select_merge: %{region_code: i.region, count: fragment("count(*)")}
+        select_merge: %{region_code: i.region, count: fragment("sum(?)", i.pageviews)}
 
     from(s in subquery(native_q),
       full_join: i in subquery(imported_q),
@@ -128,6 +144,15 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  #NOTE: rename location merge functions adding _suggestions suffix
+  def merge_imported_cities(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+    native_q
+  end
+
+  def merge_imported_cities(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+    native_q
+  end
+
   def merge_imported_cities(native_q, site, query) do
     native_q =
       native_q
@@ -139,7 +164,7 @@ defmodule Plausible.Stats.Imported do
       from i in Imported.Base.query_imported("imported_locations", site, query),
         where: i.city != 0,
         group_by: i.city,
-        select_merge: %{city_id: i.city, count: fragment("count(*)")}
+        select_merge: %{city_id: i.city, count: fragment("sum(?)", i.pageviews)}
 
     from(s in subquery(native_q),
       full_join: i in subquery(imported_q),
@@ -149,6 +174,16 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  def merge_imported_filter_suggestions(
+        native_q,
+        _site,
+        %Plausible.Stats.Query{filters: [_ | _]},
+        _filter_name,
+        _filter_search
+      ) do
+    native_q
+  end
+
   def merge_imported_filter_suggestions(
         native_q,
         _site,
@@ -159,6 +194,12 @@ defmodule Plausible.Stats.Imported do
     native_q
   end
 
+  @filter_suggestions_mapping %{
+    referrer_source: :source,
+    screen_size: :device,
+    pathname: :page
+  }
+
   def merge_imported_filter_suggestions(
         native_q,
         site,
@@ -172,7 +213,7 @@ defmodule Plausible.Stats.Imported do
       |> exclude(:select)
       |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})
 
-    db_field = Map.get(Imported.Base.db_field_mappings(), filter_name, filter_name)
+    db_field = Map.get(@filter_suggestions_mapping, filter_name, filter_name)
 
     property_field =
       case db_field do
@@ -185,12 +226,12 @@ defmodule Plausible.Stats.Imported do
     table_by_event = Map.get(@property_to_table_mappings, "event:#{property_field}")
     table = table_by_visit || table_by_event
 
-    if db_field && table do
+    if table do
       imported_q =
         from i in Imported.Base.query_imported(table, site, query),
           where: fragment("? ilike ?", field(i, ^db_field), ^filter_query),
           group_by: field(i, ^db_field),
-          select_merge: %{name: field(i, ^db_field), count: fragment("count(*)")}
+          select_merge: %{name: field(i, ^db_field), count: fragment("sum(?)", i.pageviews)}
 
       from(s in subquery(native_q),
         full_join: i in subquery(imported_q),
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index fc37312e8715..a82a2f6aa117 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -635,29 +635,484 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
   end
 
-  describe "imported data - country" do
+  describe "imported data" do
     setup [:create_user, :log_in, :create_site, :create_site_import]
 
-    test "it works", %{conn: conn, site: site, site_import: site_import} do
+    test "merges country suggestions from native and imported data", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], country_code: "US"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:30:01], country_code: "US"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:40:01], country_code: "US"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], country_code: "GB"),
+        build(:imported_locations, date: ~D[2019-01-01], country: "GB", pageviews: 3)
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q=Unit&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [
+               %{"value" => "GB", "label" => "United Kingdom"},
+               %{"value" => "US", "label" => "United States"}
+             ]
+    end
+
+    test "ignores imported data in country suggestions when filters applied", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview, country_code: "EE", referrer_source: "Bing"),
+        build(:imported_locations, country: "GB")
+      ])
+
+      filters = Jason.encode!(%{source: "Bing"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/country?filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "EE", "label" => "Estonia"}]
+    end
+
+    test "ignores imported country data when not requested", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, date: ~D[2019-01-01], country: "GB", pageviews: 3)
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q="
+        )
+
+      assert json_response(conn, 200) == []
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"H", "with filter"}] do
+      test "merges region suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, country_code: "EE", subdivision1_code: "EE-37"),
+          build(:pageview, country_code: "EE", subdivision1_code: "EE-39"),
+          build(:pageview, country_code: "EE", subdivision1_code: "EE-39"),
+          build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/region?q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "EE-37", "label" => "Harjumaa"},
+                 %{"value" => "EE-39", "label" => "Hiiumaa"}
+               ]
+      end
+    end
+
+    test "ignores imported data in region suggestions when filters applied", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
       populate_stats(site, site_import.id, [
         build(:pageview,
-          timestamp: ~N[2019-01-01 23:00:01],
-          pathname: "/",
-          country_code: "US"
+          country_code: "EE",
+          subdivision1_code: "EE-39",
+          referrer_source: "Bing"
         ),
-        build(:imported_locations,
-          date: ~D[2019-01-01],
-          country: "PL"
+        build(:imported_locations, country: "EE", region: "EE-37")
+      ])
+
+      filters = Jason.encode!(%{source: "Bing"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?filters=#{filters}&q=&with_imported=true"
         )
+
+      assert json_response(conn, 200) == [%{"value" => "EE-39", "label" => "Hiiumaa"}]
+    end
+
+    test "ignores imported region data when not requested", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2)
       ])
 
       conn =
         get(
           conn,
-          "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q=Unit"
+          "/api/stats/#{site.domain}/suggestions/region?q="
         )
 
-      assert json_response(conn, 200) == [%{"value" => "US", "label" => "United States"}]
+      assert json_response(conn, 200) == []
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"l", "with filter"}] do
+      test "merges city suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview,
+            country_code: "EE",
+            subdivision1_code: "EE-37",
+            city_geoname_id: 588_409
+          ),
+          build(:pageview,
+            country_code: "EE",
+            subdivision1_code: "EE-39",
+            city_geoname_id: 591_632
+          ),
+          build(:pageview,
+            country_code: "EE",
+            subdivision1_code: "EE-39",
+            city_geoname_id: 591_632
+          ),
+          build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, pageviews: 2)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/city?q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "588409", "label" => "Tallinn"},
+                 %{"value" => "591632", "label" => "Kärdla"}
+               ]
+      end
+    end
+
+    test "ignores imported data in city suggestions when filters applied", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          country_code: "EE",
+          subdivision1_code: "EE-39",
+          city_geoname_id: 591_632,
+          referrer_source: "Bing"
+        ),
+        build(:imported_locations, country: "EE", region: "EE-37", city: 588_409)
+      ])
+
+      filters = Jason.encode!(%{source: "Bing"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/city?filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "591632", "label" => "Kärdla"}]
+    end
+
+    test "ignores imported city data when not requested", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, pageviews: 2)
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/city?q="
+        )
+
+      assert json_response(conn, 200) == []
+    end
+
+    test "ignores imported data when asking for prop key and value suggestions", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          "meta.key": ["url"],
+          "meta.value": ["http://example1.com"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:imported_custom_events,
+          date: ~D[2022-01-01],
+          name: "Outbound Link: Click",
+          link_url: "http://example2.com"
+        ),
+        build(:imported_custom_events,
+          date: ~D[2022-01-01],
+          name: "404",
+          path: "/dev/null"
+        )
+      ])
+
+      key_conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01&with_imported=true"
+        )
+
+      assert json_response(key_conn, 200) == [%{"label" => "url", "value" => "url"}]
+
+      filters = Jason.encode!(%{props: %{url: "!(none)"}})
+
+      value_conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01&with_imported=true&filters=#{filters}"
+        )
+
+      assert json_response(value_conn, 200) == [
+               %{"label" => "http://example1.com", "value" => "http://example1.com"}
+             ]
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"g", "with filter"}] do
+      test "merges source suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Google"),
+          build(:imported_sources, date: ~D[2019-01-01], source: "Google", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "Google", "label" => "Google"},
+                 %{"value" => "Bing", "label" => "Bing"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do
+      test "merges screen suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], screen_size: "Mobile"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], screen_size: "Mobile"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], screen_size: "Mobile"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], screen_size: "Desktop"),
+          build(:imported_devices, date: ~D[2019-01-01], device: "Desktop", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "Desktop", "label" => "Desktop"},
+                 %{"value" => "Mobile", "label" => "Mobile"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do
+      test "merges page suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),
+          build(:imported_pages, date: ~D[2019-01-01], page: "/blog", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "/blog", "label" => "/blog"},
+                 %{"value" => "/welcome", "label" => "/welcome"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do
+      test "merges entry page suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),
+          build(:imported_entry_pages, date: ~D[2019-01-01], entry_page: "/blog", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/entry_page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "/blog", "label" => "/blog"},
+                 %{"value" => "/welcome", "label" => "/welcome"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do
+      test "merges exit page suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),
+          build(:imported_exit_pages, date: ~D[2019-01-01], exit_page: "/blog", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/exit_page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "/blog", "label" => "/blog"},
+                 %{"value" => "/welcome", "label" => "/welcome"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do
+      test "merges browser suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], browser: "Chrome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], browser: "Chrome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], browser: "Chrome"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], browser: "Firefox"),
+          build(:imported_browsers, date: ~D[2019-01-01], browser: "Firefox", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "Firefox", "label" => "Firefox"},
+                 %{"value" => "Chrome", "label" => "Chrome"}
+               ]
+      end
+    end
+
+    for {q, label} <- [{"", "without filter"}, {"i", "with filter"}] do
+      test "merges operating system suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], operating_system: "Linux"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], operating_system: "Linux"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], operating_system: "Linux"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], operating_system: "Windows"),
+          build(:imported_operating_systems,
+            date: ~D[2019-01-01],
+            operating_system: "Windows",
+            pageviews: 3
+          )
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "Windows", "label" => "Windows"},
+                 %{"value" => "Linux", "label" => "Linux"}
+               ]
+      end
+    end
+
+    test "does not query imported data when filters applied", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 23:00:01],
+          pathname: "/blog",
+          operating_system: "Linux"
+        ),
+        build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Windows")
+      ])
+
+      filters = Jason.encode!(%{page: "/blog"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "Linux", "label" => "Linux"}]
     end
   end
 end

From a7dadb88c118eb527a1aaebe200f97ba647cb88d Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 27 May 2024 15:49:34 +0200
Subject: [PATCH 17/41] Rename imported suggestions query helpers

---
 lib/plausible/stats/filter_suggestions.ex | 15 ++++++++------
 lib/plausible/stats/imported/imported.ex  | 25 ++++++++++++++---------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 31772493b641..64d5585580a4 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -2,9 +2,12 @@ defmodule Plausible.Stats.FilterSuggestions do
   use Plausible.Repo
   use Plausible.ClickhouseRepo
   use Plausible.Stats.Fragments
+
   import Plausible.Stats.Base
   import Ecto.Query
+
   alias Plausible.Stats.Query
+  alias Plausible.Stats.Imported
 
   def filter_suggestions(site, query, "country", filter_search) do
     matches = Location.search_country(filter_search)
@@ -16,7 +19,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.country_code
       )
-      |> Plausible.Stats.Imported.merge_imported_countries(site, query)
+      |> Imported.merge_imported_country_suggestions(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end)
@@ -38,7 +41,7 @@ defmodule Plausible.Stats.FilterSuggestions do
       select: e.subdivision1_code,
       where: e.subdivision1_code != ""
     )
-    |> Plausible.Stats.Imported.merge_imported_regions(site, query)
+    |> Imported.merge_imported_region_suggestions(site, query)
     |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
@@ -61,7 +64,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.subdivision1_code
       )
-      |> Plausible.Stats.Imported.merge_imported_regions(site, query)
+      |> Imported.merge_imported_region_suggestions(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)
@@ -83,7 +86,7 @@ defmodule Plausible.Stats.FilterSuggestions do
       select: e.city_geoname_id,
       where: e.city_geoname_id != 0
     )
-    |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+    |> Imported.merge_imported_city_suggestions(site, query)
     |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
@@ -107,7 +110,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         select: e.city_geoname_id,
         where: e.city_geoname_id != 0
       )
-      |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+      |> Imported.merge_imported_city_suggestions(site, query)
       |> limit(5000)
 
     ClickhouseRepo.all(q)
@@ -231,7 +234,7 @@ defmodule Plausible.Stats.FilterSuggestions do
       order_by: [desc: fragment("count(*)")]
     )
     |> apply_additional_filters(filter_name, site)
-    |> Plausible.Stats.Imported.merge_imported_filter_suggestions(
+    |> Imported.merge_imported_filter_suggestions(
       site,
       query,
       filter_name,
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 68c3257eab17..17b62de21150 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -87,15 +87,17 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  def merge_imported_countries(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+  def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
     native_q
   end
 
-  def merge_imported_countries(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+  def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
+        include_imported: false
+      }) do
     native_q
   end
 
-  def merge_imported_countries(native_q, site, query) do
+  def merge_imported_country_suggestions(native_q, site, query) do
     native_q =
       native_q
       |> exclude(:order_by)
@@ -115,15 +117,17 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  def merge_imported_regions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+  def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
     native_q
   end
 
-  def merge_imported_regions(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+  def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{
+        include_imported: false
+      }) do
     native_q
   end
 
-  def merge_imported_regions(native_q, site, query) do
+  def merge_imported_region_suggestions(native_q, site, query) do
     native_q =
       native_q
       |> exclude(:order_by)
@@ -144,16 +148,17 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  #NOTE: rename location merge functions adding _suggestions suffix
-  def merge_imported_cities(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
+  def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
     native_q
   end
 
-  def merge_imported_cities(native_q, _site, %Plausible.Stats.Query{include_imported: false}) do
+  def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{
+        include_imported: false
+      }) do
     native_q
   end
 
-  def merge_imported_cities(native_q, site, query) do
+  def merge_imported_city_suggestions(native_q, site, query) do
     native_q =
       native_q
       |> exclude(:order_by)

From 6039e7d8ecf239248c9e6e2413cc293e7a568b44 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 27 May 2024 15:27:22 +0100
Subject: [PATCH 18/41] fix screen size breakdown with screen size filter

---
 lib/plausible/stats/imported/base.ex          |  1 +
 .../breakdown_test.exs                        | 31 +++++++++++++++++++
 .../stats_controller/screen_sizes_test.exs    | 21 +++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index affb7f33331d..7dd0ffd643b5 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -80,6 +80,7 @@ defmodule Plausible.Stats.Imported.Base do
 
     cond do
       is_nil(property) -> table_candidate
+      property == "visit:device" and filtered_prop == "visit:screen" -> table_candidate
       property == filtered_prop -> table_candidate
       true -> nil
     end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 1125722da236..fee6f7d53ef6 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -3158,6 +3158,37 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   describe "imported data" do
+    test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          screen_size: "Mobile"
+        ),
+        build(:imported_devices,
+          device: "Mobile",
+          visitors: 3,
+          pageviews: 5,
+          date: ~D[2021-01-01]
+        )
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/breakdown", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "date" => "2021-01-01",
+          "property" => "visit:device",
+          "filters" => "visit:device==Mobile",
+          "metrics" => "visitors,pageviews",
+          "with_imported" => "true"
+        })
+
+      assert [%{"pageviews" => 6, "visitors" => 4, "device" => "Mobile"}] =
+               json_response(conn, 200)["results"]
+    end
+
     test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
       insert(:goal, site: site, event_name: "Purchase")
       insert(:goal, site: site, page_path: "/test")
diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
index 13f7736bc4c6..49c94c6e7d01 100644
--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
@@ -187,6 +187,27 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
              ]
     end
 
+    test "returns screen sizes when filtering by imported screen size", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, screen_size: "Desktop"),
+        build(:imported_devices, device: "Desktop"),
+        build(:imported_devices, device: "Laptop"),
+        build(:imported_visitors, visitors: 2)
+      ])
+
+      filters = Jason.encode!(%{screen: "Desktop"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/screen-sizes?filters=#{filters}&period=day&with_imported=true"
+        )
+
+      assert json_response(conn, 200)["results"] == [
+               %{"name" => "Desktop", "visitors" => 2, "percentage" => 100.0}
+             ]
+    end
+
     test "returns screen sizes for user making multiple sessions by no of visitors with imported data",
          %{conn: conn, site: site} do
       populate_stats(site, [

From 6982dbe231564b8e9479d1eaa4ea3a09ac1e83d2 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 27 May 2024 16:19:37 +0100
Subject: [PATCH 19/41] support filtering by the same suggestion property

---
 lib/plausible/stats/imported/imported.ex      | 68 ++++++++++---------
 .../api/stats_controller/suggestions_test.exs | 28 +++++++-
 2 files changed, 62 insertions(+), 34 deletions(-)

diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 17b62de21150..778e382cca5c 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -179,16 +179,6 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  def merge_imported_filter_suggestions(
-        native_q,
-        _site,
-        %Plausible.Stats.Query{filters: [_ | _]},
-        _filter_name,
-        _filter_search
-      ) do
-    native_q
-  end
-
   def merge_imported_filter_suggestions(
         native_q,
         _site,
@@ -199,12 +189,6 @@ defmodule Plausible.Stats.Imported do
     native_q
   end
 
-  @filter_suggestions_mapping %{
-    referrer_source: :source,
-    screen_size: :device,
-    pathname: :page
-  }
-
   def merge_imported_filter_suggestions(
         native_q,
         site,
@@ -212,26 +196,21 @@ defmodule Plausible.Stats.Imported do
         filter_name,
         filter_query
       ) do
-    native_q =
-      native_q
-      |> exclude(:order_by)
-      |> exclude(:select)
-      |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})
+    {table, db_field} = expand_suggestions_field(filter_name)
 
-    db_field = Map.get(@filter_suggestions_mapping, filter_name, filter_name)
+    supports_filter_set? =
+      Enum.all?(query.filters, fn filter ->
+        [_, filtered_prop | _] = filter
+        @property_to_table_mappings[filtered_prop] == table
+      end)
 
-    property_field =
-      case db_field do
-        :operating_system -> :os
-        :operating_system_version -> :os_version
-        other -> other
-      end
-
-    table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property_field}")
-    table_by_event = Map.get(@property_to_table_mappings, "event:#{property_field}")
-    table = table_by_visit || table_by_event
+    if supports_filter_set? do
+      native_q =
+        native_q
+        |> exclude(:order_by)
+        |> exclude(:select)
+        |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})
 
-    if table do
       imported_q =
         from i in Imported.Base.query_imported(table, site, query),
           where: fragment("? ilike ?", field(i, ^db_field), ^filter_query),
@@ -250,6 +229,29 @@ defmodule Plausible.Stats.Imported do
     end
   end
 
+  @filter_suggestions_mapping %{
+    referrer_source: :source,
+    screen_size: :device,
+    pathname: :page
+  }
+
+  defp expand_suggestions_field(filter_name) do
+    db_field = Map.get(@filter_suggestions_mapping, filter_name, filter_name)
+
+    property =
+      case db_field do
+        :operating_system -> :os
+        :operating_system_version -> :os_version
+        other -> other
+      end
+
+    table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property}")
+    table_by_event = Map.get(@property_to_table_mappings, "event:#{property}")
+    table = table_by_visit || table_by_event
+
+    {table, db_field}
+  end
+
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index a82a2f6aa117..4b6c529a75d5 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -1090,7 +1090,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       end
     end
 
-    test "does not query imported data when filters applied", %{
+    test "does not query imported data when a different property is filtered by", %{
       conn: conn,
       site: site,
       site_import: site_import
@@ -1114,5 +1114,31 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
 
       assert json_response(conn, 200) == [%{"value" => "Linux", "label" => "Linux"}]
     end
+
+    test "queries imported data when filtering by the same property", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 23:00:01],
+          pathname: "/blog",
+          operating_system: "Linux"
+        ),
+        build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Windows"),
+        build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Linux")
+      ])
+
+      filters = Jason.encode!(%{os: "!Linux"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "Windows", "label" => "Windows"}]
+    end
   end
 end

From b652cbb4fd780032a1e575b35f41d535c66313a6 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Mon, 27 May 2024 16:49:44 +0100
Subject: [PATCH 20/41] support location filters when fetching location
 suggestions

---
 lib/plausible/stats/imported/imported.ex      | 133 ++++++++++--------
 .../api/stats_controller/suggestions_test.exs |  84 +++++++++--
 2 files changed, 149 insertions(+), 68 deletions(-)

diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 778e382cca5c..8ff326559817 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -87,10 +87,6 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
-    native_q
-  end
-
   def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
         include_imported: false
       }) do
@@ -98,27 +94,34 @@ defmodule Plausible.Stats.Imported do
   end
 
   def merge_imported_country_suggestions(native_q, site, query) do
-    native_q =
-      native_q
-      |> exclude(:order_by)
-      |> exclude(:select)
-      |> select([e], %{country_code: e.country_code, count: fragment("count(*)")})
+    supports_filter_set? =
+      Enum.all?(query.filters, fn filter ->
+        [_, filtered_prop | _] = filter
+        @property_to_table_mappings[filtered_prop] == "imported_locations"
+      end)
 
-    imported_q =
-      from i in Imported.Base.query_imported("imported_locations", site, query),
-        group_by: i.country,
-        select_merge: %{country_code: i.country, count: fragment("sum(?)", i.pageviews)}
+    if supports_filter_set? do
+      native_q =
+        native_q
+        |> exclude(:order_by)
+        |> exclude(:select)
+        |> select([e], %{country_code: e.country_code, count: fragment("count(*)")})
 
-    from(s in subquery(native_q),
-      full_join: i in subquery(imported_q),
-      on: s.country_code == i.country_code,
-      select: fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code),
-      order_by: [desc: fragment("? + ?", s.count, i.count)]
-    )
-  end
+      imported_q =
+        from i in Imported.Base.query_imported("imported_locations", site, query),
+          group_by: i.country,
+          select_merge: %{country_code: i.country, count: fragment("sum(?)", i.pageviews)}
 
-  def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
-    native_q
+      from(s in subquery(native_q),
+        full_join: i in subquery(imported_q),
+        on: s.country_code == i.country_code,
+        select:
+          fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code),
+        order_by: [desc: fragment("? + ?", s.count, i.count)]
+      )
+    else
+      native_q
+    end
   end
 
   def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{
@@ -128,28 +131,34 @@ defmodule Plausible.Stats.Imported do
   end
 
   def merge_imported_region_suggestions(native_q, site, query) do
-    native_q =
-      native_q
-      |> exclude(:order_by)
-      |> exclude(:select)
-      |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")})
+    supports_filter_set? =
+      Enum.all?(query.filters, fn filter ->
+        [_, filtered_prop | _] = filter
+        @property_to_table_mappings[filtered_prop] == "imported_locations"
+      end)
 
-    imported_q =
-      from i in Imported.Base.query_imported("imported_locations", site, query),
-        where: i.region != "",
-        group_by: i.region,
-        select_merge: %{region_code: i.region, count: fragment("sum(?)", i.pageviews)}
+    if supports_filter_set? do
+      native_q =
+        native_q
+        |> exclude(:order_by)
+        |> exclude(:select)
+        |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")})
 
-    from(s in subquery(native_q),
-      full_join: i in subquery(imported_q),
-      on: s.region_code == i.region_code,
-      select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code),
-      order_by: [desc: fragment("? + ?", s.count, i.count)]
-    )
-  end
+      imported_q =
+        from i in Imported.Base.query_imported("imported_locations", site, query),
+          where: i.region != "",
+          group_by: i.region,
+          select_merge: %{region_code: i.region, count: fragment("sum(?)", i.pageviews)}
 
-  def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{filters: [_ | _]}) do
-    native_q
+      from(s in subquery(native_q),
+        full_join: i in subquery(imported_q),
+        on: s.region_code == i.region_code,
+        select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code),
+        order_by: [desc: fragment("? + ?", s.count, i.count)]
+      )
+    else
+      native_q
+    end
   end
 
   def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{
@@ -159,24 +168,34 @@ defmodule Plausible.Stats.Imported do
   end
 
   def merge_imported_city_suggestions(native_q, site, query) do
-    native_q =
-      native_q
-      |> exclude(:order_by)
-      |> exclude(:select)
-      |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")})
+    supports_filter_set? =
+      Enum.all?(query.filters, fn filter ->
+        [_, filtered_prop | _] = filter
+        @property_to_table_mappings[filtered_prop] == "imported_locations"
+      end)
 
-    imported_q =
-      from i in Imported.Base.query_imported("imported_locations", site, query),
-        where: i.city != 0,
-        group_by: i.city,
-        select_merge: %{city_id: i.city, count: fragment("sum(?)", i.pageviews)}
+    if supports_filter_set? do
+      native_q =
+        native_q
+        |> exclude(:order_by)
+        |> exclude(:select)
+        |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")})
 
-    from(s in subquery(native_q),
-      full_join: i in subquery(imported_q),
-      on: s.city_id == i.city_id,
-      select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id),
-      order_by: [desc: fragment("? + ?", s.count, i.count)]
-    )
+      imported_q =
+        from i in Imported.Base.query_imported("imported_locations", site, query),
+          where: i.city != 0,
+          group_by: i.city,
+          select_merge: %{city_id: i.city, count: fragment("sum(?)", i.pageviews)}
+
+      from(s in subquery(native_q),
+        full_join: i in subquery(imported_q),
+        on: s.city_id == i.city_id,
+        select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id),
+        order_by: [desc: fragment("? + ?", s.count, i.count)]
+      )
+    else
+      native_q
+    end
   end
 
   def merge_imported_filter_suggestions(
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index 4b6c529a75d5..bb8bfb845df1 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -663,11 +663,12 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
              ]
     end
 
-    test "ignores imported data in country suggestions when filters applied", %{
-      conn: conn,
-      site: site,
-      site_import: site_import
-    } do
+    test "ignores imported data in country suggestions when a different property is filtered by",
+         %{
+           conn: conn,
+           site: site,
+           site_import: site_import
+         } do
       populate_stats(site, site_import.id, [
         build(:pageview, country_code: "EE", referrer_source: "Bing"),
         build(:imported_locations, country: "GB")
@@ -684,6 +685,26 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       assert json_response(conn, 200) == [%{"value" => "EE", "label" => "Estonia"}]
     end
 
+    test "queries imported countries when filtering by country", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, date: ~D[2019-01-01], country: "EE")
+      ])
+
+      filters = Jason.encode!(%{country: "EE"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "EE", "label" => "Estonia"}]
+    end
+
     test "ignores imported country data when not requested", %{
       conn: conn,
       site: site,
@@ -728,11 +749,12 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       end
     end
 
-    test "ignores imported data in region suggestions when filters applied", %{
-      conn: conn,
-      site: site,
-      site_import: site_import
-    } do
+    test "ignores imported data in region suggestions when a different property is filtered by",
+         %{
+           conn: conn,
+           site: site,
+           site_import: site_import
+         } do
       populate_stats(site, site_import.id, [
         build(:pageview,
           country_code: "EE",
@@ -753,6 +775,26 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       assert json_response(conn, 200) == [%{"value" => "EE-39", "label" => "Hiiumaa"}]
     end
 
+    test "queries imported regions when filtering by region", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, date: ~D[2019-01-01], region: "EE-39")
+      ])
+
+      filters = Jason.encode!(%{region: "EE-39"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "EE-39", "label" => "Hiiumaa"}]
+    end
+
     test "ignores imported region data when not requested", %{
       conn: conn,
       site: site,
@@ -809,7 +851,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       end
     end
 
-    test "ignores imported data in city suggestions when filters applied", %{
+    test "ignores imported data in city suggestions when a different property is filtered by", %{
       conn: conn,
       site: site,
       site_import: site_import
@@ -835,6 +877,26 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       assert json_response(conn, 200) == [%{"value" => "591632", "label" => "Kärdla"}]
     end
 
+    test "queries imported cities when filtering by city", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, date: ~D[2019-01-01], city: 591_632)
+      ])
+
+      filters = Jason.encode!(%{city: "591632"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/city?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [%{"value" => "591632", "label" => "Kärdla"}]
+    end
+
     test "ignores imported city data when not requested", %{
       conn: conn,
       site: site,

From 82ba93d27e597fa18cc2a9030887ecfc99fba398 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 28 May 2024 13:08:45 +0100
Subject: [PATCH 21/41] support filtering by multiple props from the same table

---
 lib/plausible/stats/imported/base.ex          | 39 +++++++------
 .../aggregate_test.exs                        |  6 +-
 .../breakdown_test.exs                        | 58 +++++++++++++++++++
 3 files changed, 82 insertions(+), 21 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 7dd0ffd643b5..5a10425fed52 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -74,20 +74,23 @@ defmodule Plausible.Stats.Imported.Base do
     Imported.property_to_table_mappings()[property]
   end
 
-  def decide_table(%Query{filters: [filter], property: property}) do
-    [_op, filtered_prop | _] = filter
-    table_candidate = Imported.property_to_table_mappings()[filtered_prop]
-
-    cond do
-      is_nil(property) -> table_candidate
-      property == "visit:device" and filtered_prop == "visit:screen" -> table_candidate
-      property == filtered_prop -> table_candidate
-      true -> nil
+  def decide_table(%Query{filters: filters, property: property}) do
+    table_candidates =
+      filters
+      |> Enum.map(fn [_, prop | _] -> prop end)
+      |> Enum.concat(if property, do: [property], else: [])
+      |> Enum.map(fn
+        "visit:screen" -> "visit:device"
+        prop -> prop
+      end)
+      |> Enum.map(&Imported.property_to_table_mappings()[&1])
+
+    case Enum.uniq(table_candidates) do
+      [candidate] -> candidate
+      _ -> nil
     end
   end
 
-  def decide_table(_query_with_more_than_one_filter), do: nil
-
   defp apply_filter(q, %Query{filters: [[:is, "event:goal", {:event, name}]]})
        when name in @special_goals do
     where(q, [i], i.name == ^name)
@@ -98,13 +101,13 @@ defmodule Plausible.Stats.Imported.Base do
     raise "Unimplemented"
   end
 
-  defp apply_filter(q, %Query{filters: [[_, filtered_prop | _] = filter]}) do
-    db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
-    mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
-    condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)
+  defp apply_filter(q, %Query{filters: filters}) do
+    Enum.reduce(filters, q, fn [_, filtered_prop | _] = filter, q ->
+      db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
+      mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
+      condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)
 
-    where(q, ^condition)
+      where(q, ^condition)
+    end)
   end
-
-  defp apply_filter(q, _), do: q
 end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 5dcbe84601b8..523c42e2b757 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -561,14 +561,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
              }
     end
 
-    test "ignores imported data when filters are applied", %{
+    test "includes imported data in comparison when filter applied", %{
       conn: conn,
       site: site,
       site_import: site_import
     } do
       populate_stats(site, site_import.id, [
         build(:imported_visitors, date: ~D[2023-01-01]),
-        build(:imported_sources, date: ~D[2023-01-01]),
+        build(:imported_sources, source: "Google", date: ~D[2023-01-01], visitors: 3),
         build(:pageview,
           referrer_source: "Google",
           timestamp: ~N[2023-01-02 00:10:00]
@@ -587,7 +587,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
         })
 
       assert json_response(conn, 200)["results"] == %{
-               "visitors" => %{"value" => 1, "change" => 100}
+               "visitors" => %{"value" => 1, "change" => -67}
              }
     end
 
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index fee6f7d53ef6..92972c05d57d 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -3509,5 +3509,63 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
 
       refute json_response(conn, 200)["warning"]
     end
+
+    test "applies multiple filters if the properties belong to the same table", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),
+        build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),
+        build(:imported_sources,
+          source: "Facebook",
+          utm_medium: "something_else",
+          utm_term: "one"
+        )
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/breakdown", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "property" => "visit:source",
+          "filters" => "visit:utm_medium==organic;visit:utm_term==one",
+          "with_imported" => "true"
+        })
+
+      assert json_response(conn, 200) == %{
+               "results" => [%{"source" => "Google", "visitors" => 1}]
+             }
+    end
+
+    test "ignores imported data if filtered property belongs to a different table than the breakdown property", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:imported_sources, source: "Google"),
+        build(:imported_devices, device: "Desktop"),
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/breakdown", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "property" => "visit:source",
+          "filters" => "visit:device==Desktop",
+          "with_imported" => "true"
+        })
+
+      assert %{
+               "results" => [],
+               "warning" => warning
+             } = json_response(conn, 200)
+
+      assert warning =~ "Imported stats are not included in the results"
+    end
   end
 end

From 39fec37d64c6463a6608cbe0ff165007990773c1 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Tue, 28 May 2024 15:11:22 +0100
Subject: [PATCH 22/41] Implement filtering by goals

---
 lib/plausible/stats/breakdown.ex              |  22 +-
 lib/plausible/stats/imported/base.ex          |  51 ++--
 lib/plausible/stats/imported/imported.ex      | 101 +++++---
 .../breakdown_test.exs                        |  11 +-
 .../api/stats_controller/conversions_test.exs | 241 ++++++++++++++++++
 5 files changed, 364 insertions(+), 62 deletions(-)

diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index cfc2929cad30..1c7adfac2145 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -43,6 +43,11 @@ defmodule Plausible.Stats.Breakdown do
         property: "event:name"
     }
 
+    event_query =
+      struct!(event_query,
+        include_imported: event_query.include_imported && schema_supports_query?(event_query)
+      )
+
     if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
 
     no_revenue = {nil, metrics -- @revenue_metrics}
@@ -74,12 +79,19 @@ defmodule Plausible.Stats.Breakdown do
 
     page_q =
       if Enum.any?(pageview_goals) do
+        page_query = struct!(query, property: "event:page")
+
+        page_query =
+          struct!(page_query,
+            include_imported: page_query.include_imported && schema_supports_query?(page_query)
+          )
+
         page_exprs = Enum.map(pageview_goals, & &1.page_path)
         page_regexes = Enum.map(page_exprs, &page_regex/1)
 
         select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
 
-        from(e in base_event_query(site, query),
+        from(e in base_event_query(site, page_query),
           order_by: [desc: fragment("uniq(?)", e.user_id)],
           where:
             fragment(
@@ -94,7 +106,7 @@ defmodule Plausible.Stats.Breakdown do
           }
         )
         |> select_merge(^select_columns)
-        |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select)
+        |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)
         |> apply_pagination(pagination)
       else
         nil
@@ -185,6 +197,12 @@ defmodule Plausible.Stats.Breakdown do
           |> struct!(property: "visit:entry_page")
       end
 
+    entry_page_query =
+      struct!(entry_page_query,
+        include_imported:
+          entry_page_query.include_imported && schema_supports_query?(entry_page_query)
+      )
+
     if Enum.any?(event_metrics) && Enum.empty?(event_result) do
       []
     else
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 5a10425fed52..139004668285 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -8,7 +8,6 @@ defmodule Plausible.Stats.Imported.Base do
 
   @goals_with_url Imported.goals_with_url()
   @goals_with_path Imported.goals_with_path()
-  @special_goals @goals_with_path ++ @goals_with_url
 
   @db_field_mappings %{
     referrer_source: :source,
@@ -20,19 +19,19 @@ defmodule Plausible.Stats.Imported.Base do
     subdivision1_code: :region,
     city_geoname_id: :city,
     entry_page_hostname: :hostname,
-    pathname: :page
+    pathname: :page,
+    url: :link_url
   }
 
   def query_imported(site, query) do
-    query = Imported.drop_redundant_filters(query)
-
     query
+    |> Imported.transform_filters()
     |> decide_table()
     |> query_imported(site, query)
   end
 
   def query_imported(table, site, query) do
-    query = Imported.drop_redundant_filters(query)
+    query = Imported.transform_filters(query)
     import_ids = site.complete_import_ids
     %{first: date_from, last: date_to} = query.date_range
 
@@ -52,7 +51,7 @@ defmodule Plausible.Stats.Imported.Base do
 
   def decide_table(%Query{filters: filters, property: "event:props:url"}) do
     case filters do
-      [[:is, "event:goal", {:event, name}]] when name in @goals_with_url ->
+      [[:is, "event:name", name]] when name in @goals_with_url ->
         "imported_custom_events"
 
       _ ->
@@ -62,7 +61,7 @@ defmodule Plausible.Stats.Imported.Base do
 
   def decide_table(%Query{filters: filters, property: "event:props:path"}) do
     case filters do
-      [[:is, "event:goal", {:event, name}]] when name in @goals_with_path ->
+      [[:is, "event:name", name]] when name in @goals_with_path ->
         "imported_custom_events"
 
       _ ->
@@ -70,10 +69,38 @@ defmodule Plausible.Stats.Imported.Base do
     end
   end
 
+  def decide_table(%Query{filters: [], property: "event:goal"}) do
+    "imported_custom_events"
+  end
+
   def decide_table(%Query{filters: [], property: property}) do
     Imported.property_to_table_mappings()[property]
   end
 
+  def decide_table(%Query{filters: filters, property: "event:goal"}) do
+    any_event_name_filters? =
+      filters
+      |> Enum.filter(fn filter -> Enum.at(filter, 1) in ["event:name"] end)
+      |> Enum.any?()
+
+    any_page_filters? =
+      filters
+      |> Enum.filter(fn filter -> Enum.at(filter, 1) in ["event:page"] end)
+      |> Enum.any?()
+
+    any_other_filters? =
+      filters
+      |> Enum.filter(fn filter -> Enum.at(filter, 1) not in ["event:page", "event:name"] end)
+      |> Enum.any?()
+
+    cond do
+      any_other_filters? -> nil
+      any_event_name_filters? && !any_page_filters? -> "imported_custom_events"
+      any_page_filters? && !any_event_name_filters? -> "imported_pages"
+      true -> nil
+    end
+  end
+
   def decide_table(%Query{filters: filters, property: property}) do
     table_candidates =
       filters
@@ -91,16 +118,6 @@ defmodule Plausible.Stats.Imported.Base do
     end
   end
 
-  defp apply_filter(q, %Query{filters: [[:is, "event:goal", {:event, name}]]})
-       when name in @special_goals do
-    where(q, [i], i.name == ^name)
-  end
-
-  defp apply_filter(_q, %Query{filters: [[_, "event:goal" | _]]}) do
-    # TODO: implement and test.
-    raise "Unimplemented"
-  end
-
   defp apply_filter(q, %Query{filters: filters}) do
     Enum.reduce(filters, q, fn [_, filtered_prop | _] = filter, q ->
       db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 8ff326559817..3a6704a4c80f 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -33,7 +33,6 @@ defmodule Plausible.Stats.Imported do
     "event:props:path" => "imported_custom_events",
 
     # NOTE: these properties can be only filtered by
-    # TODO: add support for breaking down by translating to device breakdown
     "visit:screen" => "imported_devices",
     "event:hostname" => "imported_pages"
   }
@@ -61,14 +60,9 @@ defmodule Plausible.Stats.Imported do
   (see `@goals_with_url` and `@goals_with_path`).
   """
   def schema_supports_query?(query) do
-    query = drop_redundant_filters(query)
+    query = transform_filters(query)
 
-    # TODO: We want to be able to filter by a goal and still
-    # show a goal breakdown
-    case query.property do
-      "event:goal" -> query.filters == []
-      _ -> not is_nil(Imported.Base.decide_table(query))
-    end
+    not is_nil(Imported.Base.decide_table(query))
   end
 
   @doc """
@@ -77,14 +71,36 @@ defmodule Plausible.Stats.Imported do
   ignored when deciding whether to include imported data in the query or
   not, as well as when actually applying those filters to the query.
   """
-  def drop_redundant_filters(query) do
-    struct!(query,
-      filters:
-        Enum.reject(query.filters, fn
-          [:is, "event:name", "pageview"] -> true
-          _ -> false
-        end)
-    )
+  def transform_filters(query) do
+    new_filters =
+      query.filters
+      |> Enum.reject(fn
+        [:is, "event:name", "pageview"] -> true
+        _ -> false
+      end)
+      |> Enum.map(fn filter ->
+        case filter do
+          [:is, "event:goal", {:event, name}] ->
+            [[:is, "event:name", name]]
+
+          [:is, "event:goal", {:page, page}] ->
+            [[:is, "event:page", page]]
+
+          [:member, "event:goal", events] ->
+            events
+            |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
+            |> Enum.map(fn
+              {:event, names} -> [:member, "event:name", names]
+              {:page, pages} -> [:member, "event:page", pages]
+            end)
+
+          filter ->
+            [filter]
+        end
+      end)
+      |> Enum.concat()
+
+    struct!(query, filters: new_filters)
   end
 
   def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
@@ -370,30 +386,39 @@ defmodule Plausible.Stats.Imported do
   def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
 
   def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
-    page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
+    query_table =
+      query
+      |> transform_filters()
+      |> Imported.Base.decide_table()
 
-    imported_q =
-      "imported_pages"
-      |> Imported.Base.query_imported(site, query)
-      |> where([i], i.visitors > 0)
-      |> where(
-        [i],
-        fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
-      )
-      |> join(:array, index in fragment("indices"))
-      |> group_by([_i, index], index)
-      |> select_merge([_i, index], %{
-        name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
-      })
-      |> select_imported_metrics(metrics)
+    if query_table == "imported_pages" do
+      page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
 
-    from(s in Ecto.Query.subquery(q),
-      full_join: i in subquery(imported_q),
-      on: s.name == i.name,
-      select: %{}
-    )
-    |> select_joined_dimension(:name)
-    |> select_joined_metrics(metrics)
+      imported_q =
+        "imported_pages"
+        |> Imported.Base.query_imported(site, query)
+        |> where([i], i.visitors > 0)
+        |> where(
+          [i],
+          fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
+        )
+        |> join(:array, index in fragment("indices"))
+        |> group_by([_i, index], index)
+        |> select_merge([_i, index], %{
+          name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
+        })
+        |> select_imported_metrics(metrics)
+
+      from(s in Ecto.Query.subquery(q),
+        full_join: i in subquery(imported_q),
+        on: s.name == i.name,
+        select: %{}
+      )
+      |> select_joined_dimension(:name)
+      |> select_joined_metrics(metrics)
+    else
+      q
+    end
   end
 
   def total_imported_visitors(site, query) do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 92972c05d57d..8be42e318717 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -3540,15 +3540,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
              }
     end
 
-    test "ignores imported data if filtered property belongs to a different table than the breakdown property", %{
-      conn: conn,
-      site: site
-    } do
+    test "ignores imported data if filtered property belongs to a different table than the breakdown property",
+         %{
+           conn: conn,
+           site: site
+         } do
       site_import = insert(:site_import, site: site)
 
       populate_stats(site, site_import.id, [
         build(:imported_sources, source: "Google"),
-        build(:imported_devices, device: "Desktop"),
+        build(:imported_devices, device: "Desktop")
       ])
 
       conn =
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index 1f5125464a70..fe4c0159f0ff 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -807,6 +807,247 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
              ] = json_response(conn, 200)["results"]
     end
 
+    test "returns only custom event goals with a custom event goal filter", %{
+      conn: conn,
+      site: site
+    } do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, event_name: "Activation")
+      insert(:goal, site: site, page_path: "/test")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{goal: "Purchase"})
+      url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+      assert [
+               %{
+                 "name" => "Purchase",
+                 "visitors" => 5,
+                 "events" => 7,
+                 "conversion_rate" => 62.5
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
+    test "returns custom event goals with more than one option in goal filter", %{
+      conn: conn,
+      site: site
+    } do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, event_name: "Activation")
+      insert(:goal, site: site, page_path: "/test")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Activation",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_custom_events,
+          name: "Activation",
+          visitors: 2,
+          events: 4,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{goal: "Purchase|Activation"})
+      url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+      assert [
+               %{
+                 "name" => "Purchase",
+                 "visitors" => 5,
+                 "events" => 7,
+                 "conversion_rate" => 55.6
+               },
+               %{
+                 "name" => "Activation",
+                 "visitors" => 3,
+                 "events" => 5,
+                 "conversion_rate" => 33.3
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
+    test "returns only pageview goals with a pageview goal filter", %{
+      conn: conn,
+      site: site
+    } do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, event_name: "Activation")
+      insert(:goal, site: site, page_path: "/test")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{goal: "Visit /test"})
+      url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+      assert [
+               %{
+                 "name" => "Visit /test",
+                 "visitors" => 3,
+                 "events" => 3,
+                 "conversion_rate" => 37.5
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
+    test "returns pageview goals with more than one option in pageview goal filter", %{
+      conn: conn,
+      site: site
+    } do
+      insert(:goal, site: site, event_name: "Purchase")
+      insert(:goal, site: site, event_name: "Activation")
+      insert(:goal, site: site, page_path: "/test")
+      insert(:goal, site: site, page_path: "/blog")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/test"
+        ),
+        build(:pageview,
+          timestamp: ~N[2021-01-01 00:00:01],
+          pathname: "/blog"
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:event,
+          name: "Purchase",
+          timestamp: ~N[2021-01-01 00:00:03]
+        ),
+        build(:imported_custom_events,
+          name: "Purchase",
+          visitors: 3,
+          events: 5,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/test",
+          visitors: 2,
+          pageviews: 2,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_pages,
+          page: "/blog",
+          visitors: 1,
+          pageviews: 1,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{goal: "Visit /test|Visit /blog"})
+      url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+      assert [
+               %{
+                 "name" => "Visit /test",
+                 "visitors" => 3,
+                 "events" => 3,
+                 "conversion_rate" => 33.3
+               },
+               %{
+                 "name" => "Visit /blog",
+                 "visitors" => 2,
+                 "events" => 2,
+                 "conversion_rate" => 22.2
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
     test "calculates conversion_rate for goals with glob pattern with imported data", %{
       conn: conn,
       site: site

From a86219fbf241308ce776a03c63d792aae8619798 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Tue, 28 May 2024 22:18:58 +0200
Subject: [PATCH 23/41] Make views per visit metric work for import entry and
 exit pages

---
 lib/plausible/stats/imported/imported.ex | 26 ++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 3a6704a4c80f..0c16bf397daf 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -587,6 +587,32 @@ defmodule Plausible.Stats.Imported do
     |> select_imported_metrics(rest)
   end
 
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
+         [:views_per_visit | rest]
+       ) do
+    q
+    |> where([i], i.pageviews > 0)
+    |> select_merge([i], %{
+      pageviews: sum(i.pageviews),
+      __internal_visits: sum(i.entrances)
+    })
+    |> select_imported_metrics(rest)
+  end
+
+  defp select_imported_metrics(
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
+         [:views_per_visit | rest]
+       ) do
+    q
+    |> where([i], i.pageviews > 0)
+    |> select_merge([i], %{
+      pageviews: sum(i.pageviews),
+      __internal_visits: sum(i.exits)
+    })
+    |> select_imported_metrics(rest)
+  end
+
   defp select_imported_metrics(q, [:views_per_visit | rest]) do
     q
     |> where([i], i.pageviews > 0)

From 502b7ae351d6c39e206e34ba581add53b3d6888a Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 09:55:58 +0200
Subject: [PATCH 24/41] Get rid of circular dependencies between Stats.Imported
 and Stats.Imported.Base

---
 lib/plausible/stats/imported/base.ex     | 120 +++++++++++++++++------
 lib/plausible/stats/imported/imported.ex |  82 ++--------------
 2 files changed, 97 insertions(+), 105 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 139004668285..ac90c66f1ea8 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -2,10 +2,41 @@ defmodule Plausible.Stats.Imported.Base do
   @moduledoc """
   A module for building the base of an imported stats query
   """
-  alias Plausible.Stats.{Query, Imported, Filters}
 
   import Ecto.Query
 
+  alias Plausible.Imported
+  alias Plausible.Stats.Filters
+  alias Plausible.Stats.Query
+
+  @property_to_table_mappings %{
+    "visit:source" => "imported_sources",
+    "visit:referrer" => "imported_sources",
+    "visit:utm_source" => "imported_sources",
+    "visit:utm_medium" => "imported_sources",
+    "visit:utm_campaign" => "imported_sources",
+    "visit:utm_term" => "imported_sources",
+    "visit:utm_content" => "imported_sources",
+    "visit:entry_page" => "imported_entry_pages",
+    "visit:exit_page" => "imported_exit_pages",
+    "visit:country" => "imported_locations",
+    "visit:region" => "imported_locations",
+    "visit:city" => "imported_locations",
+    "visit:device" => "imported_devices",
+    "visit:browser" => "imported_browsers",
+    "visit:browser_version" => "imported_browsers",
+    "visit:os" => "imported_operating_systems",
+    "visit:os_version" => "imported_operating_systems",
+    "event:page" => "imported_pages",
+    "event:name" => "imported_custom_events",
+    "event:props:url" => "imported_custom_events",
+    "event:props:path" => "imported_custom_events",
+
+    # NOTE: these properties can be only filtered by
+    "visit:screen" => "imported_devices",
+    "event:hostname" => "imported_pages"
+  }
+
   @goals_with_url Imported.goals_with_url()
   @goals_with_path Imported.goals_with_path()
 
@@ -23,15 +54,17 @@ defmodule Plausible.Stats.Imported.Base do
     url: :link_url
   }
 
+  def property_to_table_mappings(), do: @property_to_table_mappings
+
   def query_imported(site, query) do
     query
-    |> Imported.transform_filters()
+    |> transform_filters()
     |> decide_table()
     |> query_imported(site, query)
   end
 
   def query_imported(table, site, query) do
-    query = Imported.transform_filters(query)
+    query = transform_filters(query)
     import_ids = site.complete_import_ids
     %{first: date_from, last: date_to} = query.date_range
 
@@ -45,11 +78,49 @@ defmodule Plausible.Stats.Imported.Base do
     |> apply_filter(query)
   end
 
-  def decide_table(%Query{filters: [], property: nil}), do: "imported_visitors"
-  def decide_table(%Query{filters: [], property: "event:props:url"}), do: nil
-  def decide_table(%Query{filters: [], property: "event:props:path"}), do: nil
+  def decide_table(query) do
+    query
+    |> transform_filters()
+    |> do_decide_table()
+  end
+
+  defp transform_filters(query) do
+    new_filters =
+      query.filters
+      |> Enum.reject(fn
+        [:is, "event:name", "pageview"] -> true
+        _ -> false
+      end)
+      |> Enum.map(fn filter ->
+        case filter do
+          [:is, "event:goal", {:event, name}] ->
+            [[:is, "event:name", name]]
+
+          [:is, "event:goal", {:page, page}] ->
+            [[:is, "event:page", page]]
+
+          [:member, "event:goal", events] ->
+            events
+            |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
+            |> Enum.map(fn
+              {:event, names} -> [:member, "event:name", names]
+              {:page, pages} -> [:member, "event:page", pages]
+            end)
+
+          filter ->
+            [filter]
+        end
+      end)
+      |> Enum.concat()
 
-  def decide_table(%Query{filters: filters, property: "event:props:url"}) do
+    struct!(query, filters: new_filters)
+  end
+
+  defp do_decide_table(%Query{filters: [], property: nil}), do: "imported_visitors"
+  defp do_decide_table(%Query{filters: [], property: "event:props:url"}), do: nil
+  defp do_decide_table(%Query{filters: [], property: "event:props:path"}), do: nil
+
+  defp do_decide_table(%Query{filters: filters, property: "event:props:url"}) do
     case filters do
       [[:is, "event:name", name]] when name in @goals_with_url ->
         "imported_custom_events"
@@ -59,7 +130,7 @@ defmodule Plausible.Stats.Imported.Base do
     end
   end
 
-  def decide_table(%Query{filters: filters, property: "event:props:path"}) do
+  defp do_decide_table(%Query{filters: filters, property: "event:props:path"}) do
     case filters do
       [[:is, "event:name", name]] when name in @goals_with_path ->
         "imported_custom_events"
@@ -69,39 +140,30 @@ defmodule Plausible.Stats.Imported.Base do
     end
   end
 
-  def decide_table(%Query{filters: [], property: "event:goal"}) do
+  defp do_decide_table(%Query{filters: [], property: "event:goal"}) do
     "imported_custom_events"
   end
 
-  def decide_table(%Query{filters: [], property: property}) do
-    Imported.property_to_table_mappings()[property]
+  defp do_decide_table(%Query{filters: [], property: property}) do
+    @property_to_table_mappings[property]
   end
 
-  def decide_table(%Query{filters: filters, property: "event:goal"}) do
-    any_event_name_filters? =
-      filters
-      |> Enum.filter(fn filter -> Enum.at(filter, 1) in ["event:name"] end)
-      |> Enum.any?()
+  defp do_decide_table(%Query{filters: filters, property: "event:goal"}) do
+    filter_props = Enum.map(filters, &Enum.at(&1, 1))
 
-    any_page_filters? =
-      filters
-      |> Enum.filter(fn filter -> Enum.at(filter, 1) in ["event:page"] end)
-      |> Enum.any?()
-
-    any_other_filters? =
-      filters
-      |> Enum.filter(fn filter -> Enum.at(filter, 1) not in ["event:page", "event:name"] end)
-      |> Enum.any?()
+    any_event_name_filters? = "event:name" in filter_props
+    any_page_filters? = "event:page" in filter_props
+    any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))
 
     cond do
       any_other_filters? -> nil
-      any_event_name_filters? && !any_page_filters? -> "imported_custom_events"
-      any_page_filters? && !any_event_name_filters? -> "imported_pages"
+      any_event_name_filters? and not any_page_filters? -> "imported_custom_events"
+      any_page_filters? and not any_event_name_filters? -> "imported_pages"
       true -> nil
     end
   end
 
-  def decide_table(%Query{filters: filters, property: property}) do
+  defp do_decide_table(%Query{filters: filters, property: property}) do
     table_candidates =
       filters
       |> Enum.map(fn [_, prop | _] -> prop end)
@@ -110,7 +172,7 @@ defmodule Plausible.Stats.Imported.Base do
         "visit:screen" -> "visit:device"
         prop -> prop
       end)
-      |> Enum.map(&Imported.property_to_table_mappings()[&1])
+      |> Enum.map(&@property_to_table_mappings[&1])
 
     case Enum.uniq(table_candidates) do
       [candidate] -> candidate
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 0c16bf397daf..dc83e3c13c1e 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -1,43 +1,18 @@
 defmodule Plausible.Stats.Imported do
   use Plausible.ClickhouseRepo
-  alias Plausible.Stats.{Query, Base, Imported}
 
   import Ecto.Query
   import Plausible.Stats.Fragments
 
+  alias Plausible.Stats.Base
+  alias Plausible.Stats.Imported
+  alias Plausible.Stats.Query
+
   @no_ref "Direct / None"
   @not_set "(not set)"
   @none "(none)"
 
-  @property_to_table_mappings %{
-    "visit:source" => "imported_sources",
-    "visit:referrer" => "imported_sources",
-    "visit:utm_source" => "imported_sources",
-    "visit:utm_medium" => "imported_sources",
-    "visit:utm_campaign" => "imported_sources",
-    "visit:utm_term" => "imported_sources",
-    "visit:utm_content" => "imported_sources",
-    "visit:entry_page" => "imported_entry_pages",
-    "visit:exit_page" => "imported_exit_pages",
-    "visit:country" => "imported_locations",
-    "visit:region" => "imported_locations",
-    "visit:city" => "imported_locations",
-    "visit:device" => "imported_devices",
-    "visit:browser" => "imported_browsers",
-    "visit:browser_version" => "imported_browsers",
-    "visit:os" => "imported_operating_systems",
-    "visit:os_version" => "imported_operating_systems",
-    "event:page" => "imported_pages",
-    "event:name" => "imported_custom_events",
-    "event:props:url" => "imported_custom_events",
-    "event:props:path" => "imported_custom_events",
-
-    # NOTE: these properties can be only filtered by
-    "visit:screen" => "imported_devices",
-    "event:hostname" => "imported_pages"
-  }
-
-  def property_to_table_mappings(), do: @property_to_table_mappings
+  @property_to_table_mappings Imported.Base.property_to_table_mappings()
 
   @imported_properties Map.keys(@property_to_table_mappings)
 
@@ -60,49 +35,9 @@ defmodule Plausible.Stats.Imported do
   (see `@goals_with_url` and `@goals_with_path`).
   """
   def schema_supports_query?(query) do
-    query = transform_filters(query)
-
     not is_nil(Imported.Base.decide_table(query))
   end
 
-  @doc """
-  This function gets rid of filters that do not make sense in the context
-  of imported data, for example `event:name == "pageview"`. They will be
-  ignored when deciding whether to include imported data in the query or
-  not, as well as when actually applying those filters to the query.
-  """
-  def transform_filters(query) do
-    new_filters =
-      query.filters
-      |> Enum.reject(fn
-        [:is, "event:name", "pageview"] -> true
-        _ -> false
-      end)
-      |> Enum.map(fn filter ->
-        case filter do
-          [:is, "event:goal", {:event, name}] ->
-            [[:is, "event:name", name]]
-
-          [:is, "event:goal", {:page, page}] ->
-            [[:is, "event:page", page]]
-
-          [:member, "event:goal", events] ->
-            events
-            |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
-            |> Enum.map(fn
-              {:event, names} -> [:member, "event:name", names]
-              {:page, pages} -> [:member, "event:page", pages]
-            end)
-
-          filter ->
-            [filter]
-        end
-      end)
-      |> Enum.concat()
-
-    struct!(query, filters: new_filters)
-  end
-
   def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
         include_imported: false
       }) do
@@ -386,12 +321,7 @@ defmodule Plausible.Stats.Imported do
   def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
 
   def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
-    query_table =
-      query
-      |> transform_filters()
-      |> Imported.Base.decide_table()
-
-    if query_table == "imported_pages" do
+    if Imported.Base.decide_table(query) == "imported_pages" do
       page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
 
       imported_q =

From 222a20d368e6a05df45f3905d1ec6b26e4b6587c Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 10:54:56 +0200
Subject: [PATCH 25/41] Clean up Query struct manipulation in Breakdown

---
 lib/plausible/stats/breakdown.ex | 29 +++++++++--------------------
 lib/plausible/stats/query.ex     | 15 +++++++++++++--
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 1c7adfac2145..08fc04b64a19 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -37,16 +37,11 @@ defmodule Plausible.Stats.Breakdown do
     {event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
     events = Enum.map(event_goals, & &1.event_name)
 
-    event_query = %Query{
-      query
-      | filters: query.filters ++ [[:member, "event:name", events]],
-        property: "event:name"
-    }
-
     event_query =
-      struct!(event_query,
-        include_imported: event_query.include_imported && schema_supports_query?(event_query)
-      )
+      query
+      |> Query.put_filter([:member, "event:name", events])
+      |> Query.set_property("event:name")
+      |> Query.refresh(site)
 
     if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
 
@@ -81,10 +76,9 @@ defmodule Plausible.Stats.Breakdown do
       if Enum.any?(pageview_goals) do
         page_query = struct!(query, property: "event:page")
 
-        page_query =
-          struct!(page_query,
-            include_imported: page_query.include_imported && schema_supports_query?(page_query)
-          )
+        query
+        |> Query.set_property("event:page")
+        |> Query.refresh(site)
 
         page_exprs = Enum.map(pageview_goals, & &1.page_path)
         page_regexes = Enum.map(page_exprs, &page_regex/1)
@@ -194,15 +188,10 @@ defmodule Plausible.Stats.Breakdown do
           query
           |> Query.remove_filters(["event:page"])
           |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])
-          |> struct!(property: "visit:entry_page")
+          |> Query.set_property("visit:entry_page")
+          |> Query.refresh(site)
       end
 
-    entry_page_query =
-      struct!(entry_page_query,
-        include_imported:
-          entry_page_query.include_imported && schema_supports_query?(entry_page_query)
-      )
-
     if Enum.any?(event_metrics) && Enum.empty?(event_result) do
       []
     else
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 3fe853e6a4f1..ad9a3591247b 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -19,6 +19,7 @@ defmodule Plausible.Stats.Query do
 
   @type t :: %__MODULE__{}
 
+  @spec from(Plausible.Site.t(), map()) :: t()
   def from(site, params) do
     now = NaiveDateTime.utc_now(:second)
 
@@ -201,12 +202,22 @@ defmodule Plausible.Stats.Query do
     struct!(query, filters: Filters.parse(params["filters"]))
   end
 
+  @spec set_property(t(), String.t()) :: t()
+  def set_property(query, property) do
+    struct!(query, property: property)
+  end
+
   def put_filter(query, filter) do
     struct!(query,
       filters: query.filters ++ [filter]
     )
   end
 
+  @spec refresh(t(), Plausible.Site.t()) :: t()
+  def refresh(query, site) do
+    put_imported_opts(query, site)
+  end
+
   def remove_filters(query, prefixes) do
     new_filters =
       Enum.reject(query.filters, fn [_, filter_key | _rest] ->
@@ -246,8 +257,8 @@ defmodule Plausible.Stats.Query do
     end
   end
 
-  defp put_imported_opts(query, site, params) do
-    requested? = params["with_imported"] == "true"
+  defp put_imported_opts(query, site, params \\ %{}) do
+    requested? = params["with_imported"] == "true" || query.imported_data_requested
 
     case ensure_include_imported(query, site, requested?) do
       :ok ->

From 08e7284f5b53653200b3734c0d34e18160b8767b Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 13:16:53 +0200
Subject: [PATCH 26/41] Rename helper function for clarity

---
 lib/plausible/stats/breakdown.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 08fc04b64a19..f63e6458391e 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -252,18 +252,18 @@ defmodule Plausible.Stats.Breakdown do
               "visit:entry_page",
               "visit:referrer"
             ] do
-    update_hostname(query, "visit:entry_page_hostname")
+    update_hostname_filter_prop(query, "visit:entry_page_hostname")
   end
 
   defp maybe_update_breakdown_filters(%Query{property: "visit:exit_page"} = query) do
-    update_hostname(query, "visit:exit_page_hostname")
+    update_hostname_filter_prop(query, "visit:exit_page_hostname")
   end
 
   defp maybe_update_breakdown_filters(query) do
     query
   end
 
-  defp update_hostname(query, visit_prop) do
+  defp update_hostname_filter_prop(query, visit_prop) do
     case Query.get_filter(query, "event:hostname") do
       nil ->
         query

From 431b4a8e22147daa0ee4f4258759879d74aae903 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 14:27:26 +0200
Subject: [PATCH 27/41] Automatically refresh query struct state after
 modifications

---
 lib/plausible/imported.ex                     |  5 ++
 lib/plausible/stats/base.ex                   | 20 ++++++-
 lib/plausible/stats/breakdown.ex              | 11 +---
 lib/plausible/stats/comparisons.ex            |  6 +-
 lib/plausible/stats/email_report.ex           |  4 +-
 lib/plausible/stats/query.ex                  | 58 +++++++++++++------
 .../api/external_stats_controller.ex          |  2 +-
 .../controllers/api/stats_controller.ex       |  2 +-
 .../aggregate_test.exs                        | 28 +++++++++
 .../custom_prop_breakdown_test.exs            |  2 +
 10 files changed, 104 insertions(+), 34 deletions(-)

diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index b8e00f19d890..35fa3489229f 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -54,6 +54,11 @@ defmodule Plausible.Imported do
     @max_complete_imports
   end
 
+  @spec imported_custom_props() :: [String.t()]
+  def imported_custom_props do
+    Plausible.Props.internal_keys()
+  end
+
   @spec goals_with_url() :: [String.t()]
   def goals_with_url() do
     @goals_with_url
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 8b30b2910adf..dde073e05c59 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -308,7 +308,10 @@ defmodule Plausible.Stats.Base do
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
-      query = struct!(query, property: nil)
+      query =
+        query
+        |> Query.set_property(nil)
+        |> Query.exclude_imported()
 
       q
       |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
@@ -334,7 +337,20 @@ defmodule Plausible.Stats.Base do
       total_query =
         query
         |> Query.remove_filters(["event:goal", "event:props"])
-        |> struct!(property: nil)
+        |> Query.set_property(nil)
+
+      has_custom_prop_filters? =
+        Enum.any?(query.filters, fn
+          [_, "event:props:" <> prop, _] -> prop not in Plausible.Imported.imported_custom_props()
+          _ -> false
+        end)
+
+      total_query =
+        if has_custom_prop_filters? do
+          Query.exclude_imported(total_query)
+        else
+          total_query
+        end
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index f63e6458391e..23db74a9e04c 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -41,7 +41,6 @@ defmodule Plausible.Stats.Breakdown do
       query
       |> Query.put_filter([:member, "event:name", events])
       |> Query.set_property("event:name")
-      |> Query.refresh(site)
 
     if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
 
@@ -74,11 +73,7 @@ defmodule Plausible.Stats.Breakdown do
 
     page_q =
       if Enum.any?(pageview_goals) do
-        page_query = struct!(query, property: "event:page")
-
-        query
-        |> Query.set_property("event:page")
-        |> Query.refresh(site)
+        page_query = Query.set_property(query, "event:page")
 
         page_exprs = Enum.map(pageview_goals, & &1.page_path)
         page_regexes = Enum.map(page_exprs, &page_regex/1)
@@ -189,7 +184,6 @@ defmodule Plausible.Stats.Breakdown do
           |> Query.remove_filters(["event:page"])
           |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])
           |> Query.set_property("visit:entry_page")
-          |> Query.refresh(site)
       end
 
     if Enum.any?(event_metrics) && Enum.empty?(event_result) do
@@ -269,7 +263,8 @@ defmodule Plausible.Stats.Breakdown do
         query
 
       [op, "event:hostname", value] ->
-        Plausible.Stats.Query.put_filter(query, [op, visit_prop, value])
+        query
+        |> Query.put_filter([op, visit_prop, value])
     end
   end
 
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
index c7bd4bf85898..dcd34bcd7907 100644
--- a/lib/plausible/stats/comparisons.ex
+++ b/lib/plausible/stats/comparisons.ex
@@ -63,7 +63,7 @@ defmodule Plausible.Stats.Comparisons do
 
     with :ok <- validate_mode(source_query, mode),
          {:ok, comparison_query} <- do_compare(source_query, mode, opts),
-         comparison_query <- maybe_include_imported(comparison_query, source_query, site),
+         comparison_query <- maybe_include_imported(comparison_query, source_query),
          do: {:ok, comparison_query}
   end
 
@@ -162,10 +162,10 @@ defmodule Plausible.Stats.Comparisons do
     Date.add(date, -days_to_subtract)
   end
 
-  defp maybe_include_imported(query, source_query, site) do
+  defp maybe_include_imported(query, source_query) do
     requested? = source_query.imported_data_requested
 
-    case Query.ensure_include_imported(query, site, requested?) do
+    case Query.ensure_include_imported(query, requested?) do
       :ok ->
         struct!(query,
           imported_data_requested: true,
diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex
index 63809e3af869..675597731b34 100644
--- a/lib/plausible/stats/email_report.ex
+++ b/lib/plausible/stats/email_report.ex
@@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport do
   end
 
   defp put_top_5_pages(stats, site, query) do
-    query = struct!(query, property: "event:page")
+    query = Query.set_property(query, "event:page")
     pages = Stats.breakdown(site, query, [:visitors], {5, 1})
     Map.put(stats, :pages, pages)
   end
@@ -49,7 +49,7 @@ defmodule Plausible.Stats.EmailReport do
     query =
       query
       |> Query.put_filter([:is_not, "visit:source", "Direct / None"])
-      |> struct!(property: "visit:source")
+      |> Query.set_property("visit:source")
 
     sources = Stats.breakdown(site, query, [:visitors], {5, 1})
 
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index ad9a3591247b..48f621093201 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -12,7 +12,8 @@ defmodule Plausible.Stats.Query do
             skip_imported_reason: nil,
             now: nil,
             experimental_session_count?: false,
-            experimental_reduced_joins?: false
+            experimental_reduced_joins?: false,
+            latest_import_end_date: nil
 
   require OpenTelemetry.Tracer, as: Tracer
   alias Plausible.Stats.{Filters, Interval, Imported}
@@ -204,18 +205,15 @@ defmodule Plausible.Stats.Query do
 
   @spec set_property(t(), String.t()) :: t()
   def set_property(query, property) do
-    struct!(query, property: property)
+    query
+    |> struct!(property: property)
+    |> refresh()
   end
 
   def put_filter(query, filter) do
-    struct!(query,
-      filters: query.filters ++ [filter]
-    )
-  end
-
-  @spec refresh(t(), Plausible.Site.t()) :: t()
-  def refresh(query, site) do
-    put_imported_opts(query, site)
+    query
+    |> struct!(filters: query.filters ++ [filter])
+    |> refresh()
   end
 
   def remove_filters(query, prefixes) do
@@ -224,7 +222,20 @@ defmodule Plausible.Stats.Query do
         Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
       end)
 
-    struct!(query, filters: new_filters)
+    query
+    |> struct!(filters: new_filters)
+    |> refresh()
+  end
+
+  def exclude_imported(query) do
+    struct!(query,
+      include_imported: false,
+      skip_imported_reason: :manual_exclusion
+    )
+  end
+
+  defp refresh(query) do
+    refresh_imported_opts(query)
   end
 
   def has_event_filters?(query) do
@@ -257,10 +268,23 @@ defmodule Plausible.Stats.Query do
     end
   end
 
-  defp put_imported_opts(query, site, params \\ %{}) do
+  defp refresh_imported_opts(query) do
+    put_imported_opts(query, nil, %{})
+  end
+
+  defp put_imported_opts(query, site, params) do
     requested? = params["with_imported"] == "true" || query.imported_data_requested
 
-    case ensure_include_imported(query, site, requested?) do
+    latest_import_end_date =
+      if site do
+        site.latest_import_end_date
+      else
+        query.latest_import_end_date
+      end
+
+    query = struct!(query, latest_import_end_date: latest_import_end_date)
+
+    case ensure_include_imported(query, requested?) do
       :ok ->
         struct!(query,
           imported_data_requested: true,
@@ -276,13 +300,13 @@ defmodule Plausible.Stats.Query do
     end
   end
 
-  @spec ensure_include_imported(t(), Plausible.Site.t(), boolean()) ::
+  @spec ensure_include_imported(t(), boolean()) ::
           :ok | {:error, :not_requested | :no_imported_data | :out_of_range | :unsupported_query}
-  def ensure_include_imported(query, site, requested?) do
+  def ensure_include_imported(query, requested?) do
     cond do
       not requested? -> {:error, :not_requested}
-      is_nil(site.latest_import_end_date) -> {:error, :no_imported_data}
-      Date.after?(query.date_range.first, site.latest_import_end_date) -> {:error, :out_of_range}
+      is_nil(query.latest_import_end_date) -> {:error, :no_imported_data}
+      Date.after?(query.date_range.first, query.latest_import_end_date) -> {:error, :out_of_range}
       not Imported.schema_supports_query?(query) -> {:error, :unsupported_query}
       query.period == "realtime" -> {:error, :unsupported_query}
       true -> :ok
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 7fcb2fca3bc6..f8145b8ccfa0 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
   end
 
   defp maybe_add_warning(payload, %{skip_imported_reason: reason})
-       when reason in [nil, :not_requested, :no_imported_data, :out_of_range] do
+       when reason in [nil, :not_requested, :no_imported_data, :out_of_range, :manual_exclusion] do
     payload
   end
 
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 9c50482d8a34..868194ffa7d6 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -872,7 +872,7 @@ defmodule PlausibleWeb.Api.StatsController do
         |> Query.remove_filters(["visit:exit_page"])
         |> Query.put_filter([:member, "event:page", pages])
         |> Query.put_filter([:is, "event:name", "pageview"])
-        |> struct!(property: "event:page")
+        |> Query.set_property("event:page")
 
       total_pageviews =
         Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 523c42e2b757..e1151c953beb 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -681,6 +681,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
 
       refute json_response(conn, 200)["warning"]
     end
+
+    test "excludes imported data from conversion rate computation when query filters by non-imported props",
+         %{conn: conn, site: site, site_import: site_import} do
+      insert(:goal, site: site, event_name: "Purchase")
+
+      populate_stats(site, site_import.id, [
+        build(:event,
+          name: "Purchase",
+          "meta.key": ["package"],
+          "meta.value": ["large"]
+        ),
+        build(:imported_visitors, visitors: 9)
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/aggregate", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "metrics" => "visitors,conversion_rate",
+          "filters" => "event:goal==Purchase;event:props:package==large",
+          "with_imported" => "true"
+        })
+
+      assert json_response(conn, 200)["results"] == %{
+               "visitors" => %{"value" => 1},
+               "conversion_rate" => %{"value" => 100.0}
+             }
+    end
   end
 
   describe "filters" do
diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
index 8f972a139e71..9155606e01cb 100644
--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
@@ -65,6 +65,8 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
                  "percentage" => 100.0
                }
              ]
+
+      refute json_response(conn, 200)["warning"]
     end
 
     test "returns (none) values in the breakdown", %{conn: conn, site: site} do

From 22d11a64dda5a5d8c71d436d7c86929d664c42ce Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 14:28:59 +0200
Subject: [PATCH 28/41] Shutup credo

---
 lib/plausible/stats/imported/imported.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index dc83e3c13c1e..2589a238734d 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -715,7 +715,7 @@ defmodule Plausible.Stats.Imported do
   end
 
   defp select_joined_metrics(q, []), do: q
-  # TODO: Reverse-engineering the native data bounces and total visit
+  # NOTE: Reverse-engineering the native data bounces and total visit
   # durations to combine with imported data is inefficient. Instead both
   # queries should fetch bounces/total_visit_duration and visits and be
   # used as subqueries to a main query that then find the bounce rate/avg

From 09bc702f1b912ce1df3b2ec8e0eadbf37c24ef4e Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Wed, 29 May 2024 17:50:42 +0100
Subject: [PATCH 29/41] display imported warning bubble in prop breakdown
 section

---
 assets/js/dashboard/stats/behaviours/index.js              | 7 ++++---
 assets/js/dashboard/stats/behaviours/props.js              | 1 +
 .../dashboard/stats/imported-query-unsupported-warning.js  | 6 ++++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js
index 85b8b48f90e9..1c898655e7ff 100644
--- a/assets/js/dashboard/stats/behaviours/index.js
+++ b/assets/js/dashboard/stats/behaviours/index.js
@@ -172,14 +172,14 @@ export default function Behaviours(props) {
     )
   }
 
-  function afterFetchGoalData(apiResponse) {
+  function afterFetchData(apiResponse) {
     const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'
     setImportedQueryUnsupported(unsupportedQuery && !isRealtime())
   }
 
   function renderConversions() {
     if (site.hasGoals) {
-      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} afterFetchData={afterFetchGoalData}/>
+      return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} afterFetchData={afterFetchData}/>
     }
     else if (adminAccess) {
       return (
@@ -231,7 +231,7 @@ export default function Behaviours(props) {
 
   function renderProps() {
     if (site.hasProps && site.propsAvailable) {
-      return <Properties site={site} query={query} />
+      return <Properties site={site} query={query} afterFetchData={afterFetchData}/>
     } else if (adminAccess) {
       let callToAction
 
@@ -342,6 +342,7 @@ export default function Behaviours(props) {
                 {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}
               </h3>
               <ImportedQueryUnsupportedWarning condition={mode === CONVERSIONS  && importedQueryUnsupported}/>
+              <ImportedQueryUnsupportedWarning condition={mode === PROPS && importedQueryUnsupported} message="Imported data is unavailable in this view"/>
             </div>
             {tabs()}
           </div>
diff --git a/assets/js/dashboard/stats/behaviours/props.js b/assets/js/dashboard/stats/behaviours/props.js
index 22d32980d322..17ebe5e245c8 100644
--- a/assets/js/dashboard/stats/behaviours/props.js
+++ b/assets/js/dashboard/stats/behaviours/props.js
@@ -75,6 +75,7 @@ export default function Properties(props) {
     return (
       <ListReport
         fetchData={fetchProps}
+        afterFetchData={props.afterFetchData}
         getFilterFor={getFilterFor}
         keyLabel={propKey}
         metrics={[
diff --git a/assets/js/dashboard/stats/imported-query-unsupported-warning.js b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
index 5c6c64adb0de..1dbe941fda4d 100644
--- a/assets/js/dashboard/stats/imported-query-unsupported-warning.js
+++ b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
@@ -1,10 +1,12 @@
 import React from "react";
 import { ExclamationCircleIcon } from '@heroicons/react/24/outline'
 
-export default function ImportedQueryUnsupportedWarning({condition}) {
+export default function ImportedQueryUnsupportedWarning({condition, message}) {
+  const tooltipMessage = message || "Imported data is excluded due to applied filters"
+  
   if (condition) {
     return (
-      <span tooltip="Imported data is excluded due to applied filters">
+      <span tooltip={tooltipMessage}>
         <ExclamationCircleIcon className="w-6 h-6" />
       </span>
     )

From c9f6a6f4d78040a48c2f85f1dba8dc91b0a15c9f Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Wed, 29 May 2024 18:36:03 +0100
Subject: [PATCH 30/41] Render warning bubble for funnels whenever imported
 data is in the view

---
 assets/js/dashboard/historical.js                | 4 ++--
 assets/js/dashboard/index.js                     | 8 +++++++-
 assets/js/dashboard/stats/behaviours/index.js    | 1 +
 assets/js/dashboard/stats/graph/visitor-graph.js | 3 +++
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/assets/js/dashboard/historical.js b/assets/js/dashboard/historical.js
index 4b93d806be53..8d946525a400 100644
--- a/assets/js/dashboard/historical.js
+++ b/assets/js/dashboard/historical.js
@@ -31,7 +31,7 @@ function Historical(props) {
           <ComparisonInput site={props.site} query={props.query} />
         </div>
       </div>
-      <VisitorGraph site={props.site} query={props.query} />
+      <VisitorGraph site={props.site} query={props.query} updateImportedDataInView={props.updateImportedDataInView}/>
 
       <div className="w-full md:flex">
         <div className={ statsBoxClass }>
@@ -51,7 +51,7 @@ function Historical(props) {
         </div>
       </div>
 
-      <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} />
+      <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} importedDataInView={props.importedDataInView}/>
     </div>
   )
 }
diff --git a/assets/js/dashboard/index.js b/assets/js/dashboard/index.js
index 3fe0f7be0261..b60d2f0f39ce 100644
--- a/assets/js/dashboard/index.js
+++ b/assets/js/dashboard/index.js
@@ -13,8 +13,10 @@ class Dashboard extends React.Component {
   constructor(props) {
     super(props)
     this.updateLastLoadTimestamp = this.updateLastLoadTimestamp.bind(this)
+    this.updateImportedDataInView = this.updateImportedDataInView.bind(this)
     this.state = {
       query: parseQuery(props.location.search, this.props.site),
+      importedDataInView: false,
       lastLoadTimestamp: new Date()
     }
   }
@@ -35,6 +37,10 @@ class Dashboard extends React.Component {
     this.setState({lastLoadTimestamp: new Date()})
   }
 
+  updateImportedDataInView(newBoolean) {
+    this.setState({importedDataInView: newBoolean})
+  }
+
   render() {
     const { site, loggedIn, currentUserRole } = this.props
     const { query, lastLoadTimestamp } = this.state
@@ -42,7 +48,7 @@ class Dashboard extends React.Component {
     if (this.state.query.period === 'realtime') {
       return <Realtime site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>
     } else {
-      return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>
+      return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp} importedDataInView={this.state.importedDataInView} updateImportedDataInView={this.updateImportedDataInView}/>
     }
   }
 }
diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js
index 1c898655e7ff..fba5997278df 100644
--- a/assets/js/dashboard/stats/behaviours/index.js
+++ b/assets/js/dashboard/stats/behaviours/index.js
@@ -343,6 +343,7 @@ export default function Behaviours(props) {
               </h3>
               <ImportedQueryUnsupportedWarning condition={mode === CONVERSIONS  && importedQueryUnsupported}/>
               <ImportedQueryUnsupportedWarning condition={mode === PROPS && importedQueryUnsupported} message="Imported data is unavailable in this view"/>
+              <ImportedQueryUnsupportedWarning condition={mode === FUNNELS && props.importedDataInView} message="Imported data is unavailable in this view"/>
             </div>
             {tabs()}
           </div>
diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.js
index a9b51f7aeb8b..c6ffa0a30a72 100644
--- a/assets/js/dashboard/stats/graph/visitor-graph.js
+++ b/assets/js/dashboard/stats/graph/visitor-graph.js
@@ -81,6 +81,9 @@ export default function VisitorGraph(props) {
   function fetchTopStatsAndGraphData() {
     fetchTopStats(site, query)
       .then((res) => {
+        if (props.updateImportedDataInView) {
+          props.updateImportedDataInView(res.includes_imported)
+        }
         setTopStatData(res)
         setTopStatsLoading(false)
       })

From 4962ba496337cb5437c67e0e5b1a37584bb1d140 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 20:50:13 +0200
Subject: [PATCH 31/41] Transform any operator on respective goal filters

---
 lib/plausible/stats/imported/base.ex | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index ac90c66f1ea8..717fcee06de2 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -93,18 +93,18 @@ defmodule Plausible.Stats.Imported.Base do
       end)
       |> Enum.map(fn filter ->
         case filter do
-          [:is, "event:goal", {:event, name}] ->
-            [[:is, "event:name", name]]
+          [op, "event:goal", {:event, name}] ->
+            [[op, "event:name", name]]
 
-          [:is, "event:goal", {:page, page}] ->
-            [[:is, "event:page", page]]
+          [op, "event:goal", {:page, page}] ->
+            [[op, "event:page", page]]
 
-          [:member, "event:goal", events] ->
+          [op, "event:goal", events] ->
             events
             |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
             |> Enum.map(fn
-              {:event, names} -> [:member, "event:name", names]
-              {:page, pages} -> [:member, "event:page", pages]
+              {:event, names} -> [op, "event:name", names]
+              {:page, pages} -> [op, "event:page", pages]
             end)
 
           filter ->

From 6596596b1eb584c84c4953c73c738ea1101de41f Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Wed, 29 May 2024 21:12:22 +0200
Subject: [PATCH 32/41] Fix percentage and conversion_rate calculation in
 presence of custom props

---
 lib/plausible/stats/base.ex  | 40 +++++++++++++++++++++++++-----------
 lib/plausible/stats/query.ex |  2 +-
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index dde073e05c59..76d88e057f56 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -308,13 +308,17 @@ defmodule Plausible.Stats.Base do
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
-      query =
-        query
-        |> Query.set_property(nil)
-        |> Query.exclude_imported()
+      total_query = Query.set_property(query, nil)
+
+      total_query =
+        if has_custom_property?(query) do
+          Query.exclude_imported(total_query)
+        else
+          total_query
+        end
 
       q
-      |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
+      |> select_merge(^%{__total_visitors: total_visitors_subquery(site, total_query)})
       |> select_merge(%{
         percentage:
           fragment(
@@ -339,14 +343,8 @@ defmodule Plausible.Stats.Base do
         |> Query.remove_filters(["event:goal", "event:props"])
         |> Query.set_property(nil)
 
-      has_custom_prop_filters? =
-        Enum.any?(query.filters, fn
-          [_, "event:props:" <> prop, _] -> prop not in Plausible.Imported.imported_custom_props()
-          _ -> false
-        end)
-
       total_query =
-        if has_custom_prop_filters? do
+        if has_custom_prop_filters?(query) do
           Query.exclude_imported(total_query)
         else
           total_query
@@ -368,4 +366,22 @@ defmodule Plausible.Stats.Base do
       q
     end
   end
+
+  defp has_custom_property?(query) do
+    case query.property do
+      "event:props:" <> prop -> prop not in Plausible.Imported.imported_custom_props()
+      _ -> false
+    end
+  end
+
+  defp has_custom_prop_filters?(query) do
+    if query.filters == [] do
+      false
+    else
+      Enum.any?(query.filters, fn
+        [_, "event:props:" <> prop, _] -> prop not in Plausible.Imported.imported_custom_props()
+        _ -> false
+      end)
+    end
+  end
 end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 48f621093201..6183c51eec4e 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -203,7 +203,7 @@ defmodule Plausible.Stats.Query do
     struct!(query, filters: Filters.parse(params["filters"]))
   end
 
-  @spec set_property(t(), String.t()) :: t()
+  @spec set_property(t(), String.t() | nil) :: t()
   def set_property(query, property) do
     query
     |> struct!(property: property)

From 17564dbab05706edfeeaac6438ac48ab70422777 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Thu, 30 May 2024 09:22:42 +0100
Subject: [PATCH 33/41] add tests for for combining page and pageview goal
 filters

---
 .../aggregate_test.exs                        | 26 +++++++++++
 .../api/stats_controller/conversions_test.exs | 43 +++++++++++++++++++
 .../api/stats_controller/pages_test.exs       | 30 +++++++++++++
 3 files changed, 99 insertions(+)

diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index e1151c953beb..68cfefd85968 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -709,6 +709,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
                "conversion_rate" => %{"value" => 100.0}
              }
     end
+
+    test "returns stats with page + pageview goal filter",
+         %{conn: conn, site: site, site_import: site_import} do
+      insert(:goal, site: site, page_path: "/blog/**")
+
+      populate_stats(site, site_import.id, [
+        build(:imported_pages, page: "/blog/1", visitors: 1, pageviews: 1),
+        build(:imported_pages, page: "/blog/2", visitors: 2, pageviews: 2),
+        build(:imported_pages, visitors: 3)
+      ])
+
+      conn =
+        get(conn, "/api/v1/stats/aggregate", %{
+          "site_id" => site.domain,
+          "period" => "day",
+          "metrics" => "visitors,events,conversion_rate",
+          "filters" => "event:page==/blog/2;event:goal==Visit /blog/**",
+          "with_imported" => "true"
+        })
+
+      assert json_response(conn, 200)["results"] == %{
+               "visitors" => %{"value" => 2},
+               "events" => %{"value" => 2},
+               "conversion_rate" => %{"value" => 100.0}
+             }
+    end
   end
 
   describe "filters" do
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index fe4c0159f0ff..24e18ed2da32 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -1048,6 +1048,49 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
              ] = json_response(conn, 200)["results"]
     end
 
+    test "returns pageview goals with a page filter", %{
+      conn: conn,
+      site: site
+    } do
+      insert(:goal, site: site, page_path: "/blog/two")
+      insert(:goal, site: site, page_path: "/blog/thr**")
+      insert(:goal, site: site, page_path: "/blog/*")
+
+      site_import = insert(:site_import, site: site)
+
+      populate_stats(site, site_import.id, [
+        build(:imported_pages, page: "/", visitors: 1, pageviews: 1, date: ~D[2021-01-01]),
+        build(:imported_pages, page: "/blog/one", visitors: 2, pageviews: 2, date: ~D[2021-01-01]),
+        build(:imported_pages, page: "/blog/two", visitors: 3, pageviews: 3, date: ~D[2021-01-01]),
+        build(:imported_pages,
+          page: "/blog/three",
+          visitors: 4,
+          pageviews: 4,
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, visitors: 10, date: ~D[2021-01-01])
+      ])
+
+      filters = Jason.encode!(%{page: "/blog/one|/blog/two"})
+      q = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/conversions#{q}")
+
+      assert [
+               %{
+                 "name" => "Visit /blog/*",
+                 "visitors" => 5,
+                 "events" => 5,
+                 "conversion_rate" => 100.0
+               },
+               %{
+                 "name" => "Visit /blog/two",
+                 "visitors" => 3,
+                 "events" => 3,
+                 "conversion_rate" => 60.0
+               }
+             ] = json_response(conn, 200)["results"]
+    end
+
     test "calculates conversion_rate for goals with glob pattern with imported data", %{
       conn: conn,
       site: site
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 0df5dc2260b5..4af5c5cd0bd5 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -877,6 +877,36 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
              ]
     end
 
+    test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} do
+      insert(:goal, site: site, page_path: "/blog**")
+
+      populate_stats(site, [
+        build(:imported_pages, page: "/blog"),
+        build(:imported_pages, page: "/not-this"),
+        build(:imported_pages, page: "/blog/post-1", visitors: 2),
+        build(:imported_visitors, visitors: 4)
+      ])
+
+      filters = Jason.encode!(%{goal: "Visit /blog**"})
+      q = "?period=day&filters=#{filters}&with_imported=true"
+      conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")
+
+      assert json_response(conn, 200)["results"] == [
+               %{
+                 "visitors" => 2,
+                 "name" => "/blog/post-1",
+                 "conversion_rate" => 100.0,
+                 "total_visitors" => 2
+               },
+               %{
+                 "visitors" => 1,
+                 "name" => "/blog",
+                 "conversion_rate" => 100.0,
+                 "total_visitors" => 1
+               }
+             ]
+    end
+
     test "calculates bounce rate and time on page for pages", %{conn: conn, site: site} do
       populate_stats(site, [
         build(:pageview,

From 95b5a9048ec5c40e2a442023c2ec368c7f10c44a Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Thu, 30 May 2024 10:30:04 +0100
Subject: [PATCH 34/41] add skip_refresh option to query tweaking functions

---
 lib/plausible/stats/base.ex  | 38 +++---------------------------------
 lib/plausible/stats/query.ex | 18 ++++++++---------
 2 files changed, 12 insertions(+), 44 deletions(-)

diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 76d88e057f56..d53d8c073a08 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -308,14 +308,7 @@ defmodule Plausible.Stats.Base do
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
-      total_query = Query.set_property(query, nil)
-
-      total_query =
-        if has_custom_property?(query) do
-          Query.exclude_imported(total_query)
-        else
-          total_query
-        end
+      total_query = Query.set_property(query, nil, skip_refresh: true)
 
       q
       |> select_merge(^%{__total_visitors: total_visitors_subquery(site, total_query)})
@@ -340,15 +333,8 @@ defmodule Plausible.Stats.Base do
     if :conversion_rate in metrics do
       total_query =
         query
-        |> Query.remove_filters(["event:goal", "event:props"])
-        |> Query.set_property(nil)
-
-      total_query =
-        if has_custom_prop_filters?(query) do
-          Query.exclude_imported(total_query)
-        else
-          total_query
-        end
+        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh: true)
+        |> Query.set_property(nil, skip_refresh: true)
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)
@@ -366,22 +352,4 @@ defmodule Plausible.Stats.Base do
       q
     end
   end
-
-  defp has_custom_property?(query) do
-    case query.property do
-      "event:props:" <> prop -> prop not in Plausible.Imported.imported_custom_props()
-      _ -> false
-    end
-  end
-
-  defp has_custom_prop_filters?(query) do
-    if query.filters == [] do
-      false
-    else
-      Enum.any?(query.filters, fn
-        [_, "event:props:" <> prop, _] -> prop not in Plausible.Imported.imported_custom_props()
-        _ -> false
-      end)
-    end
-  end
 end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 6183c51eec4e..5c9f64e20af6 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -203,11 +203,11 @@ defmodule Plausible.Stats.Query do
     struct!(query, filters: Filters.parse(params["filters"]))
   end
 
-  @spec set_property(t(), String.t() | nil) :: t()
-  def set_property(query, property) do
-    query
-    |> struct!(property: property)
-    |> refresh()
+  @spec set_property(t(), String.t() | nil, Keyword.t()) :: t()
+  def set_property(query, property, opts \\ []) do
+    query = struct!(query, property: property)
+
+    if Keyword.get(opts, :skip_refresh), do: query, else: refresh(query)
   end
 
   def put_filter(query, filter) do
@@ -216,15 +216,15 @@ defmodule Plausible.Stats.Query do
     |> refresh()
   end
 
-  def remove_filters(query, prefixes) do
+  def remove_filters(query, prefixes, opts \\ []) do
     new_filters =
       Enum.reject(query.filters, fn [_, filter_key | _rest] ->
         Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
       end)
 
-    query
-    |> struct!(filters: new_filters)
-    |> refresh()
+    query = struct!(query, filters: new_filters)
+
+    if Keyword.get(opts, :skip_refresh), do: query, else: refresh(query)
   end
 
   def exclude_imported(query) do

From a027cfe2864a3b517602086272fda1b2cd7c2f60 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Thu, 30 May 2024 11:31:00 +0100
Subject: [PATCH 35/41] add imported CR support for timeseries

---
 lib/plausible/stats/timeseries.ex             |  13 +-
 .../timeseries_test.exs                       | 139 ++++++++++++++++++
 2 files changed, 147 insertions(+), 5 deletions(-)

diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 08dfaf18bf2f..1e491af9ca75 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -1,7 +1,7 @@
 defmodule Plausible.Stats.Timeseries do
   use Plausible.ClickhouseRepo
   use Plausible
-  alias Plausible.Stats.{Query, Util}
+  alias Plausible.Stats.{Query, Util, Imported}
   import Plausible.Stats.{Base}
   import Ecto.Query
   use Plausible.Stats.Fragments
@@ -56,8 +56,8 @@ defmodule Plausible.Stats.Timeseries do
 
     from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
     |> select_bucket(:events, site, query)
+    |> Imported.merge_imported_timeseries(site, query, metrics)
     |> maybe_add_timeseries_conversion_rate(site, query, metrics)
-    |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
     |> ClickhouseRepo.all()
   end
 
@@ -67,7 +67,7 @@ defmodule Plausible.Stats.Timeseries do
     from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
     |> filter_converted_sessions(site, query)
     |> select_bucket(:sessions, site, query)
-    |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
+    |> Imported.merge_imported_timeseries(site, query, metrics)
     |> ClickhouseRepo.all()
     |> Util.keep_requested_metrics(metrics)
   end
@@ -314,13 +314,16 @@ defmodule Plausible.Stats.Timeseries do
 
   defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do
     if :conversion_rate in metrics do
-      totals_query = query |> Query.remove_filters(["event:goal", "event:props"])
+      totals_query =
+        query
+        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh: true)
 
       totals_timeseries_q =
         from(e in base_event_query(site, totals_query),
           select: ^select_event_metrics([:visitors])
         )
-        |> select_bucket(:events, site, query)
+        |> select_bucket(:events, site, totals_query)
+        |> Imported.merge_imported_timeseries(site, totals_query, [:visitors])
 
       from(e in subquery(q),
         left_join: c in subquery(totals_timeseries_q),
diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
index f4cc1081ad85..9149711aa3ab 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
@@ -1743,5 +1743,144 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
                }
              ]
     end
+
+    test "returns conversion rate timeseries with a goal filter", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      insert(:goal, site: site, event_name: "Outbound Link: Click")
+
+      populate_stats(site, site_import.id, [
+        # 2021-01-01
+        build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-01 00:00:00]),
+        build(:imported_custom_events, name: "Outbound Link: Click", date: ~D[2021-01-01]),
+        build(:imported_visitors, date: ~D[2021-01-01], visitors: 4),
+        # 2021-01-02
+        build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-02 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),
+        # 2021-01-03
+        build(:imported_custom_events, name: "Outbound Link: Click", date: ~D[2021-01-03]),
+        build(:imported_visitors, date: ~D[2021-01-03]),
+        # 2021-01-04
+        build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-04 00:00:00]),
+        build(:imported_visitors, date: ~D[2021-01-04], visitors: 2)
+      ])
+
+      results =
+        conn
+        |> get("/api/v1/stats/timeseries", %{
+          "site_id" => site.domain,
+          "period" => "custom",
+          "date" => "2021-01-01,2021-01-04",
+          "metrics" => "conversion_rate",
+          "filters" => "event:goal==Outbound Link: Click",
+          "with_imported" => "true"
+        })
+        |> json_response(200)
+        |> Map.get("results")
+
+      assert results == [
+               %{
+                 "date" => "2021-01-01",
+                 "conversion_rate" => 40.0
+               },
+               %{
+                 "date" => "2021-01-02",
+                 "conversion_rate" => 50.0
+               },
+               %{
+                 "date" => "2021-01-03",
+                 "conversion_rate" => 100.0
+               },
+               %{
+                 "date" => "2021-01-04",
+                 "conversion_rate" => 33.3
+               }
+             ]
+    end
+
+    test "returns conversion rate timeseries with a goal + custom prop filter", %{
+      conn: conn,
+      site: site
+    } do
+      site_import = insert(:site_import, site: site)
+
+      insert(:goal, site: site, event_name: "Outbound Link: Click")
+
+      populate_stats(site, site_import.id, [
+        # 2021-01-01
+        build(:event,
+          name: "Outbound Link: Click",
+          "meta.key": ["url"],
+          "meta.value": ["https://site.com"],
+          timestamp: ~N[2021-01-01 00:00:00]
+        ),
+        build(:imported_custom_events,
+          name: "Outbound Link: Click",
+          link_url: "https://site.com",
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_custom_events,
+          name: "File Download",
+          link_url: "https://site.com",
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_custom_events,
+          name: "Outbound Link: Click",
+          link_url: "https://notthis.com",
+          date: ~D[2021-01-01]
+        ),
+        build(:imported_visitors, date: ~D[2021-01-01], visitors: 4),
+        # 2021-01-03
+        build(:imported_custom_events,
+          name: "Outbound Link: Click",
+          link_url: "https://site.com",
+          date: ~D[2021-01-03]
+        ),
+        build(:imported_visitors, date: ~D[2021-01-03]),
+        # 2021-01-04
+        build(:event,
+          name: "Outbound Link: Click",
+          "meta.key": ["url"],
+          "meta.value": ["https://site.com"],
+          timestamp: ~N[2021-01-04 00:00:00]
+        ),
+        build(:imported_visitors, date: ~D[2021-01-04], visitors: 2)
+      ])
+
+      results =
+        conn
+        |> get("/api/v1/stats/timeseries", %{
+          "site_id" => site.domain,
+          "period" => "custom",
+          "date" => "2021-01-01,2021-01-04",
+          "metrics" => "conversion_rate",
+          "filters" => "event:goal==Outbound Link: Click;event:props:url==https://site.com",
+          "with_imported" => "true"
+        })
+        |> json_response(200)
+        |> Map.get("results")
+
+      assert results == [
+               %{
+                 "date" => "2021-01-01",
+                 "conversion_rate" => 40.0
+               },
+               %{
+                 "date" => "2021-01-02",
+                 "conversion_rate" => 0.0
+               },
+               %{
+                 "date" => "2021-01-03",
+                 "conversion_rate" => 100.0
+               },
+               %{
+                 "date" => "2021-01-04",
+                 "conversion_rate" => 33.3
+               }
+             ]
+    end
   end
 end

From e8c43d6acf338412799e1308a63fce5f95be4615 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Thu, 30 May 2024 12:18:27 +0100
Subject: [PATCH 36/41] still show url breakdown when special goal + url in
 filter

---
 lib/plausible/stats/imported/base.ex          | 38 ++++++-------
 .../custom_prop_breakdown_test.exs            | 55 ++++++++++++++++++-
 2 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 717fcee06de2..e89068447079 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -37,8 +37,7 @@ defmodule Plausible.Stats.Imported.Base do
     "event:hostname" => "imported_pages"
   }
 
-  @goals_with_url Imported.goals_with_url()
-  @goals_with_path Imported.goals_with_path()
+  @imported_custom_props Imported.imported_custom_props()
 
   @db_field_mappings %{
     referrer_source: :source,
@@ -117,26 +116,24 @@ defmodule Plausible.Stats.Imported.Base do
   end
 
   defp do_decide_table(%Query{filters: [], property: nil}), do: "imported_visitors"
-  defp do_decide_table(%Query{filters: [], property: "event:props:url"}), do: nil
-  defp do_decide_table(%Query{filters: [], property: "event:props:path"}), do: nil
 
-  defp do_decide_table(%Query{filters: filters, property: "event:props:url"}) do
-    case filters do
-      [[:is, "event:name", name]] when name in @goals_with_url ->
-        "imported_custom_events"
-
-      _ ->
-        nil
-    end
-  end
+  defp do_decide_table(%Query{filters: filters, property: "event:props:" <> prop_key = property})
+       when prop_key in @imported_custom_props do
+    has_required_name_filter? =
+      Enum.any?(filters, fn
+        [:is, "event:name", name] -> name in special_goals_for(prop_key)
+        _ -> false
+      end)
 
-  defp do_decide_table(%Query{filters: filters, property: "event:props:path"}) do
-    case filters do
-      [[:is, "event:name", name]] when name in @goals_with_path ->
-        "imported_custom_events"
+    has_unsupported_filters? =
+      Enum.any?(filters, fn [_, filtered_prop | _] ->
+        filtered_prop not in [property, "event:name"]
+      end)
 
-      _ ->
-        nil
+    if has_required_name_filter? && not has_unsupported_filters? do
+      "imported_custom_events"
+    else
+      nil
     end
   end
 
@@ -189,4 +186,7 @@ defmodule Plausible.Stats.Imported.Base do
       where(q, ^condition)
     end)
   end
+
+  def special_goals_for("url"), do: Imported.goals_with_url()
+  def special_goals_for("path"), do: Imported.goals_with_path()
 end
diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
index 9155606e01cb..d916347897ab 100644
--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
@@ -1141,7 +1141,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
   describe "with imported data" do
     setup [:create_user, :log_in, :create_new_site]
 
-    for goal_name <- ["Outbound Link: Click", "File Download"] do
+    for goal_name <- ["Outbound Link: Click", "File Download", "Cloaked Link: Click"] do
       test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
         insert(:goal, event_name: unquote(goal_name), site: site)
         site_import = insert(:site_import, site: site)
@@ -1196,5 +1196,58 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
                ]
       end
     end
+
+    for goal_name <- ["Outbound Link: Click", "File Download", "Cloaked Link: Click"] do
+      test "returns url breakdown for #{goal_name} goal with a url filter", %{
+        conn: conn,
+        site: site
+      } do
+        insert(:goal, event_name: unquote(goal_name), site: site)
+        site_import = insert(:site_import, site: site)
+
+        populate_stats(site, site_import.id, [
+          build(:event,
+            name: unquote(goal_name),
+            "meta.key": ["url"],
+            "meta.value": ["https://one.com"]
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 2,
+            events: 5,
+            link_url: "https://one.com"
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 5,
+            events: 10,
+            link_url: "https://two.com"
+          ),
+          build(:imported_custom_events,
+            name: "view_search_results",
+            visitors: 100,
+            events: 200
+          ),
+          build(:imported_visitors, visitors: 9)
+        ])
+
+        filters = Jason.encode!(%{goal: unquote(goal_name), props: %{url: "https://two.com"}})
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"
+          )
+
+        assert json_response(conn, 200)["results"] == [
+                 %{
+                   "visitors" => 5,
+                   "name" => "https://two.com",
+                   "events" => 10,
+                   "conversion_rate" => 50.0
+                 }
+               ]
+      end
+    end
   end
 end

From 7039fec71c4a48ec04d70285264f86703d3d26d4 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Fri, 31 May 2024 10:01:40 +0100
Subject: [PATCH 37/41] rename Query.refresh

---
 lib/plausible/stats/base.ex       |  6 +++---
 lib/plausible/stats/query.ex      | 18 +++++++++---------
 lib/plausible/stats/timeseries.ex |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index d53d8c073a08..108e29f7f974 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -308,7 +308,7 @@ defmodule Plausible.Stats.Base do
 
   def add_percentage_metric(q, site, query, metrics) do
     if :percentage in metrics do
-      total_query = Query.set_property(query, nil, skip_refresh: true)
+      total_query = Query.set_property(query, nil, skip_refresh_imported_opts: true)
 
       q
       |> select_merge(^%{__total_visitors: total_visitors_subquery(site, total_query)})
@@ -333,8 +333,8 @@ defmodule Plausible.Stats.Base do
     if :conversion_rate in metrics do
       total_query =
         query
-        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh: true)
-        |> Query.set_property(nil, skip_refresh: true)
+        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh_imported_opts: true)
+        |> Query.set_property(nil, skip_refresh_imported_opts: true)
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 5c9f64e20af6..261840c8317d 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -207,13 +207,15 @@ defmodule Plausible.Stats.Query do
   def set_property(query, property, opts \\ []) do
     query = struct!(query, property: property)
 
-    if Keyword.get(opts, :skip_refresh), do: query, else: refresh(query)
+    if Keyword.get(opts, :skip_refresh_imported_opts),
+      do: query,
+      else: refresh_imported_opts(query)
   end
 
   def put_filter(query, filter) do
     query
     |> struct!(filters: query.filters ++ [filter])
-    |> refresh()
+    |> refresh_imported_opts()
   end
 
   def remove_filters(query, prefixes, opts \\ []) do
@@ -224,7 +226,9 @@ defmodule Plausible.Stats.Query do
 
     query = struct!(query, filters: new_filters)
 
-    if Keyword.get(opts, :skip_refresh), do: query, else: refresh(query)
+    if Keyword.get(opts, :skip_refresh_imported_opts),
+      do: query,
+      else: refresh_imported_opts(query)
   end
 
   def exclude_imported(query) do
@@ -234,8 +238,8 @@ defmodule Plausible.Stats.Query do
     )
   end
 
-  defp refresh(query) do
-    refresh_imported_opts(query)
+  defp refresh_imported_opts(query) do
+    put_imported_opts(query, nil, %{})
   end
 
   def has_event_filters?(query) do
@@ -268,10 +272,6 @@ defmodule Plausible.Stats.Query do
     end
   end
 
-  defp refresh_imported_opts(query) do
-    put_imported_opts(query, nil, %{})
-  end
-
   defp put_imported_opts(query, site, params) do
     requested? = params["with_imported"] == "true" || query.imported_data_requested
 
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 1e491af9ca75..bab37c6e2272 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -316,7 +316,7 @@ defmodule Plausible.Stats.Timeseries do
     if :conversion_rate in metrics do
       totals_query =
         query
-        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh: true)
+        |> Query.remove_filters(["event:goal", "event:props"], skip_refresh_imported_opts: true)
 
       totals_timeseries_q =
         from(e in base_event_query(site, totals_query),

From 29bc847349066209e8cdf6e6e0748bf78c4f9b53 Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Fri, 31 May 2024 12:13:21 +0100
Subject: [PATCH 38/41] use flat_map instead of map and concat

---
 lib/plausible/stats/imported/base.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index e89068447079..e15e09d62cdb 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -90,7 +90,7 @@ defmodule Plausible.Stats.Imported.Base do
         [:is, "event:name", "pageview"] -> true
         _ -> false
       end)
-      |> Enum.map(fn filter ->
+      |> Enum.flat_map(fn filter ->
         case filter do
           [op, "event:goal", {:event, name}] ->
             [[op, "event:name", name]]
@@ -110,7 +110,6 @@ defmodule Plausible.Stats.Imported.Base do
             [filter]
         end
       end)
-      |> Enum.concat()
 
     struct!(query, filters: new_filters)
   end

From 905687a81532b58664fc6f9d32390d4fffb9347c Mon Sep 17 00:00:00 2001
From: Robert Joonas <robertjoonas16@gmail.com>
Date: Fri, 31 May 2024 17:40:58 +0100
Subject: [PATCH 39/41] fix darkmode color

---
 assets/js/dashboard/stats/imported-query-unsupported-warning.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assets/js/dashboard/stats/imported-query-unsupported-warning.js b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
index 1dbe941fda4d..5c01bc78a64f 100644
--- a/assets/js/dashboard/stats/imported-query-unsupported-warning.js
+++ b/assets/js/dashboard/stats/imported-query-unsupported-warning.js
@@ -7,7 +7,7 @@ export default function ImportedQueryUnsupportedWarning({condition, message}) {
   if (condition) {
     return (
       <span tooltip={tooltipMessage}>
-        <ExclamationCircleIcon className="w-6 h-6" />
+        <ExclamationCircleIcon className="w-6 h-6 dark:text-gray-100" />
       </span>
     )
   } else {

From 4e932dd214078e82ea97adb056b7b875db662999 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 3 Jun 2024 10:27:52 +0200
Subject: [PATCH 40/41] Handle invalid imported region codes in suggestions
 gracefully

---
 lib/plausible/stats/filter_suggestions.ex     | 37 +++++++++++++++---
 .../api/stats_controller/suggestions_test.exs | 38 +++++++++++++++++++
 2 files changed, 69 insertions(+), 6 deletions(-)

diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 64d5585580a4..8713919e3e4a 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -47,27 +47,52 @@ defmodule Plausible.Stats.FilterSuggestions do
     |> Enum.map(fn c ->
       subdiv = Location.get_subdivision(c)
 
-      %{
-        value: c,
-        label: subdiv.name
-      }
+      if subdiv do
+        %{
+          value: c,
+          label: subdiv.name
+        }
+      else
+        %{
+          value: c,
+          label: c
+        }
+      end
     end)
   end
 
   def filter_suggestions(site, query, "region", filter_search) do
     matches = Location.search_subdivision(filter_search)
+    filter_search = String.downcase(filter_search)
 
     q =
       from(
         e in query_sessions(site, query),
         group_by: e.subdivision1_code,
         order_by: [desc: fragment("count(*)")],
-        select: e.subdivision1_code
+        select: e.subdivision1_code,
+        where: e.subdivision1_code != ""
       )
       |> Imported.merge_imported_region_suggestions(site, query)
 
     ClickhouseRepo.all(q)
-    |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)
+    |> Enum.map(fn c ->
+      match = Enum.find(matches, fn x -> x.code == c end)
+
+      cond do
+        match ->
+          match
+
+        String.contains?(String.downcase(c), filter_search) ->
+          %{
+            code: c,
+            name: c
+          }
+
+        true ->
+          nil
+      end
+    end)
     |> Enum.filter(& &1)
     |> Enum.slice(0..24)
     |> Enum.map(fn subdiv ->
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index bb8bfb845df1..bfe7ae16d488 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -749,6 +749,44 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       end
     end
 
+    test "handles invalid region codes in imported data gracefully (GA4)", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      # NOTE: Currently, the regions imported from GA4 do not conform to region code standard
+      # we are using. Instead, literal region names are persisted. Those names often do not
+      # match the names from our region databases either. Regardless of that, we still consider
+      # them when filtering suggestions.
+
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2),
+        build(:imported_locations, country: "EE", region: "Hiiumaa", pageviews: 1)
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [
+               %{"value" => "EE-37", "label" => "Harjumaa"},
+               %{"value" => "Hiiumaa", "label" => "Hiiumaa"}
+             ]
+
+      conn2 =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?q=H&with_imported=true"
+        )
+
+      assert json_response(conn2, 200) == [
+               %{"value" => "EE-37", "label" => "Harjumaa"},
+               %{"value" => "Hiiumaa", "label" => "Hiiumaa"}
+             ]
+    end
+
     test "ignores imported data in region suggestions when a different property is filtered by",
          %{
            conn: conn,

From 7b610e33fcfca61e752c0d953d914c37163b90fc Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 3 Jun 2024 11:56:57 +0200
Subject: [PATCH 41/41] Add an entry to CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 485433e7ba92..d55b88656f97 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Snippet integration verification
+- Limited filtering support for imported data in the dashboard and via Stats API
 
 ### Removed