Implement filtering suggestions and translate filter fields for imported

lib/plausible/stats/filter_suggestions.ex

@@ -16,6 +16,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.country_code
       )
+      |> Plausible.Stats.Imported.merged_imported_countries(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end)
@@ -35,9 +36,10 @@ defmodule Plausible.Stats.FilterSuggestions do
       group_by: e.subdivision1_code,
       order_by: [desc: fragment("count(*)")],
       select: e.subdivision1_code,
-      where: e.subdivision1_code != "",
-      limit: 24
+      where: e.subdivision1_code != ""
     )
+    |> Plausible.Stats.Imported.merge_imported_regions(site, query)
+    |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
       subdiv = Location.get_subdivision(c)
@@ -59,6 +61,7 @@ defmodule Plausible.Stats.FilterSuggestions do
         order_by: [desc: fragment("count(*)")],
         select: e.subdivision1_code
       )
+      |> Plausible.Stats.Imported.merge_imported_regions(site, query)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)
@@ -78,9 +81,10 @@ defmodule Plausible.Stats.FilterSuggestions do
       group_by: e.city_geoname_id,
       order_by: [desc: fragment("count(*)")],
       select: e.city_geoname_id,
-      where: e.city_geoname_id != 0,
-      limit: 24
+      where: e.city_geoname_id != 0
     )
+    |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+    |> limit(24)
     |> ClickhouseRepo.all()
     |> Enum.map(fn c ->
       city = Location.get_city(c)
@@ -101,9 +105,10 @@ defmodule Plausible.Stats.FilterSuggestions do
         group_by: e.city_geoname_id,
         order_by: [desc: fragment("count(*)")],
         select: e.city_geoname_id,
-        where: e.city_geoname_id != 0,
-        limit: 5000
+        where: e.city_geoname_id != 0
       )
+      |> Plausible.Stats.Imported.merge_imported_cities(site, query)
+      |> limit(5000)
 
     ClickhouseRepo.all(q)
     |> Enum.map(fn c -> Location.get_city(c) end)
@@ -223,10 +228,16 @@ defmodule Plausible.Stats.FilterSuggestions do
       where: fragment("? ilike ?", field(e, ^filter_name), ^filter_query),
       select: field(e, ^filter_name),
       group_by: ^filter_name,
-      order_by: [desc: fragment("count(*)")],
-      limit: 25
+      order_by: [desc: fragment("count(*)")]
     )
     |> apply_additional_filters(filter_name, site)
+    |> Plausible.Stats.Imported.merge_imported_filter_suggestions(
+      site,
+      query,
+      filter_name,
+      filter_query
+    )
+    |> limit(25)
     |> ClickhouseRepo.all()
     |> Enum.filter(fn suggestion -> suggestion != "" end)
     |> wrap_suggestions()

lib/plausible/stats/imported/base.ex

@@ -10,6 +10,21 @@ defmodule Plausible.Stats.Imported.Base do
   @goals_with_path Imported.goals_with_path()
   @special_goals @goals_with_path ++ @goals_with_url
 
+  @db_field_mappings %{
+    referrer_source: :source,
+    screen_size: :device,
+    screen: :device,
+    os: :operating_system,
+    os_version: :operating_system_versions,
+    country_code: :country,
+    subdivision1_code: :region,
+    city_geoname_id: :city,
+    entry_page_hostname: :hostname,
+    pathname: :page
+  }
+
+  def db_field_mappings(), do: @db_field_mappings
+
   def query_imported(site, query) do
     query = Imported.drop_redundant_filters(query)
 
@@ -86,7 +101,8 @@ defmodule Plausible.Stats.Imported.Base do
 
   defp apply_filter(q, %Query{filters: [[_, filtered_prop | _] = filter]}) do
     db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)
-    condition = Filters.WhereBuilder.build_condition(db_field, filter)
+    mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
+    condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)
 
     where(q, ^condition)
   end

lib/plausible/stats/imported/imported.ex

@@ -23,6 +23,7 @@ defmodule Plausible.Stats.Imported do
     "visit:region" => "imported_locations",
     "visit:city" => "imported_locations",
     "visit:device" => "imported_devices",
+    "visit:screen" => "imported_devices",
     "visit:browser" => "imported_browsers",
     "visit:browser_version" => "imported_browsers",
     "visit:os" => "imported_operating_systems",
@@ -82,6 +83,123 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
+  def merged_imported_countries(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{country_code: e.country_code, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        group_by: i.country,
+        select_merge: %{country_code: i.country, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.country_code == i.country_code,
+      select: fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_regions(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        where: i.region != "",
+        group_by: i.region,
+        select_merge: %{region_code: i.region, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.region_code == i.region_code,
+      select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_cities(native_q, site, query) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")})
+
+    imported_q =
+      from i in Imported.Base.query_imported("imported_locations", site, query),
+        where: i.city != 0,
+        group_by: i.city,
+        select_merge: %{city_id: i.city, count: fragment("count(*)")}
+
+    from(s in subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: s.city_id == i.city_id,
+      select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id),
+      order_by: [desc: fragment("? + ?", s.count, i.count)]
+    )
+  end
+
+  def merge_imported_filter_suggestions(
+        native_q,
+        _site,
+        %Plausible.Stats.Query{include_imported: false},
+        _filter_name,
+        _filter_search
+      ) do
+    native_q
+  end
+
+  def merge_imported_filter_suggestions(
+        native_q,
+        site,
+        query,
+        filter_name,
+        filter_query
+      ) do
+    native_q =
+      native_q
+      |> exclude(:order_by)
+      |> exclude(:select)
+      |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})
+
+    db_field = Map.get(Imported.Base.db_field_mappings(), filter_name, filter_name)
+
+    property_field =
+      case db_field do
+        :operating_system -> :os
+        :operating_system_version -> :os_version
+        other -> other
+      end
+
+    table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property_field}")
+    table_by_event = Map.get(@property_to_table_mappings, "event:#{property_field}")
+    table = table_by_visit || table_by_event
+
+    if db_field && table do
+      imported_q =
+        from i in Imported.Base.query_imported(table, site, query),
+          where: fragment("? ilike ?", field(i, ^db_field), ^filter_query),
+          group_by: field(i, ^db_field),
+          select_merge: %{name: field(i, ^db_field), count: fragment("count(*)")}
+
+      from(s in subquery(native_q),
+        full_join: i in subquery(imported_q),
+        on: s.name == i.name,
+        select: fragment("if(not empty(?), ?, ?)", s.name, s.name, i.name),
+        order_by: [desc: fragment("? + ?", s.count, i.count)],
+        limit: 25
+      )
+    else
+      native_q
+    end
+  end
+
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 

lib/plausible_web/controllers/api/stats_controller.ex

@@ -1281,7 +1281,6 @@ defmodule PlausibleWeb.Api.StatsController do
         Enum.map(entry, &format_revenue_metric/1)
         |> Map.new()
       end)
-      |> IO.inspect()
 
     %{results: props, skip_imported_reason: query.skip_imported_reason}
   end