From efcb7eab7934ecb6dd339589695aad5ecbdb1351 Mon Sep 17 00:00:00 2001
From: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
Date: Mon, 3 Jun 2024 10:27:52 +0200
Subject: [PATCH] Handle invalid imported region codes in suggestions
 gracefully

---
 lib/plausible/stats/filter_suggestions.ex     | 37 +++++++++++++++---
 .../api/stats_controller/suggestions_test.exs | 38 +++++++++++++++++++
 2 files changed, 69 insertions(+), 6 deletions(-)

diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 64d5585580a4b..8713919e3e4a4 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -47,27 +47,52 @@ defmodule Plausible.Stats.FilterSuggestions do
     |> Enum.map(fn c ->
       subdiv = Location.get_subdivision(c)
 
-      %{
-        value: c,
-        label: subdiv.name
-      }
+      if subdiv do
+        %{
+          value: c,
+          label: subdiv.name
+        }
+      else
+        %{
+          value: c,
+          label: c
+        }
+      end
     end)
   end
 
   def filter_suggestions(site, query, "region", filter_search) do
     matches = Location.search_subdivision(filter_search)
+    filter_search = String.downcase(filter_search)
 
     q =
       from(
         e in query_sessions(site, query),
         group_by: e.subdivision1_code,
         order_by: [desc: fragment("count(*)")],
-        select: e.subdivision1_code
+        select: e.subdivision1_code,
+        where: e.subdivision1_code != ""
       )
       |> Imported.merge_imported_region_suggestions(site, query)
 
     ClickhouseRepo.all(q)
-    |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)
+    |> Enum.map(fn c ->
+      match = Enum.find(matches, fn x -> x.code == c end)
+
+      cond do
+        match ->
+          match
+
+        String.contains?(String.downcase(c), filter_search) ->
+          %{
+            code: c,
+            name: c
+          }
+
+        true ->
+          nil
+      end
+    end)
     |> Enum.filter(& &1)
     |> Enum.slice(0..24)
     |> Enum.map(fn subdiv ->
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index bb8bfb845df16..bfe7ae16d4882 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -749,6 +749,44 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
       end
     end
 
+    test "handles invalid region codes in imported data gracefully (GA4)", %{
+      conn: conn,
+      site: site,
+      site_import: site_import
+    } do
+      # NOTE: Currently, the regions imported from GA4 do not conform to region code standard
+      # we are using. Instead, literal region names are persisted. Those names often do not
+      # match the names from our region databases either. Regardless of that, we still consider
+      # them when filtering suggestions.
+
+      populate_stats(site, site_import.id, [
+        build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2),
+        build(:imported_locations, country: "EE", region: "Hiiumaa", pageviews: 1)
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?q=&with_imported=true"
+        )
+
+      assert json_response(conn, 200) == [
+               %{"value" => "EE-37", "label" => "Harjumaa"},
+               %{"value" => "Hiiumaa", "label" => "Hiiumaa"}
+             ]
+
+      conn2 =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/region?q=H&with_imported=true"
+        )
+
+      assert json_response(conn2, 200) == [
+               %{"value" => "EE-37", "label" => "Harjumaa"},
+               %{"value" => "Hiiumaa", "label" => "Hiiumaa"}
+             ]
+    end
+
     test "ignores imported data in region suggestions when a different property is filtered by",
          %{
            conn: conn,