Export and import custom events via CSV (#4096)

* Export and import custom events via CSV

* Add prop support of url for cloaked links and path for 404s in imported queries

* Handle custom events with empty URL and path properties gracefully

* Make events with properties logic DRY and fix missed cloaked link

* Add test for path property breakdown

* Update raw CH data fixture and extend CSV importer tests

* Fix broken query condition after rebase

* Update CHANGELOG.md

CHANGELOG.md

@@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file.
 - Import custom events from Google Analytics 4
 - Ability to filter Search Console keywords by page, country and device plausible/analytics#4077
 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
+- Add custom events support to CSV export and import
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible/exports.ex

@@ -4,6 +4,7 @@ defmodule Plausible.Exports do
   """
 
   use Plausible
+  use Plausible.Stats.Fragments
   import Ecto.Query
 
   @doc "Schedules CSV export job to S3 storage"
@@ -234,6 +235,8 @@ defmodule Plausible.Exports do
       filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
       filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
       filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
+      filename.("imported_custom_events") =>
+        export_custom_events_q(site_id, timezone, date_range),
       filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),
       filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),
       filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),
@@ -465,6 +468,43 @@ defmodule Plausible.Exports do
       ]
   end
 
+  defp export_custom_events_q(site_id, timezone, date_range) do
+    from e in sampled("events_v2"),
+      where: ^export_filter(site_id, date_range),
+      where: e.name != "pageview",
+      group_by: [
+        selected_as(:date),
+        e.name,
+        selected_as(:link_url),
+        selected_as(:path)
+      ],
+      order_by: selected_as(:date),
+      select: [
+        date(e.timestamp, ^timezone),
+        e.name,
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Imported.goals_with_url(),
+            get_by_key(e, :meta, "url")
+          ),
+          :link_url
+        ),
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Imported.goals_with_path(),
+            get_by_key(e, :meta, "path")
+          ),
+          :path
+        ),
+        visitors(e),
+        selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :events)
+      ]
+  end
+
   defp export_locations_q(site_id, timezone, date_range) do
     from s in sampled("sessions_v2"),
       where: ^export_filter(site_id, date_range),

lib/plausible/imported.ex

@@ -38,6 +38,11 @@ defmodule Plausible.Imported do
   # Maximum number of complete imports to account for when querying stats
   @max_complete_imports 5
 
+  # Goals which can be filtered by url property
+  @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+  # Goals which can be filtered by path property
+  @goals_with_path ["404"]
+
   @spec schemas() :: [module()]
   def schemas, do: @tables
 
@@ -49,6 +54,16 @@ defmodule Plausible.Imported do
     @max_complete_imports
   end
 
+  @spec goals_with_url() :: [String.t()]
+  def goals_with_url() do
+    @goals_with_url
+  end
+
+  @spec goals_with_path() :: [String.t()]
+  def goals_with_path() do
+    @goals_with_path
+  end
+
   @spec load_import_data(Site.t()) :: Site.t()
   def load_import_data(%{import_data_loaded: true} = site), do: site
 

lib/plausible/imported/csv_importer.ex

@@ -154,6 +154,8 @@ defmodule Plausible.Imported.CSVImporter do
       "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
     "imported_exit_pages" =>
       "date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
+    "imported_custom_events" =>
+      "date Date, name String, link_url String, path String, visitors UInt64, events UInt64",
     "imported_locations" =>
       "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
     "imported_operating_systems" =>

lib/plausible/imported/custom_event.ex

@@ -9,6 +9,7 @@ defmodule Plausible.Imported.CustomEvent do
     field :date, :date
     field :name, :string
     field :link_url, :string
+    field :path, :string
     field :visitors, Ch, type: "UInt64"
     field :events, Ch, type: "UInt64"
   end

lib/plausible/stats/imported.ex

@@ -7,6 +7,7 @@ defmodule Plausible.Stats.Imported do
 
   @no_ref "Direct / None"
   @not_set "(not set)"
+  @none "(none)"
 
   @property_to_table_mappings %{
     "visit:source" => "imported_sources",
@@ -28,12 +29,14 @@ defmodule Plausible.Stats.Imported do
     "visit:os_version" => "imported_operating_systems",
     "event:page" => "imported_pages",
     "event:name" => "imported_custom_events",
-    "event:props:url" => "imported_custom_events"
+    "event:props:url" => "imported_custom_events",
+    "event:props:path" => "imported_custom_events"
   }
 
   @imported_properties Map.keys(@property_to_table_mappings)
 
-  @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+  @goals_with_url Plausible.Imported.goals_with_url()
+  @goals_with_path Plausible.Imported.goals_with_path()
 
   @doc """
   Returns a boolean indicating whether the combination of filters and
@@ -52,6 +55,7 @@ defmodule Plausible.Stats.Imported do
       {0, "event:props:" <> _} -> false
       {0, _} -> true
       {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
+      {1, "event:props:path"} -> has_special_goal_filter?(query, @goals_with_path)
       {_, _} -> false
     end
   end
@@ -126,7 +130,7 @@ defmodule Plausible.Stats.Imported do
 
     join_on =
       case dim do
-        :url ->
+        _ when dim in [:url, :path] ->
           dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
 
         :os_version ->
@@ -224,6 +228,14 @@ defmodule Plausible.Stats.Imported do
     end
   end
 
+  defp maybe_apply_filter(q, query, "event:props:path", _) do
+    if name = find_special_goal_filter(query, @goals_with_path) do
+      where(q, [i], i.name == ^name)
+    else
+      q
+    end
+  end
+
   defp maybe_apply_filter(q, query, property, dim) do
     case Query.get_filter(query, property) do
       [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
@@ -469,7 +481,17 @@ defmodule Plausible.Stats.Imported do
   defp group_imported_by(q, :url) do
     q
     |> group_by([i], i.link_url)
-    |> select_merge([i], %{breakdown_prop_value: i.link_url})
+    |> select_merge([i], %{
+      breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
+    })
+  end
+
+  defp group_imported_by(q, :path) do
+    q
+    |> group_by([i], i.path)
+    |> select_merge([i], %{
+      breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
+    })
   end
 
   defp select_joined_dimension(q, :city) do
@@ -493,7 +515,7 @@ defmodule Plausible.Stats.Imported do
     })
   end
 
-  defp select_joined_dimension(q, :url) do
+  defp select_joined_dimension(q, dim) when dim in [:url, :path] do
     select_merge(q, [s, i], %{
       breakdown_prop_value:
         fragment(

test/plausible/exports_test.exs

@@ -12,6 +12,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers.csv",
+               "imported_custom_events.csv",
                "imported_devices.csv",
                "imported_entry_pages.csv",
                "imported_exit_pages.csv",
@@ -31,6 +32,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers_20230101_20240312.csv",
+               "imported_custom_events_20230101_20240312.csv",
                "imported_devices_20230101_20240312.csv",
                "imported_entry_pages_20230101_20240312.csv",
                "imported_exit_pages_20230101_20240312.csv",
@@ -50,6 +52,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers.ch",
+               "imported_custom_events.ch",
                "imported_devices.ch",
                "imported_entry_pages.ch",
                "imported_exit_pages.ch",