Skip to content

Commit

Permalink
APIv2: macros, SQL cleanup (#4286)
Browse files Browse the repository at this point in the history 
* Move fragments module under Plausible.Stats.SQL

* Introduce select_merge_as macro

This simplifies some select_merge calls

* Simplify select_join_fields

* Remove a needless dynamic

* wrap_select_columns macro

* Move metrics from base.ex to expression.ex

* Move WhereBuilder under Plausible.Stats.SQL

* Moduledoc

* Improved macros

* Wrap more code

* select_merge_as more

* Move defp to the end

* wrap_alias
  • Loading branch information
@macobo
macobo committed Jul 3, 2024
1 parent 790984e commit 05ac840
Show file tree
Hide file tree
Showing 18 changed files with 407 additions and 499 deletions.
2 changes: 1 addition & 1 deletion extra/lib/plausible/stats/funnel.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ defmodule Plausible.Stats.Funnel do
alias Plausible.Funnels

import Ecto.Query
import Plausible.Stats.Fragments
import Plausible.Stats.SQL.Fragments

alias Plausible.ClickhouseRepo
alias Plausible.Stats.Base
Expand Down
20 changes: 0 additions & 20 deletions extra/lib/plausible/stats/goal/revenue.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,6 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics
end

def total_revenue_query() do
dynamic(
[e],
selected_as(
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:total_revenue
)
)
end

def average_revenue_query() do
dynamic(
[e],
selected_as(
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:average_revenue
)
)
end

@spec get_revenue_tracking_currency(Plausible.Site.t(), Plausible.Stats.Query.t(), [atom()]) ::
{atom() | nil, [atom()]}
@doc """
Expand Down
2 changes: 1 addition & 1 deletion lib/plausible/exports.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule Plausible.Exports do
"""

use Plausible
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
import Ecto.Query

@doc "Schedules CSV export job to S3 storage"
Expand Down
5 changes: 2 additions & 3 deletions lib/plausible/stats/aggregate.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do
use Plausible
import Plausible.Stats.Base
import Ecto.Query
alias Plausible.Stats.{Query, Util}
alias Plausible.Stats.{Query, Util, SQL}

def aggregate(site, query, metrics) do
{currency, metrics} =
Expand Down Expand Up @@ -64,8 +64,7 @@ defmodule Plausible.Stats.Aggregate do
timed_page_transitions_q =
from e in Ecto.Query.subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
where:
^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),
where: ^SQL.WhereBuilder.build_condition(:pathname, event_page_filter),
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,
Expand Down
261 changes: 30 additions & 231 deletions lib/plausible/stats/base.ex
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
defmodule Plausible.Stats.Base do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments

alias Plausible.Stats.{Query, Filters, TableDecider}
alias Plausible.Stats.{Query, TableDecider, SQL}
alias Plausible.Timezones
import Ecto.Query

@uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"

def base_event_query(site, query) do
events_q = query_events(site, query)

Expand All @@ -32,7 +30,7 @@ defmodule Plausible.Stats.Base do
end

def query_events(site, query) do
q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))

on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
Expand All @@ -42,7 +40,7 @@ defmodule Plausible.Stats.Base do
end

def query_sessions(site, query) do
q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))
q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, query))

on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
Expand All @@ -53,206 +51,16 @@ defmodule Plausible.Stats.Base do

def select_event_metrics(metrics) do
metrics
|> Enum.map(&select_event_metric/1)
|> Enum.map(&SQL.Expression.event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end

defp select_event_metric(:pageviews) do
%{
pageviews:
dynamic(
[e],
selected_as(
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),
:pageviews
)
)
}
end

defp select_event_metric(:events) do
%{
events:
dynamic(
[],
selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)
)
}
end

defp select_event_metric(:visitors) do
%{
visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))
}
end

defp select_event_metric(:visits) do
%{
visits:
dynamic(
[e],
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),
:visits
)
)
}
end

on_ee do
defp select_event_metric(:total_revenue) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}
end

defp select_event_metric(:average_revenue) do
%{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}
end
end

defp select_event_metric(:sample_percent) do
%{
sample_percent:
dynamic(
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
end

defp select_event_metric(:percentage), do: %{}
defp select_event_metric(:conversion_rate), do: %{}
defp select_event_metric(:group_conversion_rate), do: %{}
defp select_event_metric(:total_visitors), do: %{}

defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")

def select_session_metrics(metrics, query) do
metrics
|> Enum.map(&select_session_metric(&1, query))
|> Enum.map(&SQL.Expression.session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end

defp select_session_metric(:bounce_rate, query) do
# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
event_page_filter = Query.get_filter(query, "event:page")
condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)

%{
bounce_rate:
dynamic(
[],
selected_as(
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
),
:bounce_rate
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end

defp select_session_metric(:visits, _query) do
%{
visits:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),
:visits
)
)
}
end

defp select_session_metric(:pageviews, _query) do
%{
pageviews:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),
:pageviews
)
)
}
end

defp select_session_metric(:events, _query) do
%{
events:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),
:events
)
)
}
end

defp select_session_metric(:visitors, _query) do
%{
visitors:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),
:visitors
)
)
}
end

defp select_session_metric(:visit_duration, _query) do
%{
visit_duration:
dynamic(
[],
selected_as(
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
:visit_duration
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end

defp select_session_metric(:views_per_visit, _query) do
%{
views_per_visit:
dynamic(
[s],
selected_as(
fragment(
"ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
s.sign,
s.pageviews,
s.sign
),
:views_per_visit
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end

defp select_session_metric(:sample_percent, _query) do
%{
sample_percent:
dynamic(
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
end

defp select_session_metric(:percentage, _query), do: %{}
defp select_session_metric(:conversion_rate, _query), do: %{}
defp select_session_metric(:group_conversion_rate, _query), do: %{}

def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do
converted_sessions =
Expand Down Expand Up @@ -334,7 +142,9 @@ defmodule Plausible.Stats.Base do

defp total_visitors(site, query) do
base_event_query(site, query)
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
|> select([e],
total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
)
end

# `total_visitors_subquery` returns a subquery which selects `total_visitors` -
Expand All @@ -350,38 +160,32 @@ defmodule Plausible.Stats.Base do
def total_visitors_subquery(site, query, include_imported)

def total_visitors_subquery(site, query, true = _include_imported) do
dynamic(
[e],
selected_as(
wrap_alias([], %{
total_visitors:
subquery(total_visitors(site, query)) +
subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)),
:__total_visitors
)
)
subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))
})
end

def total_visitors_subquery(site, query, false = _include_imported) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
wrap_alias([], %{
total_visitors: subquery(total_visitors(site, query))
})
end

def add_percentage_metric(q, site, query, metrics) do
if :percentage in metrics do
total_query = Query.set_dimensions(query, [])

q
|> select_merge(
^%{__total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}
)
|> select_merge(%{
|> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
|> select_merge_as([], %{
percentage:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
),
:percentage
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:total_visitors),
selected_as(:visitors),
selected_as(:total_visitors)
)
})
else
Expand All @@ -401,19 +205,14 @@ defmodule Plausible.Stats.Base do

# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
|> select_merge(
^%{total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}
)
|> select_merge([e], %{
|> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
|> select_merge_as([e], %{
conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
),
:conversion_rate
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:total_visitors),
e.visitors,
selected_as(:total_visitors)
)
})
else
Expand Down

0 comments on commit 05ac840

Please sign in to comment.