Merge branch 'dev' into update-es-translations-and-fix-typo

israelozarate7 · Jul 23, 2024 · e5d1765 · e5d1765
2 parents 9f3138e + 7ec845d
commit e5d1765
Show file tree

Hide file tree

Showing 71 changed files with 1,819 additions and 1,259 deletions.
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
@@ -66,4 +66,203 @@ CREATE TABLE umami.session_data
 )
     engine = MergeTree
         ORDER BY (website_id, session_id, data_key, created_at)
-        SETTINGS index_granularity = 8192;
+        SETTINGS index_granularity = 8192;
+
+-- stats hourly
+CREATE TABLE umami.website_event_stats_hourly
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    city String,
+    entry_url AggregateFunction(argMin, String, DateTime('UTC')),
+    exit_url AggregateFunction(argMax, String, DateTime('UTC')),
+    url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
+    url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
+    referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
+    page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
+    event_type UInt32,
+    event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
+    views SimpleAggregateFunction(sum, UInt64),
+    min_time SimpleAggregateFunction(min, DateTime('UTC')),
+    max_time SimpleAggregateFunction(max, DateTime('UTC')),
+    created_at Datetime('UTC')
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (
+    website_id,
+    toStartOfHour(created_at),
+    cityHash64(visit_id),
+    visit_id
+)
+SAMPLE BY cityHash64(visit_id)
+TTL created_at + INTERVAL 10 DAY;
+
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
+TO umami.website_event_stats_hourly
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    referrer_domain,
+    page_title,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    event_type,
+    timestamp);
+
+-- stats daily
+CREATE TABLE umami.website_event_stats_daily
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    city String,
+    entry_url AggregateFunction(argMin, String, DateTime('UTC')),
+    exit_url AggregateFunction(argMax, String, DateTime('UTC')),
+    url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
+    url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
+    referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
+    page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
+    event_type UInt32,
+    event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
+    views SimpleAggregateFunction(sum, UInt64),
+    min_time SimpleAggregateFunction(min, DateTime('UTC')),
+    max_time SimpleAggregateFunction(max, DateTime('UTC')),
+    created_at Datetime('UTC')
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (
+    website_id,
+    toStartOfDay(created_at),
+    cityHash64(visit_id),
+    visit_id
+)
+SAMPLE BY cityHash64(visit_id);
+
+CREATE MATERIALIZED VIEW umami.website_event_stats_daily_mv
+TO umami.website_event_stats_daily
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    referrer_domain,
+    page_title,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    toStartOfDay(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    country,
+    subdivision1,
+    city,
+    event_type,
+    timestamp);        
diff --git a/package.json b/package.json
@@ -64,9 +64,9 @@
     ".next/cache"
   ],
   "dependencies": {
-    "@clickhouse/client": "^1.0.2",
+    "@clickhouse/client": "^1.3.0",
     "@fontsource/inter": "^4.5.15",
-    "@prisma/client": "5.14.0",
+    "@prisma/client": "5.16.2",
     "@prisma/extension-read-replicas": "^0.3.0",
     "@react-spring/web": "^9.7.3",
     "@tanstack/react-query": "^5.28.6",
@@ -98,11 +98,11 @@
     "maxmind": "^4.3.6",
     "md5": "^2.3.0",
     "moment-timezone": "^0.5.35",
-    "next": "14.2.3",
+    "next": "14.2.4",
     "next-basics": "^0.39.0",
     "node-fetch": "^3.2.8",
     "npm-run-all": "^4.1.5",
-    "prisma": "5.14.0",
+    "prisma": "5.16.2",
     "react": "^18.2.0",
     "react-basics": "^0.123.0",
     "react-beautiful-dnd": "^13.1.0",
@@ -175,6 +175,6 @@
     "tar": "^6.1.2",
     "ts-jest": "^29.1.2",
     "ts-node": "^10.9.1",
-    "typescript": "^5.4.3"
+    "typescript": "^5.5.3"
   }
 }