Skip to content

Commit f6a6d2c

Browse files
franciscao633franciscao633
committed
performance updates checkpoint
1 parent 8563fe1 commit f6a6d2c

25 files changed

+488
-343
lines changed

‎db/clickhouse/migrations/09_add_performance.sql‎

Lines changed: 100 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,105 @@
1-
-- Create Performance
2-
CREATE TABLE umami.website_performance
3-
(
4-
website_id UUID,
5-
session_id UUID,
6-
visit_id UUID,
7-
url_path String,
8-
lcp Nullable(Decimal(10, 1)),
9-
inp Nullable(Decimal(10, 1)),
10-
cls Nullable(Decimal(10, 4)),
11-
fcp Nullable(Decimal(10, 1)),
12-
ttfb Nullable(Decimal(10, 1)),
13-
created_at DateTime('UTC')
14-
)
15-
ENGINE = MergeTree
16-
PARTITION BY toYYYYMM(created_at)
17-
ORDER BY (website_id, toStartOfHour(created_at), session_id)
18-
SETTINGS index_granularity = 8192;
1+
-- Add performance columns to website_event
2+
ALTER TABLE umami.website_event ADD COLUMN lcp Nullable(Decimal(10, 1)) AFTER twclid;
3+
ALTER TABLE umami.website_event ADD COLUMN inp Nullable(Decimal(10, 1)) AFTER lcp;
4+
ALTER TABLE umami.website_event ADD COLUMN cls Nullable(Decimal(10, 4)) AFTER inp;
5+
ALTER TABLE umami.website_event ADD COLUMN fcp Nullable(Decimal(10, 1)) AFTER cls;
6+
ALTER TABLE umami.website_event ADD COLUMN ttfb Nullable(Decimal(10, 1)) AFTER fcp;
197

20-
-- Performance hourly aggregation
21-
CREATE TABLE umami.website_performance_hourly
22-
(
23-
website_id UUID,
24-
url_path String,
25-
lcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
26-
lcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
27-
lcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
28-
inp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
29-
inp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
30-
inp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
31-
cls_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 4))),
32-
cls_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 4))),
33-
cls_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 4))),
34-
fcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
35-
fcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
36-
fcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
37-
ttfb_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
38-
ttfb_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
39-
ttfb_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
40-
sample_count SimpleAggregateFunction(sum, UInt64),
41-
created_at DateTime('UTC')
42-
)
43-
ENGINE = AggregatingMergeTree
44-
PARTITION BY toYYYYMM(created_at)
45-
ORDER BY (website_id, toStartOfHour(created_at), url_path)
46-
SETTINGS index_granularity = 8192;
8+
-- Update materialized view to exclude performance events from view counts
9+
DROP TABLE umami.website_event_stats_hourly_mv;
4710

48-
CREATE MATERIALIZED VIEW umami.website_performance_hourly_mv
49-
TO umami.website_performance_hourly
11+
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
12+
TO umami.website_event_stats_hourly
5013
AS
5114
SELECT
5215
website_id,
53-
url_path,
54-
quantileState(0.5)(lcp) as lcp_p50,
55-
quantileState(0.75)(lcp) as lcp_p75,
56-
quantileState(0.95)(lcp) as lcp_p95,
57-
quantileState(0.5)(inp) as inp_p50,
58-
quantileState(0.75)(inp) as inp_p75,
59-
quantileState(0.95)(inp) as inp_p95,
60-
quantileState(0.5)(cls) as cls_p50,
61-
quantileState(0.75)(cls) as cls_p75,
62-
quantileState(0.95)(cls) as cls_p95,
63-
quantileState(0.5)(fcp) as fcp_p50,
64-
quantileState(0.75)(fcp) as fcp_p75,
65-
quantileState(0.95)(fcp) as fcp_p95,
66-
quantileState(0.5)(ttfb) as ttfb_p50,
67-
quantileState(0.75)(ttfb) as ttfb_p75,
68-
quantileState(0.95)(ttfb) as ttfb_p95,
69-
count() as sample_count,
70-
toStartOfHour(created_at) as created_at
71-
FROM umami.website_performance
72-
GROUP BY website_id, url_path, created_at;
16+
session_id,
17+
visit_id,
18+
hostnames as hostname,
19+
browser,
20+
os,
21+
device,
22+
screen,
23+
language,
24+
country,
25+
region,
26+
city,
27+
entry_url,
28+
exit_url,
29+
url_paths as url_path,
30+
url_query,
31+
utm_source,
32+
utm_medium,
33+
utm_campaign,
34+
utm_content,
35+
utm_term,
36+
referrer_domain,
37+
page_title,
38+
gclid,
39+
fbclid,
40+
msclkid,
41+
ttclid,
42+
li_fat_id,
43+
twclid,
44+
event_type,
45+
event_name,
46+
views,
47+
min_time,
48+
max_time,
49+
tag,
50+
distinct_id,
51+
timestamp as created_at
52+
FROM (SELECT
53+
website_id,
54+
session_id,
55+
visit_id,
56+
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
57+
browser,
58+
os,
59+
device,
60+
screen,
61+
language,
62+
country,
63+
region,
64+
city,
65+
argMinState(url_path, created_at) entry_url,
66+
argMaxState(url_path, created_at) exit_url,
67+
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
68+
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
69+
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
70+
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
71+
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
72+
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
73+
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
74+
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
75+
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
76+
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
77+
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
78+
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
79+
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
80+
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
81+
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
82+
event_type,
83+
if(event_type = 2, groupArray(event_name), []) event_name,
84+
sumIf(1, event_type NOT IN (2, 5)) views,
85+
min(created_at) min_time,
86+
max(created_at) max_time,
87+
arrayFilter(x -> x != '', groupArray(tag)) tag,
88+
distinct_id,
89+
toStartOfHour(created_at) timestamp
90+
FROM umami.website_event
91+
GROUP BY website_id,
92+
session_id,
93+
visit_id,
94+
hostname,
95+
browser,
96+
os,
97+
device,
98+
screen,
99+
language,
100+
country,
101+
region,
102+
city,
103+
event_type,
104+
distinct_id,
105+
timestamp);

‎db/clickhouse/schema.sql‎

Lines changed: 7 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ CREATE TABLE umami.website_event
3434
ttclid String,
3535
li_fat_id String,
3636
twclid String,
37+
--performance
38+
lcp Nullable(Decimal(10, 1)),
39+
inp Nullable(Decimal(10, 1)),
40+
cls Nullable(Decimal(10, 4)),
41+
fcp Nullable(Decimal(10, 1)),
42+
ttfb Nullable(Decimal(10, 1)),
3743
--events
3844
event_type UInt32,
3945
event_name String,
@@ -209,7 +215,7 @@ FROM (SELECT
209215
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
210216
event_type,
211217
if(event_type = 2, groupArray(event_name), []) event_name,
212-
sumIf(1, event_type != 2) views,
218+
sumIf(1, event_type NOT IN (2, 5)) views,
213219
min(created_at) min_time,
214220
max(created_at) max_time,
215221
arrayFilter(x -> x != '', groupArray(tag)) tag,
@@ -282,79 +288,6 @@ JOIN (SELECT event_id, string_value as currency
282288
ON c.event_id = ed.event_id
283289
WHERE positionCaseInsensitive(data_key, 'revenue') > 0;
284290

285-
-- Create Performance
286-
CREATE TABLE umami.website_performance
287-
(
288-
website_id UUID,
289-
session_id UUID,
290-
visit_id UUID,
291-
url_path String,
292-
lcp Nullable(Decimal(10, 1)),
293-
inp Nullable(Decimal(10, 1)),
294-
cls Nullable(Decimal(10, 4)),
295-
fcp Nullable(Decimal(10, 1)),
296-
ttfb Nullable(Decimal(10, 1)),
297-
created_at DateTime('UTC')
298-
)
299-
ENGINE = MergeTree
300-
PARTITION BY toYYYYMM(created_at)
301-
ORDER BY (website_id, toStartOfHour(created_at), session_id)
302-
SETTINGS index_granularity = 8192;
303-
304-
-- Performance hourly aggregation
305-
CREATE TABLE umami.website_performance_hourly
306-
(
307-
website_id UUID,
308-
url_path String,
309-
lcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
310-
lcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
311-
lcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
312-
inp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
313-
inp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
314-
inp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
315-
cls_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 4))),
316-
cls_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 4))),
317-
cls_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 4))),
318-
fcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
319-
fcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
320-
fcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
321-
ttfb_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
322-
ttfb_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
323-
ttfb_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
324-
sample_count SimpleAggregateFunction(sum, UInt64),
325-
created_at DateTime('UTC')
326-
)
327-
ENGINE = AggregatingMergeTree
328-
PARTITION BY toYYYYMM(created_at)
329-
ORDER BY (website_id, toStartOfHour(created_at), url_path)
330-
SETTINGS index_granularity = 8192;
331-
332-
CREATE MATERIALIZED VIEW umami.website_performance_hourly_mv
333-
TO umami.website_performance_hourly
334-
AS
335-
SELECT
336-
website_id,
337-
url_path,
338-
quantileState(0.5)(lcp) as lcp_p50,
339-
quantileState(0.75)(lcp) as lcp_p75,
340-
quantileState(0.95)(lcp) as lcp_p95,
341-
quantileState(0.5)(inp) as inp_p50,
342-
quantileState(0.75)(inp) as inp_p75,
343-
quantileState(0.95)(inp) as inp_p95,
344-
quantileState(0.5)(cls) as cls_p50,
345-
quantileState(0.75)(cls) as cls_p75,
346-
quantileState(0.95)(cls) as cls_p95,
347-
quantileState(0.5)(fcp) as fcp_p50,
348-
quantileState(0.75)(fcp) as fcp_p75,
349-
quantileState(0.95)(fcp) as fcp_p95,
350-
quantileState(0.5)(ttfb) as ttfb_p50,
351-
quantileState(0.75)(ttfb) as ttfb_p75,
352-
quantileState(0.95)(ttfb) as ttfb_p95,
353-
count() as sample_count,
354-
toStartOfHour(created_at) as created_at
355-
FROM umami.website_performance
356-
GROUP BY website_id, url_path, created_at;
357-
358291
-- Create session_replay
359292
CREATE TABLE umami.session_replay
360293
(

0 commit comments

Comments
 (0)