Skip to content

Commit eeecf49

Browse files
huydhnhuydhn
authored
Add ClickHouse schemas for vllm_buildkite_builds and vllm_buildkite_jobs tables (#7001)
There are some events on dynamoDB now, so I use these event to create the schema for `vllm_buildkite_builds` and `vllm_buildkite_jobs` tables on ClickHouse. There is another for `vllm_buildkite_agents` that records events from Buildkite agents, but I will add it later in a separate PR once there are some records on dynamoDB that I can use to create that schema. ### Testing Run the two `CREATE TABLE` queries on the playground database `fortesting`. The two tables are also backfilling with data from dynamoDB because I just realize that I have been leaving the ingestion lambda #6998 running since July 16th-ish cc @simon-mo --------- Signed-off-by: Huy Do <[email protected]>
1 parent 8ceeed1 commit eeecf49

File tree

2 files changed

+283
-0
lines changed

2 files changed

+283
-0
lines changed

‎clickhouse_db_schema/vllm_buildkite_builds/schema.sql‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
CREATE TABLE vllm.vllm_buildkite_builds (
2+
dynamoKey String,
3+
build Tuple(
4+
finished_at Nullable(DateTime64(3)),
5+
graphql_id String,
6+
commit String,
7+
created_at DateTime64(3),
8+
scheduled_at Nullable(DateTime64(3)),
9+
source String,
10+
branch String,
11+
blocked_state String,
12+
cluster_url String,
13+
number UInt32,
14+
cluster_id String,
15+
blocked Bool,
16+
meta_data String,
17+
id String,
18+
state String,
19+
tag Nullable(String),
20+
creator Tuple(
21+
name Nullable(String),
22+
created_at Nullable(DateTime64(3)),
23+
id Nullable(String),
24+
avatar_url Nullable(String),
25+
graphql_id Nullable(String),
26+
email Nullable(String)
27+
),
28+
pull_request Tuple(
29+
id Nullable(String),
30+
repository Nullable(String),
31+
base Nullable(String),
32+
labels Array(String)
33+
),
34+
author Tuple(
35+
name String,
36+
email String,
37+
username String
38+
),
39+
message String,
40+
rebuilt_from Nullable(String),
41+
url String,
42+
cancel_reason Nullable(String),
43+
web_url String,
44+
started_at Nullable(DateTime64(3))
45+
),
46+
event String,
47+
pipeline Tuple(
48+
allow_rebuilds Bool,
49+
running_jobs_count UInt32,
50+
emoji String,
51+
color Nullable(String),
52+
graphql_id String,
53+
configuration String,
54+
description String,
55+
created_at DateTime64(3),
56+
skip_queued_branch_builds Bool,
57+
repository String,
58+
pipeline_template_uuid Nullable(String),
59+
cluster_url String,
60+
skip_queued_branch_builds_filter String,
61+
cluster_id String,
62+
cancel_running_branch_builds Bool,
63+
provider Tuple(
64+
webhook_url String,
65+
settings String,
66+
id String
67+
),
68+
waiting_jobs_count UInt32,
69+
id String,
70+
slug String,
71+
badge_url String,
72+
visibility String,
73+
archived_at Nullable(DateTime64(3)),
74+
scheduled_jobs_count UInt32,
75+
running_builds_count UInt32,
76+
env Nullable(String),
77+
created_by Tuple(
78+
name Nullable(String),
79+
created_at Nullable(DateTime64(3)),
80+
id Nullable(String),
81+
avatar_url Nullable(String),
82+
graphql_id Nullable(String),
83+
email Nullable(String)
84+
),
85+
steps String,
86+
url String,
87+
tags Nullable(String),
88+
scheduled_builds_count UInt32,
89+
builds_url String,
90+
web_url String,
91+
name String,
92+
default_branch String,
93+
branch_configuration Nullable(String),
94+
cancel_running_branch_builds_filter Nullable(String)
95+
),
96+
sender Tuple(
97+
name Nullable(String),
98+
id Nullable(String)
99+
)
100+
) ENGINE = SharedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
101+
ORDER BY (
102+
pipeline.repository,
103+
pipeline.name,
104+
build.number,
105+
dynamoKey
106+
)
107+
SETTINGS index_granularity = 8192

‎clickhouse_db_schema/vllm_buildkite_jobs/schema.sql‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
CREATE TABLE vllm.vllm_buildkite_jobs (
2+
dynamoKey String,
3+
build Tuple(
4+
finished_at Nullable(DateTime64(3)),
5+
graphql_id String,
6+
commit String,
7+
created_at DateTime64(3),
8+
scheduled_at Nullable(DateTime64(3)),
9+
source String,
10+
branch String,
11+
blocked_state String,
12+
cluster_url String,
13+
number UInt32,
14+
cluster_id String,
15+
blocked Bool,
16+
meta_data String,
17+
id String,
18+
state String,
19+
tag Nullable(String),
20+
creator Tuple(
21+
name Nullable(String),
22+
created_at Nullable(DateTime64(3)),
23+
id Nullable(String),
24+
avatar_url Nullable(String),
25+
graphql_id Nullable(String),
26+
email Nullable(String)
27+
),
28+
pull_request Tuple(
29+
id Nullable(String),
30+
repository Nullable(String),
31+
base Nullable(String),
32+
labels Array(String)
33+
),
34+
author Tuple(
35+
name String,
36+
email String,
37+
username String
38+
),
39+
message String,
40+
rebuilt_from Nullable(String),
41+
url String,
42+
cancel_reason Nullable(String),
43+
web_url String,
44+
started_at Nullable(DateTime64(3))
45+
),
46+
event String,
47+
job Tuple(
48+
retried Bool,
49+
agent Tuple(
50+
last_job_finished_at Nullable(DateTime64(3)),
51+
creator Nullable(String),
52+
paused Bool,
53+
created_at DateTime64(3),
54+
ip_address String,
55+
connection_state String,
56+
priority UInt32,
57+
version String,
58+
url String,
59+
cluster_url String,
60+
hostname String,
61+
paused_at Nullable(DateTime64(3)),
62+
web_url String,
63+
cluster_queue_url String,
64+
name String,
65+
meta_data Array(String),
66+
paused_timeout_in_minutes UInt32,
67+
id String,
68+
paused_note Nullable(String),
69+
job Nullable(String),
70+
user_agent String,
71+
paused_by Nullable(String)
72+
),
73+
finished_at Nullable(DateTime64(3)),
74+
parallel_group_index Nullable(UInt32),
75+
graphql_id String,
76+
artifacts_url String,
77+
step_key Nullable(String),
78+
created_at DateTime64(3),
79+
scheduled_at DateTime64(3),
80+
expired_at Nullable(DateTime64(3)),
81+
retries_count Nullable(UInt32),
82+
type String,
83+
exit_status Nullable(Int32),
84+
matrix Nullable(String),
85+
cluster_url String,
86+
cluster_id String,
87+
soft_failed Bool,
88+
retried_in_job_id Nullable(String),
89+
log_url String,
90+
retry_source Nullable(String),
91+
id String,
92+
state String,
93+
build_url String,
94+
retry_type Nullable(String),
95+
priority Tuple(
96+
number UInt32
97+
),
98+
agent_query_rules Array(String),
99+
command String,
100+
raw_log_url String,
101+
cluster_queue_id String,
102+
web_url String,
103+
cluster_queue_url String,
104+
parallel_group_total Nullable(UInt32),
105+
name String,
106+
started_at Nullable(DateTime64(3)),
107+
step Tuple(
108+
signature Nullable(String),
109+
id String
110+
),
111+
artifact_paths Nullable(String),
112+
runnable_at DateTime64(3)
113+
),
114+
pipeline Tuple(
115+
allow_rebuilds Bool,
116+
running_jobs_count UInt32,
117+
emoji String,
118+
color Nullable(String),
119+
graphql_id String,
120+
configuration String,
121+
description String,
122+
created_at DateTime64(3),
123+
skip_queued_branch_builds Bool,
124+
repository String,
125+
pipeline_template_uuid Nullable(String),
126+
cluster_url String,
127+
skip_queued_branch_builds_filter String,
128+
cluster_id String,
129+
cancel_running_branch_builds Bool,
130+
provider Tuple(
131+
webhook_url String,
132+
settings String,
133+
id String
134+
),
135+
waiting_jobs_count UInt32,
136+
id String,
137+
slug String,
138+
badge_url String,
139+
visibility String,
140+
archived_at Nullable(DateTime64(3)),
141+
scheduled_jobs_count UInt32,
142+
running_builds_count UInt32,
143+
env Nullable(String),
144+
created_by Tuple(
145+
name Nullable(String),
146+
created_at Nullable(DateTime64(3)),
147+
id Nullable(String),
148+
avatar_url Nullable(String),
149+
graphql_id Nullable(String),
150+
email Nullable(String)
151+
),
152+
steps String,
153+
url String,
154+
tags Nullable(String),
155+
scheduled_builds_count UInt32,
156+
builds_url String,
157+
web_url String,
158+
name String,
159+
default_branch String,
160+
branch_configuration Nullable(String),
161+
cancel_running_branch_builds_filter Nullable(String)
162+
),
163+
sender Tuple(
164+
name Nullable(String),
165+
id Nullable(String)
166+
)
167+
) ENGINE = SharedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
168+
ORDER BY (
169+
pipeline.repository,
170+
pipeline.name,
171+
build.number,
172+
job.created_at,
173+
job.id,
174+
dynamoKey
175+
)
176+
SETTINGS index_granularity = 8192;

0 commit comments

Comments
 (0)