Skip to content

Commit 2ec2527

Browse files
authored
Merge branch 'antalya' into s3-roles
2 parents a28ec24 + 1bd3a35 commit 2ec2527

File tree

9 files changed

+255
-20
lines changed

9 files changed

+255
-20
lines changed

.github/create_combined_ci_report.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def get_checks_known_fails(client: Client, job_url: str, known_fails: dict):
5454
len(df.columns) - 1,
5555
"reason",
5656
df["test_name"]
57-
.cat.remove_unused_categories()
57+
.astype(str)
5858
.apply(
5959
lambda test_name: known_fails[test_name].get("reason", "No reason given")
6060
),
@@ -111,12 +111,13 @@ def get_regression_fails(client: Client, job_url: str):
111111
job_name,
112112
report_url as results_link
113113
FROM `gh-data`.clickhouse_regression_results
114-
GROUP BY architecture, test_name, job_url, job_name, report_url, start_time
115-
ORDER BY start_time DESC, length(test_name) DESC
114+
GROUP BY architecture, test_name, job_url, job_name, report_url
115+
ORDER BY length(test_name) DESC
116116
)
117117
WHERE job_url='{job_url}'
118118
AND status IN ('Fail', 'Error')
119119
"""
120+
120121
df = client.query_dataframe(query)
121122
df = drop_prefix_rows(df, "test_name")
122123
df["job_name"] = df["job_name"].str.title()

.github/workflows/regression.yml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ jobs:
179179
python3
180180
-u ${{ env.SUITE }}/regression.py
181181
--clickhouse-binary-path ${{ env.clickhouse_path }}
182-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
182+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
183183
${{ env.args }} || EXITCODE=$?;
184184
.github/add_link_to_logs.sh;
185185
exit $EXITCODE
@@ -243,7 +243,7 @@ jobs:
243243
-u alter/regression.py
244244
--clickhouse-binary-path ${{ env.clickhouse_path }}
245245
--only "/alter/${{ matrix.ONLY }} partition/*"
246-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
246+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
247247
${{ env.args }} || EXITCODE=$?;
248248
.github/add_link_to_logs.sh;
249249
exit $EXITCODE
@@ -314,7 +314,7 @@ jobs:
314314
--aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
315315
--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
316316
--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
317-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
317+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
318318
${{ env.args }} || EXITCODE=$?;
319319
.github/add_link_to_logs.sh;
320320
exit $EXITCODE
@@ -374,7 +374,7 @@ jobs:
374374
-u ${{ env.SUITE }}/regression.py
375375
--ssl
376376
--clickhouse-binary-path ${{ env.clickhouse_path }}
377-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
377+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
378378
${{ env.args }} || EXITCODE=$?;
379379
.github/add_link_to_logs.sh;
380380
exit $EXITCODE
@@ -436,7 +436,7 @@ jobs:
436436
python3
437437
-u ${{ env.SUITE }}/regression.py
438438
--clickhouse-binary-path ${{ env.clickhouse_path }}
439-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
439+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
440440
${{ env.args }} || EXITCODE=$?;
441441
.github/add_link_to_logs.sh;
442442
exit $EXITCODE
@@ -494,7 +494,7 @@ jobs:
494494
python3
495495
-u ${{ env.SUITE }}/regression.py
496496
--clickhouse-binary-path ${{ env.clickhouse_path }}
497-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
497+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
498498
${{ env.args }} || EXITCODE=$?;
499499
.github/add_link_to_logs.sh;
500500
exit $EXITCODE
@@ -562,7 +562,7 @@ jobs:
562562
--aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
563563
--aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
564564
--aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
565-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
565+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
566566
${{ env.args }} || EXITCODE=$?;
567567
.github/add_link_to_logs.sh;
568568
exit $EXITCODE
@@ -636,7 +636,7 @@ jobs:
636636
--azure-account-name ${{ secrets.AZURE_ACCOUNT_NAME }}
637637
--azure-storage-key ${{ secrets.AZURE_STORAGE_KEY }}
638638
--azure-container ${{ secrets.AZURE_CONTAINER_NAME }}
639-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
639+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
640640
${{ env.args }} || EXITCODE=$?;
641641
.github/add_link_to_logs.sh;
642642
exit $EXITCODE
@@ -706,7 +706,7 @@ jobs:
706706
--gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
707707
--gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
708708
--with-${{ matrix.STORAGE }}
709-
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
709+
--attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)"
710710
${{ env.args }} || EXITCODE=$?;
711711
.github/add_link_to_logs.sh;
712712
exit $EXITCODE

.github/workflows/release_branches.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -476,23 +476,23 @@ jobs:
476476
#############################################################################################
477477
RegressionTestsRelease:
478478
needs: [BuilderDebRelease]
479-
if: ${{ !failure() && !cancelled() }}
479+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') }}
480480
uses: ./.github/workflows/regression.yml
481481
secrets: inherit
482482
with:
483483
runner_type: altinity-on-demand, altinity-type-cx52, altinity-image-x86-app-docker-ce, altinity-setup-regression
484-
commit: a170f32119a5c872e5ff209b8f39e13acc2d6626
484+
commit: bd31e738c0cedaca253d15a05ed245c41b6e0b6a
485485
arch: release
486486
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
487487
timeout_minutes: 300
488488
RegressionTestsAarch64:
489489
needs: [BuilderDebAarch64]
490-
if: ${{ !failure() && !cancelled() }}
490+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'aarch64') }}
491491
uses: ./.github/workflows/regression.yml
492492
secrets: inherit
493493
with:
494494
runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression
495-
commit: a170f32119a5c872e5ff209b8f39e13acc2d6626
495+
commit: bd31e738c0cedaca253d15a05ed245c41b6e0b6a
496496
arch: aarch64
497497
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
498498
timeout_minutes: 300
@@ -575,7 +575,7 @@ jobs:
575575
CHECKS_DATABASE_USER: ${{ secrets.CHECKS_DATABASE_USER }}
576576
CHECKS_DATABASE_PASSWORD: ${{ secrets.CHECKS_DATABASE_PASSWORD }}
577577
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
578-
PR_NUMBER: ${{ github.event.number }}
578+
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
579579
ACTIONS_RUN_URL: ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}
580580
shell: bash
581581
run: |
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
name: Repository Sanity Checks
2+
3+
on:
4+
workflow_dispatch: # Manual trigger only
5+
6+
workflow_call:
7+
8+
jobs:
9+
sanity-checks:
10+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
11+
strategy:
12+
fail-fast: false # Continue with other combinations if one fails
13+
matrix:
14+
include:
15+
# Production packages
16+
- env: prod
17+
type: deb
18+
base: ubuntu:22.04
19+
repo_url: https://builds.altinity.cloud/apt-repo
20+
- env: prod
21+
type: rpm
22+
base: centos:8
23+
repo_url: https://builds.altinity.cloud/yum-repo
24+
# FIPS Production packages
25+
- env: prod-fips
26+
type: deb
27+
base: ubuntu:22.04
28+
repo_url: https://builds.altinity.cloud/fips-apt-repo
29+
- env: prod-fips
30+
type: rpm
31+
base: centos:8
32+
repo_url: https://builds.altinity.cloud/fips-yum-repo
33+
# Staging packages
34+
- env: staging
35+
type: deb
36+
base: ubuntu:22.04
37+
repo_url: https://builds.staging.altinity.cloud/apt-repo
38+
- env: staging
39+
type: rpm
40+
base: centos:8
41+
repo_url: https://builds.staging.altinity.cloud/yum-repo
42+
# FIPS Staging packages
43+
- env: staging-fips
44+
type: deb
45+
base: ubuntu:22.04
46+
repo_url: https://builds.staging.altinity.cloud/fips-apt-repo
47+
- env: staging-fips
48+
type: rpm
49+
base: centos:8
50+
repo_url: https://builds.staging.altinity.cloud/fips-yum-repo
51+
# Hotfix packages
52+
- env: hotfix
53+
type: deb
54+
base: ubuntu:22.04
55+
repo_url: https://builds.altinity.cloud/hotfix-apt-repo
56+
- env: hotfix
57+
type: rpm
58+
base: centos:8
59+
repo_url: https://builds.altinity.cloud/hotfix-yum-repo
60+
# Antalya experimental packages
61+
- env: antalya
62+
type: deb
63+
base: ubuntu:22.04
64+
repo_url: https://builds.altinity.cloud/antalya-apt-repo
65+
- env: antalya
66+
type: rpm
67+
base: centos:8
68+
repo_url: https://builds.altinity.cloud/antalya-yum-repo
69+
# Hotfix staging packages
70+
- env: hotfix-staging
71+
type: deb
72+
base: ubuntu:22.04
73+
repo_url: https://builds.staging.altinity.cloud/hotfix-apt-repo
74+
- env: hotfix-staging
75+
type: rpm
76+
base: centos:8
77+
repo_url: https://builds.staging.altinity.cloud/hotfix-yum-repo
78+
# Antalya experimental staging packages
79+
- env: antalya-staging
80+
type: deb
81+
base: ubuntu:22.04
82+
repo_url: https://builds.staging.altinity.cloud/antalya-apt-repo
83+
- env: antalya-staging
84+
type: rpm
85+
base: centos:8
86+
repo_url: https://builds.staging.altinity.cloud/antalya-yum-repo
87+
88+
steps:
89+
- name: Run sanity check
90+
run: |
91+
cat << 'EOF' > sanity.sh
92+
#!/bin/bash
93+
set -e -x
94+
95+
# Package installation commands based on type
96+
if [ "${{ matrix.type }}" = "deb" ]; then
97+
export DEBIAN_FRONTEND=noninteractive
98+
apt-get update && apt-get install -y apt-transport-https ca-certificates curl gnupg2 dialog sudo
99+
mkdir -p /usr/share/keyrings
100+
curl -s "${REPO_URL}/pubkey.gpg" | gpg --dearmor > /usr/share/keyrings/altinity-archive-keyring.gpg
101+
echo "deb [signed-by=/usr/share/keyrings/altinity-archive-keyring.gpg] ${REPO_URL} stable main" > /etc/apt/sources.list.d/altinity.list
102+
apt-get update
103+
apt-get install -y clickhouse-server clickhouse-client
104+
else
105+
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
106+
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
107+
yum install -y curl gnupg2 sudo
108+
if [[ "${{ matrix.env }}" == *"staging"* ]]; then
109+
curl "${REPO_URL}/altinity-staging.repo" -o /etc/yum.repos.d/altinity-staging.repo
110+
else
111+
curl "${REPO_URL}/altinity.repo" -o /etc/yum.repos.d/altinity.repo
112+
fi
113+
yum install -y clickhouse-server clickhouse-client
114+
fi
115+
116+
# Ensure correct ownership
117+
chown -R clickhouse /var/lib/clickhouse/
118+
chown -R clickhouse /var/log/clickhouse-server/
119+
120+
# Check server version
121+
server_version=$(clickhouse-server --version)
122+
echo "$server_version" | grep "altinity" || FAILED_SERVER=true
123+
124+
# Start server and test
125+
sudo -u clickhouse clickhouse-server --config-file /etc/clickhouse-server/config.xml --daemon
126+
sleep 10
127+
clickhouse-client -q 'SELECT 1'
128+
129+
# Check client version
130+
client_version=$(clickhouse-client --version)
131+
echo "$client_version" | grep "altinity" || FAILED_CLIENT=true
132+
133+
# Report results
134+
if [ "$FAILED_SERVER" = true ]; then
135+
echo "::error::Server check failed - Version: $server_version"
136+
exit 1
137+
elif [ "$FAILED_CLIENT" = true ]; then
138+
echo "::error::Client check failed - Version: $client_version"
139+
exit 1
140+
else
141+
echo "All checks passed successfully!"
142+
fi
143+
EOF
144+
145+
chmod +x sanity.sh
146+
docker run --rm \
147+
-v $(pwd)/sanity.sh:/sanity.sh \
148+
-e REPO_URL="${{ matrix.repo_url }}" \
149+
${{ matrix.base }} \
150+
/sanity.sh

0 commit comments

Comments
 (0)