Merge branch 'antalya-25.6' into s3_assume_role_2

strtgbb · web-flow · commit 0daece9dd814 · 2025-07-14T17:02:41.000-04:00
diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
@@ -1366,7 +1366,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -1375,7 +1375,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -202,6 +202,47 @@ jobs:
             python3 -m praktika run 'Dockers Build (multiplatform manifest)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
           fi
 
+  fast_test:
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RmFzdCB0ZXN0') }}
+    name: "Fast test"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Setup
+        uses: ./.github/actions/runner_setup
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_masterci.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Fast test' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Fast test' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
+          fi
+
   build_amd_tidy:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
     needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -4726,7 +4767,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -4735,7 +4776,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
@@ -4788,6 +4829,7 @@ jobs:
       - dockers_build_amd
       - dockers_build_arm
       - dockers_build_multiplatform_manifest
+      - fast_test
       - build_amd_tidy
       - build_amd_debug
       - build_amd_release
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
@@ -335,7 +335,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -344,7 +344,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/nightly_fuzzers.yml b/.github/workflows/nightly_fuzzers.yml
@@ -241,7 +241,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -250,7 +250,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/nightly_jepsen.yml b/.github/workflows/nightly_jepsen.yml
@@ -241,7 +241,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -250,7 +250,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/nightly_statistics.yml b/.github/workflows/nightly_statistics.yml
@@ -113,7 +113,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -122,7 +122,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -288,7 +288,7 @@ jobs:
 
   build_amd_debug:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
     name: "Build (amd_debug)"
     outputs:
@@ -329,7 +329,7 @@ jobs:
 
   build_amd_release:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
     name: "Build (amd_release)"
     outputs:
@@ -370,7 +370,7 @@ jobs:
 
   build_amd_asan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9hc2FuKQ==') }}
     name: "Build (amd_asan)"
     outputs:
@@ -411,7 +411,7 @@ jobs:
 
   build_amd_tsan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF90c2FuKQ==') }}
     name: "Build (amd_tsan)"
     outputs:
@@ -452,7 +452,7 @@ jobs:
 
   build_amd_msan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9tc2FuKQ==') }}
     name: "Build (amd_msan)"
     outputs:
@@ -493,7 +493,7 @@ jobs:
 
   build_amd_ubsan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF91YnNhbik=') }}
     name: "Build (amd_ubsan)"
     outputs:
@@ -534,7 +534,7 @@ jobs:
 
   build_amd_binary:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
     name: "Build (amd_binary)"
     outputs:
@@ -575,7 +575,7 @@ jobs:
 
   build_arm_release:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
     name: "Build (arm_release)"
     outputs:
@@ -616,7 +616,7 @@ jobs:
 
   build_arm_asan:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }}
     name: "Build (arm_asan)"
     outputs:
@@ -657,7 +657,7 @@ jobs:
 
   build_arm_coverage:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9jb3ZlcmFnZSk=') }}
     name: "Build (arm_coverage)"
     outputs:
@@ -698,7 +698,7 @@ jobs:
 
   build_arm_binary:
     runs-on: [self-hosted, altinity-on-demand, altinity-builder]
-    needs: [config_workflow, dockers_build_amd, dockers_build_arm, fast_test, build_amd_tidy]
+    needs: [config_workflow, dockers_build_amd, dockers_build_arm]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9iaW5hcnkp') }}
     name: "Build (arm_binary)"
     outputs:
@@ -4728,7 +4728,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -4737,7 +4737,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
@@ -1774,7 +1774,7 @@ jobs:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -1783,7 +1783,7 @@ jobs:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py
@@ -75,7 +75,7 @@ class JobConfigs:
         runs_on=RunnerLabels.BUILDER_AMD,
         command="python3 ./ci/jobs/fast_test.py",
         # --network=host required for ec2 metadata http endpoint to work
-        run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log",
+        run_in_docker='altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"',
         digest_config=Job.CacheDigestConfig(
             include_paths=[
                 "./ci/jobs/fast_test.py",
diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py
@@ -42,7 +42,7 @@ class AltinityWorkflowTemplates:
     secrets: inherit
     with:
       docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+      # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
       tag-suffix: ${{ matrix.suffix }}
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
@@ -51,7 +51,7 @@ class AltinityWorkflowTemplates:
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
+        # version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_release]
diff --git a/ci/workflows/master.py b/ci/workflows/master.py
@@ -13,6 +13,7 @@
     event=Workflow.Event.PUSH,
     branches=[BASE_BRANCH, "releases/*", "antalya-*"],
     jobs=[
+        JobConfigs.fast_test,  # NOTE (strtgbb): for debugging
         *JobConfigs.tidy_build_jobs,
         *JobConfigs.build_jobs,
         *[
diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py
@@ -24,8 +24,8 @@
             job.set_dependency(
                 [
                     # JobNames.STYLE_CHECK, # NOTE (strtgbb): we don't run style check
-                    JobNames.FAST_TEST,
-                    JobConfigs.tidy_build_jobs[0].name,
+                    # JobNames.FAST_TEST, # NOTE (strtgbb): this takes too long, revisit later
+                    # JobConfigs.tidy_build_jobs[0].name, # NOTE (strtgbb): this takes too long, revisit later
                 ]
             )
             for job in JobConfigs.build_jobs
