[CI][Bench] Combine all benchmark jobs in one workflow

1. Right now changing only benchmark framework triggers all SYCL build.
  Thanks to this change only relevant changes are triggered for testing framework.
2. Nightly build was seperated. I believe keeping everything in one place makes
  it easier to maintain changes.

No changes in logic/builds were made in this commit - only minor cleanups
like names, plus changed the triggers.

Author: lukaszstolarczuk

.github/workflows/sycl-detect-changes.yml

@@ -67,6 +67,8 @@ jobs:
               - devops/dependencies-igc-dev.json
             benchmarks:
               - 'devops/scripts/benchmarks/**'
+              - 'devops/actions/run-tests/benchmark/**'
+              - '.github/workflows/sycl-ur-perf-benchmarking.yml'
             perf-tests:
               - sycl/test-e2e/PerformanceTests/**
             esimd:

.github/workflows/sycl-linux-precommit.yml

@@ -9,6 +9,7 @@ on:
     - sycl
     - sycl-rel-**
     # Do not run builds if changes are only in the following locations
+    # Note: benchmark-related paths are the same as in sycl-ur-perf-benchmarking.yml (to run there instead)
     paths-ignore:
     - '.github/ISSUE_TEMPLATE/**'
     - '.github/CODEOWNERS'
@@ -31,6 +32,9 @@ on:
     - 'unified-runtime/test/**'
     - 'unified-runtime/third_party/**'
     - 'unified-runtime/tools/**'
+    - 'devops/scripts/benchmarks/**'
+    - 'devops/actions/run-tests/benchmark/**'
+    - '.github/workflows/sycl-ur-perf-benchmarking.yml'
 
 concurrency:
   #  Cancel a currently running workflow from the same PR, branch or tag.
@@ -224,28 +228,6 @@ jobs:
       skip_run: ${{matrix.use_igc_dev && contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') || 'false'}}
       env: ${{ matrix.env || (contains(needs.detect_changes.outputs.filters, 'esimd') && '{}' || '{"LIT_FILTER_OUT":"ESIMD/"}') }}
 
-  test_benchmark_scripts:
-    needs: [build, detect_changes]
-    if: |
-      always() && !cancelled()
-      && needs.build.outputs.build_conclusion == 'success'
-      && contains(needs.detect_changes.outputs.filters, 'benchmarks')
-    uses: ./.github/workflows/sycl-linux-run-tests.yml
-    with:
-      name: Benchmark suite precommit testing
-      runner: '["PVC_PERF"]'
-      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
-      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-      target_devices: 'level_zero:gpu'
-      tests_selector: benchmarks
-      benchmark_upload_results: false
-      benchmark_preset: 'Minimal'
-      benchmark_dry_run: true
-      repo_ref: ${{ github.sha }}
-      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
-      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
-      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
-
   test-perf:
     needs: [build, detect_changes]
     if: |

.github/workflows/sycl-nightly-benchmarking.yml

@@ -1,6 +1,19 @@
+# A combined workflow for all benchmarks-related jobs for SYCL and UR.
+# Supports both manual triggering (dispatch) and nightly runs.
+# It also tests changes to benchmark scripts/framework in PR, if modified.
 name: Run Benchmarks
 
 on:
+  # Only run on pull requests, when a benchmark-related files were changed.
+  pull_request:
+    # these paths are exactly the same as in sycl-linux-precommit.yml (to ignore over there)
+    paths:
+      - 'devops/scripts/benchmarks/**'
+      - 'devops/actions/run-tests/benchmark/**'
+      - '.github/workflows/sycl-ur-perf-benchmarking.yml'
+  schedule:
+    - cron: '0 0 * * *'
+    # 3 hours ahead of SYCL nightly
   workflow_dispatch:
     inputs:
       preset:
@@ -57,12 +70,12 @@ on:
         options:
           - 'level_zero:gpu'
           - 'level_zero_v2:gpu'
-            # As of #17407, sycl-linux-build now builds v2 by default
 
 permissions: read-all
 
 jobs:
-  sanitize_inputs:
+  # Manual trigger (dispatch) path:
+  sanitize_inputs_dispatch:
     name: Sanitize inputs
     runs-on: ubuntu-latest
     env:
@@ -111,25 +124,24 @@ jobs:
           echo "Final sanitized values:"
           cat $GITHUB_OUTPUT
 
-  build_sycl:
+  build_sycl_dispatch:
     name: Build SYCL
-    needs: [ sanitize_inputs ]
+    needs: [ sanitize_inputs_dispatch ]
     uses: ./.github/workflows/sycl-linux-build.yml
     with:
-      build_ref: ${{ needs.sanitize_inputs.outputs.build_ref }}
+      build_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
       build_cache_root: "/__w/"
       build_cache_suffix: "prod_noassert"
       build_configure_extra_args: "--no-assertions"
       build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
       cc: clang
       cxx: clang++
       changes: '[]'
-
       toolchain_artifact: sycl_linux_prod_noassert
 
-  run_benchmarks_build:
-    name: Run Benchmarks on Build
-    needs: [ build_sycl, sanitize_inputs ]
+  benchmark_dispatch:
+    name: Benchmarks
+    needs: [ build_sycl_dispatch, sanitize_inputs_dispatch ]
     strategy:
       matrix:
         include:
@@ -139,16 +151,98 @@ jobs:
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     secrets: inherit
     with:
-      name: Run compute-benchmarks (${{ matrix.save_name }}, ${{ matrix.runner }}, ${{ matrix.backend }})
+      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ matrix.preset }})"
       runner: ${{ matrix.runner }}
       image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
       image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
       target_devices: ${{ matrix.backend }}
       tests_selector: benchmarks
       benchmark_upload_results: ${{ inputs.upload_results }}
-      benchmark_save_name: ${{ needs.sanitize_inputs.outputs.benchmark_save_name }}
+      benchmark_save_name: ${{ needs.sanitize_inputs_dispatch.outputs.benchmark_save_name }}
       benchmark_preset: ${{ inputs.preset }}
-      repo_ref: ${{ needs.sanitize_inputs.outputs.build_ref }}
-      toolchain_artifact: ${{ needs.build_sycl.outputs.toolchain_artifact }}
-      toolchain_artifact_filename: ${{ needs.build_sycl.outputs.toolchain_artifact_filename }}
-      toolchain_decompress_command: ${{ needs.build_sycl.outputs.toolchain_decompress_command }}
+      repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
+      toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build_sycl_dispatch.outputs.toolchain_decompress_command }}
+  # END manual trigger (dispatch) path
+
+  # Nightly benchmarking path:
+  build_nightly:
+    name: Build SYCL
+    if: github.repository == 'intel/llvm' && github.event_name == 'schedule'
+    uses: ./.github/workflows/sycl-linux-build.yml
+    secrets: inherit
+    with:
+      build_cache_root: "/__w/"
+      build_configure_extra_args: '--no-assertions'
+      build_image: ghcr.io/intel/llvm/ubuntu2404_build:latest
+
+      toolchain_artifact: sycl_linux_default
+      toolchain_artifact_filename: sycl_linux.tar.gz
+
+  benchmark_nightly:
+    name: Benchmarks
+    needs: [build_nightly]
+    if: always() && !cancelled() && needs.build_nightly.outputs.build_conclusion == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: ['["PVC_PERF"]', '["BMG_PERF"]']
+        backend: ['level_zero:gpu', 'level_zero_v2:gpu']
+        include:
+          - ref: ${{ github.sha }}
+            save_name: 'Baseline'
+            preset: 'Full'
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    secrets: inherit
+    with:
+      name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ matrix.preset }})"
+      runner: ${{ matrix.runner }}
+      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      target_devices: ${{ matrix.backend }}
+      tests_selector: benchmarks
+      benchmark_upload_results: true
+      benchmark_save_name: ${{ matrix.save_name }}
+      benchmark_preset: ${{ matrix.preset }}
+      repo_ref: ${{ matrix.ref }}
+      toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build_nightly.outputs.toolchain_decompress_command }}
+  # END nightly benchmarking path
+
+  # Benchmark framework builds and runs on PRs path:
+  build_pr:
+    name: Build SYCL
+    if: always() && !cancelled() && github.event_name == 'pull_request'
+    uses: ./.github/workflows/sycl-linux-build.yml
+    with:
+      build_ref: ${{ github.sha }}
+      build_cache_root: "/__w/"
+      build_cache_suffix: "default"
+      # Docker image has last nightly pre-installed and added to the PATH
+      build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
+      cc: clang
+      cxx: clang++
+      changes: ${{ needs.detect_changes.outputs.filters }}
+      toolchain_artifact: sycl_linux_default
+
+  test_benchmark_framework:
+    needs: [build_pr]
+    if: always() && !cancelled() && needs.build_pr.outputs.build_conclusion == 'success'
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Benchmark suite precommit testing
+      runner: '["PVC_PERF"]'
+      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      target_devices: 'level_zero:gpu'
+      tests_selector: benchmarks
+      benchmark_upload_results: false
+      benchmark_preset: 'Minimal'
+      benchmark_dry_run: true
+      repo_ref: ${{ github.sha }}
+      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
+  # END benchmark framework builds and runs on PRs path