From f6679eb208b5ec2cda56db8b579c4ab061758ee4 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 5 Aug 2025 12:37:54 +0200 Subject: [PATCH 01/17] update to mac-14 arm64 --- .github/workflows/mac.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 580e638b9b..148cc2a5d6 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,4 +1,4 @@ -name: macOS (13, Python 3.10) +name: macOS (14, Python 3.10) on: workflow_dispatch: pull_request: @@ -95,7 +95,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: CMAKE_GENERATOR: Ninja CMAKE_CXX_COMPILER_LAUNCHER: ccache @@ -215,7 +215,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: CCACHE_DIR: ${{ github.workspace }}/ccache CMAKE_CXX_COMPILER_LAUNCHER: ccache @@ -316,7 +316,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: OV_INSTALL_DIR: ${{ github.workspace }}/ov INSTALL_DIR: ${{ github.workspace }}/install @@ -372,7 +372,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: SRC_DIR: ${{ github.workspace }}/openvino.genai @@ -455,7 +455,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: INSTALL_DIR: ${{ github.workspace }}/install SRC_DIR: ${{ github.workspace }}/src @@ -532,7 +532,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: INSTALL_DIR: ${{ github.workspace }}/ov SRC_DIR: ${{ github.workspace }}/src @@ -625,7 +625,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: INSTALL_DIR: ${{ github.workspace }}/ov SRC_DIR: ${{ github.workspace }}/src @@ -682,7 +682,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-13 + runs-on: macos-14 env: SRC_DIR: ${{ github.workspace }}/openvino.genai From e7afd2c2d0a669dad060c876619b36a6e31f8a35 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Tue, 5 Aug 2025 12:39:44 +0200 Subject: [PATCH 02/17] update target arch --- .github/workflows/mac.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 148cc2a5d6..e2345bf1fe 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -147,7 +147,7 @@ jobs: with: repos: ${{ env.SRC_DIR }} product_type: ${{ env.BASE_PRODUCT_TYPE }}_${{ matrix.build-type }} - target_arch: 'x86_64' + target_arch: 'arm64' build_type: ${{ matrix.build-type }} save_to: ${{ env.MANIFEST_PATH }} @@ -264,7 +264,7 @@ jobs: with: repos: ${{ env.SRC_DIR }} product_type: ${{ env.BASE_PRODUCT_TYPE }}_Release - target_arch: 'x86_64' + target_arch: 'arm64' build_type: Release save_to: ${{ github.workspace }} From 92f235fe74637c4e7e483f689454dab7bee66ee8 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 7 Aug 2025 19:19:15 +0200 Subject: [PATCH 03/17] use dedicated product type --- .github/workflows/mac.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index e2345bf1fe..ffce277f58 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -19,7 +19,7 @@ env: MACOSX_DEPLOYMENT_TARGET: '11.0' PYTHON_VERSION: '3.10' TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} - BASE_PRODUCT_TYPE: public_darwin + BASE_PRODUCT_TYPE: public_macos_arm64 CCACHE_MAXSIZE: 500Mi HF_HOME: ~/.cache/hf OV_CACHE: ~/.cache/ov_cache @@ -160,7 +160,7 @@ jobs: cmake -DOpenVINODeveloperPackage_DIR=${{ env.OV_INSTALL_DIR }}/developer_package/cmake \ -DENABLE_PYTHON=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ - -S ${{ env.SRC_DIR}} \ + -S ${{ env.SRC_DIR }} \ -B ${{ env.BUILD_DIR }} cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} From 90cbf5cd650efac10f9b4e2ded56ea2a9f2d7f66 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 8 Aug 2025 11:15:10 +0200 Subject: [PATCH 04/17] use arm64 --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 941446cf2f..6d5e425d49 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -80,7 +80,7 @@ jobs: - uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master id: openvino_download with: - platform: macos_12_6 + platform: macos_14_7 commit_packages_to_provide: wheels,developer_package.tar.gz,openvino_node_npm_package.tar.gz revision: latest_available_commit From 2d0ba4dd571b237685c412d629453b7915cefca8 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 8 Aug 2025 09:22:21 +0200 Subject: [PATCH 05/17] provide arch --- .github/workflows/mac.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 6d5e425d49..1c929db35a 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -81,6 +81,7 @@ jobs: id: openvino_download with: platform: macos_14_7 + arch: 'arm64' commit_packages_to_provide: wheels,developer_package.tar.gz,openvino_node_npm_package.tar.gz revision: latest_available_commit From e77193bfca60a72e114102c6456781d3c94a02d7 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 8 Aug 2025 10:31:49 +0200 Subject: [PATCH 06/17] more time --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 1c929db35a..de9b7324a2 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -369,7 +369,7 @@ jobs: matrix: build-type: [Release] needs: [ openvino_download ] - timeout-minutes: 30 + timeout-minutes: 50 defaults: run: shell: bash From 292273e383b770dfc31729bdd1024b22cf66203d Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 8 Aug 2025 11:10:57 +0200 Subject: [PATCH 07/17] use larger runner --- .github/workflows/mac.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5e645867c4..fabfaedaea 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -216,7 +216,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14 + runs-on: macos-14-xlarge env: CCACHE_DIR: ${{ github.workspace }}/ccache CMAKE_CXX_COMPILER_LAUNCHER: ccache @@ -373,7 +373,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14 + runs-on: macos-14-xlarge env: SRC_DIR: ${{ github.workspace }}/openvino.genai From 603eb8b93cc938da1def8c0162b1e4d5673beea8 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 8 Aug 2025 11:50:44 +0200 Subject: [PATCH 08/17] use larger --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index fabfaedaea..1a40a2e541 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -456,7 +456,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14 + runs-on: macos-14-xlarge env: INSTALL_DIR: ${{ github.workspace }}/install SRC_DIR: ${{ github.workspace }}/src From 0139b0383a7c44f1edc069652374704628703cc3 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 11 Aug 2025 11:35:00 +0200 Subject: [PATCH 09/17] set parallel --- .github/workflows/mac.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 7b9982f3bb..a6313c11f3 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -216,7 +216,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14-xlarge + runs-on: macos-14 env: CCACHE_DIR: ${{ github.workspace }}/ccache CMAKE_CXX_COMPILER_LAUNCHER: ccache @@ -274,6 +274,7 @@ jobs: - name: Build Tokenizers Wheel run: | + export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ @@ -282,13 +283,16 @@ jobs: - name: Build GenAI Wheel run: | + export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ ${{ env.SRC_DIR }} working-directory: ${{ env.OV_INSTALL_DIR }} - name: Build WWB Wheel - run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark + run: | + export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark working-directory: ${{ env.OV_INSTALL_DIR }} - name: Save ccache @@ -373,7 +377,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14-xlarge + runs-on: macos-14 env: SRC_DIR: ${{ github.workspace }}/openvino.genai @@ -401,6 +405,7 @@ jobs: - name: Build GenAI Node.js bindings run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh + export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ -DENABLE_JS=ON -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF \ From 117d688f35ec1a778f9d675ee871ac436416462b Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 11 Aug 2025 13:01:09 +0200 Subject: [PATCH 10/17] use nproc for nodejs --- .github/workflows/mac.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index a6313c11f3..85c431a3de 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -405,12 +405,11 @@ jobs: - name: Build GenAI Node.js bindings run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ -DENABLE_JS=ON -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF \ -S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel --verbose + cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} - name: Upload Node.js bindings Build Package From 81c137d592f02f9773d49652cbe4497bbbcd6877 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Mon, 11 Aug 2025 13:38:34 +0200 Subject: [PATCH 11/17] use -1 --- .github/workflows/mac.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 85c431a3de..4b83fce7ee 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -274,7 +274,7 @@ jobs: - name: Build Tokenizers Wheel run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ @@ -283,7 +283,7 @@ jobs: - name: Build GenAI Wheel run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ ${{ env.SRC_DIR }} @@ -291,7 +291,7 @@ jobs: - name: Build WWB Wheel run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark working-directory: ${{ env.OV_INSTALL_DIR }} From 81504b0276549bb7bd3db2173d7e4ff1809293de Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 15 Aug 2025 11:36:15 +0200 Subject: [PATCH 12/17] skip some tests --- .github/workflows/mac.yml | 29 ++++++++++++++------------ tests/python_tests/test_gguf_reader.py | 3 +++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 0d7f422d9b..3c08b5943e 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -434,14 +434,16 @@ jobs: cmd: 'tests/python_tests/test_whisper_pipeline.py -k "not test_smoke[sample_from_dataset0 and not test_whisper_constructors[sample_from_dataset0 and not test_max_new_tokens[sample_from_dataset0 and not test_language_mode[language and not test_task_mode[sample_from_dataset0 and not test_language_autodetect[sample_from_dataset0 and not test_whisper_config_constructor and not test_language_autodetect[sample_from_dataset1 and not test_language_autodetect[sample_from_dataset2 and not test_initial_prompt_hotwords[sample_from_dataset0 and not test_random_sampling[sample_from_dataset0"' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).whisper.test }} timeout: 120 - - name: 'Cacheopt E2E' - cmd: 'tests/python_tests/test_kv_cache_eviction.py' - run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test }} - timeout: 240 - - name: 'LLM & VLM' - cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py tests/python_tests/test_structured_output.py' - run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} - timeout: 180 + # Only supported on X64 or ARM with SVE support + # - name: 'Cacheopt E2E' + # cmd: 'tests/python_tests/test_kv_cache_eviction.py' + # run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test }} + # timeout: 240 + # Only supported on X64 or ARM with SVE support + # - name: 'LLM & VLM' + # cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py tests/python_tests/test_structured_output.py' + # run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} + # timeout: 180 - name: 'GGUF Reader tests' cmd: 'tests/python_tests/test_gguf_reader.py' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).GGUF.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} @@ -450,10 +452,11 @@ jobs: cmd: 'tests/python_tests/test_tokenizer.py' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).tokenizers.test }} timeout: 60 - - name: 'API tests' - cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py' - run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test || fromJSON(needs.smart_ci.outputs.affected_components).sampling.test || fromJSON(needs.smart_ci.outputs.affected_components).text_streamer.test }} - timeout: 60 + # Only supported on X64 or ARM with SVE support + # - name: 'API tests' + # cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py' + # run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test || fromJSON(needs.smart_ci.outputs.affected_components).sampling.test || fromJSON(needs.smart_ci.outputs.affected_components).text_streamer.test }} + # timeout: 60 - name: 'Rag tests' cmd: 'tests/python_tests/test_rag.py' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }} @@ -461,7 +464,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14-xlarge + runs-on: macos-14 env: INSTALL_DIR: ${{ github.workspace }}/install SRC_DIR: ${{ github.workspace }}/src diff --git a/tests/python_tests/test_gguf_reader.py b/tests/python_tests/test_gguf_reader.py index 7451d77aa3..d81d5334bb 100644 --- a/tests/python_tests/test_gguf_reader.py +++ b/tests/python_tests/test_gguf_reader.py @@ -3,6 +3,7 @@ import pytest +import platform import torch import gc import sys @@ -112,6 +113,8 @@ def test_full_gguf_pipeline(pipeline_type, model_ids, enable_save_ov_model): @pytest.mark.parametrize("pipeline_type", get_gguf_pipeline_types()) @pytest.mark.parametrize("model_ids", [{"gguf_model_id": "Qwen/Qwen3-0.6B-GGUF", "gguf_filename": "Qwen3-0.6B-Q8_0.gguf"}]) +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 172335") @pytest.mark.precommit def test_full_gguf_qwen3_pipeline(pipeline_type, model_ids): # Temporal testing solution until transformers starts to support qwen3 in GGUF format From 7236ac1c6ada4fab888d6d66bc68aa4937e44ba8 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 15 Aug 2025 12:41:00 +0200 Subject: [PATCH 13/17] use large --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 3c08b5943e..995bf6ab23 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -217,7 +217,7 @@ jobs: defaults: run: shell: bash - runs-on: macos-14 + runs-on: macos-14-xlarge env: CCACHE_DIR: ${{ github.workspace }}/ccache CMAKE_CXX_COMPILER_LAUNCHER: ccache From 94802ee65a6579a403ac7c8ccc86fd4163386dbe Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 15 Aug 2025 13:09:31 +0200 Subject: [PATCH 14/17] rm core count --- .github/workflows/mac.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 995bf6ab23..722ba6e8e8 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -275,7 +275,6 @@ jobs: - name: Build Tokenizers Wheel run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ @@ -284,16 +283,13 @@ jobs: - name: Build GenAI Wheel run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ ${{ needs.openvino_download.outputs.ov_wheel_source }} \ ${{ env.SRC_DIR }} working-directory: ${{ env.OV_INSTALL_DIR }} - name: Build WWB Wheel - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc) - 1 )) - python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark + run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark working-directory: ${{ env.OV_INSTALL_DIR }} - name: Save ccache From f5dc051fe6f566b48c90e3c66cda26aaf0c71311 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 27 Aug 2025 12:45:54 +0200 Subject: [PATCH 15/17] rm unused --- .github/workflows/mac.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 106c88167e..9ce7178796 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -427,7 +427,7 @@ jobs: matrix: test: - name: 'Whisper' - # TODO: skip some tests temporary untill https://github.com/huggingface/datasets/issues/7647 dataset is fixed + # TODO: skip some tests temporary until https://github.com/huggingface/datasets/issues/7647 dataset is fixed cmd: 'tests/python_tests/test_whisper_pipeline.py -k "not test_smoke[sample_from_dataset0 and not test_whisper_constructors[sample_from_dataset0 and not test_max_new_tokens[sample_from_dataset0 and not test_language_mode[language and not test_task_mode[sample_from_dataset0 and not test_language_autodetect[sample_from_dataset0 and not test_whisper_config_constructor and not test_language_autodetect[sample_from_dataset1 and not test_language_autodetect[sample_from_dataset2 and not test_initial_prompt_hotwords[sample_from_dataset0 and not test_random_sampling[sample_from_dataset0"' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).whisper.test }} timeout: 120 @@ -441,10 +441,6 @@ jobs: # cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py tests/python_tests/test_structured_output.py' # run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} # timeout: 180 - - name: 'LLM & VLM' - cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py tests/python_tests/test_structured_output.py -p no:cacheprovider' - run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} - timeout: 180 - name: 'GGUF Reader tests' cmd: 'tests/python_tests/test_gguf_reader.py' run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).GGUF.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }} From b23f32e062486f218df9b2fca38b1d05543e55b4 Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 28 Aug 2025 11:14:39 +0200 Subject: [PATCH 16/17] rm +cpu for mac arm64 --- samples/export-requirements.txt | 2 +- tests/python_tests/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/export-requirements.txt b/samples/export-requirements.txt index f01510ea46..0d76aac169 100644 --- a/samples/export-requirements.txt +++ b/samples/export-requirements.txt @@ -10,7 +10,7 @@ diffusers==0.34.0 # For image generation pipelines timm==1.0.19 # For exporting InternVL2 # torchvision for visual language models torchvision==0.17.2; platform_system == "Darwin" and platform_machine == "x86_64" -torchvision==0.23.0+cpu; platform_system != "Darwin" or platform_machine != "x86_64" +torchvision==0.23.0; platform_system != "Darwin" or platform_machine != "x86_64" transformers==4.52.4 # For Whisper hf_transfer==0.1.9 # for faster models download, should used with env var HF_HUB_ENABLE_HF_TRANSFER=1 backoff==2.2.1 # for microsoft/Phi-3.5-vision-instruct diff --git a/tests/python_tests/requirements.txt b/tests/python_tests/requirements.txt index 686aa1ffd6..128a3330dd 100644 --- a/tests/python_tests/requirements.txt +++ b/tests/python_tests/requirements.txt @@ -18,7 +18,7 @@ langchain-core==0.3.74 einops==0.8.1 # - openbmb/MiniCPM-V-2 torchvision==0.17.2; platform_system == "Darwin" and platform_machine == "x86_64" -torchvision==0.23.0+cpu; platform_system != "Darwin" or platform_machine != "x86_64" +torchvision==0.23.0; platform_system != "Darwin" or platform_machine != "x86_64" # - openbmb/MiniCPM-V-2 timm==1.0.19 # - openai/whisper-base From 968e8e75d14ab384774f7220191cde9275bbb5eb Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Fri, 5 Sep 2025 10:53:25 +0200 Subject: [PATCH 17/17] skip memory ones --- tests/python_tests/test_whisper_pipeline.py | 16 ++++++++++++++++ tools/who_what_benchmark/tests/test_cli_image.py | 14 +++++++++++++- tools/who_what_benchmark/tests/test_cli_text.py | 2 ++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/python_tests/test_whisper_pipeline.py b/tests/python_tests/test_whisper_pipeline.py index 1e5302814d..b5ee89c531 100644 --- a/tests/python_tests/test_whisper_pipeline.py +++ b/tests/python_tests/test_whisper_pipeline.py @@ -3,7 +3,9 @@ import openvino_genai as ov_genai import functools +import platform import pytest +import sys import openvino_tokenizers import openvino import datasets @@ -457,6 +459,8 @@ def test_language_autodetect(model_descr, sample_from_dataset): @pytest.mark.parametrize("model_descr", get_whisper_models_list(tiny_only=True)) @pytest.mark.parametrize("sample_from_dataset", [*get_fixture_params_for_n_whisper_dataset_samples(n=1)], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_return_timestamps_short_form(model_descr, sample_from_dataset): run_pipeline_with_ref( model_id=model_descr[0], @@ -469,6 +473,8 @@ def test_return_timestamps_short_form(model_descr, sample_from_dataset): @pytest.mark.parametrize("model_descr", get_whisper_models_list(tiny_only=True)) @pytest.mark.parametrize("sample_from_dataset", [*get_fixture_params_for_n_whisper_dataset_samples(n=1)], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_return_timestamps_max_new_tokens_short_form(model_descr, sample_from_dataset): run_pipeline_with_ref( model_id=model_descr[0], @@ -483,6 +489,8 @@ def test_return_timestamps_max_new_tokens_short_form(model_descr, sample_from_da @pytest.mark.parametrize("model_descr", get_whisper_models_list()) @pytest.mark.parametrize("sample_from_dataset", [*get_fixture_params_for_n_whisper_dataset_samples(n=10, long_form=True)], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_longform_audio(model_descr, sample_from_dataset): _, _, hf_pipe, genai_pipe = read_whisper_model(model_descr) @@ -509,6 +517,8 @@ def test_longform_audio(model_descr, sample_from_dataset): @pytest.mark.parametrize("model_descr", get_whisper_models_list()) @pytest.mark.parametrize("sample_from_dataset", [*get_fixture_params_for_n_whisper_dataset_samples(n=2, long_form=True)], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_longform_audio_with_past(model_descr, sample_from_dataset): _, _, hf_pipe, genai_pipe = read_whisper_model(model_descr, stateful=True) @@ -534,6 +544,8 @@ def test_longform_audio_with_past(model_descr, sample_from_dataset): @pytest.mark.parametrize("model_descr", get_whisper_models_list()) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_shortform(model_descr): samples = [] ds = datasets.load_dataset( @@ -553,6 +565,8 @@ def test_shortform(model_descr): @pytest.mark.parametrize("model_descr", get_whisper_models_list(tiny_only=True)) @pytest.mark.parametrize("sample_from_dataset", [*get_fixture_params_for_n_whisper_dataset_samples(n=2, long_form=True)], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_beam_search(model_descr, sample_from_dataset): # use only 30 seconds of audio due to beam search results wrong with enabled timestamps # ticket: 167239 @@ -631,6 +645,8 @@ def test_random_sampling(model_descr, sample_from_dataset): @pytest.mark.parametrize("model_descr", get_whisper_models_list(tiny_only=True)) @pytest.mark.parametrize("sample_from_dataset", [{"language" : "en", "sample_id": 0}], indirect=True) @pytest.mark.precommit +@pytest.mark.xfail(condition=(sys.platform == "darwin" and platform.machine() in ('arm', 'armv7l', + 'aarch64', 'arm64', 'ARM64')), reason="Ticket - 173169") def test_perf_metrics(model_descr, sample_from_dataset): model_id, path, hf_pipe, genai_pipe = read_whisper_model(model_descr) diff --git a/tools/who_what_benchmark/tests/test_cli_image.py b/tools/who_what_benchmark/tests/test_cli_image.py index b354056daa..0536fc0447 100644 --- a/tools/who_what_benchmark/tests/test_cli_image.py +++ b/tools/who_what_benchmark/tests/test_cli_image.py @@ -1,7 +1,9 @@ import itertools import subprocess # nosec B404 import os +import sys import shutil +import platform import pytest import logging import tempfile @@ -108,7 +110,17 @@ def test_image_model_genai(model_id, model_type, tmp_path): pytest.skip(reason="FLUX-Fill is supported as inpainting only") if model_type == "image-inpainting": pytest.xfail("Segfault. Ticket 170877") - + + mac_arm64_skip = ('stable-diffusion-xl-image-to-image' in model_id or + 'stable-diffusion-3-tiny-random-image-to-image' in model_id or + 'stable-diffusion-3-tiny-random-text-to-image' in model_id or + 'tiny-random-flux' in model_id) + + if mac_arm64_skip \ + and (sys.platform == "darwin") \ + and (platform.machine() in ('arm', 'armv7l', 'aarch64', 'arm64', 'ARM64')): + pytest.xfail("Ticket 173169") + GT_FILE = tmp_path / "gt.csv" MODEL_PATH = os.path.join(MODEL_CACHE, model_id.replace("/", "--")) diff --git a/tools/who_what_benchmark/tests/test_cli_text.py b/tools/who_what_benchmark/tests/test_cli_text.py index f564d53c09..e40c2d4fa5 100644 --- a/tools/who_what_benchmark/tests/test_cli_text.py +++ b/tools/who_what_benchmark/tests/test_cli_text.py @@ -1,6 +1,7 @@ import os import shutil import tempfile +import platform import pandas as pd import pytest import logging @@ -49,6 +50,7 @@ def teardown_module(): shutil.rmtree(tmp_dir) +@pytest.mark.skipif((sys.platform == "darwin") and (platform.machine() in ('arm', 'armv7l', 'aarch64', 'arm64', 'ARM64')), reason='173169') def test_text_target_model(): run_wwb([ "--base-model",