metal.yml

name: Test Metal Backend

on:
  push:
    branches:
      - main
      - release/*
    tags:
      - ciflow/metal/*
  pull_request:
    paths:
      - .github/workflows/metal.yml
      - backends/apple/metal/**
      - backends/aoti/**
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
  cancel-in-progress: true

jobs:
  test-metal-builds:
    name: test-executorch-metal-build
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      runner: macos-m2-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        set -eux

        echo "::group::Test ExecuTorch Metal build"
        PYTHON_EXECUTABLE=python CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ${CONDA_RUN} --no-capture-output ./install_executorch.sh
        echo "::endgroup::"

  test-metal-modules:
    name: test-metal-backend-modules
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      runner: macos-m2-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 120
      script: |
        set -eux

        echo "::group::Setup ExecuTorch"
        PYTHON_EXECUTABLE=python ${CONDA_RUN} EXECUTORCH_BUILD_KERNELS_TORCHAO=1 TORCHAO_BUILD_EXPERIMENTAL_MPS=1 ./install_executorch.sh
        echo "::endgroup::"

        echo "::group::Build Metal Runtime"
        ${CONDA_RUN} backends/apple/metal/tests/run_metal_test.sh --build
        echo "::endgroup::"

        echo "::group::Run Metal Backend Module Tests"
        ${CONDA_RUN} python -m unittest backends.apple.metal.tests.test_modules.TestMetalBackendModules
        echo "::endgroup::"

  export-model-metal-artifact:
    name: export-model-metal-artifact
    # Skip this job if the pull request is from a fork (HuggingFace secrets are not available)
    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    secrets: inherit
    strategy:
      fail-fast: false
      matrix:
        model:
          - repo: "mistralai"
            name: "Voxtral-Mini-3B-2507"
          - repo: "mistralai"
            name: "Voxtral-Mini-4B-Realtime-2602"
          - repo: "openai"
            name: "whisper-small"
          - repo: "openai"
            name: "whisper-large-v3-turbo"
          - repo: "nvidia"
            name: "parakeet-tdt"
        quant:
          - "non-quantized"
          - "quantized-int4-metal"
        exclude:
          # Exclude non-quantized for Voxtral Realtime (too large)
          - model:
              repo: "mistralai"
              name: "Voxtral-Mini-4B-Realtime-2602"
            quant: "non-quantized"
    with:
      runner: macos-m2-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      secrets-env: EXECUTORCH_HF_TOKEN
      upload-artifact: ${{ matrix.model.repo }}-${{ matrix.model.name }}-metal-${{ matrix.quant }}
      script: |
        set -eux

        echo "::group::Setup Huggingface"
        ${CONDA_RUN} pip install -U "huggingface_hub[cli]<1.0" accelerate
        ${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
        echo "::endgroup::"

        echo "::group::Setup Optimum-ExecuTorch"
        OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
        echo "Using optimum-executorch version: ${OPTIMUM_ET_VERSION}"
        ${CONDA_RUN} pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
        echo "::endgroup::"

        echo "::group::Setup ExecuTorch"
        PYTHON_EXECUTABLE=python ${CONDA_RUN} EXECUTORCH_BUILD_KERNELS_TORCHAO=1 TORCHAO_BUILD_EXPERIMENTAL_MPS=1 ./install_executorch.sh
        echo "::endgroup::"

        echo "::group::Pip List"
        ${CONDA_RUN} pip list
        echo "::endgroup::"

        # Isolate Inductor cache and precompiled headers (PCH) per job to prevent
        # PCH mtime conflicts between parallel matrix jobs on the same runner.
        # TORCHINDUCTOR_CACHE_DIR isolates the code cache; setting TMPDIR isolates
        # the PCH dir, which PyTorch derives from tempfile.gettempdir() independently.
        export TMPDIR=$(mktemp -d "${RUNNER_TEMP}/tmpdir_XXXXXX")
        export TORCHINDUCTOR_CACHE_DIR=$(mktemp -d "${RUNNER_TEMP}/inductor_cache_XXXXXX")
        ${CONDA_RUN} bash .ci/scripts/export_model_artifact.sh metal "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}"

  test-model-metal-e2e:
    name: test-model-metal-e2e
    needs: export-model-metal-artifact
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      fail-fast: false
      matrix:
        model:
          - repo: "mistralai"
            name: "Voxtral-Mini-3B-2507"
          - repo: "mistralai"
            name: "Voxtral-Mini-4B-Realtime-2602"
          - repo: "openai"
            name: "whisper-small"
          - repo: "openai"
            name: "whisper-large-v3-turbo"
          - repo: "nvidia"
            name: "parakeet-tdt"
        quant:
          - "non-quantized"
          - "quantized-int4-metal"
        exclude:
          # Exclude non-quantized for Voxtral Realtime (too large)
          - model:
              repo: "mistralai"
              name: "Voxtral-Mini-4B-Realtime-2602"
            quant: "non-quantized"
    with:
      runner: macos-m2-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      download-artifact: ${{ matrix.model.repo }}-${{ matrix.model.name }}-metal-${{ matrix.quant }}
      script: |
        set -eux

        echo "::group::Print machine info"
        uname -a
        if [ $(uname -s) == Darwin ]; then
          sw_vers
          # Print RAM in GB
          RAM_BYTES=$(sysctl -n hw.memsize)
          RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
          echo "Available RAM (GB): $RAM_GB"
          sysctl machdep.cpu.brand_string
          sysctl machdep.cpu.core_count
          # Print number of GPU cores (Apple Silicon)
          if command -v system_profiler &> /dev/null; then
            GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
            if [ -z "$GPU_CORES" ]; then
              # Fallback: try to parse "Core Count" from Apple GPU section
              GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
            fi
            echo "GPU Cores: ${GPU_CORES:-Unknown}"
          else
            echo "system_profiler not available, cannot determine GPU cores."
          fi
        fi
        echo "::endgroup::"

        ${CONDA_RUN} bash .ci/scripts/test_model_e2e.sh metal "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}"