Merge branch 'main' into main

Author: cad-audio

.ci/docker/build.sh

@@ -40,6 +40,15 @@ case "${IMAGE_NAME}" in
     LINTRUNNER=""
     GCC_VERSION=11
     ;;
+  executorch-ubuntu-22.04-gcc11-aarch64-android)
+    LINTRUNNER=""
+    GCC_VERSION=11
+    ANDROID_NDK_VERSION=r28c
+    ;;
+  executorch-ubuntu-22.04-gcc11-aarch64-arm-sdk)
+    ARM_SDK=yes
+    GCC_VERSION=11
+    ;;
   executorch-ubuntu-22.04-linter)
     LINTRUNNER=yes
     CLANG_VERSION=12

.ci/docker/ci_commit_pins/optimum-executorch.txt

@@ -1 +1 @@
-f8aa919593cc51301ade73a2ee5491582521ab80
+5bf1aeb587e9b1f3572b0bd60265c5dafd007b73

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-7a79b41e29a790ebb4b530eb98a89381e2d7de29
+659af3c353e49b35c191cdd2dba3b3c79d0e6822

.ci/docker/common/install_android.sh

@@ -40,8 +40,14 @@ install_ndk() {
   rm -rf "${NDK_INSTALLATION_DIR}" && mkdir -p "${NDK_INSTALLATION_DIR}"
 
   pushd /tmp
-  # The NDK installation is cached on ossci-android S3 bucket
-  curl -Os --retry 3 "https://ossci-android.s3.amazonaws.com/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+  ARCH=$(uname -m)
+  if [ "${ARCH}" = "aarch64" ]; then
+    # aarch64 NDK is not cached on S3, download from Google directly
+    curl -Os --retry 3 "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+  else
+    # The NDK installation is cached on ossci-android S3 bucket
+    curl -Os --retry 3 "https://ossci-android.s3.amazonaws.com/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+  fi
   unzip -qo "android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
 
   # Print the content for manual verification
@@ -73,7 +79,10 @@ install_sdk() {
   yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "build-tools;35.0.0"
   # And some more tools for future emulator tests
   yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "platform-tools"
-  yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "tools"
+  # The 'tools' package (emulator) is not available on aarch64
+  if [ "$(uname -m)" != "aarch64" ]; then
+    yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "tools"
+  fi
 }
 
 install_prerequiresites

.ci/docker/requirements-ci.txt

@@ -30,6 +30,6 @@ sphinx-reredirects==0.1.4
 matplotlib>=3.9.4
 sphinx-copybutton==0.5.2
 # PyTorch Theme
-pytorch_sphinx_theme2==0.2.0
+pytorch_sphinx_theme2==0.4.4
 # script unit test requirements
 yaspin==3.1.0

.ci/scripts/cleanup-samsung-linux-deps.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set +ex
+
+if [[ "${DEVICE_ACQUIRED:-0}" != "1" ]]; then
+  exit 0
+fi
+
+if ! command -v devicefarm-cli >/dev/null 2>&1; then
+  echo "[WARN] Skip device disconnect (devicefarm-cli not installed)." >&2
+  exit 0
+fi
+
+echo "[INFO] Disconnecting device (-d)..."
+devicefarm-cli -d || echo "::warning::Device disconnect failed (ignored)"
+
+set -ex

.ci/scripts/export_model_artifact.sh

@@ -5,37 +5,51 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-# Export model to CUDA/Metal format with optional quantization
+# Export model to CUDA/Metal/XNNPACK format with optional quantization
 
 show_help() {
   cat << EOF
-Usage: export_model_artifact.sh <device> <hf_model> [quant_name] [output_dir]
+Usage: export_model_artifact.sh <device> <hf_model> [quant_name] [output_dir] [mode]
 
-Export a HuggingFace model to CUDA/Metal format with optional quantization.
+Export a HuggingFace model to CUDA/Metal/XNNPACK format with optional quantization.
 
 Arguments:
-  device       cuda or metal (required)
+  device       cuda, metal, or xnnpack (required)
 
   hf_model     HuggingFace model ID (required)
                Supported models:
                  - mistralai/Voxtral-Mini-3B-2507
+                 - mistralai/Voxtral-Mini-4B-Realtime-2602
                  - openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo})
                  - google/gemma-3-4b-it
                  - nvidia/parakeet-tdt
 
   quant_name   Quantization type (optional, default: non-quantized)
                Options:
                  - non-quantized
-                 - quantized-int4-tile-packed
-                 - quantized-int4-weight-only
+                 - quantized-int4-tile-packed (CUDA only)
+                 - quantized-int4-weight-only (CUDA only)
+                 - quantized-int4-metal (Metal only)
+                 - quantized-8da4w (XNNPACK only)
 
   output_dir   Output directory for artifacts (optional, default: current directory)
 
+  mode         Export mode (optional, default: auto-detect based on model and device)
+               Supported modes:
+                 - vr-streaming: Voxtral Realtime streaming mode
+                 - vr-offline: Voxtral Realtime offline mode
+
 Examples:
   export_model_artifact.sh metal "openai/whisper-small"
+  export_model_artifact.sh metal "nvidia/parakeet-tdt" "quantized-int4-metal"
+  export_model_artifact.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-int4-metal"
+  export_model_artifact.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "." "vr-streaming"
   export_model_artifact.sh cuda "mistralai/Voxtral-Mini-3B-2507" "quantized-int4-tile-packed"
   export_model_artifact.sh cuda "google/gemma-3-4b-it" "non-quantized" "./output"
   export_model_artifact.sh cuda "nvidia/parakeet-tdt" "non-quantized" "./output"
+  export_model_artifact.sh xnnpack "nvidia/parakeet-tdt" "quantized-8da4w" "./output"
+  export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-8da4w" "./output"
+  export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "./output" "vr-offline"
 EOF
 }
 
@@ -56,6 +70,26 @@ DEVICE="$1"
 HF_MODEL="$2"
 QUANT_NAME="${3:-non-quantized}"
 OUTPUT_DIR="${4:-.}"
+MODE="${5:-}"
+
+# Validate mode if specified
+if [ -n "$MODE" ]; then
+  case "$MODE" in
+    vr-streaming|vr-offline)
+      # Voxtral Realtime modes require Voxtral Realtime model
+      if [ "$HF_MODEL" != "mistralai/Voxtral-Mini-4B-Realtime-2602" ]; then
+        echo "Error: Mode '$MODE' can only be used with Voxtral Realtime model"
+        echo "Provided model: $HF_MODEL"
+        exit 1
+      fi
+      ;;
+    *)
+      echo "Error: Unsupported mode '$MODE'"
+      echo "Supported modes: vr-streaming, vr-offline"
+      exit 1
+      ;;
+  esac
+fi
 
 case "$DEVICE" in
   cuda)
@@ -64,9 +98,11 @@ case "$DEVICE" in
     ;;
   metal)
     ;;
+  xnnpack)
+    ;;
   *)
     echo "Error: Unsupported device '$DEVICE'"
-    echo "Supported devices: cuda, cuda-windows, metal"
+    echo "Supported devices: cuda, cuda-windows, metal, xnnpack"
     exit 1
     ;;
 esac
@@ -113,9 +149,17 @@ case "$HF_MODEL" in
     PREPROCESSOR_FEATURE_SIZE=""
     PREPROCESSOR_OUTPUT=""
     ;;
+  mistralai/Voxtral-Mini-4B-Realtime-2602)
+    MODEL_NAME="voxtral_realtime"
+    TASK=""
+    MAX_SEQ_LEN=""
+    EXTRA_PIP="mistral-common librosa"
+    PREPROCESSOR_FEATURE_SIZE=""
+    PREPROCESSOR_OUTPUT=""
+    ;;
   *)
     echo "Error: Unsupported model '$HF_MODEL'"
-    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, nvidia/parakeet-tdt"
+    echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, nvidia/parakeet-tdt"
     exit 1
     ;;
 esac
@@ -127,21 +171,35 @@ case "$QUANT_NAME" in
     ;;
   quantized-int4-tile-packed)
     if [ "$DEVICE" = "metal" ]; then
-      echo "Error: Metal backend does not yet support quantization '$QUANT_NAME'"
+      echo "Error: Metal backend does not support quantization '$QUANT_NAME'"
       exit 1
     fi
     EXTRA_ARGS="--qlinear 4w --qlinear_encoder 4w --qlinear_packing_format tile_packed_to_4d --qlinear_encoder_packing_format tile_packed_to_4d"
     ;;
   quantized-int4-weight-only)
     if [ "$DEVICE" = "metal" ]; then
-      echo "Error: Metal backend does not yet support quantization '$QUANT_NAME'"
+      echo "Error: Metal backend does not support quantization '$QUANT_NAME'"
       exit 1
     fi
     EXTRA_ARGS="--qlinear_encoder 4w"
     ;;
+  quantized-int4-metal)
+    if [ "$DEVICE" != "metal" ]; then
+      echo "Error: Quantization '$QUANT_NAME' only supported on Metal backend"
+      exit 1
+    fi
+    EXTRA_ARGS="--qlinear fpa4w --qlinear_encoder fpa4w"
+    ;;
+  quantized-8da4w)
+    if [ "$DEVICE" != "xnnpack" ]; then
+      echo "Error: quantized-8da4w is only supported with xnnpack device"
+      exit 1
+    fi
+    EXTRA_ARGS="--qlinear 8da4w --qlinear_group_size 32 --qlinear_encoder 8da4w --qlinear_encoder_group_size 32"
+    ;;
   *)
     echo "Error: Unsupported quantization '$QUANT_NAME'"
-    echo "Supported quantizations: non-quantized, quantized-int4-tile-packed, quantized-int4-weight-only"
+    echo "Supported quantizations: non-quantized, quantized-int4-tile-packed, quantized-int4-weight-only, quantized-int4-metal, quantized-8da4w"
     exit 1
     ;;
 esac
@@ -157,10 +215,18 @@ pip list
 if [ "$MODEL_NAME" = "parakeet" ]; then
   pip install -r examples/models/parakeet/install_requirements.txt
 
-  python examples/models/parakeet/export_parakeet_tdt.py \
+  # Set dtype based on backend (XNNPACK uses fp32, CUDA/Metal use bf16)
+  if [ "$DEVICE" = "xnnpack" ]; then
+    DTYPE_ARG=""
+  else
+    DTYPE_ARG="--dtype bf16"
+  fi
+
+  python -m executorch.examples.models.parakeet.export_parakeet_tdt \
       --backend "$DEVICE" \
       --output-dir "${OUTPUT_DIR}" \
-      --dtype bf16
+      ${DTYPE_ARG} \
+      ${EXTRA_ARGS}
 
   test -f "${OUTPUT_DIR}/model.pte"
   # CUDA saves named data to separate .ptd file, Metal embeds in .pte
@@ -173,6 +239,64 @@ if [ "$MODEL_NAME" = "parakeet" ]; then
   exit 0
 fi
 
+# Voxtral Realtime uses a custom export script
+if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
+  pip install safetensors huggingface_hub
+
+  # Download model weights from HuggingFace (requires HF_TOKEN for gated model)
+  LOCAL_MODEL_DIR="${OUTPUT_DIR}/model_weights"
+  python -c "from huggingface_hub import snapshot_download; snapshot_download('${HF_MODEL}', local_dir='${LOCAL_MODEL_DIR}')"
+
+  # Per-component quantization flags
+  VR_QUANT_ARGS=""
+  if [ "$QUANT_NAME" = "quantized-8da4w" ]; then
+    VR_QUANT_ARGS="--qlinear-encoder 8da4w --qlinear 8da4w --qlinear-group-size 32 --qembedding 8w"
+  elif [ "$QUANT_NAME" = "quantized-int4-metal" ]; then
+    VR_QUANT_ARGS="--qlinear-encoder fpa4w --qlinear fpa4w"
+  fi
+
+  # Determine streaming mode based on MODE parameter
+  USE_STREAMING="false"
+  if [ "$MODE" = "vr-streaming" ]; then
+    USE_STREAMING="true"
+  elif [ "$MODE" = "vr-offline" ]; then
+    USE_STREAMING="false"
+  elif [ -z "$MODE" ]; then
+    # Auto-detect: XNNPACK uses streaming, others use offline
+    if [ "$DEVICE" = "xnnpack" ]; then
+      USE_STREAMING="true"
+    fi
+  fi
+
+  # Configure export and preprocessor based on streaming mode
+  STREAMING_ARG=""
+  PREPROCESSOR_ARGS="--feature_size 128 --output_file ${OUTPUT_DIR}/preprocessor.pte"
+  if [ "$USE_STREAMING" = "true" ]; then
+    STREAMING_ARG="--streaming"
+    PREPROCESSOR_ARGS="$PREPROCESSOR_ARGS --streaming"
+  else
+    PREPROCESSOR_ARGS="$PREPROCESSOR_ARGS --stack_output --max_audio_len 300"
+  fi
+
+  python -m executorch.examples.models.voxtral_realtime.export_voxtral_rt \
+      --model-path "$LOCAL_MODEL_DIR" \
+      --backend "$DEVICE" \
+      ${STREAMING_ARG} \
+      --output-dir "${OUTPUT_DIR}" \
+      ${VR_QUANT_ARGS}
+
+  # Export preprocessor
+  python -m executorch.extension.audio.mel_spectrogram ${PREPROCESSOR_ARGS}
+
+  test -f "${OUTPUT_DIR}/model.pte"
+  test -f "${OUTPUT_DIR}/preprocessor.pte"
+  # Copy tokenizer from downloaded model weights
+  cp "$LOCAL_MODEL_DIR/tekken.json" "${OUTPUT_DIR}/tekken.json"
+  ls -al "${OUTPUT_DIR}"
+  echo "::endgroup::"
+  exit 0
+fi
+
 MAX_SEQ_LEN_ARG=""
 if [ -n "$MAX_SEQ_LEN" ]; then
   MAX_SEQ_LEN_ARG="--max_seq_len $MAX_SEQ_LEN"
@@ -181,6 +305,8 @@ fi
 DEVICE_ARG=""
 if [ "$DEVICE" = "cuda" ] || [ "$DEVICE" = "cuda-windows" ]; then
   DEVICE_ARG="--device cuda"
+elif [ "$DEVICE" = "metal" ]; then
+  DEVICE_ARG="--device mps"
 fi
 
 optimum-cli export executorch \