Update CI version (#2324)

Signed-off-by: Sun, Xuehao <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: XuehaoSun

.azure-pipelines/docker/Dockerfile.devel

@@ -36,10 +36,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
 
 RUN ln -sf $(which python3) /usr/bin/python
 
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN groupadd -g ${GROUP_ID} hostgroup && \
+    useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser
+
+USER hostuser
+
+ENV PATH="/home/hostuser/.local/bin:$PATH"
 RUN python -m pip install pip packaging --upgrade
 RUN python -m pip install --no-cache-dir setuptools
 
 RUN pip list
 
 WORKDIR /
-

.azure-pipelines/docker/DockerfileCodeScan.devel

@@ -30,9 +30,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
 
 RUN ln -sf $(which python3) /usr/bin/python
 
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
 RUN python -m pip install --no-cache-dir \
     bandit\
-    pyspelling\
     pydocstyle
 
 WORKDIR /

.azure-pipelines/docker/ubuntu-2404.devel

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG UBUNTU_VER=24.04
+FROM ubuntu:${UBUNTU_VER}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+    python3 \
+    python3-pip \
+    python3.12-dev \
+    autoconf \
+    build-essential \
+    cmake \
+    git \
+    libomp-dev \
+    numactl \
+    time \
+    wget \
+    bc \
+    jq \
+    vim
+
+RUN ln -sf $(which python3) /usr/bin/python
+
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN groupadd -g ${GROUP_ID} hostgroup && \
+    useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser
+
+USER hostuser
+
+ENV PATH="/home/hostuser/.local/bin:$PATH"
+RUN pip config set global.break-system-packages true
+RUN pip install pip packaging --upgrade
+RUN pip install --no-cache-dir setuptools
+RUN pip list
+
+WORKDIR /

.azure-pipelines/scripts/install_nc.sh

@@ -2,16 +2,15 @@
 
 echo -e "##[group]Install Neural Compressor ... "
 cd /neural-compressor
-pip install cmake==3.31.6
 if [[ $1 = *"3x_pt"* ]]; then
     python -m pip install --no-cache-dir -r requirements_pt.txt
     if [[ $1 = *"3x_pt_fp8"* ]]; then
         pip uninstall neural_compressor_3x_pt -y || true
         python setup.py pt bdist_wheel
     else
         echo -e "\n Install torch CPU ... "
-        pip install torch==2.7.1 torchvision --index-url https://download.pytorch.org/whl/cpu
-        python -m pip install intel-extension-for-pytorch==2.7.0 oneccl_bind_pt --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/
+        pip install torch==2.8.0 torchvision --index-url https://download.pytorch.org/whl/cpu
+        python -m pip install intel-extension-for-pytorch==2.8.0 oneccl_bind_pt --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/
         python -m pip install --no-cache-dir -r requirements.txt
         python setup.py bdist_wheel
     fi

.azure-pipelines/scripts/models/env_setup.sh

@@ -49,16 +49,7 @@ done
 
 SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models"
 log_dir="/neural-compressor/.azure-pipelines/scripts/models"
-if [[ "${inc_new_api}" == "3x"* ]]; then
-    pip install cmake==3.31.6
-    WORK_SOURCE_DIR="/neural-compressor/examples/${framework}"
-#    git clone https://github.com/intel/intel-extension-for-transformers.git /itrex
-#    cd /itrex
-#    pip install -r requirements.txt
-#    pip install -v .
-else
-    WORK_SOURCE_DIR="/neural-compressor/examples/deprecated/${framework}"
-fi
+WORK_SOURCE_DIR="/neural-compressor/examples/${framework}"
 
 $BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET
 
@@ -72,34 +63,16 @@ else
 fi
 
 $BOLD_YELLOW && echo "====== install requirements ======" && $RESET
-/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh ${inc_new_api}
-
-mkdir -p ${WORK_SOURCE_DIR}
-cd ${WORK_SOURCE_DIR}
-if [[ "${inc_new_api}" == "false" ]]; then
-    echo "copy old api examples to workspace..."
-    git clone -b old_api_examples https://github.com/intel/neural-compressor.git old-lpot-models
-    cd old-lpot-models
-    git branch
-    cd -
-    rm -rf ${model_src_dir}
-    mkdir -p ${model_src_dir}
-    cp -r old-lpot-models/examples/${framework}/${model_src_dir} ${WORK_SOURCE_DIR}/${model_src_dir}/../
-fi
+cd /neural-compressor
+bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh ${inc_new_api}
 
-cd ${model_src_dir}
+cd ${WORK_SOURCE_DIR}/${model_src_dir}
 
 if [[ "${fwk_ver}" != "latest" ]]; then
     pip install ruamel.yaml==0.17.40
     pip install psutil
     pip install protobuf==4.23.4
-    if [[ "${framework}" == "tensorflow" ]]; then
-        if [[ "${fwk_ver}" == *"-official" ]]; then
-            pip install tensorflow==${fwk_ver%-official}
-        else
-            pip install intel-tensorflow==${fwk_ver}
-        fi
-    elif [[ "${framework}" == "pytorch" ]]; then
+    if [[ "${framework}" == "pytorch" ]]; then
         pip install torch==${fwk_ver} --index-url https://download.pytorch.org/whl/cpu
         pip install torchvision==${torch_vision_ver} --index-url https://download.pytorch.org/whl/cpu
     elif [[ "${framework}" == "onnxrt" ]]; then
@@ -113,10 +86,6 @@ if [ -f "requirements.txt" ]; then
     if [ "${framework}" == "onnxrt" ]; then
         sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt
     fi
-    if [ "${framework}" == "tensorflow" ]; then
-        sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt
-        sed -i '/^intel-tensorflow/d' requirements.txt
-    fi
     if [ "${framework}" == "pytorch" ]; then
         sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt
     fi
@@ -130,19 +99,3 @@ if [ -f "requirements.txt" ]; then
 else
     $BOLD_RED && echo "Not found requirements.txt file." && $RESET
 fi
-
-if [[ "${inc_new_api}" == "false" ]]; then
-    $BOLD_YELLOW && echo "======== update yaml config ========" && $RESET
-    $BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET
-    cat ${yaml}
-    python ${SCRIPTS_PATH}/update_yaml_config.py \
-        --yaml=${yaml} \
-        --framework=${framework} \
-        --dataset_location=${dataset_location} \
-        --batch_size=${batch_size} \
-        --strategy=${strategy} \
-        --new_benchmark=${new_benchmark} \
-        --multi_instance='true'
-    $BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET
-    cat ${yaml}
-fi

.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh

@@ -30,40 +30,14 @@ new_benchmark=true
 inc_new_api=true
 benchmark_cmd=""
 # ======== set up config for pytorch models ========
-if [ "${model}" == "resnet18" ]; then
-    model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/eager"
-    dataset_location="/tf_dataset2/datasets/mini-imageraw"
-    input_model=""
-    yaml="conf.yaml"
-    strategy="bayesian"
-    batch_size=1
-    new_benchmark=false
-    inc_new_api=false
-    tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}"
-    benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=500"
-elif [ "${model}" == "resnet18_fx" ]; then
-    model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/fx/"
-    dataset_location="/tf_dataset2/datasets/mini-imageraw"
-    input_model="resnet18"
-    yaml=""
-    strategy="basic"
-    batch_size=1
-    new_benchmark=true
-    inc_new_api=true
-    tuning_cmd="bash run_quant.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}"
-    benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=performance --batch_size=${batch_size} --iters=500"
-elif [ "${model}" == "opt_125m_woq_gptq_int4" ]; then
+if [ "${model}" == "opt_125m_woq_gptq_int4" ]; then
     model_src_dir="nlp/huggingface_models/language-modeling/quantization/weight_only"
     inc_new_api=3x_pt
     tuning_cmd="bash run_quant.sh --topology=opt_125m_woq_gptq_int4"
 elif [ "${model}" == "opt_125m_woq_gptq_nf4_dq_bnb" ]; then
     model_src_dir="nlp/huggingface_models/language-modeling/quantization/weight_only"
     inc_new_api=3x_pt
     tuning_cmd="bash run_quant.sh --topology=opt_125m_woq_gptq_nf4_dq_bnb"
-elif [ "${model}" == "opt_125m_woq_gptq_int4_dq_ggml" ]; then
-    model_src_dir="nlp/huggingface_models/language-modeling/quantization/weight_only"
-    inc_new_api=3x_pt
-    tuning_cmd="bash run_quant.sh --topology=opt_125m_woq_gptq_int4_dq_ggml"
 fi
 
 echo "Specify FWs version..."
@@ -72,7 +46,7 @@ FRAMEWORK="pytorch"
 source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
 if [[ "${inc_new_api}" == "3x"* ]]; then
     FRAMEWORK_VERSION="latest"
-    export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH=${HOME}/.local/lib/:$LD_LIBRARY_PATH
 else
     FRAMEWORK_VERSION=${pytorch_version}
     TORCH_VISION_VERSION=${torchvision_version}