Merge pull request #225 from mlcommons/dev

Sync dev

Author: arjunsuresh

.github/workflows/build_wheel.yml

@@ -3,13 +3,15 @@ name: Build wheel and release into PYPI
 on:
   release:
     types: [published]
+    
   push:
     branches:
       - dev
     paths:
       - VERSION
 
 jobs:
+
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/format.yml

@@ -59,6 +59,6 @@ jobs:
             git config --global user.name github-actions[bot]
             git config --global user.email "github-actions[bot]@users.noreply.github.com"
             # Commit changes
-            git commit -m '[Automated Commit] Format Codebase'
+            git commit -m '[Automated Commit] Format Codebase [skip ci]'
             git push 
           fi

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -53,11 +53,11 @@ jobs:
     - name: Test MLPerf Inference ResNet50 (Windows)
       if: matrix.os == 'windows-latest'
       run: |
-        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
+        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
     - name: Test MLPerf Inference ResNet50 (Linux/macOS)
       if: matrix.os != 'windows-latest'
       run: |
-        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet     
+        mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet     
     # Step for Linux/MacOS
     - name: Randomly Execute Step (Linux/MacOS)
       if: runner.os != 'Windows'

.github/workflows/test-nvidia-mlperf-inference-implementations.yml

@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "35 01 * * *"
+    - cron: "27 11 * * *"
 
 jobs:
   run_nvidia:
@@ -23,8 +23,8 @@ jobs:
           model: [ "resnet50",  "retinanet",  "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9", "sdxl" ]
           exclude:
            - model: gptj-99.9
-           - system: phoenix
-           - system: GO-i9
+           - system: phoenix1
+           - system: GO-i91
 
       steps:
       - name: Test MLPerf Inference NVIDIA ${{ matrix.model }}
@@ -45,6 +45,11 @@ jobs:
             gpu_name=rtx_4090
             docker_string=" --docker"
           fi
+          if [ "${{ matrix.model }}" = "bert-99.9" ]; then
+            submission_preprocessor_args=" --noinfer-low-accuracy-results"
+          else
+            submission_preprocessor_args=""
+          fi
           category="datacenter,edge"
           if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
           python3 -m venv gh_action
@@ -53,6 +58,7 @@ jobs:
           pip install --upgrade mlcflow
           mlc pull repo mlcommons@mlperf-automations --branch=dev
           
-          mlcr --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r5.0-dev --preprocess_submission=yes --pull_changes=yes --pull_inference_changes=yes --execution_mode=valid --gpu_name=$gpu_name --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=$category --division=closed  --docker_dt --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  $docker_string --quiet
+          mlcr --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r5.0-dev --preprocess_submission=yes --pull_changes=yes --pull_inference_changes=yes --execution_mode=valid --gpu_name=$gpu_name --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="GATEOverflow" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=$category --division=closed  --docker_dt --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  $docker_string $submission_preprocessor_args --quiet
+          #mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from GH action on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name
+          mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/GATEOverflow/mlperf_inference_submissions_v5.0 --repo_branch=main --commit_message="Results from GH actions on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name
 
-          mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from GH action on NVIDIA_$hw_name" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=$hw_name

.gitignore

@@ -15,6 +15,10 @@ wheels/
 .coverage
 htmlcov
 *tmp/
-*tmp-ck-*/
+tmp-*
 local/cache/
-
+mlc-log.txt
+repos.json
+index_script.json
+index_cache.json
+index_experiment.json

CONTRIBUTORS.md

@@ -30,11 +30,13 @@ Once your contribution exceeds 50 lines of code (in total), we will:
 ## Current Contributors
 
 - **Grigori Fursin** - *Initial Development, CMind development to drive the automations, Added core automation features*
-- **Arjun Suresh** - *Initial Development, Added core automation features* 
-- **Anandhu Sooraj** - *Added multiple CM scripts for MLPerf Inference*
+- **Arjun Suresh** - *Initial Development, Added core automation features, MLCFlow development to drive the automations* 
+- **Anandhu Sooraj** - *Added multiple automation scripts for MLPerf Inference, MLCFlow development to drive the automations*
 - **Thomaz Zhu** - *Added CPP implementation for MLPerf Inference Onnxruntime*
 - **Sahil Avaran** - *Adding logging support in MLPerf script automation* 
-- **[Your Name Here]** - This could be you! 🎉  
+- **[Your Name Here]** - This could be you! 🎉
+
+* CMind is now replaced by MLCFlow in the MLPerf Automations
 
 ---
 

README.md

@@ -8,7 +8,7 @@
 
 Welcome to the **MLPerf Automations and Scripts** repository! This repository is your go-to resource for tools, automations, and scripts designed to streamline the execution of **MLPerf benchmarks**—with a strong emphasis on **MLPerf Inference benchmarks**.
 
-Starting **January 2025**, MLPerf automation scripts are built on the powerful [MLCFlow](https://github.com/mlcommons/mlcflow) automation interface. This modern interface replaces the earlier [Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm), offering a more robust and efficient framework for benchmarking workflows.
+Starting **January 2025**, MLPerf automation scripts will be powered by the advanced [MLCFlow](https://github.com/mlcommons/mlcflow) automation interface. This modern framework replaces the previous [Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm), providing a more robust, efficient, and self-contained solution for benchmarking workflows, making MLPerf automations independent of any external projects. 
 
 
 ---

VERSION

@@ -1 +1 @@
-0.0.9
+1.0.0

automation/script/module.py

@@ -2260,13 +2260,7 @@ def _update_env_from_input(self, env, i):
 
     ##########################################################################
     def _fix_cache_paths(self, env):
-        '''
-        mlc_repos_path = os.environ.get(
-            'MLC_REPOS', os.path.join(
-                os.path.expanduser("~"), "CM", "repos"))
-        current_cache_path = os.path.realpath(
-            os.path.join(mlc_repos_path, "local", "cache"))
-        '''
+
         current_cache_path = self.action_object.local_cache_path
 
         new_env = env  # just a reference
@@ -2285,7 +2279,7 @@ def _fix_cache_paths(self, env):
                 if loaded_cache_path != current_cache_path and os.path.exists(
                         current_cache_path):
                     new_env[key] = val.replace(
-                        loaded_cache_path, current_cache_path)
+                        loaded_cache_path, current_cache_path).replace(sep, "/")
 
             elif isinstance(val, list):
                 for i, val2 in enumerate(val):
@@ -2300,7 +2294,7 @@ def _fix_cache_paths(self, env):
                         if loaded_cache_path != current_cache_path and os.path.exists(
                                 current_cache_path):
                             new_env[key][i] = val2.replace(
-                                loaded_cache_path, current_cache_path)
+                                loaded_cache_path, current_cache_path).replace(sep, "/")
 
         return {'return': 0, 'new_env': new_env}
 

git_commit_hash.txt

@@ -1 +1 @@
-3dacfdc14894006f456d3b14d1b174e2e9e6e19f
+bee755068b7104201446663d41824053671a254d