Updated compliance scripts for verification

compliance/nvidia/TEST01/run_verification.py

@@ -95,49 +95,38 @@ def main():
         os.path.dirname(__file__), "verify_accuracy.py"
     )
     # run verify accuracy
-    verify_accuracy_command = (
-        "python3 "
-        + verify_accuracy_binary
-        + " --dtype "
-        + args.dtype
-        + unixmode
-        + " -r "
-        + results_dir
-        + "/accuracy/mlperf_log_accuracy.json"
-        + " -t "
-        + compliance_dir
-        + "/mlperf_log_accuracy.json | tee verify_accuracy.txt"
-    )
+    verify_accuracy_command = [
+        sys.executable,  
+        verify_accuracy_binary,
+        "--dtype", args.dtype,
+        unixmode,
+        "-r", os.path.join(results_dir, "accuracy", "mlperf_log_accuracy.json"),
+        "-t", os.path.join(compliance_dir, "mlperf_log_accuracy.json"),
+    ]
     try:
         os.system(verify_accuracy_command)
     except Exception:
         print(
             "Exception occurred trying to execute:\n  " +
             verify_accuracy_command)
     # check if verify accuracy script passes
-
-    accuracy_pass_command = "grep PASS verify_accuracy.txt"
     try:
-        accuracy_pass = "TEST PASS" in subprocess.check_output(
-            accuracy_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_accuracy.txt", "r") as file:
+            accuracy_pass = "PASS" in file.read()
     except Exception:
         accuracy_pass = False
 
     # run verify performance
     verify_performance_binary = os.path.join(
         os.path.dirname(__file__), "verify_performance.py"
     )
-    verify_performance_command = (
-        "python3 "
-        + verify_performance_binary
-        + " -r "
-        + results_dir
-        + "/performance/run_1/mlperf_log_summary.txt"
-        + " -t "
-        + compliance_dir
-        + "/mlperf_log_summary.txt | tee verify_performance.txt"
-    )
+    verify_performance_command = [
+        sys.executable,  
+        verify_performance_binary,
+        "-r", os.path.join(results_dir, "performance", "run_1", "mlperf_log_summary.txt"),
+        "-t", os.path.join(compliance_dir, "mlperf_log_summary.txt"),
+    ]
+
     try:
         os.system(verify_performance_command)
     except Exception:
@@ -146,11 +135,9 @@ def main():
             verify_performance_command)
 
     # check if verify performance script passes
-    performance_pass_command = "grep PASS verify_performance.txt"
     try:
-        performance_pass = "TEST PASS" in subprocess.check_output(
-            performance_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_performance.txt", "r") as file:
+            performance_pass = "PASS" in file.read()
     except Exception:
         performance_pass = False
 

compliance/nvidia/TEST01/verify_accuracy.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =============================================================================
+import hashlib
 import numpy as np
 import json
 import argparse
@@ -161,15 +162,11 @@ def main():
         print("Error: This script requires Python v3.3 or later")
         exit()
 
-    get_perf_lines_cmd = "wc -l " + perf_log + "| awk '{print $1}'"
-    num_perf_lines = int(
-        subprocess.check_output(get_perf_lines_cmd, shell=True).decode("utf-8")
-    )
+    with open(perf_log, "r") as file:
+        num_perf_lines = sum(1 for _ in file)
 
-    get_acc_lines_cmd = "wc -l " + acc_log + "| awk '{print $1}'"
-    num_acc_lines = int(
-        subprocess.check_output(get_acc_lines_cmd, shell=True).decode("utf-8")
-    )
+    with open(acc_log, "r") as file:
+        num_acc_lines = sum(1 for _ in file)
 
     num_acc_log_entries = num_acc_lines - 2
     num_perf_log_entries = num_perf_lines - 2
@@ -187,44 +184,31 @@ def main():
         # first and last line are brackets
         if perf_line == 0 or perf_line == int(num_perf_lines) - 1:
             continue
+
+        # read the specific line in perf_log
+        with open(perf_log, "r") as file:
+            for i, line in enumerate(file):
+                if i == perf_line:
+                    perf_md5sum_cmd = line.strip()
+                    break
 
         # calculate md5sum of line in perf mode accuracy_log
-        perf_md5sum_cmd = (
-            "head -n "
-            + str(perf_line + 1)
-            + " "
-            + perf_log
-            + "| tail -n 1| sed -r 's/,//g' | sed -r 's/\"seq_id\" : \\S+//g' | md5sum"
-        )
+        perf_md5sum_cmd = perf_md5sum_cmd.replace(",", "").replace("\"seq_id\" : ", "")
         # print(perf_md5sum_cmd)
-        perf_md5sum = subprocess.check_output(perf_md5sum_cmd, shell=True).decode(
-            "utf-8"
-        )
+        perf_md5sum = hashlib.md5(perf_md5sum_cmd.encode('utf-8')).hexdigest()
 
         # get qsl idx
-        get_qsl_idx_cmd = (
-            "head -n "
-            + str(perf_line + 1)
-            + " "
-            + perf_log
-            + "| tail -n 1| awk -F\": |,\" '{print $4}'"
-        )
-        qsl_idx = (
-            subprocess.check_output(get_qsl_idx_cmd, shell=True)
-            .decode("utf-8")
-            .rstrip()
-        )
-
+        qsl_idx = perf_md5sum_cmd.split(": ")[3].split(",")[0]
+
+        # find the corresponding line in acc_log
+        with open(acc_log, "r") as file:
+            for line in file:
+                if f'qsl_idx" : {qsl_idx},' in line:
+                    acc_md5sum_cmd = line.strip()
+                    break
         # calculate md5sum of line in acc mode accuracy_log
-        acc_md5sum_cmd = (
-            'grep "qsl_idx\\" : '
-            + qsl_idx
-            + '," '
-            + acc_log
-            + "| sed -r 's/,//g' | sed -r 's/\"seq_id\" : \\S+//g' | md5sum"
-        )
-        acc_md5sum = subprocess.check_output(
-            acc_md5sum_cmd, shell=True).decode("utf-8")
+        acc_md5sum_cmd = acc_md5sum_cmd.replace(",", "").replace("\"seq_id\" : ", "")
+        acc_md5sum = hashlib.md5(acc_md5sum_cmd.encode('utf-8')).hexdigest()
 
         if perf_md5sum != acc_md5sum:
             num_perf_log_data_mismatch += 1

compliance/nvidia/TEST04/run_verification.py

@@ -57,16 +57,12 @@ def main():
     verify_performance_binary = os.path.join(
         os.path.dirname(__file__), "verify_performance.py"
     )
-    verify_performance_command = (
-        "python3 "
-        + verify_performance_binary
-        + " -r "
-        + results_dir
-        + "/performance/run_1/mlperf_log_summary.txt"
-        + " -t "
-        + compliance_dir
-        + "/mlperf_log_summary.txt | tee verify_performance.txt"
-    )
+    verify_performance_command = [
+        sys.executable,  
+        verify_performance_binary,
+        "-r", os.path.join(results_dir, "performance", "run_1", "mlperf_log_summary.txt"),
+        "-t", os.path.join(compliance_dir, "mlperf_log_summary.txt"),
+    ]
     try:
         os.system(verify_performance_command)
     except Exception:
@@ -75,11 +71,9 @@ def main():
             verify_performance_command)
 
     # check if verify performance script passes
-    performance_pass_command = "grep PASS verify_performance.txt"
     try:
-        performance_pass = "TEST PASS" in subprocess.check_output(
-            performance_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_performance.txt", "r") as file:
+            performance_pass = "PASS" in file.read()
     except Exception:
         performance_pass = False