From c0c96f8325ec76a03d5c4866a4180bf7cc528e4c Mon Sep 17 00:00:00 2001 From: sahelib25 Date: Mon, 11 Sep 2023 12:24:30 +0100 Subject: [PATCH 1/2] Fix SingleStream and MultiStream reciprocal score comparison for TEST04. --- compliance/nvidia/TEST04/verify_performance.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compliance/nvidia/TEST04/verify_performance.py b/compliance/nvidia/TEST04/verify_performance.py index 0aec33ace9..af99e6059e 100644 --- a/compliance/nvidia/TEST04/verify_performance.py +++ b/compliance/nvidia/TEST04/verify_performance.py @@ -52,13 +52,11 @@ def main(): if ref_mode == "SingleStream": if re.match(".*Early stopping 90th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() - ref_score = 1e9 / float(ref_score) continue if ref_mode == "MultiStream": if re.match(".*Early stopping 99th percentile estimate", line): ref_score = line.split(": ",1)[1].strip() - ref_score = 1e9 / float(ref_score) continue if ref_mode == "Server": @@ -92,13 +90,11 @@ def main(): if test_mode == "SingleStream": if re.match(".*Early stopping 90th percentile estimate", line): test_score = line.split(": ",1)[1].strip() - test_score = 1e9 / float(test_score) continue if test_mode == "MultiStream": if re.match(".*Early stopping 99th percentile estimate", line): test_score = line.split(": ",1)[1].strip() - test_score = 1e9 / float(test_score) continue if test_mode == "Server": From 7694c10f42ac1014ebd6ac6dc4e68512e62c5b8b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Dec 2024 12:23:01 +0000 Subject: [PATCH 2/2] [Automated Commit] Format Codebase --- .../nvidia/TEST04/verify_performance.py | 8 +-- loadgen/issue_query_controller.cc | 12 ++--- loadgen/logging.cc | 3 +- loadgen/logging.h | 51 ++++++++++++------- 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/compliance/nvidia/TEST04/verify_performance.py b/compliance/nvidia/TEST04/verify_performance.py index 7da0338563..289cbdf701 100644 --- a/compliance/nvidia/TEST04/verify_performance.py +++ b/compliance/nvidia/TEST04/verify_performance.py @@ -54,12 +54,12 @@ def main(): if ref_mode == "SingleStream": if re.match(".*Early stopping 90th percentile estimate", line): - ref_score = line.split(": ",1)[1].strip() + ref_score = line.split(": ", 1)[1].strip() continue if ref_mode == "MultiStream": if re.match(".*Early stopping 99th percentile estimate", line): - ref_score = line.split(": ",1)[1].strip() + ref_score = line.split(": ", 1)[1].strip() continue if ref_mode == "Server": @@ -91,12 +91,12 @@ def main(): if test_mode == "SingleStream": if re.match(".*Early stopping 90th percentile estimate", line): - test_score = line.split(": ",1)[1].strip() + test_score = line.split(": ", 1)[1].strip() continue if test_mode == "MultiStream": if re.match(".*Early stopping 99th percentile estimate", line): - test_score = line.split(": ",1)[1].strip() + test_score = line.split(": ", 1)[1].strip() continue if test_mode == "Server": diff --git a/loadgen/issue_query_controller.cc b/loadgen/issue_query_controller.cc index c1abea9d14..4c5ca66f0c 100644 --- a/loadgen/issue_query_controller.cc +++ b/loadgen/issue_query_controller.cc @@ -459,8 +459,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Too many outstanding queries." << " issued " - << queries_issued_total << " outstanding " + << " Ending early: Too many outstanding queries." + << " issued " << queries_issued_total << " outstanding " << queries_outstanding; MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else @@ -499,8 +499,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Max query count reached." << " query_count " - << queries_issued; + << " Ending early: Max query count reached." + << " query_count " << queries_issued; MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else detail.Error("IssueQueryThread ", std::to_string(thread_idx), @@ -519,8 +519,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Max test duration reached." << " duration_ns " - << duration.count(); + << " Ending early: Max test duration reached." + << " duration_ns " << duration.count(); MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else detail.Error("IssueQueryThread ", std::to_string(thread_idx), diff --git a/loadgen/logging.cc b/loadgen/logging.cc index 807c1954a8..d7e83e54b9 100644 --- a/loadgen/logging.cc +++ b/loadgen/logging.cc @@ -812,7 +812,8 @@ void Logger::CollectTlsLoggerStats(TlsLogger* tls_logger) { if (max_entry_vector_size > kTlsLogReservedEntryCount) { #if USE_NEW_LOGGING_FORMAT std::stringstream msg; - msg << "Logging allocation detected:" << " tid: " << tls_logger->Tid() + msg << "Logging allocation detected:" + << " tid: " << tls_logger->Tid() << " reserved_entries: " << kTlsLogReservedEntryCount << " max_entries: " << max_entry_vector_size; MLPERF_LOG_WARNING((*this), "warning_generic_message", msg.str()); diff --git a/loadgen/logging.h b/loadgen/logging.h index c5514562a5..a557e4b9a8 100644 --- a/loadgen/logging.h +++ b/loadgen/logging.h @@ -119,10 +119,13 @@ class ChromeTracer { void AddCompleteEvent(const std::string& name, uint64_t pid, uint64_t tid, PerfClock::time_point start, PerfClock::time_point end, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"ph\":\"X\"," - << "\"pid\":" << pid << "," << "\"tid\":" << tid << "," + *out_ << "{\"name\":\"" << name << "\"," + << "\"ph\":\"X\"," + << "\"pid\":" << pid << "," + << "\"tid\":" << tid << "," << "\"ts\":" << Micros(start - origin_).count() << "," - << "\"dur\":" << Micros(end - start).count() << "," << "\"args\":{"; + << "\"dur\":" << Micros(end - start).count() << "," + << "\"args\":{"; AddArgs(args...); *out_ << "}},\n"; } @@ -130,9 +133,12 @@ class ChromeTracer { template void AddAsyncBeginEvent(const std::string& name, uint64_t pid, uint64_t id, PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"b\"," << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "," + *out_ << "{\"name\":\"" << name << "\"," + << "\"cat\":\"default\"," + << "\"ph\":\"b\"," + << "\"pid\":" << pid << "," + << "\"id\":" << id << "," + << "\"ts\":" << Micros(time - origin_).count() << "," << "\"args\":{"; AddArgs(args...); *out_ << "}},\n"; @@ -141,9 +147,12 @@ class ChromeTracer { template void AddAsyncInstantEvent(const std::string& name, uint64_t pid, uint64_t id, PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"n\"," << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "," + *out_ << "{\"name\":\"" << name << "\"," + << "\"cat\":\"default\"," + << "\"ph\":\"n\"," + << "\"pid\":" << pid << "," + << "\"id\":" << id << "," + << "\"ts\":" << Micros(time - origin_).count() << "," << "\"args\":{"; AddArgs(args...); *out_ << "}},\n"; @@ -152,15 +161,19 @@ class ChromeTracer { template void AddAsyncEndEvent(const std::string& name, uint64_t pid, uint64_t id, PerfClock::time_point time) { - *out_ << "{\"name\":\"" << name << "\"," << "\"cat\":\"default\"," - << "\"ph\":\"e\", " << "\"pid\":" << pid << "," << "\"id\":" << id - << "," << "\"ts\":" << Micros(time - origin_).count() << "},\n"; + *out_ << "{\"name\":\"" << name << "\"," + << "\"cat\":\"default\"," + << "\"ph\":\"e\", " + << "\"pid\":" << pid << "," + << "\"id\":" << id << "," + << "\"ts\":" << Micros(time - origin_).count() << "},\n"; } template void AddCounterEvent(const std::string& name, uint64_t pid, PerfClock::time_point time, const Args... args) { - *out_ << "{\"name\":\"" << name << "\"," << "\"ph\": \"C\"," + *out_ << "{\"name\":\"" << name << "\"," + << "\"ph\": \"C\"," << "\"pid\":" << pid << "," << "\"ts\":" << Micros(time - origin_).count() << "," << "\"args\":{ "; @@ -720,13 +733,15 @@ void AsyncLog::LogDetail(const std::string& key, const T& value, } auto time_ns = (log_detail_time_ - log_origin_).count(); for (auto os : detail_streams) { - *os << ":::MLLOG {" << "\"key\": " << ArgValueTransform(key) << ", " + *os << ":::MLLOG {" + << "\"key\": " << ArgValueTransform(key) << ", " << "\"value\": " << ArgValueTransform(value) << ", " << "\"time_ms\": " << ArgValueTransform(time_ns / 1000000ULL) << "." << std::setfill('0') << std::setw(6) << ArgValueTransform(time_ns % 1000000ULL) << ", " << "\"namespace\": \"mlperf::logging\", " - << "\"event_type\": \"POINT_IN_TIME\", " << "\"metadata\": {" + << "\"event_type\": \"POINT_IN_TIME\", " + << "\"metadata\": {" << "\"is_error\": " << ArgValueTransform(error_flagged_) << ", " << "\"is_warning\": " << ArgValueTransform(warning_flagged_) << ", " << "\"file\": \"" << file_name << "\", " @@ -755,9 +770,9 @@ void AsyncLog::LogDetail(const std::string& message, const Args... args) { detail_streams.pop_back(); } for (auto os : detail_streams) { - *os << "\"pid\": " << current_pid_ << ", " << "\"tid\": " << current_tid_ - << ", " << "\"ts\": " << (log_detail_time_ - log_origin_).count() - << "ns : "; + *os << "\"pid\": " << current_pid_ << ", " + << "\"tid\": " << current_tid_ << ", " + << "\"ts\": " << (log_detail_time_ - log_origin_).count() << "ns : "; if (error_flagged_) { *os << "ERROR : "; } else if (warning_flagged_) {