ORT perf test support for plugin EP

cmake/deps.txt

@@ -11,7 +11,7 @@
 #
 abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20250512.0.zip;3d6ff7e7ce144d9a53a53bef1f1bf79e1da4b8e1
 coremltools;https://github.com/apple/coremltools/archive/refs/tags/7.1.zip;f1bab0f30966f2e217d8e01207d518f230a1641a
-cxxopts;https://github.com/jarro2783/cxxopts/archive/3c73d91c0b04e2b59462f0a741be8c07024c1bc0.zip;6c6ca7f8480b26c8d00476e0e24b7184717fe4f0
+cxxopts;https://github.com/jarro2783/cxxopts/archive/refs/tags/v3.3.1.zip;B77F1CE4A03F610488BA0ED17C1BE2EFDBC15564
 date;https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.zip;2dac0c81dc54ebdd8f8d073a75c053b04b56e159
 dlpack;https://github.com/dmlc/dlpack/archive/5c210da409e7f1e51ddf445134a4376fdbd70d7d.zip;e499c86e4e5c5268a87661d7ea39c27fae10907c
 # This Eigen commit id matches the eigen archive being consumed from https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.zip

cmake/onnxruntime_unittests.cmake

@@ -1212,6 +1212,14 @@ endif()
 
 if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
   if(NOT IOS)
+    onnxruntime_fetchcontent_declare(
+      cxxopts
+      URL ${DEP_URL_cxxopts}
+      URL_HASH SHA1=${DEP_SHA1_cxxopts}
+      EXCLUDE_FROM_ALL
+    )
+    onnxruntime_fetchcontent_makeavailable(cxxopts)
+
     #perf test runner
     set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest)
     set(onnxruntime_perf_test_src_patterns
@@ -1238,7 +1246,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
     endif()
     target_include_directories(onnxruntime_perf_test PRIVATE   ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT}
           ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir}
-          ${CMAKE_CURRENT_BINARY_DIR})
+          ${CMAKE_CURRENT_BINARY_DIR} ${cxxopts_SOURCE_DIR}/include)
 
     if (WIN32)
       target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings})
@@ -1247,6 +1255,14 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
       endif()
     endif()
 
+    if(onnxruntime_MINIMAL_BUILD)
+      add_definitions(-DCXXOPTS_NO_RTTI)
+    endif()
+
+    if(onnxruntime_DISABLE_EXCEPTIONS)
+     add_definitions(-DDISABLE_EXCEPTIONS=1)
+    endif()
+
     if (onnxruntime_BUILD_SHARED_LIB)
       #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here.
       #onnxruntime_common is kind of ok because it is thin, tiny and totally stateless.

onnxruntime/test/perftest/command_args_parser.h

@@ -13,6 +13,7 @@ class CommandLineParser {
  public:
   static void ShowUsage();
   static bool ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]);
+  static bool ParseArgumentsV2(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]);
 };
 
 }  // namespace perftest

onnxruntime/test/perftest/common_utils.cc

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "test/perftest/utils.h"
+#include "test/perftest/strings_helper.h"
+#include <core/platform/path_lib.h>
+
+#include <cstdint>
+
+#include <filesystem>
+
+namespace onnxruntime {
+namespace perftest {
+namespace utils {
+
+void list_devices(Ort::Env& env) {
+  std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+
+  for (size_t i = 0; i < ep_devices.size(); ++i) {
+    auto device = ep_devices[i];
+    std::string device_info_msg = "===== device id " + std::to_string(i) + " ======\n";
+    device_info_msg += "name: " + std::string(device.EpName()) + "\n";
+    device_info_msg += "vendor: " + std::string(device.EpVendor()) + "\n";
+
+    auto metadata = device.EpMetadata();
+    std::unordered_map<std::string, std::string> metadata_entries = metadata.GetKeyValuePairs();
+    if (!metadata_entries.empty()) {
+      device_info_msg += "metadata:\n";
+    }
+
+    for (auto& entry : metadata_entries) {
+      device_info_msg += "  " + entry.first + ": " + entry.second + "\n";
+    }
+    device_info_msg += "\n";
+    fprintf(stdout, "%s", device_info_msg.c_str());
+  }
+}
+
+bool RegisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config) {
+  if (!test_config.plugin_ep_names_and_libs.empty()) {
+    std::unordered_map<std::string, std::string> ep_names_to_libs;
+    ParseSessionConfigs(ToUTF8String(test_config.plugin_ep_names_and_libs), ep_names_to_libs);
+    if (ep_names_to_libs.size() > 0) {
+      for (auto& pair : ep_names_to_libs) {
+        const std::filesystem::path library_path = pair.second;
+        const std::string registration_name = pair.first;
+        env.RegisterExecutionProviderLibrary(registration_name.c_str(), Utf8ToOrtString(library_path.string()));
+        test_config.registered_plugin_eps.push_back(registration_name);
+      }
+    }
+  }
+  return true;
+}
+
+bool UnregisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config) {
+  for (auto& registration_name : test_config.registered_plugin_eps) {
+    env.UnregisterExecutionProviderLibrary(registration_name.c_str());
+  }
+  return true;
+}
+
+}  // namespace utils
+}  // namespace perftest
+}  // namespace onnxruntime

onnxruntime/test/perftest/main.cc

@@ -5,7 +5,9 @@
 #include <core/session/onnxruntime_c_api.h>
 #include <random>
 #include "command_args_parser.h"
+#include "utils.h"
 #include "performance_runner.h"
+#include "strings_helper.h"
 #include <google/protobuf/stubs/common.h>
 
 using namespace onnxruntime;
@@ -18,7 +20,11 @@ int real_main(int argc, char* argv[]) {
 #endif
   g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION);
   perftest::PerformanceTestConfig test_config;
+#ifdef DISABLE_EXCEPTIONS
   if (!perftest::CommandLineParser::ParseArguments(test_config, argc, argv)) {
+#else
+  if (!perftest::CommandLineParser::ParseArgumentsV2(test_config, argc, argv)) {
+#endif
     perftest::CommandLineParser::ShowUsage();
     return -1;
   }
@@ -41,23 +47,73 @@ int real_main(int argc, char* argv[]) {
     if (failed)
       return -1;
   }
-  std::random_device rd;
-  perftest::PerformanceRunner perf_runner(env, test_config, rd);
 
-  // Exit if user enabled -n option so that user can measure session creation time
-  if (test_config.run_config.exit_after_session_creation) {
-    perf_runner.LogSessionCreationTime();
+  if (!test_config.plugin_ep_names_and_libs.empty()) {
+    perftest::utils::RegisterExecutionProviderLibrary(env, test_config);
+  }
+
+  if (test_config.list_available_devices) {
+    perftest::utils::list_devices(env);
+    if (test_config.registered_plugin_eps.empty()) {
+      fprintf(stdout, "No plugin execution provider libraries are registered. Please specify them using \"--plugin_ep_libs\"; otherwise, only CPU may be available.\n");
+    } else {
+      perftest::utils::UnregisterExecutionProviderLibrary(env, test_config);
+    }
     return 0;
   }
 
-  auto status = perf_runner.Run();
+  auto status = Status::OK();
+
+  ORT_TRY {
+    std::random_device rd;
+    perftest::PerformanceRunner perf_runner(env, test_config, rd);
+
+    // Exit if user enabled -n option so that user can measure session creation time
+    if (test_config.run_config.exit_after_session_creation) {
+      perf_runner.LogSessionCreationTime();
+      return 0;
+    }
+
+    status = perf_runner.Run();
+
+    if (!status.IsOK()) {
+      printf("Run failed:%s\n", status.ErrorMessage().c_str());
+    } else {
+      perf_runner.SerializeResult();
+    }
+  }
+  ORT_CATCH(const std::exception& ex) {
+    ORT_HANDLE_EXCEPTION([&]() {
+      fprintf(stderr, "%s\n", ex.what());
+    });
+  }
+  // The try/catch block above ensures the following:
+  // 1) Plugin EP libraries are unregistered if an exception occurs.
+  // 2) Objects are released in the correct order when running a plugin EP.
+  //
+  // Proper destruction order is critical to avoid use-after-free issues. The expected order of deleters is:
+  // session -> session allocator (accessed via EP factory) -> plugin EP -> env ->
+  // shared allocator (accessed via EP factory) -> plugin EP factory (owned by env)
+  //
+  // Without this order, the environment (`env`) might be destroyed first, and
+  // any subsequent access to the session allocator's deleter (which depends on the EP factory)
+  // can result in a segmentation fault because the factory has already been destroyed.
+
+  // Unregister all registered plugin EP libraries before program exits.
+  //
+  // This is necessary because unregistering the plugin EP also unregisters any associated shared allocators.
+  // If we don't do this first and program returns, the factories stored inside the environment will be destroyed when the environment goes out of scope.
+  // Later, when the shared allocator's deleter runs, it may cause a segmentation fault because it attempts to use the already-destroyed factory to call ReleaseAllocator.
+  //
+  // See "ep_device.ep_factory->ReleaseAllocator" in Environment::CreateSharedAllocatorImpl.
+  if (!test_config.registered_plugin_eps.empty()) {
+    perftest::utils::UnregisterExecutionProviderLibrary(env, test_config);
+  }
+
   if (!status.IsOK()) {
-    printf("Run failed:%s\n", status.ErrorMessage().c_str());
     return -1;
   }
 
-  perf_runner.SerializeResult();
-
   return 0;
 }
 

onnxruntime/test/perftest/ort_test_session.cc

@@ -62,6 +62,65 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
     : rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) {
   Ort::SessionOptions session_options;
 
+  bool is_plugin_ep_available = false;
+
+  // Add devices created from plugin EP
+  if (!performance_test_config.registered_plugin_eps.empty()) {
+    std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+    std::vector<Ort::ConstEpDevice> added_ep_devices;
+    std::unordered_set<int> added_ep_device_index_set;
+
+    // Select devices by provided device index
+    if (!performance_test_config.selected_devices.empty()) {
+      std::vector<int> device_list;
+      device_list.reserve(performance_test_config.selected_devices.size());
+      ParseDeviceList(performance_test_config.selected_devices, device_list);
+      for (auto index : device_list) {
+        if (static_cast<size_t>(index) > (ep_devices.size() - 1)) {
+          fprintf(stderr, "%s", "The device index provided is not correct. Will skip this device id.");
+        }
+
+        Ort::ConstEpDevice& device = ep_devices[index];
+
+        if (std::string(device.EpName()) == performance_test_config.machine_config.provider_type_name) {
+          if (added_ep_device_index_set.find(index) == added_ep_device_index_set.end()) {
+            added_ep_devices.push_back(device);
+            added_ep_device_index_set.insert(index);
+            fprintf(stdout, "[Plugin EP] Device [Index: %d, Name: %s] has been added to session.", index, device.EpName());
+          }
+        } else {
+          std::string err_msg = "[Plugin EP] [WARNING] : The device index and its corresponding OrtEpDevice is not created from " +
+                                performance_test_config.machine_config.provider_type_name + ". Will skip adding this device.\n";
+          fprintf(stderr, "%s", err_msg.c_str());
+        }
+      }
+    } else {
+      // All OrtEpDevice instances must be from the same execution provider.
+      // Find and select the OrtEpDevice associated with the execution provider provided via "-e" argument.
+      for (int index = 0; static_cast<size_t>(index) < ep_devices.size(); ++index) {
+        Ort::ConstEpDevice& device = ep_devices[index];
+        if (std::string(device.EpName()) == performance_test_config.machine_config.provider_type_name) {
+          added_ep_devices.push_back(device);
+          fprintf(stdout, "Device [Index: %d, Name: %s] has been added to session.", index, device.EpName());
+        }
+      }
+    }
+
+    if (added_ep_devices.empty()) {
+      for (auto ep_name : registered_plugin_ep_names_) {
+        env.UnregisterExecutionProviderLibrary(ep_name.c_str());
+      }
+      ORT_THROW(
+          "[ERROR] [plugin EP]: No matching devices found.");
+    }
+
+    std::string provider_option_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
+    std::unordered_map<std::string, std::string> provider_options;
+    ParseSessionConfigs(provider_option_string, provider_options);
+    session_options.AppendExecutionProvider_V2(env, added_ep_devices, provider_options);
+    is_plugin_ep_available = true;
+  }
+
   provider_name_ = performance_test_config.machine_config.provider_type_name;
   std::unordered_map<std::string, std::string> provider_options;
   if (provider_name_ == onnxruntime::kDnnlExecutionProvider) {
@@ -574,7 +633,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
 #endif
   } else if (!provider_name_.empty() &&
              provider_name_ != onnxruntime::kCpuExecutionProvider &&
-             provider_name_ != onnxruntime::kOpenVINOExecutionProvider) {
+             provider_name_ != onnxruntime::kOpenVINOExecutionProvider &&
+             !is_plugin_ep_available) {
     ORT_THROW("This backend is not included in perf test runner.\n");
   }
 

onnxruntime/test/perftest/ort_test_session.h

@@ -51,6 +51,7 @@ class OnnxRuntimeTestSession : public TestSession {
   const int input_length_;
   std::string provider_name_;
   std::string device_memory_name_;  // Device memory type name to use from the list in allocator.h
+  std::vector<std::string> registered_plugin_ep_names_;
 };
 
 }  // namespace perftest

onnxruntime/test/perftest/posix/utils.cc

@@ -58,6 +58,11 @@ std::unique_ptr<ICPUUsage> CreateICPUUsage() {
   return std::make_unique<CPUUsage>();
 }
 
+std::basic_string<ORTCHAR_T> Utf8ToOrtString(const std::string& utf8_str) {
+  // ORTCHAR_T == char -> just convert to std::basic_string<char>
+  return std::basic_string<ORTCHAR_T>(utf8_str.begin(), utf8_str.end());
+}
+
 }  // namespace utils
 }  // namespace perftest
 }  // namespace onnxruntime

onnxruntime/test/perftest/strings_helper.cc

@@ -5,6 +5,7 @@
 
 #include <iostream>
 #include <sstream>
+#include <vector>
 
 #include "strings_helper.h"
 #include "core/common/common.h"
@@ -53,5 +54,16 @@ void ParseSessionConfigs(const std::string& configs_string,
     session_configs.insert(std::make_pair(std::move(key), std::move(value)));
   }
 }
+
+void ParseDeviceList(const std::string& input, std::vector<int>& result) {
+  std::stringstream ss(input);
+  std::string item;
+
+  while (std::getline(ss, item, ';')) {
+    if (!item.empty()) {
+      result.push_back(std::stoi(item));
+    }
+  }
+}
 }  // namespace perftest
 }  // namespace onnxruntime

onnxruntime/test/perftest/strings_helper.h

@@ -5,12 +5,15 @@
 #include <string_view>
 #include <unordered_map>
 #include <unordered_set>
+#include <vector>
 
 namespace onnxruntime {
 namespace perftest {
 
 void ParseSessionConfigs(const std::string& configs_string,
                          std::unordered_map<std::string, std::string>& session_configs,
                          const std::unordered_set<std::string>& available_keys = {});
+
+void ParseDeviceList(const std::string& input, std::vector<int>& result);
 }  // namespace perftest
 }  // namespace onnxruntime

onnxruntime/test/perftest/test_configuration.h

@@ -74,6 +74,10 @@ struct PerformanceTestConfig {
   ModelInfo model_info;
   MachineConfig machine_config;
   RunConfig run_config;
+  std::basic_string<ORTCHAR_T> plugin_ep_names_and_libs;
+  std::vector<std::string> registered_plugin_eps;
+  std::string selected_devices;
+  bool list_available_devices = false;
 };
 
 }  // namespace perftest