From d6a95114d12921672aaf5b90d831589b87e98757 Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Tue, 15 Jul 2025 14:03:51 -0700 Subject: [PATCH 1/7] [EP ABI] Document in-memory external initializers for ort_graph_to_proto util --- .../core/providers/utils/ort_graph_to_proto.h | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h index 37665542f614f..6a2f4ed7bc549 100644 --- a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h +++ b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h @@ -75,6 +75,47 @@ // graph_proto stores large initializers in an external file } ``` + + EXAMPLE SNIPPET (external initializers that point to data in memory, not officially supported by ONNX spec): + + This example stores initializers externally. However, instead of storing the initializers in a separate + file, the onnx::TensorProto objects point directly to memory addresses. This requires setting the initializer's + location to a special tag like "_MEM_ADDR_" (instead of a file path). The offset is set to the pointer to the + initializer's data in memory (instead of an offset into a file). + + Because this is not standard ONNX, such a onnx::GraphProto should not be saved as an ONNX file. + However, it allows custom tools that operate directly on a onnx::GraphProto to get the initializer data + if it has already been loaded into memory. + + ```C++ + #define ORT_EP_UTILS_ORT_GRAPH_TO_PROTO_IMPL + #include "ort_graph_to_proto.h" + + OrtStatus* ORT_API_CALL GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* ort_graph, + OrtEpGraphSupportInfo* graph_support_info) { + std::string external_file_path = "weights.bin"; + std::ofstream out_file(external_file_path, std::ios::binary); + + auto handle_initializer_data = [&external_file_path, &out_file](const OrtValueInfo* value_info, + const void* data, size_t bytes, + bool& is_external, std::string& location, + int64_t& offset) -> Ort::Status { + // OrtValueInfo* could be used to query initializer's name, type, shape, consumers, etc. + (void)value_info; + + offset = reinterpret_cast(data); + location = "_MEM_ADDR_"; // Some special location tag that indicates the offset is a pointer. + is_external = true; // True if is external initializer + return Ort::Status{nullptr}; + } + + ONNX_NAMESPACE::GraphProto graph_proto; + OrtEpUtils::OrtGraphToProto(*ort_graph, graph_proto, handle_initializer_data); + + // graph_proto has initializers that look like they are stored in an external file, + // but they are actually point to the data in memory. + } + ``` */ #ifndef INCLUDE_ONNXRUNTIME_CORE_PROVIDERS_UTILS_ORT_GRAPH_TO_PROTO_H_ From 67a18da04effd51d04919356b3eb6a41acbeeb2f Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 10:19:58 -0700 Subject: [PATCH 2/7] Add unit test for external in-memory initializers --- .../core/providers/utils/ort_graph_to_proto.h | 4 +- .../core/session/onnxruntime_c_api.h | 4 +- onnxruntime/test/ep_graph/test_ep_graph.cc | 61 +++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h index 6a2f4ed7bc549..0f0bd470d1387 100644 --- a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h +++ b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h @@ -100,8 +100,8 @@ const void* data, size_t bytes, bool& is_external, std::string& location, int64_t& offset) -> Ort::Status { - // OrtValueInfo* could be used to query initializer's name, type, shape, consumers, etc. (void)value_info; + (void)bytes; offset = reinterpret_cast(data); location = "_MEM_ADDR_"; // Some special location tag that indicates the offset is a pointer. @@ -113,7 +113,7 @@ OrtEpUtils::OrtGraphToProto(*ort_graph, graph_proto, handle_initializer_data); // graph_proto has initializers that look like they are stored in an external file, - // but they are actually point to the data in memory. + // but they are actually pointing to the data in memory. } ``` */ diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 82e782112974f..4305b4e50419e 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -5459,9 +5459,11 @@ struct OrtApi { * Supports initializers defined in an outer scope (i.e., a parent graph). * * \param[in] value_info The OrtValueInfo instance. - * \param[out] initializer_value Output parameter set to the initializer value or NULL. + * \param[out] initializer_value Output parameter set to the initializer value or NULL. The OrtValue data pointer + * is stable during the lifetime of the session that owns the OrtGraph. * * \snippet{doc} snippets.dox OrtStatus Return Value + * * \since Version 1.23. */ ORT_API2_STATUS(ValueInfo_GetInitializerValue, _In_ const OrtValueInfo* value_info, diff --git a/onnxruntime/test/ep_graph/test_ep_graph.cc b/onnxruntime/test/ep_graph/test_ep_graph.cc index 17e829e37f729..57cfd23ec9c38 100644 --- a/onnxruntime/test/ep_graph/test_ep_graph.cc +++ b/onnxruntime/test/ep_graph/test_ep_graph.cc @@ -178,6 +178,67 @@ TEST(EpGraphTest, SerializeToProto_Mnist) { EXPECT_EQ(output_serialized, output_original); } +// Test serializing an OrtGraph (MNIST) to GraphProto. Initializers are configured as "external" but point to +// existing data in memory (not standard ONNX). +TEST(EpGraphTest, SerializeToProto_ExternalInitializersInMemory) { + const ORTCHAR_T* original_model_path = ORT_TSTR("testdata/mnist.onnx"); + auto test_graph = TestGraph::Load(original_model_path); + ASSERT_NE(test_graph, nullptr) << "Failed to load test model"; + + const OrtGraph& ort_graph = test_graph->GetOrtGraph(); + + auto handle_initializer_data = [](const OrtValueInfo* value_info, + const void* data, size_t bytes, + bool& is_external, std::string& location, + int64_t& offset) -> Ort::Status { + (void)value_info; + (void)bytes; + + offset = reinterpret_cast(data); + location = "_MEM_ADDR_"; + is_external = true; // True if is external initializer. + + return Ort::Status{nullptr}; + }; + + ONNX_NAMESPACE::GraphProto graph_proto; + OrtEpUtils::OrtGraphToProto(ort_graph, graph_proto, handle_initializer_data); + + // Verify that TensorProto objects within GraphProto point to memory owned by OrtValues in the OrtGraph. + const OrtApi& ort_api = Ort::GetApi(); + + size_t api_num_initializers = 0; + ASSERT_ORTSTATUS_OK(ort_api.Graph_GetNumInitializers(&ort_graph, &api_num_initializers)); + + std::vector api_initializers(api_num_initializers); + ASSERT_ORTSTATUS_OK(ort_api.Graph_GetInitializers(&ort_graph, api_initializers.data(), api_initializers.size())); + + const auto& tensor_protos = graph_proto.initializer(); + ASSERT_EQ(tensor_protos.size(), api_num_initializers); + + for (size_t i = 0; i < api_num_initializers; ++i) { + const OrtValue* ort_value = nullptr; + const void* ort_value_data = nullptr; + + ASSERT_ORTSTATUS_OK(ort_api.ValueInfo_GetInitializerValue(api_initializers[i], &ort_value)); + ASSERT_ORTSTATUS_OK(ort_api.GetTensorData(ort_value, &ort_value_data)); + + ONNX_NAMESPACE::TensorProto_DataLocation data_location = tensor_protos[static_cast(i)].data_location(); + ASSERT_EQ(data_location, ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL); + + const auto& ext_data_entries = tensor_protos[static_cast(i)].external_data(); + const ONNX_NAMESPACE::StringStringEntryProto& location_entry = ext_data_entries[0]; + const ONNX_NAMESPACE::StringStringEntryProto& offset_entry = ext_data_entries[1]; + + ASSERT_EQ(location_entry.key(), "location"); + ASSERT_EQ(location_entry.value(), "_MEM_ADDR_"); + ASSERT_EQ(offset_entry.key(), "offset"); + + long long offset_int = std::stoll(offset_entry.value()); + ASSERT_EQ(offset_int, reinterpret_cast(ort_value_data)); + } +} + static void Run3LayerModel(const ORTCHAR_T* model_path, bool input_cond, std::vector& output_data) { auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); Ort::SessionOptions sess_options; From 23ce8f45e0efed4837cbbae95cdeedc0651a7dd9 Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 10:21:19 -0700 Subject: [PATCH 3/7] change cast type to long long --- onnxruntime/test/ep_graph/test_ep_graph.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/ep_graph/test_ep_graph.cc b/onnxruntime/test/ep_graph/test_ep_graph.cc index 57cfd23ec9c38..00ad0e43613db 100644 --- a/onnxruntime/test/ep_graph/test_ep_graph.cc +++ b/onnxruntime/test/ep_graph/test_ep_graph.cc @@ -235,7 +235,7 @@ TEST(EpGraphTest, SerializeToProto_ExternalInitializersInMemory) { ASSERT_EQ(offset_entry.key(), "offset"); long long offset_int = std::stoll(offset_entry.value()); - ASSERT_EQ(offset_int, reinterpret_cast(ort_value_data)); + ASSERT_EQ(offset_int, reinterpret_cast(ort_value_data)); } } From ebedfc0a32b5070f028b48b9b195e2b401911dfe Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 10:38:07 -0700 Subject: [PATCH 4/7] update doc comment --- .../core/providers/utils/ort_graph_to_proto.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h index 0f0bd470d1387..9dfde75f2ea02 100644 --- a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h +++ b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h @@ -93,13 +93,10 @@ OrtStatus* ORT_API_CALL GetCapabilityImpl(OrtEp* this_ptr, const OrtGraph* ort_graph, OrtEpGraphSupportInfo* graph_support_info) { - std::string external_file_path = "weights.bin"; - std::ofstream out_file(external_file_path, std::ios::binary); - - auto handle_initializer_data = [&external_file_path, &out_file](const OrtValueInfo* value_info, - const void* data, size_t bytes, - bool& is_external, std::string& location, - int64_t& offset) -> Ort::Status { + auto handle_initializer_data = [](const OrtValueInfo* value_info, + const void* data, size_t bytes, + bool& is_external, std::string& location, + int64_t& offset) -> Ort::Status { (void)value_info; (void)bytes; From a695ded34a9218fe4d9a73e18c7b46c702a6dd59 Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 11:33:59 -0700 Subject: [PATCH 5/7] Fix unit tests --- onnxruntime/test/ep_graph/test_ep_graph.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/ep_graph/test_ep_graph.cc b/onnxruntime/test/ep_graph/test_ep_graph.cc index 00ad0e43613db..890bba2c52628 100644 --- a/onnxruntime/test/ep_graph/test_ep_graph.cc +++ b/onnxruntime/test/ep_graph/test_ep_graph.cc @@ -216,17 +216,27 @@ TEST(EpGraphTest, SerializeToProto_ExternalInitializersInMemory) { const auto& tensor_protos = graph_proto.initializer(); ASSERT_EQ(tensor_protos.size(), api_num_initializers); + std::unordered_map tensor_proto_map; + for (const auto& tensor_proto : tensor_protos) { + tensor_proto_map.emplace(tensor_proto.name(), &tensor_proto); + } + for (size_t i = 0; i < api_num_initializers; ++i) { const OrtValue* ort_value = nullptr; const void* ort_value_data = nullptr; + const char* value_name = nullptr; + ASSERT_ORTSTATUS_OK(ort_api.GetValueInfoName(api_initializers[i], &value_name)); ASSERT_ORTSTATUS_OK(ort_api.ValueInfo_GetInitializerValue(api_initializers[i], &ort_value)); ASSERT_ORTSTATUS_OK(ort_api.GetTensorData(ort_value, &ort_value_data)); - ONNX_NAMESPACE::TensorProto_DataLocation data_location = tensor_protos[static_cast(i)].data_location(); + auto iter = tensor_proto_map.find(value_name); + ASSERT_NE(iter, tensor_proto_map.end()); + const ONNX_NAMESPACE::TensorProto* tensor_proto = iter->second; + ONNX_NAMESPACE::TensorProto_DataLocation data_location = tensor_proto->data_location(); ASSERT_EQ(data_location, ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL); - const auto& ext_data_entries = tensor_protos[static_cast(i)].external_data(); + const auto& ext_data_entries = tensor_proto->external_data(); const ONNX_NAMESPACE::StringStringEntryProto& location_entry = ext_data_entries[0]; const ONNX_NAMESPACE::StringStringEntryProto& offset_entry = ext_data_entries[1]; From 6ef125b87ee569aad96b42143d7ec2d9b95370e1 Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 12:49:31 -0700 Subject: [PATCH 6/7] More clarification of OrtValue data lifetime --- include/onnxruntime/core/session/onnxruntime_c_api.h | 3 ++- include/onnxruntime/core/session/onnxruntime_ep_c_api.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4305b4e50419e..b04cc78b88c9e 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -5460,7 +5460,8 @@ struct OrtApi { * * \param[in] value_info The OrtValueInfo instance. * \param[out] initializer_value Output parameter set to the initializer value or NULL. The OrtValue data pointer - * is stable during the lifetime of the session that owns the OrtGraph. + * (obtained via GetTensorData) is stable during the lifetime of the OrtSession + * that owns the OrtGraph. * * \snippet{doc} snippets.dox OrtStatus Return Value * diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 5d00ce4940d02..15ae69ae74568 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -413,6 +413,8 @@ struct OrtEp { * \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the * graphs are only valid for the duration of the call to Compile. Any graph/node/input/output * names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp. + * The OrtValue data for initializers (obtained via GetTensorData), however, is stable for the lifetime of + * the owning OrtSession instance. * * \since Version 1.23. */ From e77853b9bf81cf5b3bf839899cdfa2ff9a67bfce Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 16 Jul 2025 12:59:27 -0700 Subject: [PATCH 7/7] remove extra comment to avoid confusion in case the OrtValue initializer is released by ORT after graph partitioning. --- include/onnxruntime/core/session/onnxruntime_ep_c_api.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 15ae69ae74568..5d00ce4940d02 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -413,8 +413,6 @@ struct OrtEp { * \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the * graphs are only valid for the duration of the call to Compile. Any graph/node/input/output * names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp. - * The OrtValue data for initializers (obtained via GetTensorData), however, is stable for the lifetime of - * the owning OrtSession instance. * * \since Version 1.23. */