From 3b4135224e0c4bef5b7079385d57e8e77f1ec56a Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Fri, 3 Dec 2021 09:12:55 +0000 Subject: [PATCH 01/19] init push --- tensorflow_lite_support/cc/task/processor/image_postprocessor.cc | 0 tensorflow_lite_support/cc/task/processor/image_postprocessor.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tensorflow_lite_support/cc/task/processor/image_postprocessor.cc create mode 100644 tensorflow_lite_support/cc/task/processor/image_postprocessor.h diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc new file mode 100644 index 000000000..e69de29bb diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h new file mode 100644 index 000000000..e69de29bb From 03446501b36414c90e837e5f7d602e8bd8e3c66b Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Fri, 3 Dec 2021 09:14:59 +0000 Subject: [PATCH 02/19] Write main logic. --- .../cc/task/processor/image_postprocessor.cc | 227 ++++++++++++++++++ .../cc/task/processor/image_postprocessor.h | 70 ++++++ 2 files changed, 297 insertions(+) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index e69de29bb..ab7496883 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -0,0 +1,227 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_lite_support/cc/task/processor/image_postprocessor.h" + +namespace tflite { +namespace task { +namespace processor { + +namespace { + +using ::absl::StatusCode; +using ::tflite::metadata::ModelMetadataExtractor; +using ::tflite::support::CreateStatusWithPayload; +using ::tflite::support::StatusOr; +using ::tflite::support::TfLiteSupportStatus; + +StatusOr> +GetNormalizationOptionsIfAny(const TensorMetadata& tensor_metadata) { + ASSIGN_OR_RETURN( + const tflite::ProcessUnit* normalization_process_unit, + ModelMetadataExtractor::FindFirstProcessUnit( + tensor_metadata, tflite::ProcessUnitOptions_NormalizationOptions)); + if (normalization_process_unit == nullptr) { + return {absl::nullopt}; + } + const tflite::NormalizationOptions* tf_normalization_options = + normalization_process_unit->options_as_NormalizationOptions(); + const auto mean_values = tf_normalization_options->mean(); + const auto std_values = tf_normalization_options->std(); + if (mean_values->size() != std_values->size()) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + absl::StrCat("NormalizationOptions: expected mean and std of same " + "dimension, got ", + mean_values->size(), " and ", std_values->size(), "."), + TfLiteSupportStatus::kMetadataInvalidProcessUnitsError); + } + absl::optional normalization_options; + if (mean_values->size() == 1) { + normalization_options = vision::NormalizationOptions{ + .mean_values = {mean_values->Get(0), mean_values->Get(0), + mean_values->Get(0)}, + .std_values = {std_values->Get(0), std_values->Get(0), + std_values->Get(0)}, + .num_values = 1}; + } else if (mean_values->size() == 3) { + normalization_options = vision::NormalizationOptions{ + .mean_values = {mean_values->Get(0), mean_values->Get(1), + mean_values->Get(2)}, + .std_values = {std_values->Get(0), std_values->Get(1), + std_values->Get(2)}, + .num_values = 3}; + } else { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + absl::StrCat("NormalizationOptions: only 1 or 3 mean and std " + "values are supported, got ", + mean_values->size(), "."), + TfLiteSupportStatus::kMetadataInvalidProcessUnitsError); + } + return normalization_options; +} +} // namespace + +/* static */ +tflite::support::StatusOr> +ImagePostprocessor::Create(core::TfLiteEngine* engine, + const std::initializer_list output_indices, + const std::initializer_list input_indices) { + ASSIGN_OR_RETURN(auto processor, Processor::Create(/* num_expected_tensors = */ 1, engine, output_indices, /* requires_metadata = */ false)); + + RETURN_IF_ERROR(processor->Init(input_indices)); + return processor; +} + +absl::Status ImagePostprocessor::Init(const std::vector& input_indices) { + if (core::TfLiteEngine::OutputCount(engine_->interpreter()) != 1) { + return tflite::support::CreateStatusWithPayload( + absl::StatusCode::kInvalidArgument, + absl::StrFormat( + "Image segmentation models are expected to have only 1 " + "output, found %d", + core::TfLiteEngine::OutputCount(engine_->interpreter())), + tflite::support::TfLiteSupportStatus::kInvalidNumOutputTensorsError); + } + + if (GetTensor()->type != kTfLiteUInt8 && GetTensor()->type != kTfLiteFloat32) { + return tflite::support::CreateStatusWithPayload( + absl::StatusCode::kInvalidArgument, + absl::StrFormat("Type mismatch for output tensor %s. Requested one " + "of these types: " + "kTfLiteUint8/kTfLiteFloat32, got %s.", + GetTensor()->name, TfLiteTypeGetName(GetTensor()->type)), + tflite::support::TfLiteSupportStatus::kInvalidOutputTensorTypeError); + } + + if (GetTensor()->dims->data[0] != 1 || GetTensor()->dims->data[3] != 3) { + return CreateStatusWithPayload( + absl::StatusCode::kInvalidArgument, + absl::StrCat("The input tensor should have dimensions 1 x height x " + "width x 3. Got ", + GetTensor()->dims->data[0], " x ", GetTensor()->dims->data[1], + " x ", GetTensor()->dims->data[2], " x ", + GetTensor()->dims->data[3], "."), + tflite::support::TfLiteSupportStatus:: + kInvalidInputTensorDimensionsError); + } + + // Gather metadata + const tflite::TensorMetadata* output_metadata = + engine_->metadata_extractor()->GetOutputTensorMetadata( + tensor_indices_.at(0)); + const tflite::TensorMetadata* input_metadata = engine_->metadata_extractor()->GetInputTensorMetadata( + input_indices.at(0)); + + // Use input metadata for normalization as fallback. + const tflite::TensorMetadata* processing_metadata = + GetNormalizationOptionsIfAny(*output_metadata).value().has_value() ? output_metadata : input_metadata; + + absl::optional normalization_options; + ASSIGN_OR_RETURN(normalization_options, + GetNormalizationOptionsIfAny(*processing_metadata)); + + if (GetTensor()->type == kTfLiteFloat32) { + if (!normalization_options.has_value()) { + return CreateStatusWithPayload( + absl::StatusCode::kNotFound, + "Output tensor has type kTfLiteFloat32: it requires specifying " + "NormalizationOptions metadata to preprocess output images.", + TfLiteSupportStatus::kMetadataMissingNormalizationOptionsError); + } else if (GetTensor()->bytes / sizeof(float) % + normalization_options.value().num_values != + 0) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + "The number of elements in the output tensor must be a multiple of " + "the number of normalization parameters.", + TfLiteSupportStatus::kInvalidArgumentError); + } + + options_ = std::make_unique( + normalization_options.value()); + } + + return absl::OkStatus(); +} + +absl::StatusOr ImagePostprocessor::Postprocess() { + has_uint8_outputs_ = GetTensor()->type == kTfLiteUInt8; + const int kRgbPixelBytes = 3; + + vision::FrameBuffer::Dimension to_buffer_dimension = { + GetTensor()->dims->data[2], GetTensor()->dims->data[1]}; + size_t output_byte_size = + GetBufferByteSize(to_buffer_dimension, vision::FrameBuffer::Format::kRGB); + std::vector postprocessed_data(output_byte_size / sizeof(uint8), 0); + + if (has_uint8_outputs_) { // No denormalization required. + if (GetTensor()->bytes != output_byte_size) { + return tflite::support::CreateStatusWithPayload( + absl::StatusCode::kInternal, + "Size mismatch or unsupported padding bytes between pixel data " + "and output tensor."); + } + const uint8* output_data = + core::AssertAndReturnTypedTensor(GetTensor()).value(); + postprocessed_data.insert(postprocessed_data.begin(), &output_data[0], + &output_data[output_byte_size / sizeof(uint8)]); + } else { // Denormalize to [0, 255] range. + if (GetTensor()->bytes / sizeof(float) != output_byte_size / sizeof(uint8)) { + return tflite::support::CreateStatusWithPayload( + absl::StatusCode::kInternal, + "Size mismatch or unsupported padding bytes between pixel data " + "and output tensor."); + } + + uint8* denormalized_output_data = postprocessed_data.data(); + const float* output_data = + core::AssertAndReturnTypedTensor(GetTensor()).value(); + const auto norm_options = GetNormalizationOptions(); + + if (norm_options.num_values == 1) { + float mean_value = norm_options.mean_values[0]; + float std_value = norm_options.std_values[0]; + + for (size_t i = 0; i < output_byte_size / sizeof(uint8); + ++i, ++denormalized_output_data, ++output_data) { + *denormalized_output_data = static_cast(std::round(std::min( + 255.f, std::max(0.f, (*output_data) * std_value + mean_value)))); + } + } else { + for (size_t i = 0; i < output_byte_size / sizeof(uint8); + ++i, ++denormalized_output_data, ++output_data) { + *denormalized_output_data = static_cast(std::round(std::min( + 255.f, + std::max(0.f, (*output_data) * norm_options.std_values[i % 3] + + norm_options.mean_values[i % 3])))); + } + } + } + + vision::FrameBuffer::Plane postprocessed_plane = { + /*buffer=*/postprocessed_data.data(), + /*stride=*/{GetTensor()->dims->data[2] * kRgbPixelBytes, kRgbPixelBytes}}; + auto postprocessed_frame_buffer = + vision::FrameBuffer::Create({postprocessed_plane}, to_buffer_dimension, + vision::FrameBuffer::Format::kRGB, + vision::FrameBuffer::Orientation::kTopLeft); + return *postprocessed_frame_buffer.get();; +} + +} // namespace processor +} // namespace task +} // namespace tflite diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h index e69de29bb..01be6226a 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h @@ -0,0 +1,70 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either exPostss or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_SUPPORT_CC_TASK_PROCESSOR_IMAGE_POSTPROCESSOR_H_ +#define TENSORFLOW_LITE_SUPPORT_CC_TASK_PROCESSOR_IMAGE_POSTPROCESSOR_H_ + +#include "tensorflow_lite_support/cc/port/status_macros.h" +#include "tensorflow_lite_support/cc/task/processor/processor.h" +#include "tensorflow_lite_support/cc/task/vision/core/frame_buffer.h" +#include "tensorflow_lite_support/cc/task/vision/utils/frame_buffer_utils.h" +#include "tensorflow_lite_support/cc/task/core/task_utils.h" +#include "tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h" + +namespace tflite { +namespace task { +namespace processor { + +// Process the associated output image tensor and convert it to a FrameBuffer. +// Requirement for the output tensor: +// (kTfLiteUInt8/kTfLiteFloat32) +// - image input of size `[batch x height x width x channels]`. +// - batch inference is not supported (`batch` is required to be 1). +// - only RGB inputs are supported (`channels` is required to be 3). +// - if type is kTfLiteFloat32, NormalizationOptions are required to be +// attached to the metadata for output de-normalization. Uses input metadata +// as fallback in case output metadata isn't provided. +class ImagePostprocessor : public Postprocessor { + public: + static tflite::support::StatusOr> + Create(core::TfLiteEngine* engine, + const std::initializer_list output_indices, + const std::initializer_list input_indices); + + // Processes the output tensor to an RGB of FrameBuffer type. + // If output tensor is of type kTfLiteFloat32, denormalize it into [0 - 255] + // via normalization parameters. + absl::StatusOr Postprocess(); + + private: + using Postprocessor::Postprocessor; + + // Whether the model features quantized inference type (QUANTIZED_UINT8). This + // is currently detected by checking if all output tensors data type is uint8. + bool has_uint8_outputs_; + + std::unique_ptr options_; + + absl::Status Init(const std::vector& input_indices); + + const vision::NormalizationOptions& GetNormalizationOptions() { + return *options_.get(); + } +}; +} // namespace processor +} // namespace task +} // namespace tflite + +#endif // TENSORFLOW_LITE_SUPPORT_CC_TASK_PROCESSOR_IMAGE_POSTPROCESSOR_H_ From 99e709d2fbd59065c9da1ad6412f20f809dd77fb Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Fri, 3 Dec 2021 09:19:30 +0000 Subject: [PATCH 03/19] build file updated --- .../cc/task/processor/BUILD | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tensorflow_lite_support/cc/task/processor/BUILD b/tensorflow_lite_support/cc/task/processor/BUILD index c412dee5d..b65555010 100644 --- a/tensorflow_lite_support/cc/task/processor/BUILD +++ b/tensorflow_lite_support/cc/task/processor/BUILD @@ -40,6 +40,24 @@ cc_library_with_tflite( ], ) +cc_library_with_tflite( + name = "image_postprocessor", + srcs = ["image_postprocessor.cc"], + hdrs = ["image_postprocessor.h"], + tflite_deps = [ + ":processor", + "//tensorflow_lite_support/cc/task/vision/utils:image_tensor_specs", + ], + deps = [ + "//tensorflow_lite_support/cc/port:status_macros", + "//tensorflow_lite_support/cc/port:statusor", + "//tensorflow_lite_support/cc/task/vision/core:frame_buffer", + "//tensorflow_lite_support/cc/task/vision/utils:frame_buffer_utils", + "//tensorflow_lite_support/cc/task/core:task_utils", + "@com_google_absl//absl/status", + ], +) + cc_library_with_tflite( name = "classification_postprocessor", srcs = ["classification_postprocessor.cc"], From 7c766c3e5a114b6f4707b231a30e36ac7454cec9 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Fri, 3 Dec 2021 09:39:32 +0000 Subject: [PATCH 04/19] format file --- .../cc/task/processor/image_postprocessor.cc | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index ab7496883..b81a567fa 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -79,14 +79,17 @@ GetNormalizationOptionsIfAny(const TensorMetadata& tensor_metadata) { tflite::support::StatusOr> ImagePostprocessor::Create(core::TfLiteEngine* engine, const std::initializer_list output_indices, - const std::initializer_list input_indices) { - ASSIGN_OR_RETURN(auto processor, Processor::Create(/* num_expected_tensors = */ 1, engine, output_indices, /* requires_metadata = */ false)); + const std::initializer_list input_indices) { + ASSIGN_OR_RETURN(auto processor, + Processor::Create( + /* num_expected_tensors = */ 1, engine, output_indices, + /* requires_metadata = */ false)); RETURN_IF_ERROR(processor->Init(input_indices)); return processor; } -absl::Status ImagePostprocessor::Init(const std::vector& input_indices) { +absl::Status ImagePostprocessor::Init(const std::vector &input_indices) { if (core::TfLiteEngine::OutputCount(engine_->interpreter()) != 1) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, @@ -97,13 +100,15 @@ absl::Status ImagePostprocessor::Init(const std::vector& input_indices) { tflite::support::TfLiteSupportStatus::kInvalidNumOutputTensorsError); } - if (GetTensor()->type != kTfLiteUInt8 && GetTensor()->type != kTfLiteFloat32) { + if (GetTensor()->type != kTfLiteUInt8 && + GetTensor()->type != kTfLiteFloat32) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, absl::StrFormat("Type mismatch for output tensor %s. Requested one " "of these types: " "kTfLiteUint8/kTfLiteFloat32, got %s.", - GetTensor()->name, TfLiteTypeGetName(GetTensor()->type)), + GetTensor()->name, + TfLiteTypeGetName(GetTensor()->type)), tflite::support::TfLiteSupportStatus::kInvalidOutputTensorTypeError); } @@ -112,8 +117,9 @@ absl::Status ImagePostprocessor::Init(const std::vector& input_indices) { absl::StatusCode::kInvalidArgument, absl::StrCat("The input tensor should have dimensions 1 x height x " "width x 3. Got ", - GetTensor()->dims->data[0], " x ", GetTensor()->dims->data[1], - " x ", GetTensor()->dims->data[2], " x ", + GetTensor()->dims->data[0], " x ", + GetTensor()->dims->data[1], " x ", + GetTensor()->dims->data[2], " x ", GetTensor()->dims->data[3], "."), tflite::support::TfLiteSupportStatus:: kInvalidInputTensorDimensionsError); @@ -123,12 +129,15 @@ absl::Status ImagePostprocessor::Init(const std::vector& input_indices) { const tflite::TensorMetadata* output_metadata = engine_->metadata_extractor()->GetOutputTensorMetadata( tensor_indices_.at(0)); - const tflite::TensorMetadata* input_metadata = engine_->metadata_extractor()->GetInputTensorMetadata( - input_indices.at(0)); + const tflite::TensorMetadata* input_metadata = + engine_->metadata_extractor()->GetInputTensorMetadata( + input_indices.at(0)); // Use input metadata for normalization as fallback. const tflite::TensorMetadata* processing_metadata = - GetNormalizationOptionsIfAny(*output_metadata).value().has_value() ? output_metadata : input_metadata; + GetNormalizationOptionsIfAny(*output_metadata).value().has_value() + ? output_metadata + : input_metadata; absl::optional normalization_options; ASSIGN_OR_RETURN(normalization_options, @@ -180,7 +189,8 @@ absl::StatusOr ImagePostprocessor::Postprocess() { postprocessed_data.insert(postprocessed_data.begin(), &output_data[0], &output_data[output_byte_size / sizeof(uint8)]); } else { // Denormalize to [0, 255] range. - if (GetTensor()->bytes / sizeof(float) != output_byte_size / sizeof(uint8)) { + if (GetTensor()->bytes / sizeof(float) != + output_byte_size / sizeof(uint8)) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInternal, "Size mismatch or unsupported padding bytes between pixel data " @@ -205,7 +215,7 @@ absl::StatusOr ImagePostprocessor::Postprocess() { for (size_t i = 0; i < output_byte_size / sizeof(uint8); ++i, ++denormalized_output_data, ++output_data) { *denormalized_output_data = static_cast(std::round(std::min( - 255.f, + 255.f, std::max(0.f, (*output_data) * norm_options.std_values[i % 3] + norm_options.mean_values[i % 3])))); } @@ -219,7 +229,7 @@ absl::StatusOr ImagePostprocessor::Postprocess() { vision::FrameBuffer::Create({postprocessed_plane}, to_buffer_dimension, vision::FrameBuffer::Format::kRGB, vision::FrameBuffer::Orientation::kTopLeft); - return *postprocessed_frame_buffer.get();; + return *postprocessed_frame_buffer.get(); } } // namespace processor From 4cdbbf8c5679b37b7588922f8ca6ea6cd8f4f036 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 9 Dec 2021 16:13:45 +0000 Subject: [PATCH 05/19] Use "is_input bool to switch input/output. --- .../task/vision/utils/image_tensor_specs.cc | 92 ++++++++++++------- .../cc/task/vision/utils/image_tensor_specs.h | 7 +- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc index afbe07dd9..4568be5a0 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc @@ -39,29 +39,38 @@ using ::tflite::support::StatusOr; using ::tflite::support::TfLiteSupportStatus; using ::tflite::task::core::TfLiteEngine; -StatusOr GetInputTensorMetadataIfAny( - const ModelMetadataExtractor& metadata_extractor) { +StatusOr GetTensorMetadataIfAny( + const ModelMetadataExtractor& metadata_extractor, const bool is_input) { if (metadata_extractor.GetModelMetadata() == nullptr || metadata_extractor.GetModelMetadata()->subgraph_metadata() == nullptr) { // Some models have no metadata at all (or very partial), so exit early. return nullptr; - } else if (metadata_extractor.GetInputTensorCount() != 1) { + } else if (is_input && metadata_extractor.GetInputTensorCount() != 1) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, "Models are assumed to have a single input TensorMetadata.", TfLiteSupportStatus::kInvalidNumInputTensorsError); + } else if (!is_input && metadata_extractor.GetOutputTensorCount() != 1) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + "Models are assumed to have a single output TensorMetadata.", + TfLiteSupportStatus::kInvalidNumOutputTensorsError); } - const TensorMetadata* metadata = metadata_extractor.GetInputTensorMetadata(0); + const TensorMetadata* input_metadata = + metadata_extractor.GetInputTensorMetadata(0); + const TensorMetadata* output_metadata = + metadata_extractor.GetOutputTensorMetadata(0); - if (metadata == nullptr) { + if (input_metadata == nullptr) { // Should never happen. return CreateStatusWithPayload(StatusCode::kInternal, "Input TensorMetadata is null."); } - return metadata; + return is_input ? input_metadata : output_metadata; } +} // namespace StatusOr GetImagePropertiesIfAny( const TensorMetadata& tensor_metadata) { @@ -132,13 +141,12 @@ StatusOr> GetNormalizationOptionsIfAny( return normalization_options; } -} // namespace - -StatusOr BuildInputImageTensorSpecs( +StatusOr BuildImageTensorSpecs( const TfLiteEngine::Interpreter& interpreter, - const tflite::metadata::ModelMetadataExtractor& metadata_extractor) { + const tflite::metadata::ModelMetadataExtractor& metadata_extractor, + const bool is_input) { ASSIGN_OR_RETURN(const TensorMetadata* metadata, - GetInputTensorMetadataIfAny(metadata_extractor)); + GetTensorMetadataIfAny(metadata_extractor, is_input)); const ImageProperties* props = nullptr; absl::optional normalization_options; @@ -146,41 +154,54 @@ StatusOr BuildInputImageTensorSpecs( ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*metadata)); ASSIGN_OR_RETURN(normalization_options, GetNormalizationOptionsIfAny(*metadata)); + if (!is_input && !normalization_options.has_value()) { + ASSIGN_OR_RETURN(normalization_options, + GetNormalizationOptionsIfAny( + *metadata_extractor.GetInputTensorMetadata(0))); + } } - if (TfLiteEngine::InputCount(&interpreter) != 1) { + if (is_input && TfLiteEngine::InputCount(&interpreter) != 1) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, "Models are assumed to have a single input.", TfLiteSupportStatus::kInvalidNumInputTensorsError); + } else if (!is_input && TfLiteEngine::OutputCount(&interpreter) != 1) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + "Models are assumed to have a single output.", + TfLiteSupportStatus::kInvalidNumOutputTensorsError); } - // Input-related specifications. - const TfLiteTensor* input_tensor = TfLiteEngine::GetInput(&interpreter, 0); - if (input_tensor->dims->size != 4) { + // Input/output-related specifications. + const TfLiteTensor* tensor = is_input + ? TfLiteEngine::GetInput(&interpreter, 0) + : TfLiteEngine::GetOutput(&interpreter, 0); + + if (tensor->dims->size != 4) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, "Only 4D tensors in BHWD layout are supported.", TfLiteSupportStatus::kInvalidInputTensorDimensionsError); } static constexpr TfLiteType valid_types[] = {kTfLiteUInt8, kTfLiteFloat32}; - TfLiteType input_type = input_tensor->type; - if (!absl::c_linear_search(valid_types, input_type)) { + TfLiteType tensor_type = tensor->type; + if (!absl::c_linear_search(valid_types, tensor_type)) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, absl::StrCat( - "Type mismatch for input tensor ", input_tensor->name, + "Type mismatch for tensor ", tensor->name, ". Requested one of these types: kTfLiteUint8/kTfLiteFloat32, got ", - TfLiteTypeGetName(input_type), "."), + TfLiteTypeGetName(tensor_type), "."), TfLiteSupportStatus::kInvalidInputTensorTypeError); } // The expected layout is BHWD, i.e. batch x height x width x color // See https://www.tensorflow.org/guide/tensors - const int batch = input_tensor->dims->data[0]; - const int height = input_tensor->dims->data[1]; - const int width = input_tensor->dims->data[2]; - const int depth = input_tensor->dims->data[3]; + const int batch = tensor->dims->data[0]; + const int height = tensor->dims->data[1]; + const int width = tensor->dims->data[2]; + const int depth = tensor->dims->data[3]; if (props != nullptr && props->color_space() != ColorSpaceType_RGB) { return CreateStatusWithPayload(StatusCode::kInvalidArgument, @@ -190,47 +211,47 @@ StatusOr BuildInputImageTensorSpecs( if (batch != 1 || depth != 3) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, - absl::StrCat("The input tensor should have dimensions 1 x height x " + absl::StrCat("The tensor should have dimensions 1 x height x " "width x 3. Got ", batch, " x ", height, " x ", width, " x ", depth, "."), TfLiteSupportStatus::kInvalidInputTensorDimensionsError); } - int bytes_size = input_tensor->bytes; + int bytes_size = tensor->bytes; size_t byte_depth = - input_type == kTfLiteFloat32 ? sizeof(float) : sizeof(uint8); + tensor_type == kTfLiteFloat32 ? sizeof(float) : sizeof(uint8); // Sanity checks. - if (input_type == kTfLiteFloat32) { + if (tensor_type == kTfLiteFloat32) { if (!normalization_options.has_value()) { return CreateStatusWithPayload( absl::StatusCode::kNotFound, - "Input tensor has type kTfLiteFloat32: it requires specifying " - "NormalizationOptions metadata to preprocess input images.", + "Tensor has type kTfLiteFloat32: it requires specifying " + "NormalizationOptions metadata to process images.", TfLiteSupportStatus::kMetadataMissingNormalizationOptionsError); } else if (bytes_size / sizeof(float) % normalization_options.value().num_values != 0) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, - "The number of elements in the input tensor must be a multiple of " + "The number of elements in the tensor must be a multiple of " "the number of normalization parameters.", TfLiteSupportStatus::kInvalidArgumentError); } } if (width <= 0) { return CreateStatusWithPayload( - StatusCode::kInvalidArgument, "The input width should be positive.", + StatusCode::kInvalidArgument, "The width should be positive.", TfLiteSupportStatus::kInvalidInputTensorDimensionsError); } if (height <= 0) { return CreateStatusWithPayload( - StatusCode::kInvalidArgument, "The input height should be positive.", + StatusCode::kInvalidArgument, "The height should be positive.", TfLiteSupportStatus::kInvalidInputTensorDimensionsError); } if (bytes_size != height * width * depth * byte_depth) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, - "The input size in bytes does not correspond to the expected number of " + "The tensor in bytes does not correspond to the expected number of " "pixels.", TfLiteSupportStatus::kInvalidInputTensorSizeError); } @@ -243,8 +264,9 @@ StatusOr BuildInputImageTensorSpecs( result.image_width = width; result.image_height = height; result.color_space = ColorSpaceType_RGB; - result.tensor_type = input_type; - result.normalization_options = normalization_options; + result.tensor_type = tensor_type; + result.normalization_options = + normalization_options; return result; } diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h index d15be3f8e..b4a79b324 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h @@ -73,7 +73,7 @@ struct ImageTensorSpecs { absl::optional normalization_options; }; -// Performs sanity checks on the expected input tensor including consistency +// Performs sanity checks on the expected input/output tensor including consistency // checks against model metadata, if any. For now, a single RGB input with BHWD // layout, where B = 1 and D = 3, is expected. Returns the corresponding input // specifications if they pass, or an error otherwise (too many input tensors, @@ -82,9 +82,10 @@ struct ImageTensorSpecs { // initialized before calling this function by means of (respectively): // - `tflite::InterpreterBuilder`, // - `tflite::metadata::ModelMetadataExtractor::CreateFromModelBuffer`. -tflite::support::StatusOr BuildInputImageTensorSpecs( +tflite::support::StatusOr BuildImageTensorSpecs( const tflite::task::core::TfLiteEngine::Interpreter& interpreter, - const tflite::metadata::ModelMetadataExtractor& metadata_extractor); + const tflite::metadata::ModelMetadataExtractor& metadata_extractor, + const bool is_input); } // namespace vision } // namespace task From 2a39596424a66378cd61e06f3c6d77fff45f93b7 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 9 Dec 2021 16:14:06 +0000 Subject: [PATCH 06/19] Preprocessor use new API --- .../cc/task/processor/image_preprocessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc index 7ad4ad470..57b12941a 100644 --- a/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc @@ -72,9 +72,9 @@ absl::Status ImagePreprocessor::Init( const vision::FrameBufferUtils::ProcessEngine& process_engine) { frame_buffer_utils_ = vision::FrameBufferUtils::Create(process_engine); - ASSIGN_OR_RETURN(input_specs_, vision::BuildInputImageTensorSpecs( + ASSIGN_OR_RETURN(input_specs_, vision::BuildImageTensorSpecs( *engine_->interpreter(), - *engine_->metadata_extractor())); + *engine_->metadata_extractor(), true)); if (input_specs_.color_space != tflite::ColorSpaceType_RGB) { return tflite::support::CreateStatusWithPayload( From ed455cb0220e54f8ed8d70f9336f648a93bd8ea5 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 9 Dec 2021 16:14:30 +0000 Subject: [PATCH 07/19] single index instead of multi --- .../cc/task/processor/image_postprocessor.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h index 01be6226a..e2479b198 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h @@ -40,8 +40,8 @@ class ImagePostprocessor : public Postprocessor { public: static tflite::support::StatusOr> Create(core::TfLiteEngine* engine, - const std::initializer_list output_indices, - const std::initializer_list input_indices); + const int output_index, + const int input_index = -1); // Processes the output tensor to an RGB of FrameBuffer type. // If output tensor is of type kTfLiteFloat32, denormalize it into [0 - 255] @@ -52,12 +52,12 @@ class ImagePostprocessor : public Postprocessor { using Postprocessor::Postprocessor; // Whether the model features quantized inference type (QUANTIZED_UINT8). This - // is currently detected by checking if all output tensors data type is uint8. - bool has_uint8_outputs_; + // is currently detected by checking if the output tensor data type is uint8. + bool has_uint8_output_; std::unique_ptr options_; - absl::Status Init(const std::vector& input_indices); + absl::Status Init(const int input_index); const vision::NormalizationOptions& GetNormalizationOptions() { return *options_.get(); From ae937935dc7c985a4eb55ebb2a377ddce8b3dc65 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 9 Dec 2021 16:15:08 +0000 Subject: [PATCH 08/19] Re-use logic from BuildImageTensorSpecs --- .../cc/task/processor/image_postprocessor.cc | 142 +++--------------- 1 file changed, 17 insertions(+), 125 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index b81a567fa..43df5bae1 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -27,148 +27,40 @@ using ::tflite::support::CreateStatusWithPayload; using ::tflite::support::StatusOr; using ::tflite::support::TfLiteSupportStatus; -StatusOr> -GetNormalizationOptionsIfAny(const TensorMetadata& tensor_metadata) { - ASSIGN_OR_RETURN( - const tflite::ProcessUnit* normalization_process_unit, - ModelMetadataExtractor::FindFirstProcessUnit( - tensor_metadata, tflite::ProcessUnitOptions_NormalizationOptions)); - if (normalization_process_unit == nullptr) { - return {absl::nullopt}; - } - const tflite::NormalizationOptions* tf_normalization_options = - normalization_process_unit->options_as_NormalizationOptions(); - const auto mean_values = tf_normalization_options->mean(); - const auto std_values = tf_normalization_options->std(); - if (mean_values->size() != std_values->size()) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - absl::StrCat("NormalizationOptions: expected mean and std of same " - "dimension, got ", - mean_values->size(), " and ", std_values->size(), "."), - TfLiteSupportStatus::kMetadataInvalidProcessUnitsError); - } - absl::optional normalization_options; - if (mean_values->size() == 1) { - normalization_options = vision::NormalizationOptions{ - .mean_values = {mean_values->Get(0), mean_values->Get(0), - mean_values->Get(0)}, - .std_values = {std_values->Get(0), std_values->Get(0), - std_values->Get(0)}, - .num_values = 1}; - } else if (mean_values->size() == 3) { - normalization_options = vision::NormalizationOptions{ - .mean_values = {mean_values->Get(0), mean_values->Get(1), - mean_values->Get(2)}, - .std_values = {std_values->Get(0), std_values->Get(1), - std_values->Get(2)}, - .num_values = 3}; - } else { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - absl::StrCat("NormalizationOptions: only 1 or 3 mean and std " - "values are supported, got ", - mean_values->size(), "."), - TfLiteSupportStatus::kMetadataInvalidProcessUnitsError); - } - return normalization_options; -} } // namespace /* static */ tflite::support::StatusOr> -ImagePostprocessor::Create(core::TfLiteEngine* engine, - const std::initializer_list output_indices, - const std::initializer_list input_indices) { +ImagePostprocessor::Create(core::TfLiteEngine* engine, const int output_index, + const int input_index) { ASSIGN_OR_RETURN(auto processor, Processor::Create( - /* num_expected_tensors = */ 1, engine, output_indices, + /* num_expected_tensors = */ 1, engine, {output_index}, /* requires_metadata = */ false)); - RETURN_IF_ERROR(processor->Init(input_indices)); + RETURN_IF_ERROR(processor->Init(input_index)); return processor; } -absl::Status ImagePostprocessor::Init(const std::vector &input_indices) { - if (core::TfLiteEngine::OutputCount(engine_->interpreter()) != 1) { - return tflite::support::CreateStatusWithPayload( - absl::StatusCode::kInvalidArgument, - absl::StrFormat( - "Image segmentation models are expected to have only 1 " - "output, found %d", - core::TfLiteEngine::OutputCount(engine_->interpreter())), - tflite::support::TfLiteSupportStatus::kInvalidNumOutputTensorsError); - } - - if (GetTensor()->type != kTfLiteUInt8 && - GetTensor()->type != kTfLiteFloat32) { +absl::Status ImagePostprocessor::Init(const int input_index) { + if (input_index == -1) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, - absl::StrFormat("Type mismatch for output tensor %s. Requested one " - "of these types: " - "kTfLiteUint8/kTfLiteFloat32, got %s.", - GetTensor()->name, - TfLiteTypeGetName(GetTensor()->type)), - tflite::support::TfLiteSupportStatus::kInvalidOutputTensorTypeError); + absl::StrFormat("Input image tensor not set. Input index found: %d", + input_index), + tflite::support::TfLiteSupportStatus::kInputTensorNotFoundError); } - - if (GetTensor()->dims->data[0] != 1 || GetTensor()->dims->data[3] != 3) { - return CreateStatusWithPayload( - absl::StatusCode::kInvalidArgument, - absl::StrCat("The input tensor should have dimensions 1 x height x " - "width x 3. Got ", - GetTensor()->dims->data[0], " x ", - GetTensor()->dims->data[1], " x ", - GetTensor()->dims->data[2], " x ", - GetTensor()->dims->data[3], "."), - tflite::support::TfLiteSupportStatus:: - kInvalidInputTensorDimensionsError); - } - - // Gather metadata - const tflite::TensorMetadata* output_metadata = - engine_->metadata_extractor()->GetOutputTensorMetadata( - tensor_indices_.at(0)); - const tflite::TensorMetadata* input_metadata = - engine_->metadata_extractor()->GetInputTensorMetadata( - input_indices.at(0)); - - // Use input metadata for normalization as fallback. - const tflite::TensorMetadata* processing_metadata = - GetNormalizationOptionsIfAny(*output_metadata).value().has_value() - ? output_metadata - : input_metadata; - - absl::optional normalization_options; - ASSIGN_OR_RETURN(normalization_options, - GetNormalizationOptionsIfAny(*processing_metadata)); - - if (GetTensor()->type == kTfLiteFloat32) { - if (!normalization_options.has_value()) { - return CreateStatusWithPayload( - absl::StatusCode::kNotFound, - "Output tensor has type kTfLiteFloat32: it requires specifying " - "NormalizationOptions metadata to preprocess output images.", - TfLiteSupportStatus::kMetadataMissingNormalizationOptionsError); - } else if (GetTensor()->bytes / sizeof(float) % - normalization_options.value().num_values != - 0) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - "The number of elements in the output tensor must be a multiple of " - "the number of normalization parameters.", - TfLiteSupportStatus::kInvalidArgumentError); - } - - options_ = std::make_unique( - normalization_options.value()); - } - + ASSIGN_OR_RETURN( + auto output_specs, + vision::BuildImageTensorSpecs(*engine_->interpreter(), + *engine_->metadata_extractor(), false)); + options_ = std::make_unique( + output_specs.normalization_options.value()); return absl::OkStatus(); } absl::StatusOr ImagePostprocessor::Postprocess() { - has_uint8_outputs_ = GetTensor()->type == kTfLiteUInt8; + has_uint8_output_ = GetTensor()->type == kTfLiteUInt8; const int kRgbPixelBytes = 3; vision::FrameBuffer::Dimension to_buffer_dimension = { @@ -177,7 +69,7 @@ absl::StatusOr ImagePostprocessor::Postprocess() { GetBufferByteSize(to_buffer_dimension, vision::FrameBuffer::Format::kRGB); std::vector postprocessed_data(output_byte_size / sizeof(uint8), 0); - if (has_uint8_outputs_) { // No denormalization required. + if (has_uint8_output_) { // No denormalization required. if (GetTensor()->bytes != output_byte_size) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInternal, From e24ef959dc7f1a58450490d4e1b8d30023d65952 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Sat, 11 Dec 2021 03:52:24 +0000 Subject: [PATCH 09/19] move in unknwn namespace --- .../cc/task/processor/image_postprocessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index 43df5bae1..673f61b98 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -27,6 +27,8 @@ using ::tflite::support::CreateStatusWithPayload; using ::tflite::support::StatusOr; using ::tflite::support::TfLiteSupportStatus; +constexpr int kRgbPixelBytes = 3; + } // namespace /* static */ @@ -61,8 +63,6 @@ absl::Status ImagePostprocessor::Init(const int input_index) { absl::StatusOr ImagePostprocessor::Postprocess() { has_uint8_output_ = GetTensor()->type == kTfLiteUInt8; - const int kRgbPixelBytes = 3; - vision::FrameBuffer::Dimension to_buffer_dimension = { GetTensor()->dims->data[2], GetTensor()->dims->data[1]}; size_t output_byte_size = From f4b16078d344fdba749909ae443584bc509a9543 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Sat, 11 Dec 2021 03:52:53 +0000 Subject: [PATCH 10/19] move has_uint8_output_ to init() --- .../cc/task/processor/image_postprocessor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index 673f61b98..b6cf8a255 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -58,11 +58,11 @@ absl::Status ImagePostprocessor::Init(const int input_index) { *engine_->metadata_extractor(), false)); options_ = std::make_unique( output_specs.normalization_options.value()); + has_uint8_output_ = GetTensor()->type == kTfLiteUInt8; return absl::OkStatus(); } absl::StatusOr ImagePostprocessor::Postprocess() { - has_uint8_output_ = GetTensor()->type == kTfLiteUInt8; vision::FrameBuffer::Dimension to_buffer_dimension = { GetTensor()->dims->data[2], GetTensor()->dims->data[1]}; size_t output_byte_size = From 5236b8b9175f54168fa844a8094737f197c572f1 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Sun, 19 Dec 2021 11:04:03 +0000 Subject: [PATCH 11/19] Use populate vector & rm size checks --- .../cc/task/processor/image_postprocessor.cc | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index b6cf8a255..c84ab1194 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -70,25 +70,10 @@ absl::StatusOr ImagePostprocessor::Postprocess() { std::vector postprocessed_data(output_byte_size / sizeof(uint8), 0); if (has_uint8_output_) { // No denormalization required. - if (GetTensor()->bytes != output_byte_size) { - return tflite::support::CreateStatusWithPayload( - absl::StatusCode::kInternal, - "Size mismatch or unsupported padding bytes between pixel data " - "and output tensor."); - } const uint8* output_data = core::AssertAndReturnTypedTensor(GetTensor()).value(); - postprocessed_data.insert(postprocessed_data.begin(), &output_data[0], - &output_data[output_byte_size / sizeof(uint8)]); + core::PopulateVector(output_data, postprocessed_data); } else { // Denormalize to [0, 255] range. - if (GetTensor()->bytes / sizeof(float) != - output_byte_size / sizeof(uint8)) { - return tflite::support::CreateStatusWithPayload( - absl::StatusCode::kInternal, - "Size mismatch or unsupported padding bytes between pixel data " - "and output tensor."); - } - uint8* denormalized_output_data = postprocessed_data.data(); const float* output_data = core::AssertAndReturnTypedTensor(GetTensor()).value(); From 31c2b31f4c20db4c5529658126d50cba4aa0e07b Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Wed, 22 Dec 2021 06:47:25 +0000 Subject: [PATCH 12/19] Pass tensor & metato tensor_spec. rm tensor count. --- .../task/vision/utils/image_tensor_specs.cc | 62 +++---------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc index 4568be5a0..203713911 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc @@ -40,35 +40,20 @@ using ::tflite::support::TfLiteSupportStatus; using ::tflite::task::core::TfLiteEngine; StatusOr GetTensorMetadataIfAny( - const ModelMetadataExtractor& metadata_extractor, const bool is_input) { + const ModelMetadataExtractor& metadata_extractor, + const TensorMetadata* tensor_metadata) { if (metadata_extractor.GetModelMetadata() == nullptr || metadata_extractor.GetModelMetadata()->subgraph_metadata() == nullptr) { // Some models have no metadata at all (or very partial), so exit early. return nullptr; - } else if (is_input && metadata_extractor.GetInputTensorCount() != 1) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - "Models are assumed to have a single input TensorMetadata.", - TfLiteSupportStatus::kInvalidNumInputTensorsError); - } else if (!is_input && metadata_extractor.GetOutputTensorCount() != 1) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - "Models are assumed to have a single output TensorMetadata.", - TfLiteSupportStatus::kInvalidNumOutputTensorsError); } - const TensorMetadata* input_metadata = - metadata_extractor.GetInputTensorMetadata(0); - const TensorMetadata* output_metadata = - metadata_extractor.GetOutputTensorMetadata(0); - - if (input_metadata == nullptr) { + if (tensor_metadata == nullptr) { // Should never happen. return CreateStatusWithPayload(StatusCode::kInternal, - "Input TensorMetadata is null."); + "Provided TensorMetadata is null."); } - - return is_input ? input_metadata : output_metadata; + return tensor_metadata; } } // namespace @@ -143,41 +128,15 @@ StatusOr> GetNormalizationOptionsIfAny( StatusOr BuildImageTensorSpecs( const TfLiteEngine::Interpreter& interpreter, - const tflite::metadata::ModelMetadataExtractor& metadata_extractor, - const bool is_input) { - ASSIGN_OR_RETURN(const TensorMetadata* metadata, - GetTensorMetadataIfAny(metadata_extractor, is_input)); - + const TensorMetadata* tensor_metadata, const TfLiteTensor* tensor) { const ImageProperties* props = nullptr; absl::optional normalization_options; - if (metadata != nullptr) { - ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*metadata)); + if (tensor_metadata != nullptr) { + ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*tensor_metadata)); ASSIGN_OR_RETURN(normalization_options, - GetNormalizationOptionsIfAny(*metadata)); - if (!is_input && !normalization_options.has_value()) { - ASSIGN_OR_RETURN(normalization_options, - GetNormalizationOptionsIfAny( - *metadata_extractor.GetInputTensorMetadata(0))); - } + GetNormalizationOptionsIfAny(*tensor_metadata)); } - if (is_input && TfLiteEngine::InputCount(&interpreter) != 1) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - "Models are assumed to have a single input.", - TfLiteSupportStatus::kInvalidNumInputTensorsError); - } else if (!is_input && TfLiteEngine::OutputCount(&interpreter) != 1) { - return CreateStatusWithPayload( - StatusCode::kInvalidArgument, - "Models are assumed to have a single output.", - TfLiteSupportStatus::kInvalidNumOutputTensorsError); - } - - // Input/output-related specifications. - const TfLiteTensor* tensor = is_input - ? TfLiteEngine::GetInput(&interpreter, 0) - : TfLiteEngine::GetOutput(&interpreter, 0); - if (tensor->dims->size != 4) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, @@ -265,8 +224,7 @@ StatusOr BuildImageTensorSpecs( result.image_height = height; result.color_space = ColorSpaceType_RGB; result.tensor_type = tensor_type; - result.normalization_options = - normalization_options; + result.normalization_options = normalization_options; return result; } From 213acb089597d4c5abc14261e44bad6be16bc00f Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Wed, 22 Dec 2021 15:21:02 +0000 Subject: [PATCH 13/19] GetTensorMetada() checks subgraph of given meta --- .../cc/task/vision/utils/image_tensor_specs.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc index 203713911..1a23b7747 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc @@ -127,14 +127,15 @@ StatusOr> GetNormalizationOptionsIfAny( } StatusOr BuildImageTensorSpecs( - const TfLiteEngine::Interpreter& interpreter, + const ModelMetadataExtractor& metadata_extractor, const TensorMetadata* tensor_metadata, const TfLiteTensor* tensor) { const ImageProperties* props = nullptr; absl::optional normalization_options; - if (tensor_metadata != nullptr) { - ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*tensor_metadata)); + ASSIGN_OR_RETURN(auto metadata, GetTensorMetadataIfAny(metadata_extractor, tensor_metadata)); + if (metadata != nullptr) { + ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*metadata)); ASSIGN_OR_RETURN(normalization_options, - GetNormalizationOptionsIfAny(*tensor_metadata)); + GetNormalizationOptionsIfAny(*metadata)); } if (tensor->dims->size != 4) { From 9cdcb04fa08d028467c614e7d5a316ec3d5a351c Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 06:13:05 +0000 Subject: [PATCH 14/19] NormProcessUnit check in postprocessor to fallback --- .../cc/task/processor/image_postprocessor.cc | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index c84ab1194..863600072 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -40,11 +40,12 @@ ImagePostprocessor::Create(core::TfLiteEngine* engine, const int output_index, /* num_expected_tensors = */ 1, engine, {output_index}, /* requires_metadata = */ false)); - RETURN_IF_ERROR(processor->Init(input_index)); + RETURN_IF_ERROR(processor->Init(input_index, output_index)); return processor; } -absl::Status ImagePostprocessor::Init(const int input_index) { +absl::Status ImagePostprocessor::Init(const int input_index, + const int output_index) { if (input_index == -1) { return tflite::support::CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, @@ -52,13 +53,23 @@ absl::Status ImagePostprocessor::Init(const int input_index) { input_index), tflite::support::TfLiteSupportStatus::kInputTensorNotFoundError); } + const TensorMetadata* metadata = GetTensorMetadata(output_index); ASSIGN_OR_RETURN( - auto output_specs, - vision::BuildImageTensorSpecs(*engine_->interpreter(), - *engine_->metadata_extractor(), false)); + const tflite::ProcessUnit* normalization_process_unit, + ModelMetadataExtractor::FindFirstProcessUnit( + *metadata, tflite::ProcessUnitOptions_NormalizationOptions)); + if (normalization_process_unit == nullptr) { + metadata = + engine_->metadata_extractor()->GetInputTensorMetadata(input_index); + } + + ASSIGN_OR_RETURN(auto output_specs, vision::BuildImageTensorSpecs( + *engine_->metadata_extractor(), + metadata, GetTensor(output_index))); options_ = std::make_unique( output_specs.normalization_options.value()); - has_uint8_output_ = GetTensor()->type == kTfLiteUInt8; + has_uint8_output_ = GetTensor(output_index)->type == kTfLiteUInt8; + has_float32_output_ = GetTensor(output_index)->type == kTfLiteFloat32; return absl::OkStatus(); } From 352fcdb14541c87160fbb9ad416ccca9a2e51bc1 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 06:28:29 +0000 Subject: [PATCH 15/19] store output tensor instead of its type. --- .../cc/task/processor/image_postprocessor.cc | 33 +++++++++++-------- .../cc/task/processor/image_postprocessor.h | 6 ++-- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc index 863600072..051badf22 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.cc @@ -54,6 +54,7 @@ absl::Status ImagePostprocessor::Init(const int input_index, tflite::support::TfLiteSupportStatus::kInputTensorNotFoundError); } const TensorMetadata* metadata = GetTensorMetadata(output_index); + // Fallback to input metadata if output meta doesn't have norm params. ASSIGN_OR_RETURN( const tflite::ProcessUnit* normalization_process_unit, ModelMetadataExtractor::FindFirstProcessUnit( @@ -62,32 +63,35 @@ absl::Status ImagePostprocessor::Init(const int input_index, metadata = engine_->metadata_extractor()->GetInputTensorMetadata(input_index); } - - ASSIGN_OR_RETURN(auto output_specs, vision::BuildImageTensorSpecs( - *engine_->metadata_extractor(), - metadata, GetTensor(output_index))); + if (!GetTensor(output_index)->data.raw) { + return tflite::support::CreateStatusWithPayload( + absl::StatusCode::kInternal, + absl::StrFormat("Output tensor (%s) has no raw data.", + GetTensor(output_index)->name)); + } + output_tensor_ = GetTensor(output_index); + ASSIGN_OR_RETURN(auto output_specs, + vision::BuildImageTensorSpecs(*engine_->metadata_extractor(), + metadata, output_tensor_)); options_ = std::make_unique( output_specs.normalization_options.value()); - has_uint8_output_ = GetTensor(output_index)->type == kTfLiteUInt8; - has_float32_output_ = GetTensor(output_index)->type == kTfLiteFloat32; return absl::OkStatus(); } absl::StatusOr ImagePostprocessor::Postprocess() { vision::FrameBuffer::Dimension to_buffer_dimension = { - GetTensor()->dims->data[2], GetTensor()->dims->data[1]}; + output_tensor_->dims->data[2], output_tensor_->dims->data[1]}; size_t output_byte_size = GetBufferByteSize(to_buffer_dimension, vision::FrameBuffer::Format::kRGB); std::vector postprocessed_data(output_byte_size / sizeof(uint8), 0); - if (has_uint8_output_) { // No denormalization required. - const uint8* output_data = - core::AssertAndReturnTypedTensor(GetTensor()).value(); - core::PopulateVector(output_data, postprocessed_data); - } else { // Denormalize to [0, 255] range. + if (output_tensor_->type == kTfLiteUInt8) { // No denormalization required. + core::PopulateVector(output_tensor_, &postprocessed_data); + } else if (output_tensor_->type == + kTfLiteFloat32) { // Denormalize to [0, 255] range. uint8* denormalized_output_data = postprocessed_data.data(); const float* output_data = - core::AssertAndReturnTypedTensor(GetTensor()).value(); + core::AssertAndReturnTypedTensor(output_tensor_).value(); const auto norm_options = GetNormalizationOptions(); if (norm_options.num_values == 1) { @@ -112,7 +116,8 @@ absl::StatusOr ImagePostprocessor::Postprocess() { vision::FrameBuffer::Plane postprocessed_plane = { /*buffer=*/postprocessed_data.data(), - /*stride=*/{GetTensor()->dims->data[2] * kRgbPixelBytes, kRgbPixelBytes}}; + /*stride=*/{output_tensor_->dims->data[2] * kRgbPixelBytes, + kRgbPixelBytes}}; auto postprocessed_frame_buffer = vision::FrameBuffer::Create({postprocessed_plane}, to_buffer_dimension, vision::FrameBuffer::Format::kRGB, diff --git a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h index e2479b198..b08567ecb 100644 --- a/tensorflow_lite_support/cc/task/processor/image_postprocessor.h +++ b/tensorflow_lite_support/cc/task/processor/image_postprocessor.h @@ -51,13 +51,11 @@ class ImagePostprocessor : public Postprocessor { private: using Postprocessor::Postprocessor; - // Whether the model features quantized inference type (QUANTIZED_UINT8). This - // is currently detected by checking if the output tensor data type is uint8. - bool has_uint8_output_; + const TfLiteTensor* output_tensor_; std::unique_ptr options_; - absl::Status Init(const int input_index); + absl::Status Init(const int input_index, const int output_index); const vision::NormalizationOptions& GetNormalizationOptions() { return *options_.get(); From d1fa7201128311aef077250c110c2d66d6da290b Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 06:30:39 +0000 Subject: [PATCH 16/19] Remove is_input from image_tensor_specs.h --- .../cc/task/vision/utils/image_tensor_specs.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h index b4a79b324..390b80caf 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h @@ -83,9 +83,8 @@ struct ImageTensorSpecs { // - `tflite::InterpreterBuilder`, // - `tflite::metadata::ModelMetadataExtractor::CreateFromModelBuffer`. tflite::support::StatusOr BuildImageTensorSpecs( - const tflite::task::core::TfLiteEngine::Interpreter& interpreter, - const tflite::metadata::ModelMetadataExtractor& metadata_extractor, - const bool is_input); + const ModelMetadataExtractor& metadata_extractor, + const TensorMetadata* tensor_metadata, const TfLiteTensor* tensor); } // namespace vision } // namespace task From 63846f8381fed3ed16ae0d223356d80c4d62e1f1 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 07:03:17 +0000 Subject: [PATCH 17/19] update BuildImageTensorSpec for preprocessor. --- .../cc/task/processor/image_preprocessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc b/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc index 57b12941a..7e433395f 100644 --- a/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc +++ b/tensorflow_lite_support/cc/task/processor/image_preprocessor.cc @@ -73,8 +73,8 @@ absl::Status ImagePreprocessor::Init( frame_buffer_utils_ = vision::FrameBufferUtils::Create(process_engine); ASSIGN_OR_RETURN(input_specs_, vision::BuildImageTensorSpecs( - *engine_->interpreter(), - *engine_->metadata_extractor(), true)); + *engine_->metadata_extractor(), + GetTensorMetadata(), GetTensor())); if (input_specs_.color_space != tflite::ColorSpaceType_RGB) { return tflite::support::CreateStatusWithPayload( From 8d9dc30e332fa9c6930c16460a61c8cb5d581f34 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 07:15:56 +0000 Subject: [PATCH 18/19] Fix compile error --- .../cc/task/vision/utils/image_tensor_specs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h index 390b80caf..81e6bb202 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.h @@ -83,7 +83,7 @@ struct ImageTensorSpecs { // - `tflite::InterpreterBuilder`, // - `tflite::metadata::ModelMetadataExtractor::CreateFromModelBuffer`. tflite::support::StatusOr BuildImageTensorSpecs( - const ModelMetadataExtractor& metadata_extractor, + const tflite::metadata::ModelMetadataExtractor& metadata_extractor, const TensorMetadata* tensor_metadata, const TfLiteTensor* tensor); } // namespace vision From da766a495a14f46b2a0f628e30af63a4d3ead713 Mon Sep 17 00:00:00 2001 From: Nanubala Gnana Sai <45007169+jonpsy@users.noreply.github.com> Date: Thu, 23 Dec 2021 08:06:56 +0000 Subject: [PATCH 19/19] Fix namespace --- .../cc/task/vision/utils/image_tensor_specs.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc index 1a23b7747..92ff74ac8 100644 --- a/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc +++ b/tensorflow_lite_support/cc/task/vision/utils/image_tensor_specs.cc @@ -55,7 +55,6 @@ StatusOr GetTensorMetadataIfAny( } return tensor_metadata; } -} // namespace StatusOr GetImagePropertiesIfAny( const TensorMetadata& tensor_metadata) { @@ -126,12 +125,15 @@ StatusOr> GetNormalizationOptionsIfAny( return normalization_options; } +} // namespace + StatusOr BuildImageTensorSpecs( const ModelMetadataExtractor& metadata_extractor, const TensorMetadata* tensor_metadata, const TfLiteTensor* tensor) { const ImageProperties* props = nullptr; absl::optional normalization_options; - ASSIGN_OR_RETURN(auto metadata, GetTensorMetadataIfAny(metadata_extractor, tensor_metadata)); + ASSIGN_OR_RETURN(const TensorMetadata* metadata, + GetTensorMetadataIfAny(metadata_extractor, tensor_metadata)); if (metadata != nullptr) { ASSIGN_OR_RETURN(props, GetImagePropertiesIfAny(*metadata)); ASSIGN_OR_RETURN(normalization_options, @@ -211,7 +213,8 @@ StatusOr BuildImageTensorSpecs( if (bytes_size != height * width * depth * byte_depth) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, - "The tensor in bytes does not correspond to the expected number of " + "The tensor size in bytes does not correspond to the expected number " + "of " "pixels.", TfLiteSupportStatus::kInvalidInputTensorSizeError); }