Merge branch 'master' into r0.4.0-rc0

Author: schmidt-sebastian

tensorflow_lite_support/c/task/processor/classification_result.cc

@@ -27,6 +27,7 @@ void TfLiteClassificationResultDelete(
   for (int head = 0; head < classification_result->size; ++head) {
     TfLiteClassifications classifications =
         classification_result->classifications[head];
+        free(classifications.head_name);
     for (int rank = 0; rank < classifications.size; ++rank) {
       TfLiteCategoryDelete(&(classifications.categories[rank]));
     }

tensorflow_lite_support/examples/colab/on_device_text_to_image_search_tflite.ipynb

@@ -134,6 +134,7 @@ cc_binary(
     name = "text_searcher_demo",
     srcs = ["text_searcher_demo.cc"],
     deps = [
+        ":universal_sentence_encoder_qa_op_resolver",
         "//tensorflow_lite_support/cc/port:configuration_proto_inc",
         "//tensorflow_lite_support/cc/port:status_macros",
         "//tensorflow_lite_support/cc/task/core/proto:base_options_cc_proto",
@@ -161,6 +162,7 @@ cc_binary(
     name = "text_embedder_demo",
     srcs = ["text_embedder_demo.cc"],
     deps = [
+        ":universal_sentence_encoder_qa_op_resolver",
         "//tensorflow_lite_support/cc/port:configuration_proto_inc",
         "//tensorflow_lite_support/cc/port:status_macros",
         "//tensorflow_lite_support/cc/task/core/proto:base_options_cc_proto",

tensorflow_lite_support/examples/task/text/desktop/BUILD

@@ -162,9 +162,111 @@ Output answers 2: He looks good. Score: 8.80944
 Output answers 1: Paris is the capital of France. Score: 5.63752
 ```
 
+## TextEmbedder
+
+#### Prerequisites
+
+You will need:
+
+*   a TFLite text embedder model such as the universal sentence encoder QA model
+    from [TensorFlow Hub][6].
+
+#### Usage
+
+The TextEmbedder demo tool takes two sentences as inputs, and outputs the
+[cosine similarity][7] between their embeddings.
+
+In the console, run:
+
+```bash
+# Download the model:
+curl \
+ -L 'https://tfhub.dev/google/lite-model/universal-sentence-encoder-qa-ondevice/1?lite-format=tflite' \
+ -o /tmp/universal_sentence_encoder_qa_with_metadata.tflite
+
+# Run the embedder tool:
+bazel run -c opt \
+tensorflow_lite_support/examples/task/text/desktop:text_embedder_demo -- \
+--model_path=/tmp/universal_sentence_encoder_qa_with_metadata.tflite \
+--l2_normalize \
+--first_sentence="It was a very sunny day." \
+--second_sentence="The sun was shining brightly."
+```
+
+#### Results
+
+In the console, you should get:
+
+```
+Cosine similarity: 0.952549
+```
+
+## TextSearcher
+
+#### Prerequisites
+
+TODO(b/231294362): update link to Model Maker.
+
+You will need:
+
+*   a TFLite text embedder model such as the universal sentence encoder QA model
+    from [TensorFlow Hub][6],
+*   an index built from that embedder model using [Model Maker][8].
+
+Model Maker also provides the ability to add the index directly to the embedder
+model metadata. The demo also supports this : just omit the `--index_path`
+argument.
+
+#### Usage
+
+In this example, we'll be using a test index built from the universal sentence
+encoder QA model, which only contains 5 embeddings.
+
+In the console, run:
+
+```bash
+# Download the model:
+curl \
+ -L 'https://tfhub.dev/google/lite-model/universal-sentence-encoder-qa-ondevice/1?lite-format=tflite' \
+ -o /tmp/universal_sentence_encoder_qa_with_metadata.tflite
+
+# Run the searcher tool:
+bazel run -c opt \
+tensorflow_lite_support/examples/task/text/desktop:text_searcher_demo -- \
+--model_path=/tmp/universal_sentence_encoder_qa_with_metadata.tflite \
+--l2_normalize \
+--index_path=$(pwd)/third_party/tensorflow_lite_support/cc/test/testdata/task/text/universal_sentence_encoder_index.ldb \
+--input_sentence="The sun was very bright."
+```
+
+#### Results
+
+In the console, you should get:
+
+```
+Results:
+ Rank#0:
+  metadata: The sun was shining on that day.
+  distance: 0.04618
+ Rank#1:
+  metadata: It was a sunny day.
+  distance: 0.10856
+ Rank#2:
+  metadata: The weather was excellent.
+  distance: 0.15223
+ Rank#3:
+  metadata: The cat is chasing after the mouse.
+  distance: 0.34271
+ Rank#4:
+  metadata: He was very happy with his newly bought car.
+  distance: 0.37703
+```
+
 [1]: https://tfhub.dev/tensorflow/lite-model/mobilebert/1/default/1
 [2]: https://tfhub.dev/tensorflow/lite-model/albert_lite_base/squadv1/1
 [3]: https://www.tensorflow.org/lite/models/text_classification/overview
 [4]: https://github.com/tensorflow/tflite-support/blob/fe8b69002f5416900285dc69e2baa078c91bd994/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.h#L55
 [5]: http://bert/nl/classifier/model
 [6]: https://tfhub.dev/google/lite-model/universal-sentence-encoder-qa-ondevice/1
+[7]: https://en.wikipedia.org/wiki/Cosine_similarity
+[8]: https://www.tensorflow.org/lite/guide/model_maker

tensorflow_lite_support/examples/task/text/desktop/README.md

@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow_lite_support/cc/task/processor/proto/embedding.pb.h"
 #include "tensorflow_lite_support/cc/task/text/proto/text_embedder_options.pb.h"
 #include "tensorflow_lite_support/cc/task/text/text_embedder.h"
+#include "tensorflow_lite_support/examples/task/text/desktop/universal_sentence_encoder_qa_op_resolver.h"
 
 ABSL_FLAG(std::string, model_path, "",
           "Absolute path to the '.tflite' text embedder model.");
@@ -79,8 +80,9 @@ TextEmbedderOptions BuildOptions() {
 absl::Status ComputeCosineSimilarity() {
   // Build TextEmbedder.
   const TextEmbedderOptions options = BuildOptions();
-  ASSIGN_OR_RETURN(std::unique_ptr<TextEmbedder> text_embedder,
-                   TextEmbedder::CreateFromOptions(options));
+  ASSIGN_OR_RETURN(
+      std::unique_ptr<TextEmbedder> text_embedder,
+      TextEmbedder::CreateFromOptions(options, CreateQACustomOpResolver()));
 
   // Run search and display results.
   auto start_embed = steady_clock::now();

tensorflow_lite_support/examples/task/text/desktop/text_embedder_demo.cc

@@ -37,11 +37,14 @@ limitations under the License.
 #include "tensorflow_lite_support/cc/task/processor/proto/search_result.pb.h"
 #include "tensorflow_lite_support/cc/task/text/proto/text_searcher_options.pb.h"
 #include "tensorflow_lite_support/cc/task/text/text_searcher.h"
+#include "tensorflow_lite_support/examples/task/text/desktop/universal_sentence_encoder_qa_op_resolver.h"
 
 ABSL_FLAG(std::string, model_path, "",
           "Absolute path to the '.tflite' text embedder model.");
 ABSL_FLAG(std::string, index_path, "",
-          "Absolute path to the index to search into.");
+          "Absolute path to the index to search into. Mandatory only if the "
+          "index is not attached to the output tensor metadata of the embedder "
+          "model as an AssociatedFile with type SCANN_INDEX_FILE.");
 ABSL_FLAG(std::string, input_sentence, "",
           "Input sentence whose nearest-neighbors to search for in the index.");
 ABSL_FLAG(int32, max_results, 5,
@@ -70,8 +73,10 @@ TextSearcherOptions BuildOptions() {
   if (absl::GetFlag(FLAGS_l2_normalize)) {
     options.mutable_embedding_options()->set_l2_normalize(true);
   }
-  options.mutable_search_options()->mutable_index_file()->set_file_name(
-      absl::GetFlag(FLAGS_index_path));
+  if (!absl::GetFlag(FLAGS_index_path).empty()) {
+    options.mutable_search_options()->mutable_index_file()->set_file_name(
+        absl::GetFlag(FLAGS_index_path));
+  }
   options.mutable_search_options()->set_max_results(
       absl::GetFlag(FLAGS_max_results));
   if (absl::GetFlag(FLAGS_use_coral)) {
@@ -96,8 +101,9 @@ void DisplayResults(const processor::SearchResult& result) {
 absl::Status Search() {
   // Build TextSearcher.
   const TextSearcherOptions options = BuildOptions();
-  ASSIGN_OR_RETURN(std::unique_ptr<TextSearcher> text_searcher,
-                   TextSearcher::CreateFromOptions(options));
+  ASSIGN_OR_RETURN(
+      std::unique_ptr<TextSearcher> text_searcher,
+      TextSearcher::CreateFromOptions(options, CreateQACustomOpResolver()));
 
   // Run search and display results.
   auto start_search = steady_clock::now();
@@ -128,10 +134,6 @@ int main(int argc, char** argv) {
     std::cerr << "Missing mandatory 'model_path' argument.\n";
     return 1;
   }
-  if (absl::GetFlag(FLAGS_index_path).empty()) {
-    std::cerr << "Missing mandatory 'index_path' argument.\n";
-    return 1;
-  }
   if (absl::GetFlag(FLAGS_input_sentence).empty()) {
     std::cerr << "Missing mandatory 'input_sentence' argument.\n";
     return 1;

tensorflow_lite_support/examples/task/text/desktop/text_searcher_demo.cc

@@ -259,7 +259,112 @@ And `/tmp/segmentation-output.jpg` should contain the segmentation mask:
 
 ![segmentation-output](g3doc/segmentation-output.png)
 
+## ImageEmbedder
+
+#### Prerequisites
+
+You will need:
+
+*   a TFLite image embedder model (e.g. [mobilenet v3][5], a generic image
+    embedder trained on ImageNet),
+*   two PNG, JPEG or GIF image to extract embeddings from.
+
+#### Usage
+
+The ImageEmbedder demo tool takes two images as inputs, and outputs the
+[cosine similarity][6] between their embeddings.
+
+In the console, run:
+
+```bash
+# Download the model:
+curl \
+ -L 'https://tfhub.dev/google/lite-model/imagenet/mobilenet_v3_small_100_224/feature_vector/5/metadata/1?lite-format=tflite' \
+ -o /tmp/mobilenet_v3_embedder.tflite
+
+# Run the embedder tool:
+bazel run -c opt \
+tensorflow_lite_support/examples/task/vision/desktop:image_embedder_demo -- \
+--model_path=/tmp/mobilenet_v3_embedder.tflite \
+--l2_normalize \
+--first_image_path=$(pwd)/tensorflow_lite_support/cc/test/testdata/task/vision/burger.jpg \
+--second_image_path=$(pwd)/tensorflow_lite_support/cc/test/testdata/task/vision/burger_crop.jpg
+```
+
+#### Results
+
+In the console, you should get:
+
+```
+Cosine similarity: 0.932738
+```
+
+## ImageSearcher
+
+#### Prerequisites
+
+TODO(b/231294362): update link to Model Maker.
+
+You will need:
+
+*   a TFLite image embedder model (e.g. [mobilenet v3][5], a generic image
+    embedder trained on ImageNet),
+*   an index built from that embedder model using [Model Maker][7].
+
+Model Maker also provides the ability to add the index directly to the embedder
+model metadata. The demo also supports this : just omit the `--index_path`
+argument.
+
+#### Usage
+
+In this example, we'll be using a test index built from the mobilenet v3
+embedder model, which only contains 5 embeddings extracted from images of a
+burger, a cat, a dog, a bird and a car.
+
+In the console, run:
+
+```bash
+# Download the model:
+curl \
+ -L 'https://tfhub.dev/google/lite-model/imagenet/mobilenet_v3_small_100_224/feature_vector/5/metadata/1?lite-format=tflite' \
+ -o /tmp/mobilenet_v3_embedder.tflite
+
+# Run the searcher tool:
+bazel run -c opt \
+tensorflow_lite_support/examples/task/vision/desktop:image_searcher_demo -- \
+--model_path=/tmp/mobilenet_v3_embedder.tflite \
+--l2_normalize \
+--index_path=$(pwd)/third_party/tensorflow_lite_support/cc/test/testdata/task/vision/searcher_index.ldb \
+--image_path=$(pwd)/third_party/tensorflow_lite_support/cc/test/testdata/task/vision/burger_crop.jpg
+```
+
+#### Results
+
+In the console, you should get:
+
+```
+Results:
+ Rank#0:
+  metadata: burger
+  distance: 0.13452
+ Rank#1:
+  metadata: car
+  distance: 1.81935
+ Rank#2:
+  metadata: bird
+  distance: 1.96617
+ Rank#3:
+  metadata: dog
+  distance: 2.05610
+ Rank#4:
+  metadata: cat
+  distance: 2.06347
+```
+
 [1]: https://tfhub.dev/google/lite-model/aiy/vision/classifier/birds_V1/3
 [2]: https://tfhub.dev/tensorflow/lite-model/ssd_mobilenet_v1/1/metadata/2
 [3]: https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/metadata/2
 [4]: https://coral.ai/docs/edgetpu/inference/
+[5]: https://tfhub.dev/google/lite-model/imagenet/mobilenet_v3_small_100_224/feature_vector/5/metadata/1
+[6]: https://en.wikipedia.org/wiki/Cosine_similarity
+[7]: https://www.tensorflow.org/lite/guide/model_maker

tensorflow_lite_support/examples/task/vision/desktop/README.md

@@ -45,7 +45,9 @@ limitations under the License.
 ABSL_FLAG(std::string, model_path, "",
           "Absolute path to the '.tflite' image embedder model.");
 ABSL_FLAG(std::string, index_path, "",
-          "Absolute path to the index to search into.");
+          "Absolute path to the index to search into. Mandatory only if the "
+          "index is not attached to the output tensor metadata of the embedder "
+          "model as an AssociatedFile with type SCANN_INDEX_FILE.");
 ABSL_FLAG(std::string, image_path, "",
           "Absolute path to the image to search. The image must be RGB or "
           "RGBA (grayscale is not supported). The image EXIF orientation "
@@ -76,8 +78,10 @@ ImageSearcherOptions BuildOptions() {
   if (absl::GetFlag(FLAGS_l2_normalize)) {
     options.mutable_embedding_options()->set_l2_normalize(true);
   }
-  options.mutable_search_options()->mutable_index_file()->set_file_name(
-      absl::GetFlag(FLAGS_index_path));
+  if (!absl::GetFlag(FLAGS_index_path).empty()) {
+    options.mutable_search_options()->mutable_index_file()->set_file_name(
+        absl::GetFlag(FLAGS_index_path));
+  }
   options.mutable_search_options()->set_max_results(
       absl::GetFlag(FLAGS_max_results));
   if (absl::GetFlag(FLAGS_use_coral)) {
@@ -152,10 +156,6 @@ int main(int argc, char** argv) {
     std::cerr << "Missing mandatory 'model_path' argument.\n";
     return 1;
   }
-  if (absl::GetFlag(FLAGS_index_path).empty()) {
-    std::cerr << "Missing mandatory 'index_path' argument.\n";
-    return 1;
-  }
   if (absl::GetFlag(FLAGS_image_path).empty()) {
     std::cerr << "Missing mandatory 'image_path' argument.\n";
     return 1;

tensorflow_lite_support/examples/task/vision/desktop/image_searcher_demo.cc

@@ -34,19 +34,18 @@ NS_SWIFT_NAME(ClassificationCategory)
 @property(nonatomic, readonly, nullable) NSString *displayName;
 
 /**
- * Initializes TFLCategory.
+ * Initializes a new `TFLCategory` with the given index, score, label and display name.
  *
  * @param index Index of the class in the corresponding label map, usually packed in the TFLite
  * Model Metadata.
  *
- * @param score Confidence score for this class .
+ * @param score Confidence score for this class.
  *
  * @param label Class name of the class.
  *
  * @param displayName Display name of the class.
  *
- * @return An instance of TFLCategory initialized to
- * the specified values.
+ * @return An instance of `TFLCategory` initialized with the given index, score, label and display name.
  */
 - (instancetype)initWithIndex:(NSInteger)index
                         score:(float)score

tensorflow_lite_support/ios/task/processor/sources/TFLCategory.h

@@ -20,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN
 @interface TFLClassificationResult (Helpers)
 
 /**
- * Creates and retrurns a TFLClassificationResult from a TfLiteClassificationResult returned by
+ * Creates and returns a TFLClassificationResult from a TfLiteClassificationResult returned by
  * TFLite Task C Library Classification tasks.
  *
  * @param cClassificationResult Classification results returned by TFLite Task C Library