[Dvaas] Streamline error message now that we have DVaaS Detective.

WORKSPACE.bazel

@@ -119,6 +119,15 @@ load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 
 rules_pkg_dependencies()
 
+# -- Load Yggdrasil Decision Forests -------------------------------------------
+
+load("@com_google_ydf//yggdrasil_decision_forests:library.bzl", ydf_load_deps = "load_dependencies")
+
+ydf_load_deps(
+    exclude_repo = "tensorflow",
+    repo_name = "@com_google_ydf",
+)
+
 # == Dependencies needed for testing and formatting only =======================
 
 # -- Load p4c ------------------------------------------------------------------

dvaas/BUILD.bazel

@@ -51,6 +51,7 @@ cc_library(
         "//p4_pdpi:p4_runtime_session_extras",
         "//p4_pdpi:sequencing",
         "//p4_pdpi/packetlib:packetlib_cc_proto",
+        "//p4_pdpi/string_encodings:hex_string",
         "//p4_symbolic/packet_synthesizer:coverage_goal_cc_proto",
         "//p4_symbolic/packet_synthesizer:packet_synthesizer_cc_proto",
         "//sai_p4/instantiations/google:p4_versions",
@@ -71,6 +72,86 @@ cc_library(
     ],
 )
 
+proto_library(
+    name = "dvaas_detective_proto",
+    srcs = ["dvaas_detective.proto"],
+)
+
+cc_proto_library(
+    name = "dvaas_detective_cc_proto",
+    deps = [":dvaas_detective_proto"],
+)
+
+cc_library(
+    name = "dvaas_detective",
+    srcs = ["dvaas_detective.cc"],
+    hdrs = ["dvaas_detective.h"],
+    deps = [
+        ":dvaas_detective_cc_proto",
+        ":test_vector_cc_proto",
+        "//gutil:overload",
+        "//gutil:proto",
+        "//p4_pdpi/packetlib:packetlib_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:variant",
+        "@com_google_ydf//yggdrasil_decision_forests/dataset:csv_example_reader",
+        "@com_google_ydf//yggdrasil_decision_forests/dataset:data_spec_cc_proto",
+        "@com_google_ydf//yggdrasil_decision_forests/dataset:data_spec_inference",
+        "@com_google_ydf//yggdrasil_decision_forests/learner:abstract_learner",
+        "@com_google_ydf//yggdrasil_decision_forests/learner:learner_library",
+        "@com_google_ydf//yggdrasil_decision_forests/learner/cart",
+        "@com_google_ydf//yggdrasil_decision_forests/learner/cart:cart_cc_proto",
+        "@com_google_ydf//yggdrasil_decision_forests/model:abstract_model",
+        "@com_google_ydf//yggdrasil_decision_forests/model/decision_tree",
+        "@com_google_ydf//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto",
+        "@com_google_ydf//yggdrasil_decision_forests/model/random_forest",
+        "@com_google_ydf//yggdrasil_decision_forests/utils:cast",
+        "@com_google_ydf//yggdrasil_decision_forests/utils:distribution_cc_proto",
+    ],
+)
+
+# go/golden-test-with-coverage
+cc_test(
+    name = "dvaas_detective_test",
+    srcs = ["dvaas_detective_test.cc"],
+    linkstatic = True,
+    deps = [
+        ":dvaas_detective",
+        ":dvaas_detective_cc_proto",
+        ":test_vector_cc_proto",
+        "//gutil:proto",
+        "//gutil:status",
+        "//gutil:status_matchers",
+        "//gutil:testing",
+        "//p4_pdpi/packetlib:packetlib_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/status",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_ydf//yggdrasil_decision_forests/dataset:data_spec_cc_proto",
+        "@com_google_ydf//yggdrasil_decision_forests/model/decision_tree",
+        "@com_google_ydf//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto",
+        "@com_google_ydf//yggdrasil_decision_forests/model/random_forest",
+    ],
+)
+
+cmd_diff_test(
+    name = "dvaas_detective_diff_test",
+    actual_cmd = " | ".join([
+        "$(execpath :dvaas_detective_test)",
+        # Strip unnecessary lines for golden testing.
+        "sed '1,/^\\[ RUN/d'",  # Strip everything up to a line beginning with '[ RUN'.
+        "sed '/^\\[/d'",  # Strip every line beginning with '['.
+    ]),
+    expected = "dvaas_detective_test.expected",
+    tools = [":dvaas_detective_test"],
+)
+
 cc_test(
     name = "dataplane_validation_test",
     srcs = ["dataplane_validation_test.cc"],
@@ -83,6 +164,7 @@ cc_test(
         "//gutil:test_artifact_writer",
         "//gutil:testing",
         "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/log",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:string_view",
         "@com_google_googletest//:gtest_main",
@@ -106,6 +188,7 @@ cc_library(
     srcs = ["validation_result.cc"],
     hdrs = ["validation_result.h"],
     deps = [
+        ":dvaas_detective",
         ":test_run_validation",
         ":test_vector_cc_proto",
         ":test_vector_stats",
@@ -115,6 +198,7 @@ cc_library(
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
     ],
 )
@@ -143,6 +227,7 @@ cc_library(
         "//p4_pdpi:p4_runtime_session",
         "//p4_pdpi/packetlib",
         "//p4_pdpi/packetlib:packetlib_cc_proto",
+	"//p4_pdpi/utils:ir",
         "//tests/forwarding:util",
         "@com_github_google_glog//:glog",
         "@com_github_p4lang_p4runtime//:p4runtime_cc_proto",
@@ -168,6 +253,8 @@ cc_library(
         "@com_github_google_glog//:glog",
         "@com_google_absl//absl/base:nullability",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/log",
+        "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
@@ -185,19 +272,23 @@ cc_library(
     deps = [
         ":packet_injection",
         ":port_id_map",
+	":test_insights",
         ":test_run_validation",
         ":test_vector",
         ":test_vector_cc_proto",
         ":validation_result",
         "//gutil:proto",
         "//gutil:test_artifact_writer",
+        "//lib/p4rt:p4rt_port",
         "//p4_pdpi:ir",
         "//p4_pdpi:ir_cc_proto",
         "//p4_pdpi:p4_runtime_session",
         "//p4_pdpi:p4_runtime_session_extras",
         "//sai_p4/instantiations/google/test_tools:test_entries",
-        "@com_github_google_glog//:glog",
+        "//tests/lib:switch_test_setup_helpers",
         "@com_github_p4lang_p4runtime//:p4runtime_cc_proto",
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/log",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
@@ -525,6 +616,7 @@ cc_test(
         "//gutil:testing",
         "//p4_pdpi/packetlib:packetlib_cc_proto",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:string_view",
         "@com_google_googletest//:gtest_main",
         "@com_google_protobuf//:protobuf",
     ],

dvaas/arriba_test_vector_validation.cc

@@ -14,13 +14,17 @@
 
 #include "dvaas/arriba_test_vector_validation.h"
 
+#include <string>
 #include <utility>
 #include <vector>
 
+#include "absl/container/btree_set.h"
+#include "absl/log/log.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
 #include "dvaas/packet_injection.h"
 #include "dvaas/port_id_map.h"
+#include "dvaas/test_insights.h"
 #include "dvaas/test_run_validation.h"
 #include "dvaas/test_vector.h"
 #include "dvaas/test_vector.pb.h"
@@ -29,15 +33,34 @@
 #include "gutil/proto.h"
 #include "gutil/status.h"
 #include "gutil/test_artifact_writer.h"
+#include "lib/p4rt/p4rt_port.h"
 #include "p4/v1/p4runtime.pb.h"
 #include "p4_pdpi/ir.h"
 #include "p4_pdpi/ir.pb.h"
 #include "p4_pdpi/p4_runtime_session.h"
 #include "p4_pdpi/p4_runtime_session_extras.h"
 #include "sai_p4/instantiations/google/test_tools/test_entries.h"
+#include "tests/lib/switch_test_setup_helpers.h"
 
 namespace dvaas {
 
+absl::StatusOr<absl::btree_set<pins_test::P4rtPortId>> GetUsedP4rtPortIds(
+    const ArribaTestVector& arriba_test_vector,
+    const std::vector<pdpi::IrTableEntry>& used_entries_list,
+    const pdpi::IrP4Info& ir_p4_info) {
+  ASSIGN_OR_RETURN(absl::btree_set<pins_test::P4rtPortId> used_p4rt_port_ids,
+                   pins_test::GetPortsUsed(ir_p4_info, used_entries_list));
+
+  for (const auto& [_, packet_test_vector] :
+       arriba_test_vector.packet_test_vector_by_id()) {
+    ASSIGN_OR_RETURN(pins_test::P4rtPortId p4rt_port_id,
+                     pins_test::P4rtPortId::MakeFromP4rtEncoding(
+                         packet_test_vector.input().packet().port()));
+    used_p4rt_port_ids.insert(p4rt_port_id);
+  }
+  return used_p4rt_port_ids;
+}
+
 absl::StatusOr<ValidationResult> ValidateAgainstArribaTestVector(
     pdpi::P4RuntimeSession& sut, pdpi::P4RuntimeSession& control_switch,
     const ArribaTestVector& arriba_test_vector,
@@ -64,9 +87,11 @@ absl::StatusOr<ValidationResult> ValidateAgainstArribaTestVector(
 
   // Prepare single packet test vectors.
   PacketTestVectorById test_vector_by_id;
+  std::vector<PacketTestVector> packet_test_vectors;
   for (const auto& [id, packet_test_vector] :
        arriba_test_vector.packet_test_vector_by_id()) {
     test_vector_by_id[id] = packet_test_vector;
+    packet_test_vectors.push_back(packet_test_vector);
   }
 
   PacketStatistics packet_statistics;
@@ -112,6 +137,12 @@ absl::StatusOr<ValidationResult> ValidateAgainstArribaTestVector(
   RETURN_IF_ERROR(artifact_writer.AppendToTestArtifact(
       "test_outcomes.txtpb", gutil::PrintTextProto(test_outcomes)));
 
+  // Store test insights.
+  ASSIGN_OR_RETURN(const std::string insights_csv,
+                   GetTestInsightsTableAsCsv(test_outcomes, ir_p4info));
+  RETURN_IF_ERROR(
+      artifact_writer.AppendToTestArtifact("test_insights.csv", insights_csv));
+
   ValidationResult validation_result(std::move(test_outcomes),
                                      /*packet_synthesis_result=*/{});
   RETURN_IF_ERROR(artifact_writer.AppendToTestArtifact(

dvaas/arriba_test_vector_validation.h

@@ -16,12 +16,15 @@
 #define PINS_DVAAS_ARRIBA_TEST_VECTOR_VALIDATION_H_
 
 #include <optional>
+#include <vector>
 
+#include "absl/container/btree_set.h"
 #include "absl/status/statusor.h"
 #include "dvaas/packet_injection.h"
 #include "dvaas/test_run_validation.h"
 #include "dvaas/test_vector.pb.h"
 #include "dvaas/validation_result.h"
+#include "lib/p4rt/p4rt_port.h"
 #include "p4_pdpi/p4_runtime_session.h"
 
 namespace dvaas {
@@ -54,11 +57,18 @@ struct ArribaTestVectorValidationParams {
       DefaultIsExpectedUnsolicitedPacket;
 };
 
+// Retrieves the set of P4RT ports used in the given `arriba_test_vector` (table
+// entries and test packet).
+absl::StatusOr<absl::btree_set<pins_test::P4rtPortId>> GetUsedP4rtPortIds(
+    const ArribaTestVector& arriba_test_vector,
+    const std::vector<pdpi::IrTableEntry>& used_entries_list,
+    const pdpi::IrP4Info& ir_p4_info);
+
 // Validates the `sut` in the provided mirror testbed (`sut` and
 // `control_switch`) against the given `arriba_test_vector`. It does so by
 // installing the entries in the test vector (on SUT), injecting the input
-// packets, collecting the output packets, and comparing the results with the
-// expected outputs for each input packet.
+// packets, collecting the output packets, and comparing the results with
+// the expected outputs for each input packet.
 //
 // Pre-condition:
 //   - The same P4Info and gNMI configs used in the generation of the given