Bring depthai_nodes into depthai-core

CMakeLists.txt

@@ -1014,6 +1014,53 @@ if(DEPTHAI_BASALT_SUPPORT)
     endif()
 endif()
 
+########################
+# Extra host nodes
+########################
+set(TARGET_EXTRA_ALIAS hostNodesExt)
+set(TARGET_EXTRA_NAME ${PROJECT_NAME}-${TARGET_EXTRA_ALIAS})
+if (DEPTHAI_BUILD_EXT_HOST_NODES)
+    set(TARGET_EXT_NODES_SOURCES
+            ParserGenerator.cpp
+            host_nodes_ext/ParsingNeuralNetwork.cpp
+            host_nodes_ext/parsers/BaseParser.cpp
+            host_nodes_ext/parsers/SimCCKeypointParser.cpp
+            host_nodes_ext/parsers/KeypointParser.cpp
+            host_nodes_ext/messages/Keypoints.cpp
+    )
+    list(TRANSFORM TARGET_EXT_NODES_SOURCES PREPEND "src/pipeline/node/host/contrib/")
+#
+
+    # Add depthai-extraHostNodes library and importable target/alias depthai::extraHostNodes
+    add_library(${TARGET_EXTRA_NAME} ${TARGET_EXT_NODES_SOURCES})
+    add_library("${PROJECT_NAME}::${TARGET_EXTRA_ALIAS}" ALIAS ${TARGET_EXTRA_NAME})
+    set_target_properties(${TARGET_EXTRA_NAME} PROPERTIES EXPORT_NAME ${TARGET_EXTRA_ALIAS})
+
+
+    target_include_directories(${TARGET_EXTRA_NAME}
+            PUBLIC
+            "$<INSTALL_INTERFACE:include/depthai/pipeline/node/host/contrib/host_nodes_ext>"
+
+#            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}>"
+#            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}host/>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai/pipeline/node/host/contrib/host_nodes_ext>"
+            PRIVATE
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src/pipeline/node/host/contrib/host_nodes_ext>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src>"
+    )
+
+    target_link_libraries(${TARGET_EXTRA_NAME}
+            PRIVATE
+            spdlog::spdlog
+            xtensor
+            PUBLIC
+            depthai::core
+    )
+
+    list(APPEND targets_to_export ${TARGET_EXTRA_NAME})
+endif ()
+
 ########################
 # Combined target
 ########################

cmake/depthaiOptions.cmake

@@ -34,6 +34,7 @@ option(DEPTHAI_BUILD_TESTS "Build tests" OFF)
 option(DEPTHAI_BUILD_EXAMPLES "Build examples - Requires OpenCV library to be installed" OFF)
 option(DEPTHAI_BUILD_DOCS "Build documentation - requires doxygen to be installed" OFF)
 option(DEPTHAI_BUILD_ZOO_HELPER "Build the Zoo helper" OFF)
+option(DEPTHAI_BUILD_EXT_HOST_NODES "Build the contrib host nodes lib" OFF)
 option(DEPTHAI_NEW_FIND_PYTHON "Use new FindPython module" ON)
 option(DEPTHAI_INSTALL "Enable install target for depthai-core targets" ON)
 

include/depthai/pipeline/node/host/contrib/ParserGenerator.hpp

@@ -0,0 +1,35 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+
+#pragma once
+#include <variant>
+
+#include "depthai/depthai.hpp"
+#include "parsers/BaseParser.hpp"
+
+namespace dai::node {
+
+typedef std::variant<std::shared_ptr<BaseParser>, std::shared_ptr<DetectionParser>> HostOrDeviceParser;
+template<typename V>
+constexpr bool all_alternatives_shared_ptr = false;
+template<typename... Ts>
+constexpr bool all_alternatives_shared_ptr<std::variant<Ts...>> =
+    (std::conjunction_v<std::is_same<std::shared_ptr<typename Ts::element_type>, Ts>...>);
+static_assert(all_alternatives_shared_ptr<HostOrDeviceParser>, "All alternatives must be std::shared_ptr<T>");
+
+struct ConfigModelWithHeads {
+    nn_archive::v1::Model model;
+    std::vector<nn_archive::v1::Head> heads;
+};
+
+class ParserGenerator {
+   public:
+    static std::vector<HostOrDeviceParser> generateAllParsers(Pipeline pipeline, const NNArchive& nnArchive, bool hostOnly = false);
+
+   private:
+    static ConfigModelWithHeads archiveGetModelEnsureOneHeadV1(const NNArchive& nnArchive, Platform targetPlatform);
+    static HostOrDeviceParser generateOneV1Parser(
+        Pipeline& pipeline, const NNArchive& owningArchive, const nn_archive::v1::Head& head, const nn_archive::v1::Model& model, bool hostOnly = false);
+};
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/ParsingNeuralNetwork.hpp

@@ -0,0 +1,50 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+#pragma once
+#include <variant>
+#include <vector>
+
+#include "../ParserGenerator.hpp"
+#include "depthai/depthai.hpp"
+#include "parsers/BaseParser.hpp"
+#include "parsers/KeypointParser.hpp"
+
+namespace dai::node {
+
+class ParsingNeuralNetwork : public CustomThreadedNode<ParsingNeuralNetwork> {
+   public:
+    std::shared_ptr<ParsingNeuralNetwork> build(Output& input, const NNArchive& nnArchive);
+    std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, NNModelDescription modelDesc, std::optional<float> fps = std::nullopt);
+    std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, const NNArchive& nnArchive, std::optional<float> fps = std::nullopt);
+
+    InputMap& inputs = nn->inputs;
+    Input& input = nn->input;
+    std::optional<std::reference_wrapper<Output>> out = std::nullopt;
+    Output& passthrough = nn->passthrough;
+    OutputMap& passthroughs = nn->passthroughs;
+
+    template <typename T>
+    std::optional<size_t> getIndexOfFirstParserOfType() const {
+        const auto which = std::find_if(parsers.begin(), parsers.end(), [](const auto& p) {
+            return std::visit([](auto& anyP) { return std::dynamic_pointer_cast<T>(anyP) != nullptr; }, p);;
+        });
+        return which == parsers.end() ? std::nullopt : static_cast<std::optional<size_t>>(std::distance(parsers.begin(), which));
+    }
+
+    void run() override;
+
+   private:
+    std::vector<HostOrDeviceParser> getParserNodes(const NNArchive& nnArchive);
+
+    void updateParsers(const NNArchive& nnArchive);
+
+    void removeOldParserNodes();
+    Subnode<NeuralNetwork> nn{*this, "nn"};
+    std::optional<Subnode<Sync>> parserSync = std::nullopt;
+
+   protected:
+    std::vector<HostOrDeviceParser> parsers;
+};
+
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/messages/Keypoints.hpp

@@ -0,0 +1,56 @@
+//
+// Created by thwdpc on 7/28/25.
+//
+
+#pragma once
+#include <depthai/common/ImgTransformations.hpp>
+#include <depthai/pipeline/datatype/Buffer.hpp>
+#include <depthai/pipeline/datatype/NNData.hpp>
+
+namespace dai {
+
+struct ValueWithConfidence {
+    float_t value;
+    float_t confidence;
+};
+
+// Per-dimension confidence, strictly [v0, c0, v1, c1, ...]
+template <std::size_t D>
+struct KeypointPerDimConfidence {
+    ValueWithConfidence data[D];                 // value, confidence, value, confidence, ...
+    static constexpr std::size_t value = 2 * D;  // value,conf xD
+};
+
+// Per-keypoint confidence, strictly [v0, v1, v2 ... confidence]
+template <std::size_t D>
+struct KeypointPerKeypointConfidence {
+    float_t values[D];
+    float_t confidence;
+    static constexpr std::size_t value = D + 1;  // D values + 1 confidence
+};
+
+using Keypoint2D2C = KeypointPerDimConfidence<2>;
+using Keypoint2D1C = KeypointPerKeypointConfidence<2>;
+using Keypoint3D3C = KeypointPerDimConfidence<3>;
+using Keypoint3D1C = KeypointPerKeypointConfidence<3>;
+
+template <typename KP>
+class Keypoints : public Buffer {
+   public:
+    std::optional<ImgTransformation> transformation;
+
+    std::vector<KP> kpVec;
+
+    Keypoints(std::shared_ptr<NNData>&& other, xt::xarray<float>&& planarStackedKeypoints);
+};
+
+template class Keypoints<Keypoint2D1C>;
+typedef Keypoints<Keypoint2D1C> Keypoints2D;
+template class Keypoints<Keypoint2D2C>;
+typedef Keypoints<Keypoint2D2C> Keypoints2D2C;
+
+template class Keypoints<Keypoint3D1C>;
+typedef Keypoints<Keypoint3D1C> Keypoints3D;
+template class Keypoints<Keypoint3D3C>;
+typedef Keypoints<Keypoint3D3C> Keypoints3D3C;
+}  // namespace dai

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/BaseParser.hpp

@@ -0,0 +1,20 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+
+#pragma once
+#include "depthai/depthai.hpp"
+
+namespace dai::node {
+class BaseParser : public CustomThreadedNode<BaseParser> {
+    const char* getName() const override = 0;
+
+   public:
+    Input input{*this, {"in", DEFAULT_GROUP, true, 5, {{{DatatypeEnum::NNData, true}}}, true}};
+    Output out{*this, {"out", DEFAULT_GROUP, {{{DatatypeEnum::Buffer, true}}}}};
+    virtual std::shared_ptr<BaseParser> build(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model);
+
+   protected:
+    virtual void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) = 0;
+};
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/KeypointParser.hpp

@@ -0,0 +1,30 @@
+//
+// Created by thwdpc on 7/25/25.
+//
+
+#pragma once
+#include "BaseParser.hpp"
+
+namespace dai::node {
+
+enum class ValuesPerKeypoint: uint8_t {
+    Two = 2,
+    Three = 3
+};
+
+class KeypointParser : public BaseParser {
+public:
+    const char* getName() const override{ return "KeypointParser"; };
+
+protected:
+    void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
+    void run() override;
+
+    std::vector<nn_archive::v1::Output> keypointsOutputs{};
+    uint16_t nKeypoints = 17;
+    // dimensionality: 2D or 3D
+    ValuesPerKeypoint valuesPerKeypoint = ValuesPerKeypoint::Two;
+    std::vector<std::string> keypointNames{};
+    std::vector<std::pair<uint8_t, uint8_t>> skeletonEdges{};
+};
+}

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/SimCCKeypointParser.hpp

@@ -0,0 +1,32 @@
+
+#pragma once
+#include "KeypointParser.hpp"
+#include "parsers/BaseParser.hpp"
+
+namespace dai::node {
+
+
+class SimCCKeypointParser final : public KeypointParser {
+   public:
+    const char* getName() const override{ return "SimCCKeypointParser"; };
+
+   protected:
+    void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
+    void run() override;
+    void foggyGuessesForOneDim(const nn_archive::v1::Head& head,
+                               const nn_archive::v1::Model& model,
+                               const nn_archive::v1::Input& imgInput,
+                               const std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWHMaybe);
+    void inferConfigFromMultipleOutputs(const nn_archive::v1::Head& head,
+                                        const nn_archive::v1::Model& model,
+                                        const nn_archive::v1::Input& imgInput,
+                                        std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWidthHeight);
+
+    uint8_t pixelSubdivisions = 2;
+    // Populated if the keypoint # dim and the XY(Z) dimensionality are collapsed(like yolo). Stores whether collapsed dim is interleaved(x1, x2 .. y1, y2 .. z1, z2)
+    //  or planar(x1, y1, z1, x2, y2, z2)
+    std::optional<bool> collapsedDimsAreInterleaved = std::nullopt;
+    bool replicateXDimToZDim = true;
+    std::vector<uint16_t> simCCDimLengths;
+};
+}  // namespace dai::node

src/pipeline/node/host/contrib/ParserGenerator.cpp

@@ -0,0 +1,82 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+
+#include "../ParserGenerator.hpp"
+
+#include <iosfwd>
+#include <variant>
+#include <vector>
+
+#include "utility/ErrorMacros.hpp"
+#include "parsers/SimCCKeypointParser.hpp"
+
+namespace dai::node {
+
+// Utility for device parser names
+const std::vector<std::string> DEVICE_PARSERS = {"YOLO", "SSD"};
+
+static const std::unordered_map<std::string, std::function<std::shared_ptr<BaseParser>()>> parserMap = {
+    // { "YOLOExtendedParser", [](){ return std::make_shared<YOLOExtendedParser>(); } },
+    { "SimCCKeypointParser", [](){ return std::make_shared<SimCCKeypointParser>(); } }
+};
+
+
+std::string getHostParserName(const std::string& parserName) {
+    assert(false); // TODO MobileNet
+    if(parserName == "YOLO") {
+        return "YOLOExtendedParser";
+    }
+    throw std::runtime_error("Parser " + parserName + " is not supported for host only mode.");
+}
+
+std::vector<HostOrDeviceParser> ParserGenerator::generateAllParsers(Pipeline pipeline, const NNArchive& nnArchive, const bool hostOnly) {
+    auto [model, heads] = archiveGetModelEnsureOneHeadV1(nnArchive, pipeline.getDefaultDevice()->getPlatform());
+
+    std::vector<HostOrDeviceParser> parsers;
+
+    for(int i = 0; i < heads.size(); i++) {
+        HostOrDeviceParser parser = generateOneV1Parser(pipeline, nnArchive, heads[i], model, hostOnly);
+        parsers.push_back(std::move(parser));
+    }
+    return parsers;
+}
+
+
+ConfigModelWithHeads ParserGenerator::archiveGetModelEnsureOneHeadV1(const NNArchive& nnArchive, const Platform targetPlatform) {
+    const auto& nnArchiveCfg = nnArchive.getVersionedConfig();
+
+    DAI_CHECK_V(nnArchiveCfg.getVersion() == NNArchiveConfigVersion::V1, "Only V1 configs are supported for NeuralNetwork.build method");
+    auto supportedPlatforms = nnArchive.getSupportedPlatforms();
+    bool platformSupported = std::find(supportedPlatforms.begin(), supportedPlatforms.end(), targetPlatform) != supportedPlatforms.end();
+    DAI_CHECK_V(platformSupported, "Platform not supported by the neural network model");
+
+    // Get model heads
+    auto [_, model] = nnArchive.getConfig<nn_archive::v1::Config>();
+
+    if(const auto headsOpt = model.heads) {
+        if(const auto headsV1 = *headsOpt; !headsV1.empty()) {
+            return ConfigModelWithHeads{.model = model, .heads = headsV1};
+        }
+    }
+    throw std::runtime_error(fmt::format("No heads defined in the NN Archive."));
+}
+
+HostOrDeviceParser ParserGenerator::generateOneV1Parser(
+    Pipeline& pipeline, const NNArchive& owningArchive, const nn_archive::v1::Head& head, const nn_archive::v1::Model& model, const bool hostOnly) {
+    std::string parser_name = head.parser;
+
+    // If this *could* be an on-device parser(currently just DetectionParser) then check whether that's allowed by !hostOnly
+    if(std::find(DEVICE_PARSERS.begin(), DEVICE_PARSERS.end(), parser_name) != DEVICE_PARSERS.end() && !hostOnly) {
+        // Device parser handling
+        auto device_parser = pipeline.create<DetectionParser>();
+        device_parser->setNNArchive(owningArchive);
+        return device_parser;
+    }
+    parser_name = getHostParserName(parser_name);
+    DAI_CHECK(parserMap.find(parser_name) != parserMap.end(), "Parser " + parser_name + " not found");
+    auto parser = parserMap.find(parser_name)->second()->build(head, model);
+    return parser;
+}
+
+}  // namespace dai::node