Skip to content
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1014,6 +1014,53 @@ if(DEPTHAI_BASALT_SUPPORT)
endif()
endif()

########################
# Extra host nodes
########################
set(TARGET_EXTRA_ALIAS hostNodesExt)
set(TARGET_EXTRA_NAME ${PROJECT_NAME}-${TARGET_EXTRA_ALIAS})
if (DEPTHAI_BUILD_EXT_HOST_NODES)
set(TARGET_EXT_NODES_SOURCES
ParserGenerator.cpp
host_nodes_ext/ParsingNeuralNetwork.cpp
host_nodes_ext/parsers/BaseParser.cpp
host_nodes_ext/parsers/SimCCKeypointParser.cpp
host_nodes_ext/parsers/KeypointParser.cpp
host_nodes_ext/messages/Keypoints.cpp
)
list(TRANSFORM TARGET_EXT_NODES_SOURCES PREPEND "src/pipeline/node/host/contrib/")
#

# Add depthai-extraHostNodes library and importable target/alias depthai::extraHostNodes
add_library(${TARGET_EXTRA_NAME} ${TARGET_EXT_NODES_SOURCES})
add_library("${PROJECT_NAME}::${TARGET_EXTRA_ALIAS}" ALIAS ${TARGET_EXTRA_NAME})
set_target_properties(${TARGET_EXTRA_NAME} PROPERTIES EXPORT_NAME ${TARGET_EXTRA_ALIAS})


target_include_directories(${TARGET_EXTRA_NAME}
PUBLIC
"$<INSTALL_INTERFACE:include/depthai/pipeline/node/host/contrib/host_nodes_ext>"

# "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}>"
# "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}host/>"
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai/pipeline/node/host/contrib/host_nodes_ext>"
PRIVATE
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai>"
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src/pipeline/node/host/contrib/host_nodes_ext>"
"$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src>"
)

target_link_libraries(${TARGET_EXTRA_NAME}
PRIVATE
spdlog::spdlog
xtensor
PUBLIC
depthai::core
)

list(APPEND targets_to_export ${TARGET_EXTRA_NAME})
endif ()

########################
# Combined target
########################
Expand Down
1 change: 1 addition & 0 deletions cmake/depthaiOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ option(DEPTHAI_BUILD_TESTS "Build tests" OFF)
option(DEPTHAI_BUILD_EXAMPLES "Build examples - Requires OpenCV library to be installed" OFF)
option(DEPTHAI_BUILD_DOCS "Build documentation - requires doxygen to be installed" OFF)
option(DEPTHAI_BUILD_ZOO_HELPER "Build the Zoo helper" OFF)
option(DEPTHAI_BUILD_EXT_HOST_NODES "Build the contrib host nodes lib" OFF)
option(DEPTHAI_NEW_FIND_PYTHON "Use new FindPython module" ON)
option(DEPTHAI_INSTALL "Enable install target for depthai-core targets" ON)

Expand Down
35 changes: 35 additions & 0 deletions include/depthai/pipeline/node/host/contrib/ParserGenerator.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
//
// Created by thwdpc on 7/24/25.
//

#pragma once
#include <variant>

#include "depthai/depthai.hpp"
#include "parsers/BaseParser.hpp"

namespace dai::node {

typedef std::variant<std::shared_ptr<BaseParser>, std::shared_ptr<DetectionParser>> HostOrDeviceParser;
template<typename V>
constexpr bool all_alternatives_shared_ptr = false;
template<typename... Ts>
constexpr bool all_alternatives_shared_ptr<std::variant<Ts...>> =
(std::conjunction_v<std::is_same<std::shared_ptr<typename Ts::element_type>, Ts>...>);
static_assert(all_alternatives_shared_ptr<HostOrDeviceParser>, "All alternatives must be std::shared_ptr<T>");

struct ConfigModelWithHeads {
nn_archive::v1::Model model;
std::vector<nn_archive::v1::Head> heads;
};

class ParserGenerator {
public:
static std::vector<HostOrDeviceParser> generateAllParsers(Pipeline pipeline, const NNArchive& nnArchive, bool hostOnly = false);

private:
static ConfigModelWithHeads archiveGetModelEnsureOneHeadV1(const NNArchive& nnArchive, Platform targetPlatform);
static HostOrDeviceParser generateOneV1Parser(
Pipeline& pipeline, const NNArchive& owningArchive, const nn_archive::v1::Head& head, const nn_archive::v1::Model& model, bool hostOnly = false);
};
} // namespace dai::node
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
//
// Created by thwdpc on 7/24/25.
//
#pragma once
#include <variant>
#include <vector>

#include "../ParserGenerator.hpp"
#include "depthai/depthai.hpp"
#include "parsers/BaseParser.hpp"
#include "parsers/KeypointParser.hpp"

namespace dai::node {

class ParsingNeuralNetwork : public CustomThreadedNode<ParsingNeuralNetwork> {
public:
std::shared_ptr<ParsingNeuralNetwork> build(Output& input, const NNArchive& nnArchive);
std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, NNModelDescription modelDesc, std::optional<float> fps = std::nullopt);
std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, const NNArchive& nnArchive, std::optional<float> fps = std::nullopt);

InputMap& inputs = nn->inputs;
Input& input = nn->input;
std::optional<std::reference_wrapper<Output>> out = std::nullopt;
Output& passthrough = nn->passthrough;
OutputMap& passthroughs = nn->passthroughs;

template <typename T>
std::optional<size_t> getIndexOfFirstParserOfType() const {
const auto which = std::find_if(parsers.begin(), parsers.end(), [](const auto& p) {
return std::visit([](auto& anyP) { return std::dynamic_pointer_cast<T>(anyP) != nullptr; }, p);;
});
return which == parsers.end() ? std::nullopt : static_cast<std::optional<size_t>>(std::distance(parsers.begin(), which));
}

void run() override;

private:
std::vector<HostOrDeviceParser> getParserNodes(const NNArchive& nnArchive);

void updateParsers(const NNArchive& nnArchive);

void removeOldParserNodes();
Subnode<NeuralNetwork> nn{*this, "nn"};
std::optional<Subnode<Sync>> parserSync = std::nullopt;

protected:
std::vector<HostOrDeviceParser> parsers;
};

} // namespace dai::node
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
//
// Created by thwdpc on 7/28/25.
//

#pragma once
#include <depthai/common/ImgTransformations.hpp>
#include <depthai/pipeline/datatype/Buffer.hpp>
#include <depthai/pipeline/datatype/NNData.hpp>

namespace dai {

struct ValueWithConfidence {
float_t value;
float_t confidence;
};

// Per-dimension confidence, strictly [v0, c0, v1, c1, ...]
template <std::size_t D>
struct KeypointPerDimConfidence {
ValueWithConfidence data[D]; // value, confidence, value, confidence, ...
static constexpr std::size_t value = 2 * D; // value,conf xD
};

// Per-keypoint confidence, strictly [v0, v1, v2 ... confidence]
template <std::size_t D>
struct KeypointPerKeypointConfidence {
float_t values[D];
float_t confidence;
static constexpr std::size_t value = D + 1; // D values + 1 confidence
};

using Keypoint2D2C = KeypointPerDimConfidence<2>;
using Keypoint2D1C = KeypointPerKeypointConfidence<2>;
using Keypoint3D3C = KeypointPerDimConfidence<3>;
using Keypoint3D1C = KeypointPerKeypointConfidence<3>;

template <typename KP>
class Keypoints : public Buffer {
public:
std::optional<ImgTransformation> transformation;

std::vector<KP> kpVec;

Keypoints(std::shared_ptr<NNData>&& other, xt::xarray<float>&& planarStackedKeypoints);
};

template class Keypoints<Keypoint2D1C>;
typedef Keypoints<Keypoint2D1C> Keypoints2D;
template class Keypoints<Keypoint2D2C>;
typedef Keypoints<Keypoint2D2C> Keypoints2D2C;

template class Keypoints<Keypoint3D1C>;
typedef Keypoints<Keypoint3D1C> Keypoints3D;
template class Keypoints<Keypoint3D3C>;
typedef Keypoints<Keypoint3D3C> Keypoints3D3C;
} // namespace dai
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
//
// Created by thwdpc on 7/24/25.
//

#pragma once
#include "depthai/depthai.hpp"

namespace dai::node {
class BaseParser : public CustomThreadedNode<BaseParser> {
const char* getName() const override = 0;

public:
Input input{*this, {"in", DEFAULT_GROUP, true, 5, {{{DatatypeEnum::NNData, true}}}, true}};
Output out{*this, {"out", DEFAULT_GROUP, {{{DatatypeEnum::Buffer, true}}}}};
virtual std::shared_ptr<BaseParser> build(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model);

protected:
virtual void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) = 0;
};
} // namespace dai::node
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
//
// Created by thwdpc on 7/25/25.
//

#pragma once
#include "BaseParser.hpp"

namespace dai::node {

enum class ValuesPerKeypoint: uint8_t {
Two = 2,
Three = 3
};

class KeypointParser : public BaseParser {
public:
const char* getName() const override{ return "KeypointParser"; };

protected:
void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
void run() override;

std::vector<nn_archive::v1::Output> keypointsOutputs{};
uint16_t nKeypoints = 17;
// dimensionality: 2D or 3D
ValuesPerKeypoint valuesPerKeypoint = ValuesPerKeypoint::Two;
std::vector<std::string> keypointNames{};
std::vector<std::pair<uint8_t, uint8_t>> skeletonEdges{};
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@

#pragma once
#include "KeypointParser.hpp"
#include "parsers/BaseParser.hpp"

namespace dai::node {


class SimCCKeypointParser final : public KeypointParser {
public:
const char* getName() const override{ return "SimCCKeypointParser"; };

protected:
void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
void run() override;
void foggyGuessesForOneDim(const nn_archive::v1::Head& head,
const nn_archive::v1::Model& model,
const nn_archive::v1::Input& imgInput,
const std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWHMaybe);
void inferConfigFromMultipleOutputs(const nn_archive::v1::Head& head,
const nn_archive::v1::Model& model,
const nn_archive::v1::Input& imgInput,
std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWidthHeight);

uint8_t pixelSubdivisions = 2;
// Populated if the keypoint # dim and the XY(Z) dimensionality are collapsed(like yolo). Stores whether collapsed dim is interleaved(x1, x2 .. y1, y2 .. z1, z2)
// or planar(x1, y1, z1, x2, y2, z2)
std::optional<bool> collapsedDimsAreInterleaved = std::nullopt;
bool replicateXDimToZDim = true;
std::vector<uint16_t> simCCDimLengths;
};
} // namespace dai::node
82 changes: 82 additions & 0 deletions src/pipeline/node/host/contrib/ParserGenerator.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
//
// Created by thwdpc on 7/24/25.
//

#include "../ParserGenerator.hpp"

#include <iosfwd>
#include <variant>
#include <vector>

#include "utility/ErrorMacros.hpp"
#include "parsers/SimCCKeypointParser.hpp"

namespace dai::node {

// Utility for device parser names
const std::vector<std::string> DEVICE_PARSERS = {"YOLO", "SSD"};

static const std::unordered_map<std::string, std::function<std::shared_ptr<BaseParser>()>> parserMap = {
// { "YOLOExtendedParser", [](){ return std::make_shared<YOLOExtendedParser>(); } },
{ "SimCCKeypointParser", [](){ return std::make_shared<SimCCKeypointParser>(); } }
};


std::string getHostParserName(const std::string& parserName) {
assert(false); // TODO MobileNet
if(parserName == "YOLO") {
return "YOLOExtendedParser";
}
throw std::runtime_error("Parser " + parserName + " is not supported for host only mode.");
}

std::vector<HostOrDeviceParser> ParserGenerator::generateAllParsers(Pipeline pipeline, const NNArchive& nnArchive, const bool hostOnly) {
auto [model, heads] = archiveGetModelEnsureOneHeadV1(nnArchive, pipeline.getDefaultDevice()->getPlatform());

std::vector<HostOrDeviceParser> parsers;

for(int i = 0; i < heads.size(); i++) {
HostOrDeviceParser parser = generateOneV1Parser(pipeline, nnArchive, heads[i], model, hostOnly);
parsers.push_back(std::move(parser));
}
return parsers;
}


ConfigModelWithHeads ParserGenerator::archiveGetModelEnsureOneHeadV1(const NNArchive& nnArchive, const Platform targetPlatform) {
const auto& nnArchiveCfg = nnArchive.getVersionedConfig();

DAI_CHECK_V(nnArchiveCfg.getVersion() == NNArchiveConfigVersion::V1, "Only V1 configs are supported for NeuralNetwork.build method");
auto supportedPlatforms = nnArchive.getSupportedPlatforms();
bool platformSupported = std::find(supportedPlatforms.begin(), supportedPlatforms.end(), targetPlatform) != supportedPlatforms.end();
DAI_CHECK_V(platformSupported, "Platform not supported by the neural network model");

// Get model heads
auto [_, model] = nnArchive.getConfig<nn_archive::v1::Config>();

if(const auto headsOpt = model.heads) {
if(const auto headsV1 = *headsOpt; !headsV1.empty()) {
return ConfigModelWithHeads{.model = model, .heads = headsV1};
}
}
throw std::runtime_error(fmt::format("No heads defined in the NN Archive."));
}

HostOrDeviceParser ParserGenerator::generateOneV1Parser(
Pipeline& pipeline, const NNArchive& owningArchive, const nn_archive::v1::Head& head, const nn_archive::v1::Model& model, const bool hostOnly) {
std::string parser_name = head.parser;

// If this *could* be an on-device parser(currently just DetectionParser) then check whether that's allowed by !hostOnly
if(std::find(DEVICE_PARSERS.begin(), DEVICE_PARSERS.end(), parser_name) != DEVICE_PARSERS.end() && !hostOnly) {
// Device parser handling
auto device_parser = pipeline.create<DetectionParser>();
device_parser->setNNArchive(owningArchive);
return device_parser;
}
parser_name = getHostParserName(parser_name);
DAI_CHECK(parserMap.find(parser_name) != parserMap.end(), "Parser " + parser_name + " not found");
auto parser = parserMap.find(parser_name)->second()->build(head, model);
return parser;
}

} // namespace dai::node
Loading