Added support for a pybind11 embedded module target

CMakeLists.txt

@@ -1125,7 +1125,15 @@ if (EMSCRIPTEN)
 endif()
 
 if (DEPTHAI_BUILD_PYTHON)
-  add_subdirectory(bindings/python)
+    if(DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        set(TARGET_EMBED_ALIAS "embed")
+        set(TARGET_EMBED_NAME ${PROJECT_NAME}-${TARGET_EMBED_ALIAS} CACHE INTERNAL "Embedded Python module target name")
+        add_subdirectory(bindings/python)
+        set_target_properties(${TARGET_EMBED_NAME} PROPERTIES EXPORT_NAME ${TARGET_EMBED_ALIAS})
+        list(APPEND targets_to_export ${TARGET_EMBED_NAME})
+    else ()
+        add_subdirectory(bindings/python)
+    endif ()
 endif()
 
 ########################

bindings/python/CMakeLists.txt

@@ -39,6 +39,14 @@ option(DEPTHAI_PYTHON_ENABLE_TESTS "Enable tests" OFF)
 option(DEPTHAI_PYTHON_ENABLE_EXAMPLES "Enable examples" OFF)
 option(DEPTHAI_PYTHON_BUILD_DOCSTRINGS "Generate docstrings from header files if module 'pybind11_mkdoc' available" ON)
 option(DEPTHAI_PYTHON_EMBEDDED_MODULE "Create an embeddable module" OFF)
+cmake_dependent_option(DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET "Create an additional target with pybind embedded module" OFF DEPTHAI_PYTHON_EMBEDDED_MODULE;TARGET_EMBED_NAME OFF)
+
+find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy)
+set(PYTHON_EXECUTABLE Python_EXECUTABLE)
+# Backward Compatability for xtensor to find numpy
+if(NOT NUMPY_INCLUDE_DIRS)
+    set(NUMPY_INCLUDE_DIRS ${Python_NumPy_INCLUDE_DIRS})
+endif()
 
 # Add pybind11 dependency
 #add_subdirectory(pybind11-2.5.0)
@@ -171,6 +179,9 @@ endif()
 
 # Add files for python module
 if(DEPTHAI_PYTHON_EMBEDDED_MODULE)
+    if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        add_library(${TARGET_EMBED_NAME} SHARED ${SOURCE_LIST})
+    endif ()
     add_library(${TARGET_NAME} ${SOURCE_LIST})
 else()
     pybind11_add_module(${TARGET_NAME} ${SOURCE_LIST})
@@ -273,8 +284,15 @@ endif()
 target_include_directories(${TARGET_NAME} PRIVATE src ${DOCSTRINGS_INCLUDE_PLACEHOLDER_DIR} "${CMAKE_CURRENT_LIST_DIR}/../../src")
 if(DEPTHAI_MERGED_TARGET)
     target_include_directories(${TARGET_NAME} PRIVATE external/pybind11_opencv_numpy)
+    if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        target_include_directories(${TARGET_EMBED_NAME} PRIVATE external/pybind11_opencv_numpy)
+    endif ()
 endif()
 
+if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+    target_include_directories(${TARGET_EMBED_NAME} PRIVATE src ${DOCSTRINGS_INCLUDE_PLACEHOLDER_DIR} "${CMAKE_CURRENT_LIST_DIR}/../../src")
+endif ()
+
 set(DEPTHAI_LINK_TARGET depthai::core)
 if(NOT DEPTHAI_MERGED_TARGET)
     list(APPEND DEPTHAI_LINK_TARGET depthai::opencv)
@@ -299,16 +317,48 @@ target_link_libraries(${TARGET_NAME}
         spdlog::spdlog
 )
 
+if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+    target_link_libraries(${TARGET_EMBED_NAME}
+            PUBLIC
+            # pybind11
+            pybind11::embed
+            depthai::core
+            PRIVATE
+            hedley
+            pybind11_json
+            xtensor-python
+            spdlog::spdlog
+    )
+endif ()
+
+string(TIMESTAMP BUILD_DATETIME "%Y-%m-%d %H:%M:%S +0000" UTC)
+
 # Add embedded module option, otherwise link to pybind11 as usual
 if(DEPTHAI_PYTHON_EMBEDDED_MODULE)
     target_compile_definitions(${TARGET_NAME} PRIVATE DEPTHAI_PYTHON_EMBEDDED_MODULE)
+    if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        target_compile_definitions(${TARGET_EMBED_NAME}
+                PRIVATE
+                DEPTHAI_PYTHON_VERSION="${DEPTHAI_PYTHON_VERSION}"
+                DEPTHAI_PYTHON_COMMIT_HASH="${BUILD_COMMIT}"
+                DEPTHAI_PYTHON_COMMIT_DATETIME="${BUILD_COMMIT_DATETIME}"
+                DEPTHAI_PYTHON_BUILD_DATETIME="${BUILD_DATETIME}"
+                DEPTHAI_PYTHON_EMBEDDED_MODULE
+        )
+    endif ()
 endif()
 
 if(DEPTHAI_ENABLE_REMOTE_CONNECTION)
     target_compile_definitions(${TARGET_NAME} PRIVATE DEPTHAI_ENABLE_REMOTE_CONNECTION)
+    if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        target_compile_definitions(${TARGET_EMBED_NAME} PRIVATE DEPTHAI_ENABLE_REMOTE_CONNECTION)
+    endif ()
 endif()
 if(DEPTHAI_ENABLE_EVENTS_MANAGER)
     target_compile_definitions(${TARGET_NAME} PRIVATE DEPTHAI_ENABLE_EVENTS_MANAGER)
+    if (DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET)
+        target_compile_definitions(${TARGET_EMBED_NAME} PRIVATE DEPTHAI_ENABLE_EVENTS_MANAGER)
+    endif ()
 endif()
 
 # Add the clang-format target
@@ -356,7 +406,6 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} "-c" "${version_command}"
 )
 
 
-string(TIMESTAMP BUILD_DATETIME "%Y-%m-%d %H:%M:%S +0000" UTC)
 target_compile_definitions(${TARGET_NAME}
     PRIVATE
         DEPTHAI_PYTHON_VERSION="${DEPTHAI_PYTHON_VERSION}"

bindings/python/src/py_bindings.cpp

@@ -43,7 +43,9 @@
 
 #ifdef DEPTHAI_PYTHON_EMBEDDED_MODULE
     #include <pybind11/embed.h>
+// Don't think this symbol ever exists? Not finding it using `objdump -T libdepthai-core.so | grep bindngs`
 extern "C" void depthai_bindings_init() {}  // to force inclusion
+extern "C" PyObject* pybind11_init_impl_depthai();
 #endif
 
 // Specify module

examples/cpp/Misc/CMakeLists.txt

@@ -2,4 +2,5 @@ project(misc_examples)
 cmake_minimum_required(VERSION 3.10)
 
 add_subdirectory(AutoReconnect)
+add_subdirectory(EmbeddedPython)
 add_subdirectory(Projectors)

examples/cpp/Misc/EmbeddedPython/CMakeLists.txt

@@ -0,0 +1,31 @@
+project(python_from_cpp_examples)
+cmake_minimum_required(VERSION 3.10)
+
+## function: dai_add_example(example_name example_src enable_test use_pcl)
+## function: dai_set_example_test_labels(example_name ...)
+
+dai_add_example(build_only_pipeline_from_cpp build_only_pipeline_from_cpp.cpp OFF OFF)
+target_link_libraries(build_only_pipeline_from_cpp PRIVATE depthai-embed
+			pybind11::pybind11
+			pybind11::module
+			pybind11::embed)
+
+dai_add_example(run_pipeline_from_cpp run_pipeline_from_cpp.cpp OFF OFF)
+target_link_libraries(run_pipeline_from_cpp PRIVATE depthai-embed
+		pybind11::pybind11
+		pybind11::module
+		pybind11::embed)
+
+# Copy Python file to the same directory as the executable
+add_custom_command(TARGET build_only_pipeline_from_cpp POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E copy_if_different
+		${CMAKE_CURRENT_SOURCE_DIR}/build_pipeline.py
+		$<TARGET_FILE_DIR:build_only_pipeline_from_cpp>/build_pipeline.py
+		COMMENT "Copying Python script to build directory"
+)
+add_custom_command(TARGET run_pipeline_from_cpp POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E copy_if_different
+		${CMAKE_CURRENT_SOURCE_DIR}/build_pipeline.py
+		$<TARGET_FILE_DIR:run_pipeline_from_cpp>/build_pipeline.py
+		COMMENT "Copying Python script to build directory"
+)

examples/cpp/Misc/EmbeddedPython/README.md

@@ -0,0 +1,7 @@
+Examples of how to use target built by CMake configuration `DEPTHAI_PYTHON_EMBEDDED_MODULE_TARGET`, which is installed as a separate shared lib(dependent on core).
+
+This allows other C++ projects to include the additional lib and use python to build its pipeline.
+
+This example set encompasses 2 scenarios:
+- build_pipeline_from_script.cpp: Embedding python in C++ projects and using it to build the pipeline only, then disgarding the interpreter. Note that this does not allow depthai_nodes or ANY other python-derived classes to survive and be used.
+- TODO: Embedding python in C++ projects and using it to build the pipeline *and* keep it running such that it can be used to run depthai_nodes and other python-derived classes. Note that this has a performance penalty because python is slow.

examples/cpp/Misc/EmbeddedPython/build_only_pipeline_from_cpp.cpp

@@ -0,0 +1,54 @@
+#include "depthai/depthai.hpp"
+#include "pybind11/embed.h"
+#include "pybind11/pytypes.h"
+#include "pybind11/stl.h"
+
+extern "C" PyObject* pybind11_init_impl_depthai(void);
+[[maybe_unused]] ::pybind11::detail::embedded_module pybind11_module_depthai("depthai", pybind11_init_impl_depthai);
+
+int main() {
+    namespace py = pybind11;
+    try {
+        auto device = std::make_shared<dai::Device>();
+        auto pipeline = std::make_shared<dai::Pipeline>(device);
+        std::shared_ptr<dai::MessageQueue> queue;
+        {
+            py::scoped_interpreter guard{};  // start interpreter, dies when out of scope
+            try {
+                const char* build_script_name = "build_pipeline";
+                // Just a simple function that makes a single camera node and returns it's output
+                const char* build_function_name = "build_fn";
+                // Note that pybind adds the directory that this executable is running from to PYTHONPATH for you;
+                //  else you can add it manually via putenv
+                const py::module_ build_script = py::module_::import(build_script_name);
+                const py::function build_function = build_script.attr(build_function_name);
+                // Because the depthai pybind11 bindings do not specify std::shared_ptr as a holder type,
+                //  you must be explicit and unwrap the shared_ptr, or pybind11 will destroy the pipeline when finalizing the interpreter
+                py::object pipeline_obj = py::cast(*pipeline, py::return_value_policy::reference);
+                const py::object ret = build_function(pipeline_obj);
+                queue = ret.cast<std::shared_ptr<dai::MessageQueue>>();
+            } catch (const py::error_already_set& e) {
+                std::cerr << "Py Error: " << e.what() << std::endl;
+            }
+        } // The interpreter dies here. Starting a pipeline with Python classes such as classes that derive HostNode leads to unverbose termination
+        // Start the pipeline
+        pipeline->start();
+
+        while(true) {
+            auto videoIn = queue->get<dai::ImgFrame>();
+            if(videoIn == nullptr) continue;
+
+            cv::imshow("video", videoIn->getCvFrame());
+
+            if(cv::waitKey(1) == 'q') {
+                break;
+            }
+        }
+        device->close();
+    } catch(const std::exception& e) {
+        std::cerr << "Error: " << e.what() << std::endl;
+        return 1;
+    }
+
+    return 0;
+}

examples/cpp/Misc/EmbeddedPython/build_pipeline.py

@@ -0,0 +1,35 @@
+import depthai as dai
+
+def build_fn(p: dai.Pipeline):
+    # Build a whole complicated pipeline here, for the example just build a camera node
+    cam = p.create(dai.node.Camera).build()
+    return cam.requestFullResolutionOutput().createOutputQueue()
+
+class PyHostPassthrough(dai.node.ThreadedHostNode):
+    def __init__(self):
+        super().__init__()
+        self.name = "PyHostPassthrough"
+        self.input = self.createInput("py_input")
+        self.output = self.createOutput("py_output")
+
+        self.input.setPossibleDatatypes([
+            (dai.DatatypeEnum.ImgFrame, True),
+            (dai.DatatypeEnum.Buffer, True)
+        ])
+        self.output.setPossibleDatatypes([
+            (dai.DatatypeEnum.ImgFrame, True),
+            (dai.DatatypeEnum.Buffer, True)
+        ])
+    def run(self):
+        while self.isRunning():
+            buffer = self.input.get()
+            print("The passthrough node received a buffer!")
+            self.output.send(buffer)
+
+
+def build_with_py_nodes(p: dai.Pipeline):
+    cam = p.create(dai.node.Camera).build()
+    py_node = p.create(PyHostPassthrough)
+    cam.requestFullResolutionOutput().link(py_node.input)
+    return py_node.output.createOutputQueue()
+

examples/cpp/Misc/EmbeddedPython/run_pipeline_from_cpp.cpp

@@ -0,0 +1,59 @@
+#include "depthai/depthai.hpp"
+#include "pybind11/embed.h"
+#include "pybind11/pytypes.h"
+#include "pybind11/stl.h"
+
+extern "C" PyObject* pybind11_init_impl_depthai(void);
+[[maybe_unused]] ::pybind11::detail::embedded_module pybind11_module_depthai("depthai", pybind11_init_impl_depthai);
+
+int main() {
+    namespace py = pybind11;
+    try {
+        auto device = std::make_shared<dai::Device>();
+        auto pipeline = std::make_shared<dai::Pipeline>(device);
+        std::shared_ptr<dai::MessageQueue> queue;
+        // Starting the interpreter explicitly rather than using py::scoped_interpreter, we need to keep it running for the derived HostNode
+        py::initialize_interpreter();
+        try {
+            const char* build_script_name = "build_pipeline";
+            // This function builds a pipeline using a custom Python node. The custom node is the distinction between
+            //  this script and the build_only_pipeline_from_cpp example
+            const char* build_function_name = "build_with_py_nodes";
+            // Note that pybind adds the directory that this executable is running from to PYTHONPATH for you;
+            //  else you can add it manually via putenv
+            const py::module_ build_script = py::module_::import(build_script_name);
+            const py::function build_function = build_script.attr(build_function_name);
+            // Because the depthai pybind11 bindings do not specify std::shared_ptr as a holder type,
+            //  you must be explicit and unwrap the shared_ptr, or pybind11 will destroy the pipeline when finalizing the interpreter
+            py::object pipeline_obj = py::cast(*pipeline, py::return_value_policy::reference);
+            const py::object ret = build_function(pipeline_obj);
+            queue = ret.cast<std::shared_ptr<dai::MessageQueue>>();
+        } catch (const py::error_already_set& e) {
+            std::cerr << "Py Error: " << e.what() << std::endl;
+        }
+        assert(queue != nullptr);
+        // Start the pipeline
+        pipeline->start();
+
+        while(true) {
+            auto videoIn = queue->get();
+            std::cout << "back" << std::endl;
+            if(videoIn == nullptr) continue;
+
+            // cv::imshow("video", videoIn->getCvFrame());
+            std::cout << "Got frame" << std::endl;
+
+            if(cv::waitKey(1) == 'q') {
+                break;
+            }
+        }
+        device->close();
+    } catch(const std::exception& e) {
+        std::cerr << "Error: " << e.what() << std::endl;
+        py::finalize_interpreter();
+        return 1;
+    }
+    py::finalize_interpreter();
+
+    return 0;
+}

include/depthai/utility/ImageManipImpl.hpp

@@ -2855,7 +2855,7 @@ bool ImageManipOperations<ImageManipBuffer, ImageManipData, WarpBackend>::apply(
             switch(base.colormap) {  // TODO(asahtik): set correct stereo colormaps
                 case Colormap::TURBO:
                 case Colormap::STEREO_TURBO:
-                    cvColormap = cv::COLORMAP_TURBO;
+                    // cvColormap = cv::COLORMAP_TURBO;
                     break;
                 case Colormap::STEREO_JET:
                 case Colormap::JET: