Video: Rework buffer management

cmake/linker_script/common/common-ram.cmake

@@ -127,6 +127,7 @@ endif()
 
 if(CONFIG_VIDEO)
   zephyr_iterable_section(NAME video_device GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT})
+  zephyr_iterable_section(NAME video_interface GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT})
 endif()
 
 if(CONFIG_LOG)

drivers/video/CMakeLists.txt

@@ -2,6 +2,7 @@
 
 zephyr_library()
 
+zephyr_library_sources(video_buffer.c)
 zephyr_library_sources(video_common.c)
 zephyr_library_sources(video_ctrls.c)
 zephyr_library_sources(video_device.c)

drivers/video/Kconfig

@@ -8,6 +8,7 @@
 #
 menuconfig VIDEO
 	bool "Video drivers"
+	imply RTIO
 	help
 	  Enable support for the VIDEO.
 

drivers/video/video.ld

@@ -1,3 +1,4 @@
 #include <zephyr/linker/iterable_sections.h>
 
 ITERABLE_SECTION_RAM(video_device, Z_LINK_ITERABLE_SUBALIGN)
+ITERABLE_SECTION_RAM(video_interface, Z_LINK_ITERABLE_SUBALIGN)

drivers/video/video_buffer.c

@@ -0,0 +1,224 @@
+/*
+ * Copyright (c) 2019, Linaro Limited
+ * Copyright (c) 2024-2025, tinyVision.ai Inc.
+ * Copyright 2024 NXP
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/logging/log.h>
+#include <zephyr/drivers/video.h>
+#include <zephyr/rtio/rtio.h>
+
+#include "video_buffer.h"
+#include "video_device.h"
+
+LOG_MODULE_REGISTER(video_buffer, CONFIG_VIDEO_LOG_LEVEL);
+
+#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP)
+#include <zephyr/multi_heap/shared_multi_heap.h>
+
+#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout)                                              \
+	shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size)
+#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block)
+#else
+K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX *CONFIG_VIDEO_BUFFER_POOL_NUM_MAX);
+#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout)                                              \
+	k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout);
+#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block)
+#endif
+
+static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
+
+struct mem_block {
+	void *data;
+};
+
+static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
+
+struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout)
+{
+	struct video_buffer *vbuf = NULL;
+	struct mem_block *block;
+	int i;
+
+	/* find available video buffer */
+	for (i = 0; i < ARRAY_SIZE(video_buf); i++) {
+		if (video_buf[i].buffer == NULL) {
+			vbuf = &video_buf[i];
+			vbuf->index = i;
+			block = &video_block[i];
+			break;
+		}
+	}
+
+	if (vbuf == NULL) {
+		return NULL;
+	}
+
+	/* Alloc buffer memory */
+	block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout);
+	if (block->data == NULL) {
+		return NULL;
+	}
+
+	vbuf->buffer = block->data;
+
+	return vbuf;
+}
+
+struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout)
+{
+	return video_buffer_aligned_alloc(size, sizeof(void *), timeout);
+}
+
+void video_buffer_release(struct video_buffer *vbuf)
+{
+	struct mem_block *block = NULL;
+	int i;
+
+	__ASSERT_NO_MSG(vbuf != NULL);
+
+	/* vbuf to block */
+	for (i = 0; i < ARRAY_SIZE(video_block); i++) {
+		if (video_block[i].data == vbuf->buffer) {
+			block = &video_block[i];
+			break;
+		}
+	}
+
+	vbuf->buffer = NULL;
+	if (block) {
+		VIDEO_COMMON_FREE(block->data);
+	}
+}
+
+/* To be completed */
+int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type,
+			  enum video_buf_memory memory)
+{
+	struct video_buffer *buffer;
+
+	/* TODO: Input request vs Output request for m2m devices */
+	if (count > CONFIG_VIDEO_BUFFER_POOL_NUM_MAX) {
+		return -EINVAL;
+	}
+
+	/* Allocate buffers */
+	for (uint8_t i = 0; i < count; i++) {
+		if (memory == VIDEO_MEMORY_INTERNAL) {
+			buffer = video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN,
+							    K_FOREVER);
+		} else {
+			/* find available video buffer */
+			for (uint8_t j = 0; j < ARRAY_SIZE(video_buf); j++) {
+				if (video_buf[j].buffer == NULL) {
+					buffer = &video_buf[j];
+					buffer->index = j;
+					break;
+				}
+			}
+		}
+
+		buffer->type = type;
+		buffer->size = size;
+		buffer->bytesused = 0;
+		buffer->state = VIDEO_BUF_STATE_DONE;
+	}
+
+	return 0;
+}
+
+RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX);
+
+int video_enqueue(const struct device *dev, struct video_buffer *buf)
+{
+	__ASSERT_NO_MSG(dev != NULL);
+
+	if (video_buf[buf->index].type != buf->type ||
+	    video_buf[buf->index].memory != buf->memory ||
+	    video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) {
+		return -EINVAL;
+	}
+
+	if (buf->memory == VIDEO_MEMORY_EXTERNAL) {
+		if (buf->size != video_buf[buf->index].size) {
+			return -EINVAL;
+		}
+
+		video_buf[buf->index].buffer = buf->buffer;
+	}
+
+	/* RTIO submission */
+	struct rtio_iodev *ri = video_find_iodev(dev);
+	struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio);
+
+	__ASSERT_NO_MSG(ri != NULL);
+	__ASSERT_NO_MSG(sqe != NULL);
+
+	rtio_sqe_prep_read(sqe, ri, RTIO_PRIO_NORM, video_buf[index].buffer, video_buf[index].size,
+			   &video_buf[index]);
+
+	sqe->flags |= RTIO_SQE_MULTISHOT;
+
+	/* Do not wait for complete */
+	rtio_submit(&rtio, 0);
+
+	video_buf[index].state = VIDEO_BUF_STATE_QUEUED;
+
+	return 0;
+}
+
+struct rtio_cqe *video_dequeue(void)
+{
+	return rtio_cqe_consume_block(&rtio);
+}
+
+void video_release_buf(struct rtio_cqe *cqe)
+{
+	/* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */
+	rtio_cqe_release(&rtio, cqe);
+}
+
+struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q)
+{
+	struct mpsc_node *node;
+	struct rtio_iodev_sqe *iodev_sqe;
+	struct video_buffer *vbuf;
+
+	node = mpsc_pop(io_q);
+	if (node == NULL) {
+		return NULL;
+	}
+
+	iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
+	vbuf = iodev_sqe->sqe.userdata;
+
+	__ASSERT_NO_MSG(vbuf != NULL);
+
+	if ((vbuf->type == VIDEO_BUF_TYPE_OUTPUT && iodev_sqe->sqe.op == RTIO_OP_RX) ||
+	    (vbuf->type == VIDEO_BUF_TYPE_INPUT && iodev_sqe->sqe.op == RTIO_OP_TX)) {
+		return iodev_sqe;
+	} else {
+		LOG_ERR("Unsupported RTIO operation (%d) or video buffer type (%d)",
+			iodev_sqe->sqe.op, vbuf->type);
+		rtio_iodev_sqe_err(iodev_sqe, -EINVAL);
+		return NULL;
+	}
+}
+
+static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe)
+{
+	struct video_interface *vi = iodev_sqe->sqe.iodev->data;
+	const struct video_driver_api *api = vi->dev->api;
+
+	if (api->iodev_submit != NULL) {
+		api->iodev_submit(vi->dev, iodev_sqe);
+	}
+
+	mpsc_push(vi->io_q, &iodev_sqe->q);
+}
+
+const struct rtio_iodev_api _video_iodev_api = {
+	.submit = video_iodev_submit,
+};

drivers/video/video_buffer.h

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2025 NXP
+ * Copyright (c) 2024-2025, tinyVision.ai Inc.
+ * 
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q);

drivers/video/video_common.c

@@ -17,88 +17,10 @@
 #include <zephyr/sys/util.h>
 
 #include "video_common.h"
+#include "video_device.h"
 
 LOG_MODULE_REGISTER(video_common, CONFIG_VIDEO_LOG_LEVEL);
 
-#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP)
-#include <zephyr/multi_heap/shared_multi_heap.h>
-
-#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout)                                              \
-	shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size)
-#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block)
-#else
-K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX*CONFIG_VIDEO_BUFFER_POOL_NUM_MAX);
-#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout)                                              \
-	k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout);
-#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block)
-#endif
-
-static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
-
-struct mem_block {
-	void *data;
-};
-
-static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
-
-struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout)
-{
-	struct video_buffer *vbuf = NULL;
-	struct mem_block *block;
-	int i;
-
-	/* find available video buffer */
-	for (i = 0; i < ARRAY_SIZE(video_buf); i++) {
-		if (video_buf[i].buffer == NULL) {
-			vbuf = &video_buf[i];
-			block = &video_block[i];
-			break;
-		}
-	}
-
-	if (vbuf == NULL) {
-		return NULL;
-	}
-
-	/* Alloc buffer memory */
-	block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout);
-	if (block->data == NULL) {
-		return NULL;
-	}
-
-	vbuf->buffer = block->data;
-	vbuf->size = size;
-	vbuf->bytesused = 0;
-
-	return vbuf;
-}
-
-struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout)
-{
-	return video_buffer_aligned_alloc(size, sizeof(void *), timeout);
-}
-
-void video_buffer_release(struct video_buffer *vbuf)
-{
-	struct mem_block *block = NULL;
-	int i;
-
-	__ASSERT_NO_MSG(vbuf != NULL);
-
-	/* vbuf to block */
-	for (i = 0; i < ARRAY_SIZE(video_block); i++) {
-		if (video_block[i].data == vbuf->buffer) {
-			block = &video_block[i];
-			break;
-		}
-	}
-
-	vbuf->buffer = NULL;
-	if (block) {
-		VIDEO_COMMON_FREE(block->data);
-	}
-}
-
 int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt,
 			    size_t *idx)
 {

drivers/video/video_device.c

@@ -4,6 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+#include <zephyr/rtio/rtio.h>
+
 #include "video_device.h"
 
 struct video_device *video_find_vdev(const struct device *dev)
@@ -20,3 +22,21 @@ struct video_device *video_find_vdev(const struct device *dev)
 
 	return NULL;
 }
+
+struct rtio_iodev *video_find_iodev(const struct device *dev)
+{
+	struct video_interface *vi;
+
+	if (!dev) {
+		return NULL;
+	}
+
+	STRUCT_SECTION_FOREACH(rtio_iodev, ri) {
+		vi = ri->data;
+		if (vi->dev == dev) {
+			return ri;
+		}
+	}
+
+	return NULL;
+}