From e3336743989e8f35c71199ebbb43bbb0a66e4ce0 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Mon, 7 Jul 2025 21:36:33 +0200 Subject: [PATCH 01/12] drivers: video: Enqueue and track internal buffers via index Currently, the video_enqueue() API enqueues the whole external video buffer container structure from the application and the driver stores this container in its FIFO queue. While it works in simple applications where the enqueued video_buffer container persists for the whole program lifecycle, it does not work in situations where we cannot keep this container, e.g. enqueuing a buffer inside a function, the local variable will be destroyed when the function returns and hence the buffer is no longer valid. Video buffers are maintained and tracked by the via their indices. Set the index field when buffers are allocated and enqueue the internal buffer rather than the external container will fix the issue. Signed-off-by: Phi Bang Nguyen --- drivers/video/video_common.c | 17 +++++++++++++++++ include/zephyr/drivers/video.h | 18 ++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index 37ed899b8029a..f7e307abdc7a0 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -51,6 +51,7 @@ struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_tim for (i = 0; i < ARRAY_SIZE(video_buf); i++) { if (video_buf[i].buffer == NULL) { vbuf = &video_buf[i]; + vbuf->index = i; block = &video_block[i]; break; } @@ -99,6 +100,22 @@ void video_buffer_release(struct video_buffer *vbuf) } } +int video_enqueue(const struct device *dev, struct video_buffer *buf) +{ + const struct video_driver_api *api = (const struct video_driver_api *)dev->api; + + __ASSERT_NO_MSG(dev != NULL); + __ASSERT_NO_MSG(buf != NULL); + __ASSERT_NO_MSG(buf->buffer != NULL); + + api = (const struct video_driver_api *)dev->api; + if (api->enqueue == NULL) { + return -ENOSYS; + } + + return api->enqueue(dev, &video_buf[buf->index]); +} + int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, size_t *idx) { diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index 20de3754887ab..b5b02d3b011e0 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -150,7 +150,7 @@ struct video_buffer { void *driver_data; /** pointer to the start of the buffer. */ uint8_t *buffer; - /** index of the buffer, optionally set by the application */ + /** index of the buffer, read-only */ uint8_t index; /** size of the buffer in bytes. */ uint32_t size; @@ -571,21 +571,7 @@ static inline int video_enum_frmival(const struct device *dev, struct video_frmi * @retval -EINVAL If parameters are invalid. * @retval -EIO General input / output error. */ -static inline int video_enqueue(const struct device *dev, struct video_buffer *buf) -{ - const struct video_driver_api *api = (const struct video_driver_api *)dev->api; - - __ASSERT_NO_MSG(dev != NULL); - __ASSERT_NO_MSG(buf != NULL); - __ASSERT_NO_MSG(buf->buffer != NULL); - - api = (const struct video_driver_api *)dev->api; - if (api->enqueue == NULL) { - return -ENOSYS; - } - - return api->enqueue(dev, buf); -} +int video_enqueue(const struct device *dev, struct video_buffer *buf); /** * @brief Dequeue a video buffer. From 3717a99083b79ab8f7af4dfe94373a3d8c2d5c5d Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Mon, 30 Jun 2025 00:59:49 +0200 Subject: [PATCH 02/12] drivers: video: Add video buffer state Add a video_buf_state field to track the current state of a buffer when it is flowing. This also helps to check whether a buffer can be safely enqueued or dequeued, data are valid or not, etc. Signed-off-by: Phi Bang Nguyen --- drivers/video/video_common.c | 7 +++++++ include/zephyr/drivers/video.h | 27 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index f7e307abdc7a0..6c348269a0f92 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -70,6 +70,7 @@ struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_tim vbuf->buffer = block->data; vbuf->size = size; vbuf->bytesused = 0; + vbuf->state = VIDEO_BUF_STATE_DONE; return vbuf; } @@ -113,6 +114,12 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf) return -ENOSYS; } + if (video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { + return -EINVAL; + } + + video_buf[buf->index].state = VIDEO_BUF_STATE_QUEUED; + return api->enqueue(dev, &video_buf[buf->index]); } diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index b5b02d3b011e0..65948c7d86865 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -56,6 +56,22 @@ enum video_buf_type { VIDEO_BUF_TYPE_OUTPUT, }; +/** + * @brief video_buf_state enum + * + * Current state of a video buffer. + * This helps to keep track of state of a buffer for debugging or to do some guard + * checks, e.g. prevents re-enqueuing a buffer already queued, etc. + */ +enum video_buf_state { + /** buffer queued in video subsystem and driver */ + VIDEO_BUF_STATE_QUEUED, + /** buffer filled and can be dequeued by application */ + VIDEO_BUF_STATE_DONE, + /** buffer returned from the driver with an error */ + VIDEO_BUF_STATE_ERROR, +}; + /** * @struct video_format * @brief Video format structure @@ -146,6 +162,8 @@ struct video_caps { struct video_buffer { /** type of the buffer */ enum video_buf_type type; + /** current state of the buffer */ + enum video_buf_state state; /** pointer to driver specific data. */ void *driver_data; /** pointer to the start of the buffer. */ @@ -591,6 +609,7 @@ static inline int video_dequeue(const struct device *dev, struct video_buffer ** k_timeout_t timeout) { const struct video_driver_api *api; + int ret; __ASSERT_NO_MSG(dev != NULL); __ASSERT_NO_MSG(buf != NULL); @@ -600,7 +619,13 @@ static inline int video_dequeue(const struct device *dev, struct video_buffer ** return -ENOSYS; } - return api->dequeue(dev, buf, timeout); + ret = api->dequeue(dev, buf, timeout); + + if (ret == 0) { + (*buf)->state = VIDEO_BUF_STATE_DONE; + } + + return ret; } /** From 2caea53e2047845aa9eabaf2285895ab443fabd3 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Mon, 30 Jun 2025 02:23:18 +0200 Subject: [PATCH 03/12] drivers: video: Request buffers instead of allocating The video subsystem holds an internal buffer pool and allocate buffers of this pool on its own heap. The application hence requests the subsystem to allocate buffers on behalf of it. Application does not need to get handles on the allocated buffers, just read back the first buffer index and the number of allocated buffer. Signed-off-by: Phi Bang Nguyen --- drivers/video/video_common.c | 30 +++++++++++++++++----- include/zephyr/drivers/video.h | 32 +++--------------------- samples/drivers/video/capture/src/main.c | 23 +++++++---------- 3 files changed, 36 insertions(+), 49 deletions(-) diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index 6c348269a0f92..2affdddf97e31 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -101,26 +101,44 @@ void video_buffer_release(struct video_buffer *vbuf) } } -int video_enqueue(const struct device *dev, struct video_buffer *buf) +/* To be completed */ +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type) +{ + struct video_buffer *buffer; + + /* TODO: Input request vs Output request for m2m devices */ + if (count > CONFIG_VIDEO_BUFFER_POOL_NUM_MAX) { + return -EINVAL; + } + + /* Allocate buffers */ + for (uint8_t i = 0; i < count; i++) { + buffer = + video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); + buffer->type = type; + } + + return 0; +} + +int video_enqueue(const struct device *dev, uint8_t index) { const struct video_driver_api *api = (const struct video_driver_api *)dev->api; __ASSERT_NO_MSG(dev != NULL); - __ASSERT_NO_MSG(buf != NULL); - __ASSERT_NO_MSG(buf->buffer != NULL); api = (const struct video_driver_api *)dev->api; if (api->enqueue == NULL) { return -ENOSYS; } - if (video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { + if (video_buf[index].state != VIDEO_BUF_STATE_DONE) { return -EINVAL; } - video_buf[buf->index].state = VIDEO_BUF_STATE_QUEUED; + video_buf[index].state = VIDEO_BUF_STATE_QUEUED; - return api->enqueue(dev, &video_buf[buf->index]); + return api->enqueue(dev, &video_buf[index]); } int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index 65948c7d86865..68d9eab3681cf 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -583,13 +583,13 @@ static inline int video_enum_frmival(const struct device *dev, struct video_frmi * endpoint incoming queue. * * @param dev Pointer to the device structure for the driver instance. - * @param buf Pointer to the video buffer. + * @param index Index of the buffer to be queued * * @retval 0 Is successful. * @retval -EINVAL If parameters are invalid. * @retval -EIO General input / output error. */ -int video_enqueue(const struct device *dev, struct video_buffer *buf); +int video_enqueue(const struct device *dev, uint8_t index); /** * @brief Dequeue a video buffer. @@ -895,33 +895,7 @@ static inline int video_get_selection(const struct device *dev, struct video_sel return api->get_selection(dev, sel); } -/** - * @brief Allocate aligned video buffer. - * - * @param size Size of the video buffer (in bytes). - * @param align Alignment of the requested memory, must be a power of two. - * @param timeout Timeout duration or K_NO_WAIT - * - * @retval pointer to allocated video buffer - */ -struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout); - -/** - * @brief Allocate video buffer. - * - * @param size Size of the video buffer (in bytes). - * @param timeout Timeout duration or K_NO_WAIT - * - * @retval pointer to allocated video buffer - */ -struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout); - -/** - * @brief Release a video buffer. - * - * @param buf Pointer to the video buffer to release. - */ -void video_buffer_release(struct video_buffer *buf); +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type); /** * @brief Search for a format that matches in a list of capabilities diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index d744c1604de40..151fde219fe5b 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -279,20 +279,15 @@ int main(void) bsize = fmt.pitch * caps.min_line_count; } - /* Alloc video buffers and enqueue for capture */ + err = video_request_buffers(ARRAY_SIZE(buffers), bsize, type); + if (err) { + LOG_ERR("Unable to request video buf"); + return 0; + } + + /* Enqueue buffers for capture */ for (i = 0; i < ARRAY_SIZE(buffers); i++) { - /* - * For some hardwares, such as the PxP used on i.MX RT1170 to do image rotation, - * buffer alignment is needed in order to achieve the best performance - */ - buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, - K_FOREVER); - if (buffers[i] == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return 0; - } - buffers[i]->type = type; - video_enqueue(video_dev, buffers[i]); + video_enqueue(video_dev, i); } /* Start video capture */ @@ -325,7 +320,7 @@ int main(void) video_display_frame(display_dev, vbuf, fmt); #endif - err = video_enqueue(video_dev, vbuf); + err = video_enqueue(video_dev, vbuf->index); if (err) { LOG_ERR("Unable to requeue video buf"); return 0; From 6ea13f2662fd6ea389944550b074f29bb7b5c0a5 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Fri, 4 Jul 2025 17:06:11 +0200 Subject: [PATCH 04/12] drivers: video: Add support for user-provided memory Add support to allow user provide their own allocated buffers. Signed-off-by: Phi Bang Nguyen --- drivers/video/video_common.c | 50 +++++++++++++++++++----- include/zephyr/drivers/video.h | 18 +++++++-- samples/drivers/video/capture/src/main.c | 9 +++-- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index 2affdddf97e31..90c6e9b0ae4ac 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -68,9 +68,6 @@ struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_tim } vbuf->buffer = block->data; - vbuf->size = size; - vbuf->bytesused = 0; - vbuf->state = VIDEO_BUF_STATE_DONE; return vbuf; } @@ -102,7 +99,8 @@ void video_buffer_release(struct video_buffer *vbuf) } /* To be completed */ -int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type) +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, + enum video_buf_memory memory) { struct video_buffer *buffer; @@ -113,16 +111,33 @@ int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type) /* Allocate buffers */ for (uint8_t i = 0; i < count; i++) { - buffer = - video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); + if (memory == VIDEO_MEMORY_INTERNAL) { + buffer = + video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); + } else { + /* find available video buffer */ + for (uint8_t j = 0; j < ARRAY_SIZE(video_buf); j++) { + if (video_buf[j].buffer == NULL) { + buffer = &video_buf[j]; + buffer->index = j; + break; + } + } + + } + buffer->type = type; + buffer->size = size; + buffer->bytesused = 0; + buffer->state = VIDEO_BUF_STATE_DONE; } return 0; } -int video_enqueue(const struct device *dev, uint8_t index) +int video_enqueue(const struct device *dev, struct video_buffer *buf) { + int ret; const struct video_driver_api *api = (const struct video_driver_api *)dev->api; __ASSERT_NO_MSG(dev != NULL); @@ -132,13 +147,28 @@ int video_enqueue(const struct device *dev, uint8_t index) return -ENOSYS; } - if (video_buf[index].state != VIDEO_BUF_STATE_DONE) { + if (video_buf[buf->index].type != buf->type || + video_buf[buf->index].memory != buf->memory || + video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { return -EINVAL; } - video_buf[index].state = VIDEO_BUF_STATE_QUEUED; + if (buf->memory == VIDEO_MEMORY_EXTERNAL) { + if (buf->size != video_buf[buf->index].size) { + return -EINVAL; + } + + video_buf[buf->index].buffer = buf->buffer; + } - return api->enqueue(dev, &video_buf[index]); + ret = api->enqueue(dev, &video_buf[buf->index]); + if (ret < 0) { + return ret; + } + + video_buf[buf->index].state = VIDEO_BUF_STATE_QUEUED; + + return 0; } int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index 68d9eab3681cf..ebdc5ec026eb8 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -72,6 +72,13 @@ enum video_buf_state { VIDEO_BUF_STATE_ERROR, }; +enum video_buf_memory { + /** buffer memory is on the video heap */ + VIDEO_MEMORY_INTERNAL = 1, + /** buffer memory is provided by user */ + VIDEO_MEMORY_EXTERNAL = 2, +}; + /** * @struct video_format * @brief Video format structure @@ -164,6 +171,8 @@ struct video_buffer { enum video_buf_type type; /** current state of the buffer */ enum video_buf_state state; + /** the backend memory of the buffer */ + enum video_buf_memory memory; /** pointer to driver specific data. */ void *driver_data; /** pointer to the start of the buffer. */ @@ -583,13 +592,13 @@ static inline int video_enum_frmival(const struct device *dev, struct video_frmi * endpoint incoming queue. * * @param dev Pointer to the device structure for the driver instance. - * @param index Index of the buffer to be queued + * @param buf Pointer to a video buffer structure. * * @retval 0 Is successful. * @retval -EINVAL If parameters are invalid. * @retval -EIO General input / output error. */ -int video_enqueue(const struct device *dev, uint8_t index); +int video_enqueue(const struct device *dev, struct video_buffer *buf); /** * @brief Dequeue a video buffer. @@ -598,7 +607,7 @@ int video_enqueue(const struct device *dev, uint8_t index); * endpoint outgoing queue. * * @param dev Pointer to the device structure for the driver instance. - * @param buf Pointer a video buffer pointer. + * @param buf Pointer to a video buffer structure. * @param timeout Timeout * * @retval 0 Is successful. @@ -895,7 +904,8 @@ static inline int video_get_selection(const struct device *dev, struct video_sel return api->get_selection(dev, sel); } -int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type); +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, + enum video_buf_memory memory); /** * @brief Search for a format that matches in a list of capabilities diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index 151fde219fe5b..af4ff6d6c11e3 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -279,15 +279,17 @@ int main(void) bsize = fmt.pitch * caps.min_line_count; } - err = video_request_buffers(ARRAY_SIZE(buffers), bsize, type); + err = video_request_buffers(ARRAY_SIZE(buffers), bsize, type, VIDEO_MEMORY_INTERNAL); if (err) { LOG_ERR("Unable to request video buf"); return 0; } /* Enqueue buffers for capture */ + vbuf->type = type; for (i = 0; i < ARRAY_SIZE(buffers); i++) { - video_enqueue(video_dev, i); + vbuf->index = i; + video_enqueue(video_dev, vbuf); } /* Start video capture */ @@ -299,7 +301,6 @@ int main(void) LOG_INF("Capture started"); /* Grab video frames */ - vbuf->type = type; while (1) { err = video_dequeue(video_dev, &vbuf, K_FOREVER); if (err) { @@ -320,7 +321,7 @@ int main(void) video_display_frame(display_dev, vbuf, fmt); #endif - err = video_enqueue(video_dev, vbuf->index); + err = video_enqueue(video_dev, vbuf); if (err) { LOG_ERR("Unable to requeue video buf"); return 0; From 72e0d3fb9568cb46d730a02c23e3aa9b322ce535 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Thu, 3 Jul 2025 19:07:08 +0200 Subject: [PATCH 05/12] drivers: video: Introduce video interface device The main device that interface with application on memory. Signed-off-by: Phi Bang Nguyen --- cmake/linker_script/common/common-ram.cmake | 1 + drivers/video/video.ld | 1 + drivers/video/video_device.h | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/cmake/linker_script/common/common-ram.cmake b/cmake/linker_script/common/common-ram.cmake index be70a2a065d3d..4bd870a8c42d3 100644 --- a/cmake/linker_script/common/common-ram.cmake +++ b/cmake/linker_script/common/common-ram.cmake @@ -127,6 +127,7 @@ endif() if(CONFIG_VIDEO) zephyr_iterable_section(NAME video_device GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT}) + zephyr_iterable_section(NAME video_interface GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT}) endif() if(CONFIG_LOG) diff --git a/drivers/video/video.ld b/drivers/video/video.ld index 935ee1ed302b3..716bc897ab166 100644 --- a/drivers/video/video.ld +++ b/drivers/video/video.ld @@ -1,3 +1,4 @@ #include ITERABLE_SECTION_RAM(video_device, Z_LINK_ITERABLE_SUBALIGN) +ITERABLE_SECTION_RAM(video_interface, Z_LINK_ITERABLE_SUBALIGN) diff --git a/drivers/video/video_device.h b/drivers/video/video_device.h index 3ba2ee3486cda..ee8056d2303f3 100644 --- a/drivers/video/video_device.h +++ b/drivers/video/video_device.h @@ -16,6 +16,10 @@ struct video_device { sys_dlist_t ctrls; }; +struct video_interface { + const struct device *dev; +}; + #define VIDEO_DEVICE_DEFINE(name, device, source) \ static STRUCT_SECTION_ITERABLE(video_device, name) = { \ .dev = device, \ @@ -23,6 +27,11 @@ struct video_device { .ctrls = SYS_DLIST_STATIC_INIT(&name.ctrls), \ } +#define VIDEO_INTERFACE_DEFINE(name, device) \ + static STRUCT_SECTION_ITERABLE(video_interface, name) = { \ + .dev = device, \ + } + struct video_device *video_find_vdev(const struct device *dev); #endif /* ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ */ From dccee295934b40815964680441f8efe823fc16db Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Fri, 4 Jul 2025 02:01:18 +0200 Subject: [PATCH 06/12] [WIP]drivers: video: Implement buffer management with RTIO backend - Application side nearly unchanged, except a call to video_release_buf() after finishing with the full buffer to queue it back to sqe. RTIO is opaque to application. - Driver just needs to hold an io_q and deal a bit with the buffer from sqe in isr. No need to implement rtio_iodev_api's submit() - The video subsystem does everything needed for buffer management. Signed-off-by: Phi Bang Nguyen Co-developed-by: Josuah Demangeon --- drivers/video/Kconfig | 1 + drivers/video/video_common.c | 63 ++++++++- drivers/video/video_device.c | 31 +++++ drivers/video/video_device.h | 15 ++- drivers/video/video_mcux_csi.c | 157 +++++------------------ include/zephyr/drivers/video.h | 41 +----- samples/drivers/video/capture/src/main.c | 8 +- 7 files changed, 146 insertions(+), 170 deletions(-) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 924e2156273c4..cd780ae9b43cc 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -8,6 +8,7 @@ # menuconfig VIDEO bool "Video drivers" + imply RTIO help Enable support for the VIDEO. diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index 90c6e9b0ae4ac..b7987afc09a5d 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -13,10 +13,12 @@ #include #include #include +#include #include #include #include "video_common.h" +#include "video_device.h" LOG_MODULE_REGISTER(video_common, CONFIG_VIDEO_LOG_LEVEL); @@ -135,14 +137,15 @@ int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, return 0; } +RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); + int video_enqueue(const struct device *dev, struct video_buffer *buf) { int ret; - const struct video_driver_api *api = (const struct video_driver_api *)dev->api; __ASSERT_NO_MSG(dev != NULL); - api = (const struct video_driver_api *)dev->api; + const struct video_driver_api *api = (const struct video_driver_api *)dev->api; if (api->enqueue == NULL) { return -ENOSYS; } @@ -166,11 +169,65 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf) return ret; } - video_buf[buf->index].state = VIDEO_BUF_STATE_QUEUED; + /* RTIO submission */ + struct rtio_iodev *ri = video_find_iodev(dev); + struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio); + + __ASSERT_NO_MSG(ri != NULL); + __ASSERT_NO_MSG(sqe != NULL); + + rtio_sqe_prep_read(sqe, ri, RTIO_PRIO_NORM, video_buf[index].buffer, + video_buf[index].size, &video_buf[index]); + + sqe->flags |= RTIO_SQE_MULTISHOT; + + /* Do not wait for complete */ + rtio_submit(&rtio, 0); + + video_buf[index].state = VIDEO_BUF_STATE_QUEUED; + + return 0; +} + +int video_dequeue(struct video_buffer **buf) +{ + struct rtio_cqe *cqe = rtio_cqe_consume_block(&rtio); + *buf = cqe->userdata; + + if (cqe->result < 0) { + LOG_ERR("I/O operation failed"); + return cqe->result; + } return 0; } +void video_release_buf() +{ + /* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */ + rtio_cqe_release(&rtio, rtio_cqe_consume_block(&rtio)); +} + +struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) +{ + struct mpsc_node *node = mpsc_pop(io_q); + if (node == NULL) { + return NULL; + } + + struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + struct rtio_sqe *sqe = &iodev_sqe->sqe; + + if (sqe->op != RTIO_OP_RX) { + LOG_ERR("Invalid operation %d of length %u for submission %p", sqe->op, + sqe->rx.buf_len, (void *)iodev_sqe); + rtio_iodev_sqe_err(iodev_sqe, -EINVAL); + return NULL; + } + + return sqe->userdata; +} + int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, size_t *idx) { diff --git a/drivers/video/video_device.c b/drivers/video/video_device.c index a895fed42f144..d8c86d3b5e364 100644 --- a/drivers/video/video_device.c +++ b/drivers/video/video_device.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include "video_device.h" struct video_device *video_find_vdev(const struct device *dev) @@ -20,3 +22,32 @@ struct video_device *video_find_vdev(const struct device *dev) return NULL; } + +struct rtio_iodev *video_find_iodev(const struct device *dev) +{ + struct video_interface *vi; + + if (!dev) { + return NULL; + } + + STRUCT_SECTION_FOREACH(rtio_iodev, ri) { + vi = ri->data; + if (vi->dev == dev) { + return ri; + } + } + + return NULL; +} + +static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe) +{ + struct video_interface *vi = iodev_sqe->sqe.iodev->data; + + mpsc_push(vi->io_q, &iodev_sqe->q); +} + +const struct rtio_iodev_api _video_iodev_api = { + .submit = video_iodev_submit, +}; diff --git a/drivers/video/video_device.h b/drivers/video/video_device.h index ee8056d2303f3..6fc2e90fd825b 100644 --- a/drivers/video/video_device.h +++ b/drivers/video/video_device.h @@ -8,6 +8,7 @@ #define ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ #include +#include #include struct video_device { @@ -18,8 +19,11 @@ struct video_device { struct video_interface { const struct device *dev; + struct mpsc *io_q; }; +extern const struct rtio_iodev_api _video_iodev_api; + #define VIDEO_DEVICE_DEFINE(name, device, source) \ static STRUCT_SECTION_ITERABLE(video_device, name) = { \ .dev = device, \ @@ -27,11 +31,16 @@ struct video_interface { .ctrls = SYS_DLIST_STATIC_INIT(&name.ctrls), \ } -#define VIDEO_INTERFACE_DEFINE(name, device) \ - static STRUCT_SECTION_ITERABLE(video_interface, name) = { \ +#define VIDEO_INTERFACE_DEFINE(name, device, io_queue) \ + static STRUCT_SECTION_ITERABLE(video_interface, name) = { \ .dev = device, \ - } + .io_q = io_queue, \ + }; \ + RTIO_IODEV_DEFINE(name##_iodev, &_video_iodev_api, &name) struct video_device *video_find_vdev(const struct device *dev); +struct rtio_iodev *video_find_iodev(const struct device *dev); + +struct video_buffer *video_get_buf_sqe(struct mpsc *io_q); #endif /* ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ */ diff --git a/drivers/video/video_mcux_csi.c b/drivers/video/video_mcux_csi.c index bb935a916be42..66fb62792922c 100644 --- a/drivers/video/video_mcux_csi.c +++ b/drivers/video/video_mcux_csi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -30,9 +31,7 @@ struct video_mcux_csi_data { const struct device *dev; csi_config_t csi_config; csi_handle_t csi_handle; - struct k_fifo fifo_in; - struct k_fifo fifo_out; - struct k_poll_signal *sig; + struct mpsc io_q; }; static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t status, void *user_data) @@ -41,7 +40,7 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu const struct device *dev = data->dev; const struct video_mcux_csi_config *config = dev->config; enum video_signal_result result = VIDEO_BUF_DONE; - struct video_buffer *vbuf, *vbuf_first = NULL; + // struct video_buffer *vbuf, *vbuf_first = NULL; uint32_t buffer_addr; /* IRQ context */ @@ -53,33 +52,13 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu status = CSI_TransferGetFullBuffer(config->base, &(data->csi_handle), &buffer_addr); if (status != kStatus_Success) { result = VIDEO_BUF_ERROR; - goto done; + return; } - /* Get matching vbuf by addr */ - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - if ((uint32_t)vbuf->buffer == buffer_addr) { - break; - } - - /* should never happen on ordered stream, except on capture - * start/restart, requeue the frame and continue looking for - * the right buffer. - */ - k_fifo_put(&data->fifo_in, vbuf); - - /* prevent infinite loop */ - if (vbuf_first == NULL) { - vbuf_first = vbuf; - } else if (vbuf_first == vbuf) { - vbuf = NULL; - break; - } - } + struct video_buffer *vbuf = video_get_buf_sqe(&data->io_q); - if (vbuf == NULL) { - result = VIDEO_BUF_ERROR; - goto done; + if ((uint32_t)vbuf->buffer != buffer_addr) { + return; } vbuf->timestamp = k_uptime_get_32(); @@ -88,13 +67,12 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu DCACHE_InvalidateByRange(buffer_addr, vbuf->bytesused); #endif - k_fifo_put(&data->fifo_out, vbuf); + // k_fifo_put(&data->fifo_out, vbuf); -done: - /* Trigger Event */ - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, result); - } + struct mpsc_node *node = mpsc_pop(&data->io_q); + struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + + rtio_iodev_sqe_ok(iodev_sqe, 0); return; } @@ -217,72 +195,6 @@ static int video_mcux_csi_set_stream(const struct device *dev, bool enable, return 0; } -static int video_mcux_csi_flush(const struct device *dev, bool cancel) -{ - const struct video_mcux_csi_config *config = dev->config; - struct video_mcux_csi_data *data = dev->data; - struct video_buf *vbuf; - uint32_t buffer_addr; - status_t ret; - - if (!cancel) { - /* wait for all buffer to be processed */ - do { - k_sleep(K_MSEC(1)); - } while (!k_fifo_is_empty(&data->fifo_in)); - } else { - /* Flush driver output queue */ - do { - ret = CSI_TransferGetFullBuffer(config->base, &(data->csi_handle), - &buffer_addr); - } while (ret == kStatus_Success); - - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - k_fifo_put(&data->fifo_out, vbuf); - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_ABORTED); - } - } - } - - return 0; -} - -static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) -{ - const struct video_mcux_csi_config *config = dev->config; - struct video_mcux_csi_data *data = dev->data; - unsigned int to_read; - status_t ret; - - to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; - vbuf->bytesused = to_read; - vbuf->line_offset = 0; - - ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, - (uint32_t)vbuf->buffer); - if (ret != kStatus_Success) { - return -EIO; - } - - k_fifo_put(&data->fifo_in, vbuf); - - return 0; -} - -static int video_mcux_csi_dequeue(const struct device *dev, struct video_buffer **vbuf, - k_timeout_t timeout) -{ - struct video_mcux_csi_data *data = dev->data; - - *vbuf = k_fifo_get(&data->fifo_out, timeout); - if (*vbuf == NULL) { - return -EAGAIN; - } - - return 0; -} - static int video_mcux_csi_get_caps(const struct device *dev, struct video_caps *caps) { const struct video_mcux_csi_config *config = dev->config; @@ -343,9 +255,6 @@ static int video_mcux_csi_init(const struct device *dev) struct video_mcux_csi_data *data = dev->data; int err; - k_fifo_init(&data->fifo_in); - k_fifo_init(&data->fifo_out); - CSI_GetDefaultConfig(&data->csi_config); /* check if there is any source device (video ctrl device) @@ -360,23 +269,10 @@ static int video_mcux_csi_init(const struct device *dev) return err; } - return 0; -} - -#ifdef CONFIG_POLL -static int video_mcux_csi_set_signal(const struct device *dev, struct k_poll_signal *sig) -{ - struct video_mcux_csi_data *data = dev->data; - - if (data->sig && sig != NULL) { - return -EALREADY; - } - - data->sig = sig; + mpsc_init(&data->io_q); return 0; } -#endif static int video_mcux_csi_set_frmival(const struct device *dev, struct video_frmival *frmival) { @@ -411,20 +307,35 @@ static int video_mcux_csi_enum_frmival(const struct device *dev, struct video_fr return ret; } +static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) +{ + const struct video_mcux_csi_config *config = dev->config; + struct video_mcux_csi_data *data = dev->data; + unsigned int to_read; + status_t ret; + + to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; + vbuf->bytesused = to_read; + vbuf->line_offset = 0; + + ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, + (uint32_t)vbuf->buffer); + if (ret != kStatus_Success) { + return -EIO; + } + + return 0; +} + static DEVICE_API(video, video_mcux_csi_driver_api) = { .set_format = video_mcux_csi_set_fmt, .get_format = video_mcux_csi_get_fmt, .set_stream = video_mcux_csi_set_stream, - .flush = video_mcux_csi_flush, .enqueue = video_mcux_csi_enqueue, - .dequeue = video_mcux_csi_dequeue, .get_caps = video_mcux_csi_get_caps, .set_frmival = video_mcux_csi_set_frmival, .get_frmival = video_mcux_csi_get_frmival, .enum_frmival = video_mcux_csi_enum_frmival, -#ifdef CONFIG_POLL - .set_signal = video_mcux_csi_set_signal, -#endif }; #if 1 /* Unique Instance */ @@ -459,4 +370,6 @@ DEVICE_DT_INST_DEFINE(0, &video_mcux_csi_init_0, NULL, &video_mcux_csi_data_0, VIDEO_DEVICE_DEFINE(csi, DEVICE_DT_INST_GET(0), SOURCE_DEV(0)); +VIDEO_INTERFACE_DEFINE(csi_intf, DEVICE_DT_INST_GET(0), &(video_mcux_csi_data_0.io_q)); + #endif diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index ebdc5ec026eb8..ff8c261fe4998 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -23,10 +23,10 @@ * @{ */ -#include #include -#include +#include +#include #include #ifdef __cplusplus @@ -350,15 +350,6 @@ typedef int (*video_api_enum_frmival_t)(const struct device *dev, struct video_f */ typedef int (*video_api_enqueue_t)(const struct device *dev, struct video_buffer *buf); -/** - * @typedef video_api_dequeue_t - * @brief Dequeue a buffer from the driver’s outgoing queue. - * - * See video_dequeue() for argument descriptions. - */ -typedef int (*video_api_dequeue_t)(const struct device *dev, struct video_buffer **buf, - k_timeout_t timeout); - /** * @typedef video_api_flush_t * @brief Flush endpoint buffers, buffer are moved from incoming queue to @@ -424,7 +415,6 @@ __subsystem struct video_driver_api { video_api_get_caps_t get_caps; /* optional callbacks */ video_api_enqueue_t enqueue; - video_api_dequeue_t dequeue; video_api_flush_t flush; video_api_ctrl_t set_ctrl; video_api_ctrl_t get_volatile_ctrl; @@ -606,36 +596,15 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf); * Dequeue a filled (capturing) or displayed (output) buffer from the driver’s * endpoint outgoing queue. * - * @param dev Pointer to the device structure for the driver instance. - * @param buf Pointer to a video buffer structure. - * @param timeout Timeout + * @param buf Pointer a video buffer pointer. * * @retval 0 Is successful. * @retval -EINVAL If parameters are invalid. * @retval -EIO General input / output error. */ -static inline int video_dequeue(const struct device *dev, struct video_buffer **buf, - k_timeout_t timeout) -{ - const struct video_driver_api *api; - int ret; - - __ASSERT_NO_MSG(dev != NULL); - __ASSERT_NO_MSG(buf != NULL); +int video_dequeue(struct video_buffer **buf); - api = (const struct video_driver_api *)dev->api; - if (api->dequeue == NULL) { - return -ENOSYS; - } - - ret = api->dequeue(dev, buf, timeout); - - if (ret == 0) { - (*buf)->state = VIDEO_BUF_STATE_DONE; - } - - return ret; -} +void video_release_buf(); /** * @brief Flush endpoint buffers. diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index af4ff6d6c11e3..8733812ddf033 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -302,7 +302,7 @@ int main(void) /* Grab video frames */ while (1) { - err = video_dequeue(video_dev, &vbuf, K_FOREVER); + err = video_dequeue(&vbuf); if (err) { LOG_ERR("Unable to dequeue video buf"); return 0; @@ -321,10 +321,6 @@ int main(void) video_display_frame(display_dev, vbuf, fmt); #endif - err = video_enqueue(video_dev, vbuf); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } + video_release_buf(); } } From b9d7f6d385c1a0bd2603a2d123cefdc6562e2a62 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Fri, 4 Jul 2025 17:29:41 +0200 Subject: [PATCH 07/12] drivers: video: Move buffer management stuffs into a separate file Move buffer management stuffs into a separate video_buffer.c file Signed-off-by: Phi Bang Nguyen --- drivers/video/CMakeLists.txt | 1 + drivers/video/video_buffer.c | 221 +++++++++++++++++++++++++++++++++++ drivers/video/video_buffer.h | 7 ++ drivers/video/video_common.c | 207 -------------------------------- drivers/video/video_device.c | 11 -- drivers/video/video_device.h | 2 - 6 files changed, 229 insertions(+), 220 deletions(-) create mode 100644 drivers/video/video_buffer.c create mode 100644 drivers/video/video_buffer.h diff --git a/drivers/video/CMakeLists.txt b/drivers/video/CMakeLists.txt index 083df586b0033..966ac637177ee 100644 --- a/drivers/video/CMakeLists.txt +++ b/drivers/video/CMakeLists.txt @@ -2,6 +2,7 @@ zephyr_library() +zephyr_library_sources(video_buffer.c) zephyr_library_sources(video_common.c) zephyr_library_sources(video_ctrls.c) zephyr_library_sources(video_device.c) diff --git a/drivers/video/video_buffer.c b/drivers/video/video_buffer.c new file mode 100644 index 0000000000000..98c84f4a94f94 --- /dev/null +++ b/drivers/video/video_buffer.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2024-2025, tinyVision.ai Inc. + * Copyright 2024 NXP + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include "video_buffer.h" +#include "video_device.h" + +LOG_MODULE_REGISTER(video_buffer, CONFIG_VIDEO_LOG_LEVEL); + +#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP) +#include + +#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ + shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size) +#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block) +#else +K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX *CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); +#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ + k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout); +#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block) +#endif + +static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; + +struct mem_block { + void *data; +}; + +static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; + +struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout) +{ + struct video_buffer *vbuf = NULL; + struct mem_block *block; + int i; + + /* find available video buffer */ + for (i = 0; i < ARRAY_SIZE(video_buf); i++) { + if (video_buf[i].buffer == NULL) { + vbuf = &video_buf[i]; + vbuf->index = i; + block = &video_block[i]; + break; + } + } + + if (vbuf == NULL) { + return NULL; + } + + /* Alloc buffer memory */ + block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout); + if (block->data == NULL) { + return NULL; + } + + vbuf->buffer = block->data; + + return vbuf; +} + +struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout) +{ + return video_buffer_aligned_alloc(size, sizeof(void *), timeout); +} + +void video_buffer_release(struct video_buffer *vbuf) +{ + struct mem_block *block = NULL; + int i; + + __ASSERT_NO_MSG(vbuf != NULL); + + /* vbuf to block */ + for (i = 0; i < ARRAY_SIZE(video_block); i++) { + if (video_block[i].data == vbuf->buffer) { + block = &video_block[i]; + break; + } + } + + vbuf->buffer = NULL; + if (block) { + VIDEO_COMMON_FREE(block->data); + } +} + +/* To be completed */ +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, + enum video_buf_memory memory) +{ + struct video_buffer *buffer; + + /* TODO: Input request vs Output request for m2m devices */ + if (count > CONFIG_VIDEO_BUFFER_POOL_NUM_MAX) { + return -EINVAL; + } + + /* Allocate buffers */ + for (uint8_t i = 0; i < count; i++) { + if (memory == VIDEO_MEMORY_INTERNAL) { + buffer = video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, + K_FOREVER); + } else { + /* find available video buffer */ + for (uint8_t j = 0; j < ARRAY_SIZE(video_buf); j++) { + if (video_buf[j].buffer == NULL) { + buffer = &video_buf[j]; + buffer->index = j; + break; + } + } + } + + buffer->type = type; + buffer->size = size; + buffer->bytesused = 0; + buffer->state = VIDEO_BUF_STATE_DONE; + } + + return 0; +} + +RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); + +int video_enqueue(const struct device *dev, struct video_buffer *buf) +{ + int ret; + + __ASSERT_NO_MSG(dev != NULL); + + const struct video_driver_api *api = (const struct video_driver_api *)dev->api; + if (api->enqueue == NULL) { + return -ENOSYS; + } + + if (video_buf[buf->index].type != buf->type || + video_buf[buf->index].memory != buf->memory || + video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { + return -EINVAL; + } + + if (buf->memory == VIDEO_MEMORY_EXTERNAL) { + if (buf->size != video_buf[buf->index].size) { + return -EINVAL; + } + + video_buf[buf->index].buffer = buf->buffer; + } + + ret = api->enqueue(dev, &video_buf[buf->index]); + if (ret < 0) { + return ret; + } + + /* RTIO submission */ + struct rtio_iodev *ri = video_find_iodev(dev); + struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio); + + __ASSERT_NO_MSG(ri != NULL); + __ASSERT_NO_MSG(sqe != NULL); + + rtio_sqe_prep_read(sqe, ri, RTIO_PRIO_NORM, video_buf[index].buffer, video_buf[index].size, + &video_buf[index]); + + sqe->flags |= RTIO_SQE_MULTISHOT; + + /* Do not wait for complete */ + rtio_submit(&rtio, 0); + + video_buf[index].state = VIDEO_BUF_STATE_QUEUED; + + return 0; +} + +int video_dequeue(struct video_buffer **buf) +{ + struct rtio_cqe *cqe = rtio_cqe_consume_block(&rtio); + *buf = cqe->userdata; + + if (cqe->result < 0) { + LOG_ERR("I/O operation failed"); + return cqe->result; + } + + return 0; +} + +void video_release_buf() +{ + /* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */ + rtio_cqe_release(&rtio, rtio_cqe_consume_block(&rtio)); +} + +struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) +{ + struct mpsc_node *node = mpsc_pop(io_q); + if (node == NULL) { + return NULL; + } + + struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + struct rtio_sqe *sqe = &iodev_sqe->sqe; + + if (sqe->op != RTIO_OP_RX) { + LOG_ERR("Invalid operation %d of length %u for submission %p", sqe->op, + sqe->rx.buf_len, (void *)iodev_sqe); + rtio_iodev_sqe_err(iodev_sqe, -EINVAL); + return NULL; + } + + return sqe->userdata; +} diff --git a/drivers/video/video_buffer.h b/drivers/video/video_buffer.h new file mode 100644 index 0000000000000..29ae6787f4d04 --- /dev/null +++ b/drivers/video/video_buffer.h @@ -0,0 +1,7 @@ +/* + * Copyright 2025 NXP + * + * SPDX-License-Identifier: Apache-2.0 + */ + +struct video_buffer *video_get_buf_sqe(struct mpsc *io_q); diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index b7987afc09a5d..6bd8ebfcc35c9 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -22,212 +21,6 @@ LOG_MODULE_REGISTER(video_common, CONFIG_VIDEO_LOG_LEVEL); -#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP) -#include - -#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ - shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size) -#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block) -#else -K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX*CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); -#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ - k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout); -#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block) -#endif - -static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; - -struct mem_block { - void *data; -}; - -static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; - -struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout) -{ - struct video_buffer *vbuf = NULL; - struct mem_block *block; - int i; - - /* find available video buffer */ - for (i = 0; i < ARRAY_SIZE(video_buf); i++) { - if (video_buf[i].buffer == NULL) { - vbuf = &video_buf[i]; - vbuf->index = i; - block = &video_block[i]; - break; - } - } - - if (vbuf == NULL) { - return NULL; - } - - /* Alloc buffer memory */ - block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout); - if (block->data == NULL) { - return NULL; - } - - vbuf->buffer = block->data; - - return vbuf; -} - -struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout) -{ - return video_buffer_aligned_alloc(size, sizeof(void *), timeout); -} - -void video_buffer_release(struct video_buffer *vbuf) -{ - struct mem_block *block = NULL; - int i; - - __ASSERT_NO_MSG(vbuf != NULL); - - /* vbuf to block */ - for (i = 0; i < ARRAY_SIZE(video_block); i++) { - if (video_block[i].data == vbuf->buffer) { - block = &video_block[i]; - break; - } - } - - vbuf->buffer = NULL; - if (block) { - VIDEO_COMMON_FREE(block->data); - } -} - -/* To be completed */ -int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, - enum video_buf_memory memory) -{ - struct video_buffer *buffer; - - /* TODO: Input request vs Output request for m2m devices */ - if (count > CONFIG_VIDEO_BUFFER_POOL_NUM_MAX) { - return -EINVAL; - } - - /* Allocate buffers */ - for (uint8_t i = 0; i < count; i++) { - if (memory == VIDEO_MEMORY_INTERNAL) { - buffer = - video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER); - } else { - /* find available video buffer */ - for (uint8_t j = 0; j < ARRAY_SIZE(video_buf); j++) { - if (video_buf[j].buffer == NULL) { - buffer = &video_buf[j]; - buffer->index = j; - break; - } - } - - } - - buffer->type = type; - buffer->size = size; - buffer->bytesused = 0; - buffer->state = VIDEO_BUF_STATE_DONE; - } - - return 0; -} - -RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); - -int video_enqueue(const struct device *dev, struct video_buffer *buf) -{ - int ret; - - __ASSERT_NO_MSG(dev != NULL); - - const struct video_driver_api *api = (const struct video_driver_api *)dev->api; - if (api->enqueue == NULL) { - return -ENOSYS; - } - - if (video_buf[buf->index].type != buf->type || - video_buf[buf->index].memory != buf->memory || - video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { - return -EINVAL; - } - - if (buf->memory == VIDEO_MEMORY_EXTERNAL) { - if (buf->size != video_buf[buf->index].size) { - return -EINVAL; - } - - video_buf[buf->index].buffer = buf->buffer; - } - - ret = api->enqueue(dev, &video_buf[buf->index]); - if (ret < 0) { - return ret; - } - - /* RTIO submission */ - struct rtio_iodev *ri = video_find_iodev(dev); - struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio); - - __ASSERT_NO_MSG(ri != NULL); - __ASSERT_NO_MSG(sqe != NULL); - - rtio_sqe_prep_read(sqe, ri, RTIO_PRIO_NORM, video_buf[index].buffer, - video_buf[index].size, &video_buf[index]); - - sqe->flags |= RTIO_SQE_MULTISHOT; - - /* Do not wait for complete */ - rtio_submit(&rtio, 0); - - video_buf[index].state = VIDEO_BUF_STATE_QUEUED; - - return 0; -} - -int video_dequeue(struct video_buffer **buf) -{ - struct rtio_cqe *cqe = rtio_cqe_consume_block(&rtio); - *buf = cqe->userdata; - - if (cqe->result < 0) { - LOG_ERR("I/O operation failed"); - return cqe->result; - } - - return 0; -} - -void video_release_buf() -{ - /* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */ - rtio_cqe_release(&rtio, rtio_cqe_consume_block(&rtio)); -} - -struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) -{ - struct mpsc_node *node = mpsc_pop(io_q); - if (node == NULL) { - return NULL; - } - - struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); - struct rtio_sqe *sqe = &iodev_sqe->sqe; - - if (sqe->op != RTIO_OP_RX) { - LOG_ERR("Invalid operation %d of length %u for submission %p", sqe->op, - sqe->rx.buf_len, (void *)iodev_sqe); - rtio_iodev_sqe_err(iodev_sqe, -EINVAL); - return NULL; - } - - return sqe->userdata; -} - int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, size_t *idx) { diff --git a/drivers/video/video_device.c b/drivers/video/video_device.c index d8c86d3b5e364..64d7ce531474c 100644 --- a/drivers/video/video_device.c +++ b/drivers/video/video_device.c @@ -40,14 +40,3 @@ struct rtio_iodev *video_find_iodev(const struct device *dev) return NULL; } - -static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe) -{ - struct video_interface *vi = iodev_sqe->sqe.iodev->data; - - mpsc_push(vi->io_q, &iodev_sqe->q); -} - -const struct rtio_iodev_api _video_iodev_api = { - .submit = video_iodev_submit, -}; diff --git a/drivers/video/video_device.h b/drivers/video/video_device.h index 6fc2e90fd825b..64c391177c514 100644 --- a/drivers/video/video_device.h +++ b/drivers/video/video_device.h @@ -41,6 +41,4 @@ extern const struct rtio_iodev_api _video_iodev_api; struct video_device *video_find_vdev(const struct device *dev); struct rtio_iodev *video_find_iodev(const struct device *dev); -struct video_buffer *video_get_buf_sqe(struct mpsc *io_q); - #endif /* ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ */ From eb5c69806437c44212736e22b84dd684b599d2f1 Mon Sep 17 00:00:00 2001 From: Josuah Demangeon Date: Sun, 6 Jul 2025 12:53:18 +0000 Subject: [PATCH 08/12] drivers: video: modifications needed to get it to work This contains only the mandatory modifications to get the drivers to work: - expose `struct rtio_iodev_sqe` to the drivers so that the functions `rtio_iodev_sqe_ok()` and `rtio_iodev_sqe_err()` can be called. - expose `stuct rtio_cqe` to the application so that the functions `rtio_cqe_release()` can be called with the same completion event (cqe) that was just dequeued. The implementation can be different, but these points seems to be mandatory to operate RTIO end-to-end. --- drivers/video/video_buffer.c | 54 ++++++++++++++++++------ drivers/video/video_buffer.h | 1 + include/zephyr/drivers/video.h | 9 ++-- samples/drivers/video/capture/src/main.c | 13 +++--- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/drivers/video/video_buffer.c b/drivers/video/video_buffer.c index 98c84f4a94f94..1038ab7e22c10 100644 --- a/drivers/video/video_buffer.c +++ b/drivers/video/video_buffer.c @@ -181,23 +181,15 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf) return 0; } -int video_dequeue(struct video_buffer **buf) +struct rtio_cqe *video_dequeue(void) { - struct rtio_cqe *cqe = rtio_cqe_consume_block(&rtio); - *buf = cqe->userdata; - - if (cqe->result < 0) { - LOG_ERR("I/O operation failed"); - return cqe->result; - } - - return 0; + return rtio_cqe_consume_block(&rtio); } -void video_release_buf() +void video_release_buf(struct rtio_cqe *cqe) { /* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */ - rtio_cqe_release(&rtio, rtio_cqe_consume_block(&rtio)); + rtio_cqe_release(&rtio, cqe); } struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) @@ -219,3 +211,41 @@ struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) return sqe->userdata; } + +struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q) +{ + struct mpsc_node *node; + struct rtio_iodev_sqe *iodev_sqe; + struct video_buffer *vbuf; + + node = mpsc_pop(io_q); + if (node == NULL) { + return NULL; + } + + iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + vbuf = iodev_sqe->sqe.userdata; + + __ASSERT_NO_MSG(vbuf != NULL); + + if ((vbuf->type == VIDEO_BUF_TYPE_OUTPUT && iodev_sqe->sqe.op == RTIO_OP_RX) || + (vbuf->type == VIDEO_BUF_TYPE_INPUT && iodev_sqe->sqe.op == RTIO_OP_TX)) { + return iodev_sqe; + } else { + LOG_ERR("Unsupported RTIO operation (%d) or video buffer type (%d)", + iodev_sqe->sqe.op, vbuf->type); + rtio_iodev_sqe_err(iodev_sqe, -EINVAL); + return NULL; + } +} + +static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe) +{ + struct video_interface *vi = iodev_sqe->sqe.iodev->data; + + mpsc_push(vi->io_q, &iodev_sqe->q); +} + +const struct rtio_iodev_api _video_iodev_api = { + .submit = video_iodev_submit, +}; diff --git a/drivers/video/video_buffer.h b/drivers/video/video_buffer.h index 29ae6787f4d04..4b30ff894654a 100644 --- a/drivers/video/video_buffer.h +++ b/drivers/video/video_buffer.h @@ -5,3 +5,4 @@ */ struct video_buffer *video_get_buf_sqe(struct mpsc *io_q); +struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q); diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index ff8c261fe4998..be7cb8169b7ff 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -598,13 +599,11 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf); * * @param buf Pointer a video buffer pointer. * - * @retval 0 Is successful. - * @retval -EINVAL If parameters are invalid. - * @retval -EIO General input / output error. + * @retval The RTIO completion event with the completed buffer. */ -int video_dequeue(struct video_buffer **buf); +struct rtio_cqe *video_dequeue(void); -void video_release_buf(); +void video_release_buf(struct rtio_cqe *cqe); /** * @brief Flush endpoint buffers. diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index 8733812ddf033..a496e05c4d695 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -19,7 +19,7 @@ LOG_MODULE_REGISTER(main, LOG_LEVEL_DBG); #else -LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); +LOG_MODULE_REGISTER(main, LOG_LEVEL_DBG); #endif #if DT_HAS_CHOSEN(zephyr_display) @@ -302,12 +302,15 @@ int main(void) /* Grab video frames */ while (1) { - err = video_dequeue(&vbuf); - if (err) { - LOG_ERR("Unable to dequeue video buf"); + struct rtio_cqe *cqe = video_dequeue(); + struct video_buffer *vbuf = cqe->userdata; + + if (cqe->result != 0) { + LOG_ERR("The video buffer of size %u completed with errors", vbuf->size); return 0; } + LOG_DBG("Got frame %u! size: %u; timestamp %u ms", frame++, vbuf->bytesused, vbuf->timestamp); @@ -321,6 +324,6 @@ int main(void) video_display_frame(display_dev, vbuf, fmt); #endif - video_release_buf(); + video_release_buf(cqe); } } From ebd3526d2f8c1d555de658f2c0bc42cffd125eb2 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Sun, 6 Jul 2025 23:35:55 +0200 Subject: [PATCH 09/12] drivers: video: Drivers implement submit() instead of video_enqueue() API Signed-off-by: Phi Bang Nguyen --- drivers/video/video_buffer.c | 37 +++++----------------------------- drivers/video/video_buffer.h | 4 ++-- include/zephyr/drivers/video.h | 11 +++------- 3 files changed, 10 insertions(+), 42 deletions(-) diff --git a/drivers/video/video_buffer.c b/drivers/video/video_buffer.c index 1038ab7e22c10..be413b52960e2 100644 --- a/drivers/video/video_buffer.c +++ b/drivers/video/video_buffer.c @@ -133,15 +133,8 @@ RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM int video_enqueue(const struct device *dev, struct video_buffer *buf) { - int ret; - __ASSERT_NO_MSG(dev != NULL); - const struct video_driver_api *api = (const struct video_driver_api *)dev->api; - if (api->enqueue == NULL) { - return -ENOSYS; - } - if (video_buf[buf->index].type != buf->type || video_buf[buf->index].memory != buf->memory || video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { @@ -156,11 +149,6 @@ int video_enqueue(const struct device *dev, struct video_buffer *buf) video_buf[buf->index].buffer = buf->buffer; } - ret = api->enqueue(dev, &video_buf[buf->index]); - if (ret < 0) { - return ret; - } - /* RTIO submission */ struct rtio_iodev *ri = video_find_iodev(dev); struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio); @@ -192,26 +180,6 @@ void video_release_buf(struct rtio_cqe *cqe) rtio_cqe_release(&rtio, cqe); } -struct video_buffer *video_get_buf_sqe(struct mpsc *io_q) -{ - struct mpsc_node *node = mpsc_pop(io_q); - if (node == NULL) { - return NULL; - } - - struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); - struct rtio_sqe *sqe = &iodev_sqe->sqe; - - if (sqe->op != RTIO_OP_RX) { - LOG_ERR("Invalid operation %d of length %u for submission %p", sqe->op, - sqe->rx.buf_len, (void *)iodev_sqe); - rtio_iodev_sqe_err(iodev_sqe, -EINVAL); - return NULL; - } - - return sqe->userdata; -} - struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q) { struct mpsc_node *node; @@ -242,6 +210,11 @@ struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q) static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe) { struct video_interface *vi = iodev_sqe->sqe.iodev->data; + const struct video_driver_api *api = vi->dev->api; + + if (api->iodev_submit != NULL) { + api->iodev_submit(vi->dev, iodev_sqe); + } mpsc_push(vi->io_q, &iodev_sqe->q); } diff --git a/drivers/video/video_buffer.h b/drivers/video/video_buffer.h index 4b30ff894654a..c797ce53ce8d5 100644 --- a/drivers/video/video_buffer.h +++ b/drivers/video/video_buffer.h @@ -1,8 +1,8 @@ /* * Copyright 2025 NXP - * + * Copyright (c) 2024-2025, tinyVision.ai Inc. + * * SPDX-License-Identifier: Apache-2.0 */ -struct video_buffer *video_get_buf_sqe(struct mpsc *io_q); struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q); diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index be7cb8169b7ff..9d1edca7851bd 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -343,13 +343,8 @@ typedef int (*video_api_frmival_t)(const struct device *dev, struct video_frmiva */ typedef int (*video_api_enum_frmival_t)(const struct device *dev, struct video_frmival_enum *fie); -/** - * @typedef video_api_enqueue_t - * @brief Enqueue a buffer in the driver’s incoming queue. - * - * See video_enqueue() for argument descriptions. - */ -typedef int (*video_api_enqueue_t)(const struct device *dev, struct video_buffer *buf); +typedef void (*video_api_iodev_submit_t)(const struct device *dev, + struct rtio_iodev_sqe *iodev_sqe); /** * @typedef video_api_flush_t @@ -415,7 +410,6 @@ __subsystem struct video_driver_api { video_api_set_stream_t set_stream; video_api_get_caps_t get_caps; /* optional callbacks */ - video_api_enqueue_t enqueue; video_api_flush_t flush; video_api_ctrl_t set_ctrl; video_api_ctrl_t get_volatile_ctrl; @@ -425,6 +419,7 @@ __subsystem struct video_driver_api { video_api_enum_frmival_t enum_frmival; video_api_selection_t set_selection; video_api_selection_t get_selection; + video_api_iodev_submit_t iodev_submit; }; /** From b80849e1e54ba91c8e3d572b3e43be4133d66a7a Mon Sep 17 00:00:00 2001 From: Josuah Demangeon Date: Sun, 6 Jul 2025 13:58:07 +0000 Subject: [PATCH 10/12] convert the video_sw_generator --- drivers/video/video_sw_generator.c | 95 +++++------------------------- 1 file changed, 16 insertions(+), 79 deletions(-) diff --git a/drivers/video/video_sw_generator.c b/drivers/video/video_sw_generator.c index ff176e0ad3480..1f21199244861 100644 --- a/drivers/video/video_sw_generator.c +++ b/drivers/video/video_sw_generator.c @@ -10,11 +10,13 @@ #include #include #include +#include #include #include #include "video_ctrls.h" #include "video_device.h" +#include "video_buffer.h" LOG_MODULE_REGISTER(video_sw_generator, CONFIG_VIDEO_LOG_LEVEL); @@ -38,11 +40,9 @@ struct video_sw_generator_data { const struct device *dev; struct sw_ctrls ctrls; struct video_format fmt; - struct k_fifo fifo_in; - struct k_fifo fifo_out; + struct mpsc io_q; struct k_work_delayable work; int pattern; - struct k_poll_signal *sig; uint32_t frame_rate; }; @@ -289,74 +289,29 @@ static int video_sw_generator_fill(const struct device *const dev, struct video_ static void video_sw_generator_worker(struct k_work *work) { struct k_work_delayable *dwork = k_work_delayable_from_work(work); - struct video_sw_generator_data *data; - struct video_buffer *vbuf; - - data = CONTAINER_OF(dwork, struct video_sw_generator_data, work); + struct video_sw_generator_data *data = + CONTAINER_OF(dwork, struct video_sw_generator_data, work); + const struct device *dev = data->dev; + struct rtio_iodev_sqe *iodev_sqe; k_work_reschedule(&data->work, K_MSEC(1000 / data->frame_rate)); - vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); - if (vbuf == NULL) { + iodev_sqe = video_pop_io_q(&data->io_q); + if (iodev_sqe == NULL) { return; } switch (data->pattern) { case VIDEO_PATTERN_COLOR_BAR: - video_sw_generator_fill(data->dev, vbuf); + video_sw_generator_fill(dev, iodev_sqe->sqe.userdata); break; } - k_fifo_put(&data->fifo_out, vbuf); - - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_DONE); - } - - k_yield(); + rtio_iodev_sqe_ok(iodev_sqe, 0); } static int video_sw_generator_enqueue(const struct device *dev, struct video_buffer *vbuf) { - struct video_sw_generator_data *data = dev->data; - - k_fifo_put(&data->fifo_in, vbuf); - - return 0; -} - -static int video_sw_generator_dequeue(const struct device *dev, struct video_buffer **vbuf, - k_timeout_t timeout) -{ - struct video_sw_generator_data *data = dev->data; - - *vbuf = k_fifo_get(&data->fifo_out, timeout); - if (*vbuf == NULL) { - return -EAGAIN; - } - - return 0; -} - -static int video_sw_generator_flush(const struct device *dev, bool cancel) -{ - struct video_sw_generator_data *data = dev->data; - struct video_buffer *vbuf; - - if (!cancel) { - /* wait for all buffer to be processed */ - do { - k_sleep(K_MSEC(1)); - } while (!k_fifo_is_empty(&data->fifo_in)); - } else { - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - k_fifo_put(&data->fifo_out, vbuf); - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_ABORTED); - } - } - } - return 0; } @@ -371,21 +326,6 @@ static int video_sw_generator_get_caps(const struct device *dev, struct video_ca return 0; } -#ifdef CONFIG_POLL -static int video_sw_generator_set_signal(const struct device *dev, struct k_poll_signal *sig) -{ - struct video_sw_generator_data *data = dev->data; - - if (data->sig && sig != NULL) { - return -EALREADY; - } - - data->sig = sig; - - return 0; -} -#endif - static int video_sw_generator_set_frmival(const struct device *dev, struct video_frmival *frmival) { struct video_sw_generator_data *data = dev->data; @@ -439,16 +379,11 @@ static DEVICE_API(video, video_sw_generator_driver_api) = { .set_format = video_sw_generator_set_fmt, .get_format = video_sw_generator_get_fmt, .set_stream = video_sw_generator_set_stream, - .flush = video_sw_generator_flush, .enqueue = video_sw_generator_enqueue, - .dequeue = video_sw_generator_dequeue, .get_caps = video_sw_generator_get_caps, .set_frmival = video_sw_generator_set_frmival, .get_frmival = video_sw_generator_get_frmival, .enum_frmival = video_sw_generator_enum_frmival, -#ifdef CONFIG_POLL - .set_signal = video_sw_generator_set_signal, -#endif }; static int video_sw_generator_init_controls(const struct device *dev) @@ -464,8 +399,7 @@ static int video_sw_generator_init(const struct device *dev) struct video_sw_generator_data *data = dev->data; data->dev = dev; - k_fifo_init(&data->fifo_in); - k_fifo_init(&data->fifo_out); + mpsc_init(&data->io_q); k_work_init_delayable(&data->work, video_sw_generator_worker); return video_sw_generator_init_controls(dev); @@ -484,6 +418,9 @@ static int video_sw_generator_init(const struct device *dev) NULL, POST_KERNEL, CONFIG_VIDEO_INIT_PRIORITY, \ &video_sw_generator_driver_api); \ \ - VIDEO_DEVICE_DEFINE(video_sw_generator_##n, DEVICE_DT_INST_GET(n), NULL); + VIDEO_DEVICE_DEFINE(video_sw_generator_##n, DEVICE_DT_INST_GET(n), NULL); \ + \ + VIDEO_INTERFACE_DEFINE(video_sw_generator_intf, DEVICE_DT_INST_GET(n), \ + &video_sw_generator_data_##n.io_q); DT_INST_FOREACH_STATUS_OKAY(VIDEO_SW_GENERATOR_DEFINE) From f291c5e4268b8deffad5a1e587b21d5ef1861ac3 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Sun, 6 Jul 2025 23:35:16 +0200 Subject: [PATCH 11/12] sw_generator: Remove enqueue() Signed-off-by: Phi Bang Nguyen --- drivers/video/video_sw_generator.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/video/video_sw_generator.c b/drivers/video/video_sw_generator.c index 1f21199244861..ca30ba40a76f0 100644 --- a/drivers/video/video_sw_generator.c +++ b/drivers/video/video_sw_generator.c @@ -310,11 +310,6 @@ static void video_sw_generator_worker(struct k_work *work) rtio_iodev_sqe_ok(iodev_sqe, 0); } -static int video_sw_generator_enqueue(const struct device *dev, struct video_buffer *vbuf) -{ - return 0; -} - static int video_sw_generator_get_caps(const struct device *dev, struct video_caps *caps) { caps->format_caps = fmts; @@ -379,7 +374,6 @@ static DEVICE_API(video, video_sw_generator_driver_api) = { .set_format = video_sw_generator_set_fmt, .get_format = video_sw_generator_get_fmt, .set_stream = video_sw_generator_set_stream, - .enqueue = video_sw_generator_enqueue, .get_caps = video_sw_generator_get_caps, .set_frmival = video_sw_generator_set_frmival, .get_frmival = video_sw_generator_get_frmival, From bfc86f6ecb8f79238086b969abb9bbb35c149de3 Mon Sep 17 00:00:00 2001 From: Phi Bang Nguyen Date: Sun, 6 Jul 2025 18:51:53 +0200 Subject: [PATCH 12/12] Convert NXP's CSI Signed-off-by: Phi Bang Nguyen --- drivers/video/video_mcux_csi.c | 66 ++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/drivers/video/video_mcux_csi.c b/drivers/video/video_mcux_csi.c index 66fb62792922c..d0fe067d3ee1b 100644 --- a/drivers/video/video_mcux_csi.c +++ b/drivers/video/video_mcux_csi.c @@ -19,6 +19,7 @@ #include #endif +#include "video_buffer.h" #include "video_device.h" struct video_mcux_csi_config { @@ -40,11 +41,9 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu const struct device *dev = data->dev; const struct video_mcux_csi_config *config = dev->config; enum video_signal_result result = VIDEO_BUF_DONE; - // struct video_buffer *vbuf, *vbuf_first = NULL; uint32_t buffer_addr; /* IRQ context */ - if (status != kStatus_CSI_FrameDone) { return; } @@ -55,22 +54,24 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu return; } - struct video_buffer *vbuf = video_get_buf_sqe(&data->io_q); + struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); - if ((uint32_t)vbuf->buffer != buffer_addr) { + if (iodev_sqe == NULL) { return; } - vbuf->timestamp = k_uptime_get_32(); + + struct video_buffer *vbuf = iodev_sqe->sqe.userdata; #ifdef CONFIG_HAS_MCUX_CACHE DCACHE_InvalidateByRange(buffer_addr, vbuf->bytesused); #endif - // k_fifo_put(&data->fifo_out, vbuf); + if ((uint32_t)vbuf->buffer != buffer_addr) { + return; + } - struct mpsc_node *node = mpsc_pop(&data->io_q); - struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + vbuf->timestamp = k_uptime_get_32(); rtio_iodev_sqe_ok(iodev_sqe, 0); @@ -307,35 +308,70 @@ static int video_mcux_csi_enum_frmival(const struct device *dev, struct video_fr return ret; } -static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) +// static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) +// { +// const struct video_mcux_csi_config *config = dev->config; +// struct video_mcux_csi_data *data = dev->data; +// unsigned int to_read; +// status_t ret; + +// struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); + +// if (iodev_sqe == NULL) { +// return -EIO; +// } + +// struct video_buffer *vbuf = iodev_sqe->sqe.userdata; + +// to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; +// vbuf->bytesused = to_read; +// vbuf->line_offset = 0; + +// ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, +// (uint32_t)vbuf->buffer); +// if (ret != kStatus_Success) { +// return -EIO; +// } + +// return 0; +// } + +static void video_mcux_csi_iodev_submit(const struct device *dev, struct rtio_iodev_sqe *iodev_sqe) { + const struct video_mcux_csi_config *config = dev->config; struct video_mcux_csi_data *data = dev->data; - unsigned int to_read; + struct video_buffer *vbuf = iodev_sqe->sqe.userdata; status_t ret; - to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; - vbuf->bytesused = to_read; + // struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); + + // if (iodev_sqe == NULL) { + // return -EIO; + // } + + vbuf->bytesused = data->csi_config.linePitch_Bytes * data->csi_config.height; vbuf->line_offset = 0; ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, (uint32_t)vbuf->buffer); if (ret != kStatus_Success) { - return -EIO; + return; } - return 0; + return; } static DEVICE_API(video, video_mcux_csi_driver_api) = { .set_format = video_mcux_csi_set_fmt, .get_format = video_mcux_csi_get_fmt, .set_stream = video_mcux_csi_set_stream, - .enqueue = video_mcux_csi_enqueue, + // .enqueue = video_mcux_csi_enqueue, .get_caps = video_mcux_csi_get_caps, .set_frmival = video_mcux_csi_set_frmival, .get_frmival = video_mcux_csi_get_frmival, .enum_frmival = video_mcux_csi_enum_frmival, + .iodev_submit = video_mcux_csi_iodev_submit, }; #if 1 /* Unique Instance */