diff --git a/cmake/linker_script/common/common-ram.cmake b/cmake/linker_script/common/common-ram.cmake index be70a2a065d3d..4bd870a8c42d3 100644 --- a/cmake/linker_script/common/common-ram.cmake +++ b/cmake/linker_script/common/common-ram.cmake @@ -127,6 +127,7 @@ endif() if(CONFIG_VIDEO) zephyr_iterable_section(NAME video_device GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT}) + zephyr_iterable_section(NAME video_interface GROUP DATA_REGION ${XIP_ALIGN_WITH_INPUT}) endif() if(CONFIG_LOG) diff --git a/drivers/video/CMakeLists.txt b/drivers/video/CMakeLists.txt index 083df586b0033..966ac637177ee 100644 --- a/drivers/video/CMakeLists.txt +++ b/drivers/video/CMakeLists.txt @@ -2,6 +2,7 @@ zephyr_library() +zephyr_library_sources(video_buffer.c) zephyr_library_sources(video_common.c) zephyr_library_sources(video_ctrls.c) zephyr_library_sources(video_device.c) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 924e2156273c4..cd780ae9b43cc 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -8,6 +8,7 @@ # menuconfig VIDEO bool "Video drivers" + imply RTIO help Enable support for the VIDEO. diff --git a/drivers/video/video.ld b/drivers/video/video.ld index 935ee1ed302b3..716bc897ab166 100644 --- a/drivers/video/video.ld +++ b/drivers/video/video.ld @@ -1,3 +1,4 @@ #include ITERABLE_SECTION_RAM(video_device, Z_LINK_ITERABLE_SUBALIGN) +ITERABLE_SECTION_RAM(video_interface, Z_LINK_ITERABLE_SUBALIGN) diff --git a/drivers/video/video_buffer.c b/drivers/video/video_buffer.c new file mode 100644 index 0000000000000..be413b52960e2 --- /dev/null +++ b/drivers/video/video_buffer.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019, Linaro Limited + * Copyright (c) 2024-2025, tinyVision.ai Inc. + * Copyright 2024 NXP + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include "video_buffer.h" +#include "video_device.h" + +LOG_MODULE_REGISTER(video_buffer, CONFIG_VIDEO_LOG_LEVEL); + +#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP) +#include + +#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ + shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size) +#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block) +#else +K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX *CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); +#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ + k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout); +#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block) +#endif + +static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; + +struct mem_block { + void *data; +}; + +static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; + +struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout) +{ + struct video_buffer *vbuf = NULL; + struct mem_block *block; + int i; + + /* find available video buffer */ + for (i = 0; i < ARRAY_SIZE(video_buf); i++) { + if (video_buf[i].buffer == NULL) { + vbuf = &video_buf[i]; + vbuf->index = i; + block = &video_block[i]; + break; + } + } + + if (vbuf == NULL) { + return NULL; + } + + /* Alloc buffer memory */ + block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout); + if (block->data == NULL) { + return NULL; + } + + vbuf->buffer = block->data; + + return vbuf; +} + +struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout) +{ + return video_buffer_aligned_alloc(size, sizeof(void *), timeout); +} + +void video_buffer_release(struct video_buffer *vbuf) +{ + struct mem_block *block = NULL; + int i; + + __ASSERT_NO_MSG(vbuf != NULL); + + /* vbuf to block */ + for (i = 0; i < ARRAY_SIZE(video_block); i++) { + if (video_block[i].data == vbuf->buffer) { + block = &video_block[i]; + break; + } + } + + vbuf->buffer = NULL; + if (block) { + VIDEO_COMMON_FREE(block->data); + } +} + +/* To be completed */ +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, + enum video_buf_memory memory) +{ + struct video_buffer *buffer; + + /* TODO: Input request vs Output request for m2m devices */ + if (count > CONFIG_VIDEO_BUFFER_POOL_NUM_MAX) { + return -EINVAL; + } + + /* Allocate buffers */ + for (uint8_t i = 0; i < count; i++) { + if (memory == VIDEO_MEMORY_INTERNAL) { + buffer = video_buffer_aligned_alloc(size, CONFIG_VIDEO_BUFFER_POOL_ALIGN, + K_FOREVER); + } else { + /* find available video buffer */ + for (uint8_t j = 0; j < ARRAY_SIZE(video_buf); j++) { + if (video_buf[j].buffer == NULL) { + buffer = &video_buf[j]; + buffer->index = j; + break; + } + } + } + + buffer->type = type; + buffer->size = size; + buffer->bytesused = 0; + buffer->state = VIDEO_BUF_STATE_DONE; + } + + return 0; +} + +RTIO_DEFINE(rtio, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); + +int video_enqueue(const struct device *dev, struct video_buffer *buf) +{ + __ASSERT_NO_MSG(dev != NULL); + + if (video_buf[buf->index].type != buf->type || + video_buf[buf->index].memory != buf->memory || + video_buf[buf->index].state != VIDEO_BUF_STATE_DONE) { + return -EINVAL; + } + + if (buf->memory == VIDEO_MEMORY_EXTERNAL) { + if (buf->size != video_buf[buf->index].size) { + return -EINVAL; + } + + video_buf[buf->index].buffer = buf->buffer; + } + + /* RTIO submission */ + struct rtio_iodev *ri = video_find_iodev(dev); + struct rtio_sqe *sqe = rtio_sqe_acquire(&rtio); + + __ASSERT_NO_MSG(ri != NULL); + __ASSERT_NO_MSG(sqe != NULL); + + rtio_sqe_prep_read(sqe, ri, RTIO_PRIO_NORM, video_buf[index].buffer, video_buf[index].size, + &video_buf[index]); + + sqe->flags |= RTIO_SQE_MULTISHOT; + + /* Do not wait for complete */ + rtio_submit(&rtio, 0); + + video_buf[index].state = VIDEO_BUF_STATE_QUEUED; + + return 0; +} + +struct rtio_cqe *video_dequeue(void) +{ + return rtio_cqe_consume_block(&rtio); +} + +void video_release_buf(struct rtio_cqe *cqe) +{ + /* Buffer will be re-queued thanks to RTIO_SQE_MULTISHOT */ + rtio_cqe_release(&rtio, cqe); +} + +struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q) +{ + struct mpsc_node *node; + struct rtio_iodev_sqe *iodev_sqe; + struct video_buffer *vbuf; + + node = mpsc_pop(io_q); + if (node == NULL) { + return NULL; + } + + iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q); + vbuf = iodev_sqe->sqe.userdata; + + __ASSERT_NO_MSG(vbuf != NULL); + + if ((vbuf->type == VIDEO_BUF_TYPE_OUTPUT && iodev_sqe->sqe.op == RTIO_OP_RX) || + (vbuf->type == VIDEO_BUF_TYPE_INPUT && iodev_sqe->sqe.op == RTIO_OP_TX)) { + return iodev_sqe; + } else { + LOG_ERR("Unsupported RTIO operation (%d) or video buffer type (%d)", + iodev_sqe->sqe.op, vbuf->type); + rtio_iodev_sqe_err(iodev_sqe, -EINVAL); + return NULL; + } +} + +static void video_iodev_submit(struct rtio_iodev_sqe *iodev_sqe) +{ + struct video_interface *vi = iodev_sqe->sqe.iodev->data; + const struct video_driver_api *api = vi->dev->api; + + if (api->iodev_submit != NULL) { + api->iodev_submit(vi->dev, iodev_sqe); + } + + mpsc_push(vi->io_q, &iodev_sqe->q); +} + +const struct rtio_iodev_api _video_iodev_api = { + .submit = video_iodev_submit, +}; diff --git a/drivers/video/video_buffer.h b/drivers/video/video_buffer.h new file mode 100644 index 0000000000000..c797ce53ce8d5 --- /dev/null +++ b/drivers/video/video_buffer.h @@ -0,0 +1,8 @@ +/* + * Copyright 2025 NXP + * Copyright (c) 2024-2025, tinyVision.ai Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +struct rtio_iodev_sqe *video_pop_io_q(struct mpsc *io_q); diff --git a/drivers/video/video_common.c b/drivers/video/video_common.c index 37ed899b8029a..6bd8ebfcc35c9 100644 --- a/drivers/video/video_common.c +++ b/drivers/video/video_common.c @@ -17,88 +17,10 @@ #include #include "video_common.h" +#include "video_device.h" LOG_MODULE_REGISTER(video_common, CONFIG_VIDEO_LOG_LEVEL); -#if defined(CONFIG_VIDEO_BUFFER_USE_SHARED_MULTI_HEAP) -#include - -#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ - shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, align, size) -#define VIDEO_COMMON_FREE(block) shared_multi_heap_free(block) -#else -K_HEAP_DEFINE(video_buffer_pool, CONFIG_VIDEO_BUFFER_POOL_SZ_MAX*CONFIG_VIDEO_BUFFER_POOL_NUM_MAX); -#define VIDEO_COMMON_HEAP_ALLOC(align, size, timeout) \ - k_heap_aligned_alloc(&video_buffer_pool, align, size, timeout); -#define VIDEO_COMMON_FREE(block) k_heap_free(&video_buffer_pool, block) -#endif - -static struct video_buffer video_buf[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; - -struct mem_block { - void *data; -}; - -static struct mem_block video_block[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX]; - -struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout) -{ - struct video_buffer *vbuf = NULL; - struct mem_block *block; - int i; - - /* find available video buffer */ - for (i = 0; i < ARRAY_SIZE(video_buf); i++) { - if (video_buf[i].buffer == NULL) { - vbuf = &video_buf[i]; - block = &video_block[i]; - break; - } - } - - if (vbuf == NULL) { - return NULL; - } - - /* Alloc buffer memory */ - block->data = VIDEO_COMMON_HEAP_ALLOC(align, size, timeout); - if (block->data == NULL) { - return NULL; - } - - vbuf->buffer = block->data; - vbuf->size = size; - vbuf->bytesused = 0; - - return vbuf; -} - -struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout) -{ - return video_buffer_aligned_alloc(size, sizeof(void *), timeout); -} - -void video_buffer_release(struct video_buffer *vbuf) -{ - struct mem_block *block = NULL; - int i; - - __ASSERT_NO_MSG(vbuf != NULL); - - /* vbuf to block */ - for (i = 0; i < ARRAY_SIZE(video_block); i++) { - if (video_block[i].data == vbuf->buffer) { - block = &video_block[i]; - break; - } - } - - vbuf->buffer = NULL; - if (block) { - VIDEO_COMMON_FREE(block->data); - } -} - int video_format_caps_index(const struct video_format_cap *fmts, const struct video_format *fmt, size_t *idx) { diff --git a/drivers/video/video_device.c b/drivers/video/video_device.c index a895fed42f144..64d7ce531474c 100644 --- a/drivers/video/video_device.c +++ b/drivers/video/video_device.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include "video_device.h" struct video_device *video_find_vdev(const struct device *dev) @@ -20,3 +22,21 @@ struct video_device *video_find_vdev(const struct device *dev) return NULL; } + +struct rtio_iodev *video_find_iodev(const struct device *dev) +{ + struct video_interface *vi; + + if (!dev) { + return NULL; + } + + STRUCT_SECTION_FOREACH(rtio_iodev, ri) { + vi = ri->data; + if (vi->dev == dev) { + return ri; + } + } + + return NULL; +} diff --git a/drivers/video/video_device.h b/drivers/video/video_device.h index 3ba2ee3486cda..64c391177c514 100644 --- a/drivers/video/video_device.h +++ b/drivers/video/video_device.h @@ -8,6 +8,7 @@ #define ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ #include +#include #include struct video_device { @@ -16,6 +17,13 @@ struct video_device { sys_dlist_t ctrls; }; +struct video_interface { + const struct device *dev; + struct mpsc *io_q; +}; + +extern const struct rtio_iodev_api _video_iodev_api; + #define VIDEO_DEVICE_DEFINE(name, device, source) \ static STRUCT_SECTION_ITERABLE(video_device, name) = { \ .dev = device, \ @@ -23,6 +31,14 @@ struct video_device { .ctrls = SYS_DLIST_STATIC_INIT(&name.ctrls), \ } +#define VIDEO_INTERFACE_DEFINE(name, device, io_queue) \ + static STRUCT_SECTION_ITERABLE(video_interface, name) = { \ + .dev = device, \ + .io_q = io_queue, \ + }; \ + RTIO_IODEV_DEFINE(name##_iodev, &_video_iodev_api, &name) + struct video_device *video_find_vdev(const struct device *dev); +struct rtio_iodev *video_find_iodev(const struct device *dev); #endif /* ZEPHYR_INCLUDE_DRIVERS_VIDEO_VIDEO_DEVICE_H_ */ diff --git a/drivers/video/video_mcux_csi.c b/drivers/video/video_mcux_csi.c index bb935a916be42..d0fe067d3ee1b 100644 --- a/drivers/video/video_mcux_csi.c +++ b/drivers/video/video_mcux_csi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -18,6 +19,7 @@ #include #endif +#include "video_buffer.h" #include "video_device.h" struct video_mcux_csi_config { @@ -30,9 +32,7 @@ struct video_mcux_csi_data { const struct device *dev; csi_config_t csi_config; csi_handle_t csi_handle; - struct k_fifo fifo_in; - struct k_fifo fifo_out; - struct k_poll_signal *sig; + struct mpsc io_q; }; static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t status, void *user_data) @@ -41,11 +41,9 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu const struct device *dev = data->dev; const struct video_mcux_csi_config *config = dev->config; enum video_signal_result result = VIDEO_BUF_DONE; - struct video_buffer *vbuf, *vbuf_first = NULL; uint32_t buffer_addr; /* IRQ context */ - if (status != kStatus_CSI_FrameDone) { return; } @@ -53,49 +51,30 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu status = CSI_TransferGetFullBuffer(config->base, &(data->csi_handle), &buffer_addr); if (status != kStatus_Success) { result = VIDEO_BUF_ERROR; - goto done; + return; } - /* Get matching vbuf by addr */ - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - if ((uint32_t)vbuf->buffer == buffer_addr) { - break; - } + struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); - /* should never happen on ordered stream, except on capture - * start/restart, requeue the frame and continue looking for - * the right buffer. - */ - k_fifo_put(&data->fifo_in, vbuf); - - /* prevent infinite loop */ - if (vbuf_first == NULL) { - vbuf_first = vbuf; - } else if (vbuf_first == vbuf) { - vbuf = NULL; - break; - } + if (iodev_sqe == NULL) { + return; } - if (vbuf == NULL) { - result = VIDEO_BUF_ERROR; - goto done; - } - vbuf->timestamp = k_uptime_get_32(); + struct video_buffer *vbuf = iodev_sqe->sqe.userdata; #ifdef CONFIG_HAS_MCUX_CACHE DCACHE_InvalidateByRange(buffer_addr, vbuf->bytesused); #endif - k_fifo_put(&data->fifo_out, vbuf); - -done: - /* Trigger Event */ - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, result); + if ((uint32_t)vbuf->buffer != buffer_addr) { + return; } + vbuf->timestamp = k_uptime_get_32(); + + rtio_iodev_sqe_ok(iodev_sqe, 0); + return; } @@ -217,72 +196,6 @@ static int video_mcux_csi_set_stream(const struct device *dev, bool enable, return 0; } -static int video_mcux_csi_flush(const struct device *dev, bool cancel) -{ - const struct video_mcux_csi_config *config = dev->config; - struct video_mcux_csi_data *data = dev->data; - struct video_buf *vbuf; - uint32_t buffer_addr; - status_t ret; - - if (!cancel) { - /* wait for all buffer to be processed */ - do { - k_sleep(K_MSEC(1)); - } while (!k_fifo_is_empty(&data->fifo_in)); - } else { - /* Flush driver output queue */ - do { - ret = CSI_TransferGetFullBuffer(config->base, &(data->csi_handle), - &buffer_addr); - } while (ret == kStatus_Success); - - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - k_fifo_put(&data->fifo_out, vbuf); - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_ABORTED); - } - } - } - - return 0; -} - -static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) -{ - const struct video_mcux_csi_config *config = dev->config; - struct video_mcux_csi_data *data = dev->data; - unsigned int to_read; - status_t ret; - - to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; - vbuf->bytesused = to_read; - vbuf->line_offset = 0; - - ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, - (uint32_t)vbuf->buffer); - if (ret != kStatus_Success) { - return -EIO; - } - - k_fifo_put(&data->fifo_in, vbuf); - - return 0; -} - -static int video_mcux_csi_dequeue(const struct device *dev, struct video_buffer **vbuf, - k_timeout_t timeout) -{ - struct video_mcux_csi_data *data = dev->data; - - *vbuf = k_fifo_get(&data->fifo_out, timeout); - if (*vbuf == NULL) { - return -EAGAIN; - } - - return 0; -} - static int video_mcux_csi_get_caps(const struct device *dev, struct video_caps *caps) { const struct video_mcux_csi_config *config = dev->config; @@ -343,9 +256,6 @@ static int video_mcux_csi_init(const struct device *dev) struct video_mcux_csi_data *data = dev->data; int err; - k_fifo_init(&data->fifo_in); - k_fifo_init(&data->fifo_out); - CSI_GetDefaultConfig(&data->csi_config); /* check if there is any source device (video ctrl device) @@ -360,23 +270,10 @@ static int video_mcux_csi_init(const struct device *dev) return err; } - return 0; -} - -#ifdef CONFIG_POLL -static int video_mcux_csi_set_signal(const struct device *dev, struct k_poll_signal *sig) -{ - struct video_mcux_csi_data *data = dev->data; - - if (data->sig && sig != NULL) { - return -EALREADY; - } - - data->sig = sig; + mpsc_init(&data->io_q); return 0; } -#endif static int video_mcux_csi_set_frmival(const struct device *dev, struct video_frmival *frmival) { @@ -411,20 +308,70 @@ static int video_mcux_csi_enum_frmival(const struct device *dev, struct video_fr return ret; } +// static int video_mcux_csi_enqueue(const struct device *dev, struct video_buffer *vbuf) +// { +// const struct video_mcux_csi_config *config = dev->config; +// struct video_mcux_csi_data *data = dev->data; +// unsigned int to_read; +// status_t ret; + +// struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); + +// if (iodev_sqe == NULL) { +// return -EIO; +// } + +// struct video_buffer *vbuf = iodev_sqe->sqe.userdata; + +// to_read = data->csi_config.linePitch_Bytes * data->csi_config.height; +// vbuf->bytesused = to_read; +// vbuf->line_offset = 0; + +// ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, +// (uint32_t)vbuf->buffer); +// if (ret != kStatus_Success) { +// return -EIO; +// } + +// return 0; +// } + +static void video_mcux_csi_iodev_submit(const struct device *dev, struct rtio_iodev_sqe *iodev_sqe) +{ + + const struct video_mcux_csi_config *config = dev->config; + struct video_mcux_csi_data *data = dev->data; + struct video_buffer *vbuf = iodev_sqe->sqe.userdata; + status_t ret; + + // struct rtio_iodev_sqe *iodev_sqe = video_pop_io_q(&data->io_q); + + // if (iodev_sqe == NULL) { + // return -EIO; + // } + + vbuf->bytesused = data->csi_config.linePitch_Bytes * data->csi_config.height; + vbuf->line_offset = 0; + + ret = CSI_TransferSubmitEmptyBuffer(config->base, &data->csi_handle, + (uint32_t)vbuf->buffer); + if (ret != kStatus_Success) { + return; + } + + return; +} + static DEVICE_API(video, video_mcux_csi_driver_api) = { .set_format = video_mcux_csi_set_fmt, .get_format = video_mcux_csi_get_fmt, .set_stream = video_mcux_csi_set_stream, - .flush = video_mcux_csi_flush, - .enqueue = video_mcux_csi_enqueue, - .dequeue = video_mcux_csi_dequeue, + // .enqueue = video_mcux_csi_enqueue, .get_caps = video_mcux_csi_get_caps, .set_frmival = video_mcux_csi_set_frmival, .get_frmival = video_mcux_csi_get_frmival, .enum_frmival = video_mcux_csi_enum_frmival, -#ifdef CONFIG_POLL - .set_signal = video_mcux_csi_set_signal, -#endif + .iodev_submit = video_mcux_csi_iodev_submit, }; #if 1 /* Unique Instance */ @@ -459,4 +406,6 @@ DEVICE_DT_INST_DEFINE(0, &video_mcux_csi_init_0, NULL, &video_mcux_csi_data_0, VIDEO_DEVICE_DEFINE(csi, DEVICE_DT_INST_GET(0), SOURCE_DEV(0)); +VIDEO_INTERFACE_DEFINE(csi_intf, DEVICE_DT_INST_GET(0), &(video_mcux_csi_data_0.io_q)); + #endif diff --git a/drivers/video/video_sw_generator.c b/drivers/video/video_sw_generator.c index ff176e0ad3480..ca30ba40a76f0 100644 --- a/drivers/video/video_sw_generator.c +++ b/drivers/video/video_sw_generator.c @@ -10,11 +10,13 @@ #include #include #include +#include #include #include #include "video_ctrls.h" #include "video_device.h" +#include "video_buffer.h" LOG_MODULE_REGISTER(video_sw_generator, CONFIG_VIDEO_LOG_LEVEL); @@ -38,11 +40,9 @@ struct video_sw_generator_data { const struct device *dev; struct sw_ctrls ctrls; struct video_format fmt; - struct k_fifo fifo_in; - struct k_fifo fifo_out; + struct mpsc io_q; struct k_work_delayable work; int pattern; - struct k_poll_signal *sig; uint32_t frame_rate; }; @@ -289,75 +289,25 @@ static int video_sw_generator_fill(const struct device *const dev, struct video_ static void video_sw_generator_worker(struct k_work *work) { struct k_work_delayable *dwork = k_work_delayable_from_work(work); - struct video_sw_generator_data *data; - struct video_buffer *vbuf; - - data = CONTAINER_OF(dwork, struct video_sw_generator_data, work); + struct video_sw_generator_data *data = + CONTAINER_OF(dwork, struct video_sw_generator_data, work); + const struct device *dev = data->dev; + struct rtio_iodev_sqe *iodev_sqe; k_work_reschedule(&data->work, K_MSEC(1000 / data->frame_rate)); - vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); - if (vbuf == NULL) { + iodev_sqe = video_pop_io_q(&data->io_q); + if (iodev_sqe == NULL) { return; } switch (data->pattern) { case VIDEO_PATTERN_COLOR_BAR: - video_sw_generator_fill(data->dev, vbuf); + video_sw_generator_fill(dev, iodev_sqe->sqe.userdata); break; } - k_fifo_put(&data->fifo_out, vbuf); - - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_DONE); - } - - k_yield(); -} - -static int video_sw_generator_enqueue(const struct device *dev, struct video_buffer *vbuf) -{ - struct video_sw_generator_data *data = dev->data; - - k_fifo_put(&data->fifo_in, vbuf); - - return 0; -} - -static int video_sw_generator_dequeue(const struct device *dev, struct video_buffer **vbuf, - k_timeout_t timeout) -{ - struct video_sw_generator_data *data = dev->data; - - *vbuf = k_fifo_get(&data->fifo_out, timeout); - if (*vbuf == NULL) { - return -EAGAIN; - } - - return 0; -} - -static int video_sw_generator_flush(const struct device *dev, bool cancel) -{ - struct video_sw_generator_data *data = dev->data; - struct video_buffer *vbuf; - - if (!cancel) { - /* wait for all buffer to be processed */ - do { - k_sleep(K_MSEC(1)); - } while (!k_fifo_is_empty(&data->fifo_in)); - } else { - while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { - k_fifo_put(&data->fifo_out, vbuf); - if (IS_ENABLED(CONFIG_POLL) && data->sig) { - k_poll_signal_raise(data->sig, VIDEO_BUF_ABORTED); - } - } - } - - return 0; + rtio_iodev_sqe_ok(iodev_sqe, 0); } static int video_sw_generator_get_caps(const struct device *dev, struct video_caps *caps) @@ -371,21 +321,6 @@ static int video_sw_generator_get_caps(const struct device *dev, struct video_ca return 0; } -#ifdef CONFIG_POLL -static int video_sw_generator_set_signal(const struct device *dev, struct k_poll_signal *sig) -{ - struct video_sw_generator_data *data = dev->data; - - if (data->sig && sig != NULL) { - return -EALREADY; - } - - data->sig = sig; - - return 0; -} -#endif - static int video_sw_generator_set_frmival(const struct device *dev, struct video_frmival *frmival) { struct video_sw_generator_data *data = dev->data; @@ -439,16 +374,10 @@ static DEVICE_API(video, video_sw_generator_driver_api) = { .set_format = video_sw_generator_set_fmt, .get_format = video_sw_generator_get_fmt, .set_stream = video_sw_generator_set_stream, - .flush = video_sw_generator_flush, - .enqueue = video_sw_generator_enqueue, - .dequeue = video_sw_generator_dequeue, .get_caps = video_sw_generator_get_caps, .set_frmival = video_sw_generator_set_frmival, .get_frmival = video_sw_generator_get_frmival, .enum_frmival = video_sw_generator_enum_frmival, -#ifdef CONFIG_POLL - .set_signal = video_sw_generator_set_signal, -#endif }; static int video_sw_generator_init_controls(const struct device *dev) @@ -464,8 +393,7 @@ static int video_sw_generator_init(const struct device *dev) struct video_sw_generator_data *data = dev->data; data->dev = dev; - k_fifo_init(&data->fifo_in); - k_fifo_init(&data->fifo_out); + mpsc_init(&data->io_q); k_work_init_delayable(&data->work, video_sw_generator_worker); return video_sw_generator_init_controls(dev); @@ -484,6 +412,9 @@ static int video_sw_generator_init(const struct device *dev) NULL, POST_KERNEL, CONFIG_VIDEO_INIT_PRIORITY, \ &video_sw_generator_driver_api); \ \ - VIDEO_DEVICE_DEFINE(video_sw_generator_##n, DEVICE_DT_INST_GET(n), NULL); + VIDEO_DEVICE_DEFINE(video_sw_generator_##n, DEVICE_DT_INST_GET(n), NULL); \ + \ + VIDEO_INTERFACE_DEFINE(video_sw_generator_intf, DEVICE_DT_INST_GET(n), \ + &video_sw_generator_data_##n.io_q); DT_INST_FOREACH_STATUS_OKAY(VIDEO_SW_GENERATOR_DEFINE) diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index 20de3754887ab..9d1edca7851bd 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -23,11 +23,12 @@ * @{ */ -#include #include -#include +#include +#include #include +#include #ifdef __cplusplus extern "C" { @@ -56,6 +57,29 @@ enum video_buf_type { VIDEO_BUF_TYPE_OUTPUT, }; +/** + * @brief video_buf_state enum + * + * Current state of a video buffer. + * This helps to keep track of state of a buffer for debugging or to do some guard + * checks, e.g. prevents re-enqueuing a buffer already queued, etc. + */ +enum video_buf_state { + /** buffer queued in video subsystem and driver */ + VIDEO_BUF_STATE_QUEUED, + /** buffer filled and can be dequeued by application */ + VIDEO_BUF_STATE_DONE, + /** buffer returned from the driver with an error */ + VIDEO_BUF_STATE_ERROR, +}; + +enum video_buf_memory { + /** buffer memory is on the video heap */ + VIDEO_MEMORY_INTERNAL = 1, + /** buffer memory is provided by user */ + VIDEO_MEMORY_EXTERNAL = 2, +}; + /** * @struct video_format * @brief Video format structure @@ -146,11 +170,15 @@ struct video_caps { struct video_buffer { /** type of the buffer */ enum video_buf_type type; + /** current state of the buffer */ + enum video_buf_state state; + /** the backend memory of the buffer */ + enum video_buf_memory memory; /** pointer to driver specific data. */ void *driver_data; /** pointer to the start of the buffer. */ uint8_t *buffer; - /** index of the buffer, optionally set by the application */ + /** index of the buffer, read-only */ uint8_t index; /** size of the buffer in bytes. */ uint32_t size; @@ -315,22 +343,8 @@ typedef int (*video_api_frmival_t)(const struct device *dev, struct video_frmiva */ typedef int (*video_api_enum_frmival_t)(const struct device *dev, struct video_frmival_enum *fie); -/** - * @typedef video_api_enqueue_t - * @brief Enqueue a buffer in the driver’s incoming queue. - * - * See video_enqueue() for argument descriptions. - */ -typedef int (*video_api_enqueue_t)(const struct device *dev, struct video_buffer *buf); - -/** - * @typedef video_api_dequeue_t - * @brief Dequeue a buffer from the driver’s outgoing queue. - * - * See video_dequeue() for argument descriptions. - */ -typedef int (*video_api_dequeue_t)(const struct device *dev, struct video_buffer **buf, - k_timeout_t timeout); +typedef void (*video_api_iodev_submit_t)(const struct device *dev, + struct rtio_iodev_sqe *iodev_sqe); /** * @typedef video_api_flush_t @@ -396,8 +410,6 @@ __subsystem struct video_driver_api { video_api_set_stream_t set_stream; video_api_get_caps_t get_caps; /* optional callbacks */ - video_api_enqueue_t enqueue; - video_api_dequeue_t dequeue; video_api_flush_t flush; video_api_ctrl_t set_ctrl; video_api_ctrl_t get_volatile_ctrl; @@ -407,6 +419,7 @@ __subsystem struct video_driver_api { video_api_enum_frmival_t enum_frmival; video_api_selection_t set_selection; video_api_selection_t get_selection; + video_api_iodev_submit_t iodev_submit; }; /** @@ -565,27 +578,13 @@ static inline int video_enum_frmival(const struct device *dev, struct video_frmi * endpoint incoming queue. * * @param dev Pointer to the device structure for the driver instance. - * @param buf Pointer to the video buffer. + * @param buf Pointer to a video buffer structure. * * @retval 0 Is successful. * @retval -EINVAL If parameters are invalid. * @retval -EIO General input / output error. */ -static inline int video_enqueue(const struct device *dev, struct video_buffer *buf) -{ - const struct video_driver_api *api = (const struct video_driver_api *)dev->api; - - __ASSERT_NO_MSG(dev != NULL); - __ASSERT_NO_MSG(buf != NULL); - __ASSERT_NO_MSG(buf->buffer != NULL); - - api = (const struct video_driver_api *)dev->api; - if (api->enqueue == NULL) { - return -ENOSYS; - } - - return api->enqueue(dev, buf); -} +int video_enqueue(const struct device *dev, struct video_buffer *buf); /** * @brief Dequeue a video buffer. @@ -593,29 +592,13 @@ static inline int video_enqueue(const struct device *dev, struct video_buffer *b * Dequeue a filled (capturing) or displayed (output) buffer from the driver’s * endpoint outgoing queue. * - * @param dev Pointer to the device structure for the driver instance. * @param buf Pointer a video buffer pointer. - * @param timeout Timeout * - * @retval 0 Is successful. - * @retval -EINVAL If parameters are invalid. - * @retval -EIO General input / output error. + * @retval The RTIO completion event with the completed buffer. */ -static inline int video_dequeue(const struct device *dev, struct video_buffer **buf, - k_timeout_t timeout) -{ - const struct video_driver_api *api; +struct rtio_cqe *video_dequeue(void); - __ASSERT_NO_MSG(dev != NULL); - __ASSERT_NO_MSG(buf != NULL); - - api = (const struct video_driver_api *)dev->api; - if (api->dequeue == NULL) { - return -ENOSYS; - } - - return api->dequeue(dev, buf, timeout); -} +void video_release_buf(struct rtio_cqe *cqe); /** * @brief Flush endpoint buffers. @@ -884,33 +867,8 @@ static inline int video_get_selection(const struct device *dev, struct video_sel return api->get_selection(dev, sel); } -/** - * @brief Allocate aligned video buffer. - * - * @param size Size of the video buffer (in bytes). - * @param align Alignment of the requested memory, must be a power of two. - * @param timeout Timeout duration or K_NO_WAIT - * - * @retval pointer to allocated video buffer - */ -struct video_buffer *video_buffer_aligned_alloc(size_t size, size_t align, k_timeout_t timeout); - -/** - * @brief Allocate video buffer. - * - * @param size Size of the video buffer (in bytes). - * @param timeout Timeout duration or K_NO_WAIT - * - * @retval pointer to allocated video buffer - */ -struct video_buffer *video_buffer_alloc(size_t size, k_timeout_t timeout); - -/** - * @brief Release a video buffer. - * - * @param buf Pointer to the video buffer to release. - */ -void video_buffer_release(struct video_buffer *buf); +int video_request_buffers(uint8_t count, size_t size, enum video_buf_type type, + enum video_buf_memory memory); /** * @brief Search for a format that matches in a list of capabilities diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index d744c1604de40..a496e05c4d695 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -19,7 +19,7 @@ LOG_MODULE_REGISTER(main, LOG_LEVEL_DBG); #else -LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); +LOG_MODULE_REGISTER(main, LOG_LEVEL_DBG); #endif #if DT_HAS_CHOSEN(zephyr_display) @@ -279,20 +279,17 @@ int main(void) bsize = fmt.pitch * caps.min_line_count; } - /* Alloc video buffers and enqueue for capture */ + err = video_request_buffers(ARRAY_SIZE(buffers), bsize, type, VIDEO_MEMORY_INTERNAL); + if (err) { + LOG_ERR("Unable to request video buf"); + return 0; + } + + /* Enqueue buffers for capture */ + vbuf->type = type; for (i = 0; i < ARRAY_SIZE(buffers); i++) { - /* - * For some hardwares, such as the PxP used on i.MX RT1170 to do image rotation, - * buffer alignment is needed in order to achieve the best performance - */ - buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN, - K_FOREVER); - if (buffers[i] == NULL) { - LOG_ERR("Unable to alloc video buffer"); - return 0; - } - buffers[i]->type = type; - video_enqueue(video_dev, buffers[i]); + vbuf->index = i; + video_enqueue(video_dev, vbuf); } /* Start video capture */ @@ -304,14 +301,16 @@ int main(void) LOG_INF("Capture started"); /* Grab video frames */ - vbuf->type = type; while (1) { - err = video_dequeue(video_dev, &vbuf, K_FOREVER); - if (err) { - LOG_ERR("Unable to dequeue video buf"); + struct rtio_cqe *cqe = video_dequeue(); + struct video_buffer *vbuf = cqe->userdata; + + if (cqe->result != 0) { + LOG_ERR("The video buffer of size %u completed with errors", vbuf->size); return 0; } + LOG_DBG("Got frame %u! size: %u; timestamp %u ms", frame++, vbuf->bytesused, vbuf->timestamp); @@ -325,10 +324,6 @@ int main(void) video_display_frame(display_dev, vbuf, fmt); #endif - err = video_enqueue(video_dev, vbuf); - if (err) { - LOG_ERR("Unable to requeue video buf"); - return 0; - } + video_release_buf(cqe); } }