From 4841df0698a8debe6d3e71c34154e474853aff67 Mon Sep 17 00:00:00 2001 From: Armin Kessler Date: Mon, 1 Jul 2024 09:34:57 +0000 Subject: [PATCH 1/5] drivers: video: clk: esp32s3: add camera clock device This commit creates a new node for the lcd_cam interface. It enables the lcd_cam pheripheral clock and adds the possibiliy to controll the camera clock frequency trough the `cam-clk` property. This is its own device for two reasons: 1. It enables to use the camera clock as a standalone clock. 2. It enables to initialize xclk before the sensor driver. Signed-off-by: Armin Kessler --- drivers/clock_control/CMakeLists.txt | 1 + drivers/clock_control/Kconfig | 2 + drivers/clock_control/Kconfig.esp32_cam | 4 + .../clock_control/clock_control_esp32_cam.c | 99 +++++++++++++++++++ .../clock/espressif,esp32-cam-clk.yaml | 21 ++++ .../espressif/esp32s3/esp32s3_common.dtsi | 7 ++ 6 files changed, 134 insertions(+) create mode 100644 drivers/clock_control/Kconfig.esp32_cam create mode 100644 drivers/clock_control/clock_control_esp32_cam.c create mode 100644 dts/bindings/clock/espressif,esp32-cam-clk.yaml diff --git a/drivers/clock_control/CMakeLists.txt b/drivers/clock_control/CMakeLists.txt index 43f86e8c7d38cf..fad3c7ecb6e745 100644 --- a/drivers/clock_control/CMakeLists.txt +++ b/drivers/clock_control/CMakeLists.txt @@ -5,6 +5,7 @@ zephyr_library() zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_BEETLE beetle_clock_control.c) zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_ADSP clock_control_adsp.c) zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_ESP32 clock_control_esp32.c) +zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_ESP32_CAM clock_control_esp32_cam.c) zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_FIXED_RATE_CLOCK clock_control_fixed_rate.c) zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_GD32 clock_control_gd32.c) zephyr_library_sources_ifdef(CONFIG_CLOCK_CONTROL_LITEX clock_control_litex.c) diff --git a/drivers/clock_control/Kconfig b/drivers/clock_control/Kconfig index 558210560479fd..4a1fe1526506c1 100644 --- a/drivers/clock_control/Kconfig +++ b/drivers/clock_control/Kconfig @@ -56,6 +56,8 @@ source "drivers/clock_control/Kconfig.rv32m1" source "drivers/clock_control/Kconfig.esp32" +source "drivers/clock_control/Kconfig.esp32_cam" + source "drivers/clock_control/Kconfig.litex" source "drivers/clock_control/Kconfig.rcar" diff --git a/drivers/clock_control/Kconfig.esp32_cam b/drivers/clock_control/Kconfig.esp32_cam new file mode 100644 index 00000000000000..f7b9b0e5861d23 --- /dev/null +++ b/drivers/clock_control/Kconfig.esp32_cam @@ -0,0 +1,4 @@ +config CLOCK_CONTROL_ESP32_CAM + bool "master clock for esp32 camera interface" + help + This option enables the pheriphery and cam clock for the lcd_cam module. diff --git a/drivers/clock_control/clock_control_esp32_cam.c b/drivers/clock_control/clock_control_esp32_cam.c new file mode 100644 index 00000000000000..01524e279d628f --- /dev/null +++ b/drivers/clock_control/clock_control_esp32_cam.c @@ -0,0 +1,99 @@ +#define DT_DRV_COMPAT espressif_esp32_cam_clk + +#include +LOG_MODULE_REGISTER(esp32_lcd_cam, CONFIG_CLOCK_CONTROL_LOG_LEVEL); + +#include +#include +#include +#include +#include + +#define ESP32_CAM_PLL_F160M 160000000UL +#define ESP32_CAM_PLL_F160M_SEL 3 +#define ESP32_CAM_CLK_OFF_SEL 0 + +struct clock_control_esp32_cam_config { + const struct pinctrl_dev_config *pcfg; + const struct device *clk_dev; + struct device *clk_subsys; + uint32_t cam_clk; + uint8_t clock_sel; +}; + +static int enable_pheripheral_clock(const struct device *dev) +{ + const struct clock_control_esp32_cam_config *cfg = dev->config; + int ret = 0; + + /* Enable peripheral */ + if (!device_is_ready(cfg->clk_dev)) { + return -ENODEV; + } + + return clock_control_on(cfg->clk_dev, cfg->clk_subsys); +} + +static int set_camera_clock(uint32_t cam_clk) +{ + int ret = 0; + + if (0 == cam_clk) { + LCD_CAM.cam_ctrl.cam_clk_sel = + ESP32_CAM_CLK_OFF_SEL; + LOG_DBG("Disabled CAM_CLK"); + return -EINVAL; + } + + if (ESP32_CAM_PLL_F160M % cam_clk) { + LOG_WRN("MCLK is not a devider of 160MHz"); + } + + LCD_CAM.cam_ctrl.cam_clk_sel = ESP32_CAM_PLL_F160M_SEL; + LCD_CAM.cam_ctrl.cam_clkm_div_num = ESP32_CAM_PLL_F160M / cam_clk; + LCD_CAM.cam_ctrl.cam_clkm_div_b = 0; + LCD_CAM.cam_ctrl.cam_clkm_div_a = 0; + LOG_DBG("MCLK set to %ld", ESP32_CAM_PLL_F160M / LCD_CAM.cam_ctrl.cam_clkm_div_num); + + return ret; +} + +static int clock_control_esp32_cam_init(const struct device *dev) +{ + const struct clock_control_esp32_cam_config *cfg = dev->config; + int ret = 0; + + ret = pinctrl_apply_state(cfg->pcfg, PINCTRL_STATE_DEFAULT); + if (ret < 0) { + LOG_ERR("video pinctrl setup failed (%d)", ret); + return ret; + } + + ret = enable_pheripheral_clock(dev); + if (ret < 0) { + LOG_ERR("Failed to enable peripheral clock"); + return ret; + } + + ret = set_camera_clock(cfg->cam_clk); + if (ret < 0) { + LOG_ERR("Failed to set camera clock"); + return ret; + } + + LOG_DBG("cam clock initialized"); + + return 0; +} + +PINCTRL_DT_INST_DEFINE(0); + +static const struct clock_control_esp32_cam_config clock_control_esp32_cam_config = { + .pcfg = PINCTRL_DT_INST_DEV_CONFIG_GET(0), + .clk_dev = DEVICE_DT_GET(DT_INST_CLOCKS_CTLR(0)), + .clk_subsys = (clock_control_subsys_t)DT_INST_CLOCKS_CELL(0, offset), + .cam_clk = DT_INST_PROP_OR(0, cam_clk, 0), +}; + +DEVICE_DT_INST_DEFINE(0, &clock_control_esp32_cam_init, NULL, NULL, &clock_control_esp32_cam_config, + PRE_KERNEL_1, CONFIG_CLOCK_CONTROL_INIT_PRIORITY, NULL); diff --git a/dts/bindings/clock/espressif,esp32-cam-clk.yaml b/dts/bindings/clock/espressif,esp32-cam-clk.yaml new file mode 100644 index 00000000000000..f4a234ec236378 --- /dev/null +++ b/dts/bindings/clock/espressif,esp32-cam-clk.yaml @@ -0,0 +1,21 @@ +# +# Copyright (c) 2019, Linaro Limited +# +# SPDX-License-Identifier: Apache-2.0 +# + +description: NXP MCUX CMOS sensor interface + +compatible: "espressif,esp32-cam-clk" + +include: [base.yaml, pinctrl-device.yaml] + +properties: + pinctrl-0: + required: true + + pinctrl-names: + required: true + + cam_clk: + type: int diff --git a/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi b/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi index 2b571a65960876..d648f3b7843161 100644 --- a/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi +++ b/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi @@ -323,6 +323,13 @@ status = "disabled"; }; + lcd_cam: lcd_cam@60041000 { + compatible = "espressif,esp32-cam-clk"; + reg = <0x60041000 DT_SIZE_K(4)>; + clocks = <&rtc ESP32_LCD_CAM_MODULE>; + status = "disabled"; + }; + usb_serial: uart@60038000 { compatible = "espressif,esp32-usb-serial"; reg = <0x60038000 DT_SIZE_K(4)>; From 4cb56a15486dad34b21e49354d8082c1c54ce418 Mon Sep 17 00:00:00 2001 From: Armin Kessler Date: Wed, 26 Jun 2024 15:24:41 +0000 Subject: [PATCH 2/5] drivers: video: esp32s3: add support for cam interface Adding support for the esp32s3 camera interface. Signed-off-by: Armin Kessler --- drivers/video/CMakeLists.txt | 1 + drivers/video/Kconfig | 2 + drivers/video/Kconfig.esp32_dvp | 6 + drivers/video/video_esp32_dvp.c | 461 ++++++++++++++++++ .../clock/espressif,esp32-cam-clk.yaml | 4 +- dts/bindings/video/espressif,esp32-cam.yaml | 51 ++ .../espressif/esp32s3/esp32s3_common.dtsi | 8 + 7 files changed, 531 insertions(+), 2 deletions(-) create mode 100644 drivers/video/Kconfig.esp32_dvp create mode 100644 drivers/video/video_esp32_dvp.c create mode 100644 dts/bindings/video/espressif,esp32-cam.yaml diff --git a/drivers/video/CMakeLists.txt b/drivers/video/CMakeLists.txt index 2e52d1f66ea889..2feae9ec6a8db6 100644 --- a/drivers/video/CMakeLists.txt +++ b/drivers/video/CMakeLists.txt @@ -4,6 +4,7 @@ zephyr_library() zephyr_library_sources(video_common.c) +zephyr_library_sources_ifdef(CONFIG_VIDEO_ESP32S3 video_esp32_dvp.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_MCUX_CSI video_mcux_csi.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_MCUX_MIPI_CSI2RX video_mcux_mipi_csi2rx.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_SW_GENERATOR video_sw_generator.c) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index d6a6bd3ccce669..8458695e143bc7 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -31,6 +31,8 @@ config VIDEO_BUFFER_POOL_ALIGN int "Alignment of the video pool’s buffer" default 64 +source "drivers/video/Kconfig.esp32_dvp" + source "drivers/video/Kconfig.mcux_csi" source "drivers/video/Kconfig.mcux_mipi_csi2rx" diff --git a/drivers/video/Kconfig.esp32_dvp b/drivers/video/Kconfig.esp32_dvp new file mode 100644 index 00000000000000..dc947a1c91e7f6 --- /dev/null +++ b/drivers/video/Kconfig.esp32_dvp @@ -0,0 +1,6 @@ +config VIDEO_ESP32S3 + bool "Video interface driver" + default n + depends on DMA + help + This option enables the video interface for the esp32s3. diff --git a/drivers/video/video_esp32_dvp.c b/drivers/video/video_esp32_dvp.c new file mode 100644 index 00000000000000..e253c35a9cfbfd --- /dev/null +++ b/drivers/video/video_esp32_dvp.c @@ -0,0 +1,461 @@ +/* + * Copyright (c) 2024 espros photonics Co. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#define DT_DRV_COMPAT espressif_esp32_cam + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +LOG_MODULE_REGISTER(esp32_cam, LOG_LEVEL_INF); + +#define VIDEO_ESP32_F160M 160000000UL +#define VIDEO_ESP32_DMA_BUFFER_MAX_SIZE 4095 + +enum video_esp32_cam_clk_sel_values { + VIDEO_ESP32_CAM_CLK_SEL_NONE = 0, + VIDEO_ESP32_CAM_CLK_SEL_XTAL = 1, + VIDEO_ESP32_CAM_CLK_SEL_PLL_DIV2 = 2, + VIDEO_ESP32_CAM_CLK_SEL_PLL_F160M = 3, +}; + +struct video_esp32_config { + const struct pinctrl_dev_config *pcfg; + const struct device *source_dev; + const struct device *clock_dev; + const clock_control_subsys_t clock_subsys; + const struct device *dma_dev; + uint8_t rx_dma_channel; + uint32_t clock_mclk; + int irq_source; + int irq_priority; + uint8_t enable_16_bit; + uint8_t invert_de; + uint8_t invert_byte_order; + uint8_t invert_bit_order; + uint8_t invert_pclk; + uint8_t invert_hsync; + uint8_t invert_vsync; +}; + +struct video_esp32_data { + const struct video_esp32_config *config; + struct video_format video_format; + struct video_buffer *active_vbuf; + bool is_streaming; + struct k_fifo fifo_in; + struct k_fifo fifo_out; + struct dma_block_config dma_blocks[CONFIG_DMA_ESP32_MAX_DESCRIPTOR_NUM]; +}; + +static int video_esp32_reload_dma(struct video_esp32_data *data); + +static void video_esp32_irq_handler(void *arg) +{ + const struct device *dev = (const struct device *)arg; + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + lcd_cam_lc_dma_int_st_reg_t status = LCD_CAM.lc_dma_int_st; + + if (!status.cam_vsync_int_st) { + LOG_WRN("Unexpected interrupt. status = %x", status.val); + return; + } + + LCD_CAM.lc_dma_int_clr.val = status.val; + data->active_vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); + + if (data->active_vbuf == NULL) { + LOG_WRN("Frame dropped. No buffer available"); + return; + } + + video_esp32_reload_dma(data); +} + +void video_esp32_dma_rx_done(const struct device *dev, void *user_data, uint32_t channel, + int status) +{ + struct video_esp32_data *data = user_data; + int ret = 0; + + if (status == DMA_STATUS_BLOCK) { + LOG_DBG("received block"); + return; + } + + if (status != DMA_STATUS_COMPLETE) { + LOG_ERR("DMA error: %d", status); + return; + } + + if (data->active_vbuf == NULL) { + LOG_ERR("No video buffer available. Enque some buffers first."); + return; + } + + k_fifo_put(&data->fifo_out, data->active_vbuf); + data->active_vbuf = NULL; +} + +static int video_esp32_reload_dma(struct video_esp32_data *data) +{ + const struct video_esp32_config *cfg = data->config; + int ret = 0; + + if (data->active_vbuf == NULL) { + LOG_ERR("No video buffer available. Enque some buffers first."); + return -EAGAIN; + } + + ret = dma_reload(cfg->dma_dev, cfg->rx_dma_channel, 0, (uint32_t)data->active_vbuf->buffer, + data->active_vbuf->bytesused); + if (ret) { + LOG_ERR("Unable to reload DMA (%d)", ret); + return ret; + } + + ret = dma_start(cfg->dma_dev, cfg->rx_dma_channel); + if (ret) { + LOG_ERR("Unable to start DMA (%d)", ret); + return ret; + } + + return 0; +} + +static int video_esp32_stream_start(const struct device *dev) +{ + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + struct dma_status dma_status = {0}; + struct dma_config dma_cfg = {0}; + struct dma_block_config *dma_block_iter = data->dma_blocks; + uint32_t buffer_size = 0; + int error = 0; + + if (data->is_streaming) { + return -EBUSY; + } + + LOG_DBG("Start streaming"); + + error = dma_get_status(cfg->dma_dev, cfg->rx_dma_channel, &dma_status); + + if (error) { + LOG_ERR("Unable to get Rx status (%d)", error); + return error; + } + + if (dma_status.busy) { + LOG_ERR("Rx DMA Channel %d is busy", cfg->rx_dma_channel); + return -EBUSY; + } + + data->active_vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); + if (!data->active_vbuf) { + LOG_ERR("No enqueued video buffers available."); + return -EAGAIN; + } + + buffer_size = data->active_vbuf->bytesused; + memset(data->dma_blocks, 0, sizeof(data->dma_blocks)); + for (int i = 0; i < CONFIG_DMA_ESP32_MAX_DESCRIPTOR_NUM; ++i) { + dma_block_iter->dest_address = + (uint32_t)data->active_vbuf->buffer + i * VIDEO_ESP32_DMA_BUFFER_MAX_SIZE; + if (buffer_size < VIDEO_ESP32_DMA_BUFFER_MAX_SIZE) { + dma_block_iter->block_size = buffer_size; + dma_block_iter->next_block = NULL; + dma_cfg.block_count = i + 1; + break; + } + dma_block_iter->block_size = VIDEO_ESP32_DMA_BUFFER_MAX_SIZE; + dma_block_iter->next_block = dma_block_iter + 1; + dma_block_iter++; + buffer_size -= VIDEO_ESP32_DMA_BUFFER_MAX_SIZE; + } + + if (dma_block_iter->next_block) { + LOG_ERR("Not enough descriptors available. Increase DMA_ESP32_DESCRIPTOR_NUM"); + return -ENOBUFS; + } + + dma_cfg.channel_direction = PERIPHERAL_TO_MEMORY; + dma_cfg.dma_callback = video_esp32_dma_rx_done; + dma_cfg.user_data = data; + dma_cfg.dma_slot = SOC_GDMA_TRIG_PERIPH_CAM0; + dma_cfg.complete_callback_en = 1; + dma_cfg.head_block = &data->dma_blocks[0]; + + error = dma_config(cfg->dma_dev, cfg->rx_dma_channel, &dma_cfg); + if (error) { + LOG_ERR("Unable to configure DMA (%d)", error); + return error; + } + + LCD_CAM.cam_ctrl1.cam_reset = 1; + LCD_CAM.cam_ctrl1.cam_reset = 0; + LCD_CAM.cam_ctrl1.cam_afifo_reset = 1; + LCD_CAM.cam_ctrl1.cam_afifo_reset = 0; + + error = dma_start(cfg->dma_dev, cfg->rx_dma_channel); + if (error) { + LOG_ERR("Unable to start DMA (%d)", error); + return error; + } + + LCD_CAM.cam_ctrl.cam_update = 1; + LCD_CAM.cam_ctrl.cam_update = 0; + LCD_CAM.cam_ctrl1.cam_start = 1; + + data->is_streaming = true; + + return 0; +} + +static int video_esp32_stream_stop(const struct device *dev) +{ + int ret = 0; + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + + LOG_DBG("Stop streaming"); + + data->is_streaming = false; + dma_stop(cfg->dma_dev, cfg->rx_dma_channel); + LCD_CAM.cam_ctrl1.cam_start = 0; + return ret; +} + +static int video_esp32_get_caps(const struct device *dev, enum video_endpoint_id ep, + struct video_caps *caps) +{ + const struct video_esp32_config *config = dev->config; + int ret = -ENODEV; + + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + /* Forward the message to the source device */ + ret = video_get_caps(config->source_dev, ep, caps); + + return ret; +} + +static int video_esp32_get_fmt(const struct device *dev, enum video_endpoint_id ep, + struct video_format *fmt) +{ + const struct video_esp32_config *cfg = dev->config; + int ret = 0; + + LOG_DBG("Get format"); + + if (fmt == NULL || ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + ret = video_get_format(cfg->source_dev, ep, fmt); + if (ret) { + LOG_ERR("Failed to get format from source"); + return ret; + } + + return ret; +} + +static int video_esp32_set_fmt(const struct device *dev, enum video_endpoint_id ep, + struct video_format *fmt) +{ + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + + if (fmt == NULL || ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + data->video_format = *fmt; + + return video_set_format(cfg->source_dev, ep, fmt); +} + +static int video_esp32_enqueue(const struct device *dev, enum video_endpoint_id ep, + struct video_buffer *vbuf) +{ + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + int ret = 0; + unsigned int key; + + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + vbuf->bytesused = data->video_format.pitch * data->video_format.height; + + k_fifo_put(&data->fifo_in, vbuf); + + return ret; +} + +static int video_esp32_dequeue(const struct device *dev, enum video_endpoint_id ep, + struct video_buffer **vbuf, k_timeout_t timeout) +{ + struct video_esp32_data *data = dev->data; + int ret = 0; + uint8_t key; + + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + *vbuf = k_fifo_get(&data->fifo_out, timeout); + LOG_DBG("Dequeue done, vbuf = %p", *vbuf); + if (*vbuf == NULL) { + return -EAGAIN; + } + + return 0; +} + +static int video_esp32_set_ctrl(const struct device *dev, unsigned int cid, void *value) +{ + const struct video_esp32_config *cfg = dev->config; + + return video_set_ctrl(cfg->source_dev, cid, value); +} + +static int video_esp32_get_ctrl(const struct device *dev, unsigned int cid, void *value) +{ + const struct video_esp32_config *cfg = dev->config; + + return video_get_ctrl(cfg->source_dev, cid, value); +} + +static void video_esp32_cam_ctrl_init(const struct device *dev) +{ + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + + LCD_CAM.cam_ctrl.cam_stop_en = 0; + LCD_CAM.cam_ctrl.cam_vsync_filter_thres = 4; + LCD_CAM.cam_ctrl.cam_byte_order = cfg->invert_byte_order; + LCD_CAM.cam_ctrl.cam_bit_order = cfg->invert_bit_order; + LCD_CAM.cam_ctrl.cam_line_int_en = 0; + LCD_CAM.cam_ctrl.cam_vs_eof_en = 1; + + LCD_CAM.cam_ctrl1.val = 0; + LCD_CAM.cam_ctrl1.cam_rec_data_bytelen = 0; + LCD_CAM.cam_ctrl1.cam_line_int_num = 0; + LCD_CAM.cam_ctrl1.cam_clk_inv = cfg->invert_pclk; + LCD_CAM.cam_ctrl1.cam_vsync_filter_en = 1; + LCD_CAM.cam_ctrl1.cam_2byte_en = cfg->enable_16_bit; + LCD_CAM.cam_ctrl1.cam_de_inv = cfg->invert_de; + LCD_CAM.cam_ctrl1.cam_hsync_inv = cfg->invert_hsync; + LCD_CAM.cam_ctrl1.cam_vsync_inv = cfg->invert_vsync; + LCD_CAM.cam_ctrl1.cam_vh_de_mode_en = 0; + + LCD_CAM.cam_rgb_yuv.val = 0; +} + +static int esp32_interrupt_init(const struct device *dev) +{ + const struct video_esp32_config *cfg = dev->config; + int ret = 0; + + LCD_CAM.lc_dma_int_clr.cam_hs_int_clr = 1; + LCD_CAM.lc_dma_int_clr.cam_vsync_int_clr = 1; + LCD_CAM.lc_dma_int_ena.cam_vsync_int_ena = 1; + LCD_CAM.lc_dma_int_ena.cam_hs_int_ena = 1; + + ret = esp_intr_alloc(cfg->irq_source, cfg->irq_priority, + (intr_handler_t)video_esp32_irq_handler, (void *)dev, NULL); + + return ret; +} + +static int esp32_init(const struct device *dev) +{ + const struct video_esp32_config *cfg = dev->config; + struct video_esp32_data *data = dev->data; + int ret = 0; + + data->config = cfg; + + k_fifo_init(&data->fifo_in); + k_fifo_init(&data->fifo_out); + + ret = pinctrl_apply_state(cfg->pcfg, PINCTRL_STATE_DEFAULT); + if (ret < 0) { + LOG_ERR("video pinctrl setup failed (%d)", ret); + return ret; + } + + video_esp32_cam_ctrl_init(dev); + + ret = esp32_interrupt_init(dev); + if (ret) { + LOG_ERR("Failed to initialize interrupt"); + return ret; + } + + if (!device_is_ready(cfg->dma_dev)) { + return -ENODEV; + } + + LOG_DBG("esp32 video driver loaded"); + + return ret; +} + +static const struct video_driver_api esp32_driver_api = { + /* mandatory callbacks */ + .set_format = video_esp32_set_fmt, + .get_format = video_esp32_get_fmt, + .stream_start = video_esp32_stream_start, + .stream_stop = video_esp32_stream_stop, + .get_caps = video_esp32_get_caps, + /* optional callbacks */ + .enqueue = video_esp32_enqueue, + .dequeue = video_esp32_dequeue, + .flush = NULL, + .set_ctrl = video_esp32_set_ctrl, + .get_ctrl = video_esp32_get_ctrl, + .set_signal = NULL, +}; + +PINCTRL_DT_INST_DEFINE(0); + +static const struct video_esp32_config esp32_config = { + .pcfg = PINCTRL_DT_INST_DEV_CONFIG_GET(0), + .source_dev = DEVICE_DT_GET(DT_INST_PHANDLE(0, source)), + .dma_dev = ESP32_DT_INST_DMA_CTLR(0, rx), + .rx_dma_channel = DT_INST_DMAS_CELL_BY_NAME(0, rx, channel), + .irq_source = DT_INST_IRQN(0), + .irq_priority = ESP_INTR_FLAG_LOWMED, + .enable_16_bit = DT_INST_PROP(0, enable_16bit_mode), + .invert_bit_order = DT_INST_PROP(0, invert_bit_order), + .invert_byte_order = DT_INST_PROP(0, invert_byte_order), + .invert_pclk = DT_INST_PROP(0, invert_pclk), + .invert_de = DT_INST_PROP(0, invert_de), + .invert_hsync = DT_INST_PROP(0, invert_hsync), + .invert_vsync = DT_INST_PROP(0, invert_vsync)}; + +static struct video_esp32_data esp32_data = {0}; + +DEVICE_DT_INST_DEFINE(0, &esp32_init, PM_DEVICE_DT_INST_GET(idx), &esp32_data, &esp32_config, + POST_KERNEL, CONFIG_VIDEO_INIT_PRIORITY, &esp32_driver_api); diff --git a/dts/bindings/clock/espressif,esp32-cam-clk.yaml b/dts/bindings/clock/espressif,esp32-cam-clk.yaml index f4a234ec236378..e4f534abd19eef 100644 --- a/dts/bindings/clock/espressif,esp32-cam-clk.yaml +++ b/dts/bindings/clock/espressif,esp32-cam-clk.yaml @@ -1,10 +1,10 @@ # -# Copyright (c) 2019, Linaro Limited +# Copyright (c) 2024 espros photonics Co. # # SPDX-License-Identifier: Apache-2.0 # -description: NXP MCUX CMOS sensor interface +description: esp32 camera clock compatible: "espressif,esp32-cam-clk" diff --git a/dts/bindings/video/espressif,esp32-cam.yaml b/dts/bindings/video/espressif,esp32-cam.yaml new file mode 100644 index 00000000000000..83a4c437963e4d --- /dev/null +++ b/dts/bindings/video/espressif,esp32-cam.yaml @@ -0,0 +1,51 @@ +# +# Copyright (c) 2024 espros photonics Co. +# +# SPDX-License-Identifier: Apache-2.0 +# + +description: esp32 camera interface + +compatible: "espressif,esp32-cam" + +include: [base.yaml, pinctrl-device.yaml] + +properties: + pinctrl-0: + required: true + + pinctrl-names: + required: true + + source: + required: true + type: phandle + description: phandle of connected source device + + enable-16bit-mode: + type: boolean + description: enable 16bit interface + + invert-byte-order: + type: boolean + description: invert byte order in 16bit mode + + invert-bit-order: + type: boolean + description: invert bit order + + invert-pclk: + type: boolean + description: invert pixel clock signal + + invert-de: + type: boolean + description: invert data enable signal + + invert-hsync: + type: boolean + description: invert hsync signal + + invert-vsync: + type: boolean + description: invert vsync signal diff --git a/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi b/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi index d648f3b7843161..5de762111d4116 100644 --- a/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi +++ b/dts/xtensa/espressif/esp32s3/esp32s3_common.dtsi @@ -328,6 +328,14 @@ reg = <0x60041000 DT_SIZE_K(4)>; clocks = <&rtc ESP32_LCD_CAM_MODULE>; status = "disabled"; + camera0: camera { + compatible = "espressif,esp32-cam"; + interrupts = ; + interrupt-parent = <&intc>; + dmas = <&dma 2>; + dma-names = "rx"; + status = "disabled"; + }; }; usb_serial: uart@60038000 { From 866c2c0cba5edd9528a8389816b164e315121bc7 Mon Sep 17 00:00:00 2001 From: Armin Kessler Date: Tue, 2 Jul 2024 12:21:15 +0000 Subject: [PATCH 3/5] drivers: video: esp32s3: remove VSync interrupt Remove the VSync interrupt handle, as DMA captures VSync through the in_suc_eof event. Signed-off-by: Armin Kessler ake@espros.com --- drivers/video/video_esp32_dvp.c | 57 ++++----------------------------- 1 file changed, 7 insertions(+), 50 deletions(-) diff --git a/drivers/video/video_esp32_dvp.c b/drivers/video/video_esp32_dvp.c index e253c35a9cfbfd..4a3cb6cb8cd737 100644 --- a/drivers/video/video_esp32_dvp.c +++ b/drivers/video/video_esp32_dvp.c @@ -13,10 +13,8 @@ #include #include #include -#include #include #include -#include #include #include @@ -64,29 +62,6 @@ struct video_esp32_data { static int video_esp32_reload_dma(struct video_esp32_data *data); -static void video_esp32_irq_handler(void *arg) -{ - const struct device *dev = (const struct device *)arg; - const struct video_esp32_config *cfg = dev->config; - struct video_esp32_data *data = dev->data; - lcd_cam_lc_dma_int_st_reg_t status = LCD_CAM.lc_dma_int_st; - - if (!status.cam_vsync_int_st) { - LOG_WRN("Unexpected interrupt. status = %x", status.val); - return; - } - - LCD_CAM.lc_dma_int_clr.val = status.val; - data->active_vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); - - if (data->active_vbuf == NULL) { - LOG_WRN("Frame dropped. No buffer available"); - return; - } - - video_esp32_reload_dma(data); -} - void video_esp32_dma_rx_done(const struct device *dev, void *user_data, uint32_t channel, int status) { @@ -109,7 +84,13 @@ void video_esp32_dma_rx_done(const struct device *dev, void *user_data, uint32_t } k_fifo_put(&data->fifo_out, data->active_vbuf); - data->active_vbuf = NULL; + data->active_vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT); + + if (data->active_vbuf == NULL) { + LOG_WRN("Frame dropped. No buffer available"); + return; + } + video_esp32_reload_dma(data); } static int video_esp32_reload_dma(struct video_esp32_data *data) @@ -372,22 +353,6 @@ static void video_esp32_cam_ctrl_init(const struct device *dev) LCD_CAM.cam_rgb_yuv.val = 0; } -static int esp32_interrupt_init(const struct device *dev) -{ - const struct video_esp32_config *cfg = dev->config; - int ret = 0; - - LCD_CAM.lc_dma_int_clr.cam_hs_int_clr = 1; - LCD_CAM.lc_dma_int_clr.cam_vsync_int_clr = 1; - LCD_CAM.lc_dma_int_ena.cam_vsync_int_ena = 1; - LCD_CAM.lc_dma_int_ena.cam_hs_int_ena = 1; - - ret = esp_intr_alloc(cfg->irq_source, cfg->irq_priority, - (intr_handler_t)video_esp32_irq_handler, (void *)dev, NULL); - - return ret; -} - static int esp32_init(const struct device *dev) { const struct video_esp32_config *cfg = dev->config; @@ -407,12 +372,6 @@ static int esp32_init(const struct device *dev) video_esp32_cam_ctrl_init(dev); - ret = esp32_interrupt_init(dev); - if (ret) { - LOG_ERR("Failed to initialize interrupt"); - return ret; - } - if (!device_is_ready(cfg->dma_dev)) { return -ENODEV; } @@ -445,8 +404,6 @@ static const struct video_esp32_config esp32_config = { .source_dev = DEVICE_DT_GET(DT_INST_PHANDLE(0, source)), .dma_dev = ESP32_DT_INST_DMA_CTLR(0, rx), .rx_dma_channel = DT_INST_DMAS_CELL_BY_NAME(0, rx, channel), - .irq_source = DT_INST_IRQN(0), - .irq_priority = ESP_INTR_FLAG_LOWMED, .enable_16_bit = DT_INST_PROP(0, enable_16bit_mode), .invert_bit_order = DT_INST_PROP(0, invert_bit_order), .invert_byte_order = DT_INST_PROP(0, invert_byte_order), From 8ebcb4534a7a5e161ed1b6d3e463ce19d3235e04 Mon Sep 17 00:00:00 2001 From: Armin Kessler Date: Mon, 5 Aug 2024 17:17:23 +0200 Subject: [PATCH 4/5] sample: video: esp32s3: add support for xiao_esp32s3 adding xiao_esp32s3 support to video capture sample Signed-off-by: Armin Kessler ake@espros.com --- .../capture/boards/xiao_esp32s3_procpu.conf | 5 ++ .../boards/xiao_esp32s3_procpu.overlay | 85 +++++++++++++++++++ samples/drivers/video/capture/src/main.c | 14 +++ 3 files changed, 104 insertions(+) create mode 100644 samples/drivers/video/capture/boards/xiao_esp32s3_procpu.conf create mode 100644 samples/drivers/video/capture/boards/xiao_esp32s3_procpu.overlay diff --git a/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.conf b/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.conf new file mode 100644 index 00000000000000..d804d2f7c9c100 --- /dev/null +++ b/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.conf @@ -0,0 +1,5 @@ +CONFIG_DMA=y +CONFIG_VIDEO_ESP32S3=y +CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=40000 +CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=3 +CONFIG_CLOCK_CONTROL_ESP32_CAM=y \ No newline at end of file diff --git a/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.overlay b/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.overlay new file mode 100644 index 00000000000000..cde6569a89705a --- /dev/null +++ b/samples/drivers/video/capture/boards/xiao_esp32s3_procpu.overlay @@ -0,0 +1,85 @@ + #include + #include + #include + + / { + chosen { + zephyr_camera = &camera0; + }; + }; + +&dma { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + clock-frequency = ; + pinctrl-0 = <&i2c1_default>; + pinctrl-names = "default"; + ov2640: ov2640@30 { + compatible = "ovti,ov2640"; + reg = <0x30>; + status = "okay"; + clock-rate-control = <0x80>; + port { + ov2640_ep_out: endpoint { + remote-endpoint = <&dvp_ep_in>; + }; + }; + }; +}; + +&lcd_cam { + status = "okay"; + cam_clk = < 10000000 >; + pinctrl-0 = <&lcd_cam_default>; + pinctrl-names = "default"; +}; + +&camera0 { + status = "okay"; + pinctrl-0 = <&cam_default>; + pinctrl-names = "default"; + source = <&ov2640>; + port { + dvp_ep_in: endpoint { + remote-endpoint = <&ov2640_ep_out>; + }; + }; +}; + +&pinctrl { + lcd_cam_default: lcd_cam_default{ + group1 { + pinmux = ; + output-enable; + }; + }; + cam_default: cam_default { + group1 { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + input-enable; + bias-disable; + }; + }; + i2c1_default: i2c1_default { + group1 { + pinmux = , + ; + bias-pull-up; + drive-open-drain; + output-high; + }; + }; + }; diff --git a/samples/drivers/video/capture/src/main.c b/samples/drivers/video/capture/src/main.c index 862dff287f7aa5..64fae79f7de536 100644 --- a/samples/drivers/video/capture/src/main.c +++ b/samples/drivers/video/capture/src/main.c @@ -144,6 +144,20 @@ int main(void) } #endif + fmt.width = 160; + fmt.height = 120; + fmt.pitch = fmt.width * 2; + fmt.pixelformat = VIDEO_PIX_FMT_RGB565; + + if(video_set_format(video_dev, VIDEO_EP_OUT, &fmt)){ + LOG_ERR("Unable to set format"); + return 0; + } + + printk("- Set format: %c%c%c%c %ux%u\n", (char)fmt.pixelformat, + (char)(fmt.pixelformat >> 8), (char)(fmt.pixelformat >> 16), + (char)(fmt.pixelformat >> 24), fmt.width, fmt.height); + /* Size to allocate for each buffer */ bsize = fmt.pitch * fmt.height; From 045ad83cbc4e85f2b3b99b2999011da1d0a379d4 Mon Sep 17 00:00:00 2001 From: Armin Kessler Date: Thu, 15 Aug 2024 15:10:58 +0200 Subject: [PATCH 5/5] sample: video: esp32s3: implement hal_api The driver now calls the `` API instead of writing directly into the registers. Signed-off-by: Armin Kessler ake@espros.com --- drivers/video/video_esp32_dvp.c | 68 ++++++++++++++++----------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/drivers/video/video_esp32_dvp.c b/drivers/video/video_esp32_dvp.c index 4a3cb6cb8cd737..98e698f5f1fa2f 100644 --- a/drivers/video/video_esp32_dvp.c +++ b/drivers/video/video_esp32_dvp.c @@ -7,8 +7,6 @@ #define DT_DRV_COMPAT espressif_esp32_cam #include -#include -#include #include #include #include @@ -17,7 +15,8 @@ #include #include #include - +#include +#include #include LOG_MODULE_REGISTER(esp32_cam, LOG_LEVEL_INF); @@ -41,7 +40,8 @@ struct video_esp32_config { uint32_t clock_mclk; int irq_source; int irq_priority; - uint8_t enable_16_bit; + uint8_t data_width; + uint8_t vsync_filter; uint8_t invert_de; uint8_t invert_byte_order; uint8_t invert_bit_order; @@ -52,6 +52,7 @@ struct video_esp32_config { struct video_esp32_data { const struct video_esp32_config *config; + cam_hal_context_t cam_hal_context; struct video_format video_format; struct video_buffer *active_vbuf; bool is_streaming; @@ -155,7 +156,7 @@ static int video_esp32_stream_start(const struct device *dev) buffer_size = data->active_vbuf->bytesused; memset(data->dma_blocks, 0, sizeof(data->dma_blocks)); - for (int i = 0; i < CONFIG_DMA_ESP32_MAX_DESCRIPTOR_NUM; ++i) { + for (size_t i = 0; i < CONFIG_DMA_ESP32_MAX_DESCRIPTOR_NUM; ++i) { dma_block_iter->dest_address = (uint32_t)data->active_vbuf->buffer + i * VIDEO_ESP32_DMA_BUFFER_MAX_SIZE; if (buffer_size < VIDEO_ESP32_DMA_BUFFER_MAX_SIZE) { @@ -171,7 +172,7 @@ static int video_esp32_stream_start(const struct device *dev) } if (dma_block_iter->next_block) { - LOG_ERR("Not enough descriptors available. Increase DMA_ESP32_DESCRIPTOR_NUM"); + LOG_ERR("Not enough descriptors available. Increase CONFIG_DMA_ESP32_MAX_DESCRIPTOR_NUM by %d", buffer_size / VIDEO_ESP32_DMA_BUFFER_MAX_SIZE+1); return -ENOBUFS; } @@ -188,20 +189,13 @@ static int video_esp32_stream_start(const struct device *dev) return error; } - LCD_CAM.cam_ctrl1.cam_reset = 1; - LCD_CAM.cam_ctrl1.cam_reset = 0; - LCD_CAM.cam_ctrl1.cam_afifo_reset = 1; - LCD_CAM.cam_ctrl1.cam_afifo_reset = 0; - error = dma_start(cfg->dma_dev, cfg->rx_dma_channel); if (error) { LOG_ERR("Unable to start DMA (%d)", error); return error; } - LCD_CAM.cam_ctrl.cam_update = 1; - LCD_CAM.cam_ctrl.cam_update = 0; - LCD_CAM.cam_ctrl1.cam_start = 1; + cam_hal_start_streaming(&data->cam_hal_context); data->is_streaming = true; @@ -218,7 +212,7 @@ static int video_esp32_stream_stop(const struct device *dev) data->is_streaming = false; dma_stop(cfg->dma_dev, cfg->rx_dma_channel); - LCD_CAM.cam_ctrl1.cam_start = 0; + cam_hal_stop_streaming(&data->cam_hal_context); return ret; } @@ -331,26 +325,29 @@ static void video_esp32_cam_ctrl_init(const struct device *dev) { const struct video_esp32_config *cfg = dev->config; struct video_esp32_data *data = dev->data; + cam_dev_t *cam_dev; + + data->cam_hal_context.hw = CAM_LL_GET_HW(0); + cam_dev = data->cam_hal_context.hw; + + cam_ll_enable_stop_signal(cam_dev, 0); + cam_ll_swap_dma_data_byte_order(cam_dev, cfg->invert_byte_order); + cam_ll_reverse_dma_data_bit_order(cam_dev, cfg->invert_bit_order); + cam_ll_enable_vsync_generate_eof(cam_dev, 1); + + cam_ll_enable_hs_line_int(cam_dev, 0); + cam_ll_set_line_int_num(cam_dev, 0); + cam_ll_enable_vsync_filter(cam_dev, cfg->vsync_filter != 0); + cam_ll_set_vsync_filter_thres(cam_dev, cfg->vsync_filter); + + cam_ll_set_input_data_width(cam_dev, cfg->data_width); + cam_ll_enable_invert_pclk(cam_dev, cfg->invert_pclk); + cam_ll_enable_invert_de(cam_dev, cfg->invert_de); + cam_ll_enable_invert_vsync(cam_dev, cfg->invert_vsync); + cam_ll_enable_invert_hsync(cam_dev, cfg->invert_hsync); + cam_ll_set_vh_de_mode(cam_dev, 0); // Disable vh_de mode default + cam_ll_enable_rgb_yuv_convert(cam_dev, 0); // bypass conv module - LCD_CAM.cam_ctrl.cam_stop_en = 0; - LCD_CAM.cam_ctrl.cam_vsync_filter_thres = 4; - LCD_CAM.cam_ctrl.cam_byte_order = cfg->invert_byte_order; - LCD_CAM.cam_ctrl.cam_bit_order = cfg->invert_bit_order; - LCD_CAM.cam_ctrl.cam_line_int_en = 0; - LCD_CAM.cam_ctrl.cam_vs_eof_en = 1; - - LCD_CAM.cam_ctrl1.val = 0; - LCD_CAM.cam_ctrl1.cam_rec_data_bytelen = 0; - LCD_CAM.cam_ctrl1.cam_line_int_num = 0; - LCD_CAM.cam_ctrl1.cam_clk_inv = cfg->invert_pclk; - LCD_CAM.cam_ctrl1.cam_vsync_filter_en = 1; - LCD_CAM.cam_ctrl1.cam_2byte_en = cfg->enable_16_bit; - LCD_CAM.cam_ctrl1.cam_de_inv = cfg->invert_de; - LCD_CAM.cam_ctrl1.cam_hsync_inv = cfg->invert_hsync; - LCD_CAM.cam_ctrl1.cam_vsync_inv = cfg->invert_vsync; - LCD_CAM.cam_ctrl1.cam_vh_de_mode_en = 0; - - LCD_CAM.cam_rgb_yuv.val = 0; } static int esp32_init(const struct device *dev) @@ -404,7 +401,8 @@ static const struct video_esp32_config esp32_config = { .source_dev = DEVICE_DT_GET(DT_INST_PHANDLE(0, source)), .dma_dev = ESP32_DT_INST_DMA_CTLR(0, rx), .rx_dma_channel = DT_INST_DMAS_CELL_BY_NAME(0, rx, channel), - .enable_16_bit = DT_INST_PROP(0, enable_16bit_mode), + .data_width = DT_INST_PROP(0, data_width), + .vsync_filter = DT_INST_PROP(0, vsync_filter), .invert_bit_order = DT_INST_PROP(0, invert_bit_order), .invert_byte_order = DT_INST_PROP(0, invert_byte_order), .invert_pclk = DT_INST_PROP(0, invert_pclk),