Compare commits

...

5 Commits

Author SHA1 Message Date
Kwabena W. Agyeman
0c1d134a84 ports/stm32: Enable gamma correction. 2025-11-02 22:36:32 -08:00
Kwabena W. Agyeman
0307402a57 ports/alif: Add gamma correction to PAG7936 video.
Gamma correction massively improves the PAG7936 image quality
by increasing the image brightness. Brightness and contrast adjustment
support is also exposed for future automatic control.
2025-11-02 22:35:57 -08:00
Kwabena W. Agyeman
4745e2999e drivers/sensors: Add basic awb controls for sensors using an isp. 2025-11-02 22:35:48 -08:00
Kwabena W. Agyeman
579334fdff ports/stm32: Use 250ms rgb moving average for stats.
This also resolves issues with the luminace target flickering as
the average cannot update instantly which filters out flicking
on the PS5520 updating its settings quickly.
2025-11-02 22:35:48 -08:00
Kwabena W. Agyeman
6d927a91f9 ports/alif: Add 250ms time limited moving average.
The image turning green previously was caused by the rgb stats not being
sampled continously (only every 100 frames) coupled with only happening
on snapshot. As such, any algorithm causing the fps to slow down would
lower the update freq. The new average solution ensures that the rgb
average is smoothed over many frames and only keeps the last 250ms of
history.
2025-11-02 22:35:47 -08:00
12 changed files with 134 additions and 29 deletions

View File

@ -65,6 +65,8 @@
#define OMV_CSI_CLK_TOLERANCE (500000)
#endif
#define OMV_CSI_RGB_AVG_MAX_MS (250)
#ifndef __weak
#define __weak __attribute__((weak))
#endif
@ -322,6 +324,12 @@ __weak int omv_csi_reset(omv_csi_t *csi, bool hard) {
NULL, NULL
};
#if defined(OMV_CSI_RGB_AVG_SIZE)
// Setup AWB moving average.
csi->awb_enabled = true;
rgb_moving_average_init(&csi->rgb_avg, OMV_CSI_RGB_AVG_MAX_MS, OMV_CSI_RGB_AVG_SIZE, csi->rgb_avg_buf);
#endif // defined(OMV_CSI_RGB_AVG_SIZE)
// Restore shutdown state on reset.
if (!csi->power_on) {
omv_csi_shutdown(csi, false);

View File

@ -652,6 +652,23 @@ static int get_exposure_us(omv_csi_t *csi, int *exposure_us) {
return ret;
}
static int set_auto_whitebal(omv_csi_t *csi, int enable, float r_gain_db, float g_gain_db, float b_gain_db) {
csi->awb_enabled = enable;
return 0;
}
static int get_rgb_gain_db(omv_csi_t *csi, float *r_gain_db, float *g_gain_db, float *b_gain_db) {
uint32_t r_avg, g_avg, b_avg;
rgb_moving_average_get(&csi->rgb_avg, &r_avg, &g_avg, &b_avg);
float g_avg_f = (float) g_avg;
*r_gain_db = 20.0f * log10f(g_avg_f / r_avg);
*g_gain_db = 0.0f;
*b_gain_db = 20.0f * log10f(g_avg_f / b_avg);
return 0;
}
static int set_hmirror(omv_csi_t *csi, int enable) {
uint8_t reg;
int ret = omv_i2c_readb2(csi->i2c, csi->slv_addr, TG_FLIP, &reg);
@ -723,6 +740,8 @@ int pag7936_init(omv_csi_t *csi) {
csi->get_gain_db = get_gain_db;
csi->set_auto_exposure = set_auto_exposure;
csi->get_exposure_us = get_exposure_us;
csi->set_auto_whitebal = set_auto_whitebal;
csi->get_rgb_gain_db = get_rgb_gain_db;
csi->set_hmirror = set_hmirror;
csi->set_vflip = set_vflip;

View File

@ -1442,6 +1442,23 @@ static int get_exposure_us(omv_csi_t *csi, int *exposure_us) {
return ret;
}
static int set_auto_whitebal(omv_csi_t *csi, int enable, float r_gain_db, float g_gain_db, float b_gain_db) {
csi->awb_enabled = enable;
return 0;
}
static int get_rgb_gain_db(omv_csi_t *csi, float *r_gain_db, float *g_gain_db, float *b_gain_db) {
uint32_t r_avg, g_avg, b_avg;
rgb_moving_average_get(&csi->rgb_avg, &r_avg, &g_avg, &b_avg);
float g_avg_f = (float) g_avg;
*r_gain_db = 20.0f * log10f(g_avg_f / r_avg);
*g_gain_db = 0.0f;
*b_gain_db = 20.0f * log10f(g_avg_f / b_avg);
return 0;
}
static int set_hmirror(omv_csi_t *csi, int enable) {
uint8_t reg1, reg2;
uint16_t hsize;
@ -1590,6 +1607,8 @@ int ps5520_init(omv_csi_t *csi) {
csi->get_gain_db = get_gain_db;
csi->set_auto_exposure = set_auto_exposure;
csi->get_exposure_us = get_exposure_us;
csi->set_auto_whitebal = set_auto_whitebal;
csi->get_rgb_gain_db = get_rgb_gain_db;
csi->set_hmirror = set_hmirror;
csi->set_vflip = set_vflip;

View File

@ -559,6 +559,10 @@ static inline v128_t vdebayer_to_cr(vrgb_pixels_t pixels) {
return pixels.r;
}
// FIXME: This needs to be placed in the DTCM. .bss is in the DTCM on Alif. But, this is not portable.
// Move this to a proper DTCM section for other platforms.
static uint8_t gamma_lut[256];
// In the case of vectors larger than 32-bits the pattern is repeated for every 32-bits.
//
// pixels.r = MSB [0, R1, 0, R0] LSB pixels where each pixel is 8-bits.
@ -569,6 +573,9 @@ static inline v128_t vdebayer_to_cr(vrgb_pixels_t pixels) {
static inline vrgb_pixels_t vdebayer_apply_rb_gain(vrgb_pixels_t pixels, uint32_t red_gain, uint32_t blue_gain) {
pixels.r = vusat_s16_narrow_u8_lo(pixels.r, vmul_n_u32(pixels.r, red_gain), 5);
pixels.b = vusat_s16_narrow_u8_lo(pixels.b, vmul_n_u32(pixels.b, blue_gain), 5);
pixels.r = vldr_u8_gather(gamma_lut, pixels.r);
pixels.g = vldr_u8_gather(gamma_lut, pixels.g);
pixels.b = vldr_u8_gather(gamma_lut, pixels.b);
return pixels;
}
@ -2291,13 +2298,19 @@ static void vdebayer_rggb_to_rgb565_awb_quarter(image_t *src, image_t *dst, uint
// GRAYSCALE: src->data == dst->data
// RGB565: src->data == dst->data + image_size(src)
// YUV422: Not supported
void imlib_debayer_image_awb(image_t *dst, image_t *src, bool fast, uint32_t r_out, uint32_t g_out, uint32_t b_out) {
void imlib_debayer_image_awb(image_t *dst, image_t *src, bool fast, uint32_t r_out, uint32_t g_out, uint32_t b_out,
float brightness, float contrast, float gamma) {
uint32_t red_gain = IM_DIV(g_out * 32, r_out);
red_gain = IM_MIN(red_gain, 128U);
uint32_t blue_gain = IM_DIV(g_out * 32, b_out);
blue_gain = IM_MIN(blue_gain, 128U);
for (int i = 0; i < 256; i++) {
int v = fast_roundf(((fast_powf(i * (1.0f / 255.0f), gamma) * contrast) + brightness) * 255.0f);
gamma_lut[i] = IM_CLAMP(v, 0, 255);
}
if (fast) {
switch (src->pixfmt) {
case PIXFORMAT_BAYER_BGGR: {

View File

@ -1207,7 +1207,8 @@ pixformat_t imlib_bayer_shift(pixformat_t pixfmt, int x, int y, bool transpose);
void imlib_debayer_ycbcr(image_t *src, rectangle_t *roi, int8_t *Y0, int8_t *CB, int8_t *CR);
void imlib_debayer_line(int x_start, int x_end, int y_row, void *dst_row_ptr, pixformat_t pixfmt, image_t *src);
void imlib_debayer_image(image_t *dst, image_t *src);
void imlib_debayer_image_awb(image_t *dst, image_t *src, bool fast, uint32_t r_out, uint32_t g_out, uint32_t b_out);
void imlib_debayer_image_awb(image_t *dst, image_t *src, bool fast, uint32_t r_out, uint32_t g_out, uint32_t b_out,
float brightness, float contrast, float gamma);
// YUV Image Processing
pixformat_t imlib_yuv_shift(pixformat_t pixfmt, int x);

View File

@ -1514,6 +1514,19 @@ static inline v128_t vldr_u8_pred(const uint8_t *p, v128_predicate_t pred) {
#endif
}
static inline v128_t vldr_u8_gather(const uint8_t *p, v128_t offsets) {
#if (__ARM_ARCH >= 8)
return (v128_t) vldrbq_gather_offset(p, offsets.u8);
#else
v128_t v0;
v0.u8[0] = *(p + offsets.u8[0]);
v0.u8[1] = *(p + offsets.u8[1]);
v0.u8[2] = *(p + offsets.u8[2]);
v0.u8[3] = *(p + offsets.u8[3]);
return v0;
#endif
}
static inline v128_t vldr_u8_widen_u32_gather_pred(const uint8_t *p,
v128_t offsets,
v128_predicate_t pred) {

View File

@ -48,6 +48,7 @@
#include "omv_gpu.h"
#include "omv_i2c.h"
#include "omv_csi.h"
#include "average.h"
#include "unaligned_memcpy.h"
// Bits missing from cpi.h
@ -230,10 +231,7 @@ static uint32_t omv_csi_get_fb_offset(omv_csi_t *csi) {
int alif_csi_snapshot(omv_csi_t *csi, image_t *dst_image, uint32_t flags) {
CPI_Type *cpi = csi->base;
framebuffer_t *fb = csi->fb;
vbuffer_t *buffer = NULL;
static uint32_t frames = 0;
static uint32_t r_stat, gb_stat, gr_stat, b_stat;
// Configure and re/start the capture if it's not alrady active
// and there are no pending buffers (from non-blocking capture).
@ -390,13 +388,23 @@ int alif_csi_snapshot(omv_csi_t *csi, image_t *dst_image, uint32_t flags) {
// Set the target pixel format before debayer.
dst_image->pixfmt = fb->pixfmt;
// Update AWB stats every n frames.
if ((frames++ % 100) == 0) {
omv_csi_ioctl(csi, OMV_CSI_IOCTL_GET_RGB_STATS, &r_stat, &gb_stat, &gr_stat, &b_stat);
uint32_t r_stat, g_stat, b_stat;
if (csi->awb_enabled) {
uint32_t gb_stat, gr_stat;
int ret = omv_csi_ioctl(csi, OMV_CSI_IOCTL_GET_RGB_STATS, &r_stat, &gb_stat, &gr_stat, &b_stat);
if (ret != 0) {
return ret;
}
g_stat = (gb_stat + gr_stat) / 2;
rgb_moving_average_update(&csi->rgb_avg, &r_stat, &g_stat, &b_stat, mp_hal_ticks_ms());
} else {
rgb_moving_average_get(&csi->rgb_avg, &r_stat, &g_stat, &b_stat);
}
// Debayer frame.
imlib_debayer_image_awb(dst_image, &src_image, false, r_stat, (gb_stat + gr_stat) / 2, b_stat);
imlib_debayer_image_awb(dst_image, &src_image, false, r_stat, g_stat, b_stat, 0.0f, 1.0f, 0.5f);
}
return 0;
}

View File

@ -34,6 +34,7 @@
#define __OMV_PORTCONFIG_H__
#include <stdlib.h>
#include "average.h"
#include "clk.h"
#include "i2c.h"
#include "i3c.h"
@ -137,9 +138,14 @@ struct { \
uint8_t datasize; \
};
#define OMV_CSI_PORT_BITS \
struct { \
CPI_Type *base; \
#define OMV_CSI_RGB_AVG_SIZE (60)
#define OMV_CSI_PORT_BITS \
struct { \
CPI_Type *base; \
bool awb_enabled; \
rgb_moving_average_t rgb_avg; \
rgb_moving_average_val_t rgb_avg_buf[OMV_CSI_RGB_AVG_SIZE]; \
};
#endif // __OMV_PORTCONFIG_H__

View File

@ -751,7 +751,7 @@ static int stm_csi_snapshot(omv_csi_t *csi, image_t *image, uint32_t flags) {
#if USE_DCMIPP
if (csi->raw_output) {
float luminance = stm_isp_update_awb(&csi->dcmipp, DCMIPP_PIPE, fb->u * fb->v);
float luminance = stm_isp_update_awb(csi, DCMIPP_PIPE, fb->u * fb->v);
if (csi->ioctl) {
omv_csi_ioctl(csi, OMV_CSI_IOCTL_UPDATE_AGC_AEC, fast_floorf(luminance));
}

View File

@ -27,6 +27,8 @@
#define __OMV_PORTCONFIG_H__
#include STM32_HAL_H
#include "average.h"
// GPIO speeds.
#define OMV_GPIO_SPEED_LOW GPIO_SPEED_FREQ_LOW
#define OMV_GPIO_SPEED_MED GPIO_SPEED_FREQ_MEDIUM
@ -147,12 +149,15 @@ typedef I2C_HandleTypeDef *omv_i2c_dev_t;
#define OMV_CSI_PORT_BITS_MDMA
#endif
#if defined(STM32N6)
#define OMV_CSI_PORT_BITS_DCMIPP \
struct { \
DCMIPP_HandleTypeDef dcmipp; \
DMA_QListTypeDef dma_queue; \
#define OMV_CSI_RGB_AVG_SIZE (120)
#define OMV_CSI_PORT_BITS_DCMIPP \
struct { \
DCMIPP_HandleTypeDef dcmipp; \
DMA_QListTypeDef dma_queue; \
bool awb_enabled; \
rgb_moving_average_t rgb_avg; \
rgb_moving_average_val_t rgb_avg_buf[OMV_CSI_RGB_AVG_SIZE]; \
};
#else
#define OMV_CSI_PORT_BITS_DCMIPP

View File

@ -33,24 +33,32 @@
#include <string.h>
#include "imlib.h"
#include "average.h"
#include "py/mphal.h"
#include "stm_isp.h"
#include "omv_boardconfig.h"
#ifdef DCMIPP
float stm_isp_update_awb(DCMIPP_HandleTypeDef *dcmipp, uint32_t pipe, uint32_t n_pixels) {
float stm_isp_update_awb(omv_csi_t *csi, uint32_t pipe, uint32_t n_pixels) {
uint32_t avg[3];
uint32_t shift[3];
uint32_t multi[3];
for (int i = 0; i < 3; i++) {
// DCMIPP_STATEXT_MODULE1
HAL_DCMIPP_PIPE_GetISPAccumulatedStatisticsCounter(dcmipp, pipe, i + 1, &avg[i]);
}
if (csi->awb_enabled) {
for (int i = 0; i < 3; i++) {
// DCMIPP_STATEXT_MODULE1
HAL_DCMIPP_PIPE_GetISPAccumulatedStatisticsCounter(&csi->dcmipp, pipe, i + 1, &avg[i]);
}
// Averages are collected from bayer components (4R 2G 4B).
avg[0] = OMV_MAX((avg[0] * 256 * 4) / n_pixels, 1);
avg[1] = OMV_MAX((avg[1] * 256 * 2) / n_pixels, 1);
avg[2] = OMV_MAX((avg[2] * 256 * 4) / n_pixels, 1);
// Averages are collected from bayer components (4R 2G 4B).
avg[0] = OMV_MAX((avg[0] * 256 * 4) / n_pixels, 1);
avg[1] = OMV_MAX((avg[1] * 256 * 2) / n_pixels, 1);
avg[2] = OMV_MAX((avg[2] * 256 * 4) / n_pixels, 1);
rgb_moving_average_update(&csi->rgb_avg, &avg[0], &avg[1], &avg[2], mp_hal_ticks_ms());
} else {
rgb_moving_average_get(&csi->rgb_avg, &avg[0], &avg[1], &avg[2]);
}
// Compute global luminance
float luminance = avg[0] * 0.299 + avg[1] * 0.587 + avg[2] * 0.114;
@ -75,7 +83,7 @@ float stm_isp_update_awb(DCMIPP_HandleTypeDef *dcmipp, uint32_t pipe, uint32_t n
.MultiplierBlue = multi[2],
};
HAL_DCMIPP_PIPE_SetISPExposureConfig(dcmipp, pipe, &expcfg);
HAL_DCMIPP_PIPE_SetISPExposureConfig(&csi->dcmipp, pipe, &expcfg);
return luminance;
}
@ -159,6 +167,10 @@ int stm_isp_config_pipeline(DCMIPP_HandleTypeDef *dcmipp, uint32_t pipe,
}
}
if (HAL_DCMIPP_PIPE_EnableGammaConversion(dcmipp, pipe) != HAL_OK) {
return -1;
}
return 0;
}
#endif // DCMIPP

View File

@ -35,11 +35,12 @@
#include <stdint.h>
#include <stdbool.h>
#include STM32_HAL_H
#include "omv_csi.h"
#ifdef DCMIPP
int stm_isp_config_pipeline(DCMIPP_HandleTypeDef *dcmipp, uint32_t pipe,
pixformat_t pixformat, bool raw_output);
float stm_isp_update_awb(DCMIPP_HandleTypeDef *dcmipp, uint32_t pipe, uint32_t n_pixels);
float stm_isp_update_awb(omv_csi_t *csi, uint32_t pipe, uint32_t n_pixels);
#endif
#endif // __STM_ISP_H__