From f7ed570231950007af5fb4c7710975fb16b1d483 Mon Sep 17 00:00:00 2001 From: "Kwabena W. Agyeman" Date: Tue, 2 Apr 2024 17:13:30 -0700 Subject: [PATCH] modules/py_image: Refactor get_similarity() to use draw_image backend. --- src/omv/imlib/imlib.h | 13 ++- src/omv/imlib/stats.c | 194 +++++++++++++++++++++---------------- src/omv/modules/py_image.c | 61 +++++++++--- 3 files changed, 167 insertions(+), 101 deletions(-) diff --git a/src/omv/imlib/imlib.h b/src/omv/imlib/imlib.h index cd46a92e6..7739e2ee3 100644 --- a/src/omv/imlib/imlib.h +++ b/src/omv/imlib/imlib.h @@ -1440,9 +1440,18 @@ void imlib_rotation_corr(image_t *img, float x_rotation, float y_rotation, float zoom, float fov, float *corners); // Statistics void imlib_get_similarity(image_t *img, - const char *path, image_t *other, - int scalar, + int x_start, + int y_start, + float x_scale, + float y_scale, + rectangle_t *roi, + int rgb_channel, + int alpha, + const uint16_t *color_palette, + const uint8_t *alpha_palette, + image_hint_t hint, + bool dssim, float *avg, float *std, float *min, diff --git a/src/omv/imlib/stats.c b/src/omv/imlib/stats.c index cfb096040..6dbde974e 100644 --- a/src/omv/imlib/stats.c +++ b/src/omv/imlib/stats.c @@ -11,69 +11,68 @@ #include "imlib.h" #ifdef IMLIB_ENABLE_GET_SIMILARITY -typedef struct imlib_similatiry_line_op_state { +typedef struct imlib_similarity_line_op_state { + bool dssim; int *sumBucketsOfX, *sumBucketsOfY, *sum2BucketsOfX, *sum2BucketsOfY, *sum2Buckets; float similarity_sum, similarity_sum_2, similarity_min, similarity_max; - int lines_processed; -} imlib_similatiry_line_op_state_t; + int lines_processed, lines; +} imlib_similarity_line_op_state_t; -void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, bool vflipped) { - imlib_similatiry_line_op_state_t *state = (imlib_similatiry_line_op_state_t *) data; vflipped = vflipped; +static void imlib_similarity_line_op(int x, int x_end, int y_row, imlib_draw_row_data_t *data) { + imlib_similarity_line_op_state_t *state = data->callback_arg; float c1 = 0, c2 = 0; + int x_start = x; - switch (img->pixfmt) { + switch (data->dst_img->pixfmt) { case PIXFORMAT_BINARY: { - uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(img, line); - uint32_t *other_row_ptr = (uint32_t *) other; - for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { - for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { - int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x + i); - int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x + i); - state->sumBucketsOfX[x] += pixel; - state->sumBucketsOfY[x] += other_pixel; - state->sum2BucketsOfX[x] += pixel * pixel; - state->sum2BucketsOfY[x] += other_pixel * other_pixel; - state->sum2Buckets[x] += pixel * other_pixel; - } + uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(data->dst_img, y_row); + uint32_t *other_row_ptr = (uint32_t *) data->dst_row_override; + for (; x < x_end; x++) { + int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x); + int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x); + int bucket = (x - x_start) / 8; + state->sumBucketsOfX[bucket] += pixel; + state->sumBucketsOfY[bucket] += other_pixel; + state->sum2BucketsOfX[bucket] += pixel * pixel; + state->sum2BucketsOfY[bucket] += other_pixel * other_pixel; + state->sum2Buckets[bucket] += pixel * other_pixel; } - c1 = COLOR_BINARY_MAX * 0.01f; - c2 = COLOR_BINARY_MAX * 0.03f; + c1 = (COLOR_BINARY_MAX * 0.01f) * (COLOR_BINARY_MAX * 0.01f); + c2 = (COLOR_BINARY_MAX * 0.03f) * (COLOR_BINARY_MAX * 0.03f); break; } case PIXFORMAT_GRAYSCALE: { - uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(img, line); - uint8_t *other_row_ptr = (uint8_t *) other; - for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { - for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { - int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x + i); - int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x + i); - state->sumBucketsOfX[x] += pixel; - state->sumBucketsOfY[x] += other_pixel; - state->sum2BucketsOfX[x] += pixel * pixel; - state->sum2BucketsOfY[x] += other_pixel * other_pixel; - state->sum2Buckets[x] += pixel * other_pixel; - } + uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(data->dst_img, y_row); + uint8_t *other_row_ptr = (uint8_t *) data->dst_row_override; + for (; x < x_end; x++) { + int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x); + int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x); + int bucket = (x - x_start) / 8; + state->sumBucketsOfX[bucket] += pixel; + state->sumBucketsOfY[bucket] += other_pixel; + state->sum2BucketsOfX[bucket] += pixel * pixel; + state->sum2BucketsOfY[bucket] += other_pixel * other_pixel; + state->sum2Buckets[bucket] += pixel * other_pixel; } - c1 = COLOR_GRAYSCALE_MAX * 0.01f; - c2 = COLOR_GRAYSCALE_MAX * 0.03f; + c1 = (COLOR_GRAYSCALE_MAX * 0.01f) * (COLOR_GRAYSCALE_MAX * 0.01f); + c2 = (COLOR_GRAYSCALE_MAX * 0.03f) * (COLOR_GRAYSCALE_MAX * 0.03f); break; } case PIXFORMAT_RGB565: { - uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(img, line); - uint16_t *other_row_ptr = (uint16_t *) other; - for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { - for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { - int pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x + i)); - int other_pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x + i)); - state->sumBucketsOfX[x] += pixel; - state->sumBucketsOfY[x] += other_pixel; - state->sum2BucketsOfX[x] += pixel * pixel; - state->sum2BucketsOfY[x] += other_pixel * other_pixel; - state->sum2Buckets[x] += pixel * other_pixel; - } + uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(data->dst_img, y_row); + uint16_t *other_row_ptr = (uint16_t *) data->dst_row_override; + for (; x < x_end; x++) { + int pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x)); + int other_pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x)); + int bucket = (x - x_start) / 8; + state->sumBucketsOfX[bucket] += pixel; + state->sumBucketsOfY[bucket] += other_pixel; + state->sum2BucketsOfX[bucket] += pixel * pixel; + state->sum2BucketsOfY[bucket] += other_pixel * other_pixel; + state->sum2Buckets[bucket] += pixel * other_pixel; } - c1 = COLOR_L_MAX * 0.01f; - c2 = COLOR_L_MAX * 0.03f; + c1 = (COLOR_Y_MAX * 0.01f) * (COLOR_Y_MAX * 0.01f); + c2 = (COLOR_Y_MAX * 0.03f) * (COLOR_Y_MAX * 0.03f); break; } default: { @@ -82,33 +81,41 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b } // https://en.wikipedia.org/wiki/Structural_similarity - if (((state->lines_processed + 1) == img->h) || (!((state->lines_processed + 1) % 8))) { - for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { - int w = IM_MIN((img->w - (x * 8)), 8); - int h = IM_MIN((img->h - ((state->lines_processed / 8) * 8)), 8); - int size = w * h; + if ((!((state->lines_processed + 1) % 8)) || ((state->lines_processed + 1) == state->lines)) { + for (x = x_start; x < x_end; x += 8) { + int bucket = (x - x_start) / 8; + int w = IM_MIN((x_end - x), 8); + int h = IM_MIN((state->lines - state->lines_processed), 8); + float size = w * h; - int mx = state->sumBucketsOfX[x] / size; - int my = state->sumBucketsOfY[x] / size; - int vx = state->sum2BucketsOfX[x] - ((mx * state->sumBucketsOfX[x]) + (mx * state->sumBucketsOfX[x])) + - (size * mx * mx); - int vy = state->sum2BucketsOfY[x] - ((my * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfY[x])) + - (size * my * my); - int vxy = state->sum2Buckets[x] - ((mx * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfX[x])) + - (size * mx * my); + // Dividng the sum squared buckets by size causes a loss of accuracy which results in + // the single pass standard deviation formula giving the wrong answer. To bypass this + // vx, vy, vxy have been multiplied by size which will be divided back out in the final + // ssim calculation (given c1/c2 ~= 0). - float ssim = ( ((2 * mx * my) + c1) * ((2 * vxy) + c2) ) / ( ((mx * mx) + (my * my) + c1) * (vx + vy + c2) ); + float mx = state->sumBucketsOfX[bucket] / size; + float my = state->sumBucketsOfY[bucket] / size; + float vx = state->sum2BucketsOfX[bucket] - (size * mx * mx); + float vy = state->sum2BucketsOfY[bucket] - (size * my * my); + float vxy = state->sum2Buckets[bucket] - (size * mx * my); + + float ssim = (((2 * mx * my) + c1) * ((2 * vxy) + c2)) / + (((mx * mx) + (my * my) + c1) * (vx + vy + c2)); + + if (state->dssim) { + ssim = (1.0f - ssim) / 2.0f; + } state->similarity_sum += ssim; state->similarity_sum_2 += ssim * ssim; state->similarity_min = IM_MIN(state->similarity_min, ssim); state->similarity_max = IM_MAX(state->similarity_max, ssim); - state->sumBucketsOfX[x] = 0; - state->sumBucketsOfY[x] = 0; - state->sum2BucketsOfX[x] = 0; - state->sum2BucketsOfY[x] = 0; - state->sum2Buckets[x] = 0; + state->sumBucketsOfX[bucket] = 0; + state->sumBucketsOfY[bucket] = 0; + state->sum2BucketsOfX[bucket] = 0; + state->sum2BucketsOfY[bucket] = 0; + state->sum2Buckets[bucket] = 0; } } @@ -116,43 +123,62 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b } void imlib_get_similarity(image_t *img, - const char *path, image_t *other, - int scalar, + int x_start, + int y_start, + float x_scale, + float y_scale, + rectangle_t *roi, + int rgb_channel, + int alpha, + const uint16_t *color_palette, + const uint8_t *alpha_palette, + image_hint_t hint, + bool dssim, float *avg, float *std, float *min, float *max) { - int h_blocks = (img->w + 7) / 8; - int v_blocks = (img->h + 7) / 8; + point_t p0, p1; + imlib_draw_image_get_bounds(img, other, x_start, y_start, x_scale, y_scale, roi, + alpha, alpha_palette, hint, &p0, &p1); + int h_blocks = ((p1.x - p0.x) + 7) / 8; + int v_blocks = ((p1.y - p0.y) + 7) / 8; int blocks = h_blocks * v_blocks; + if (!blocks) { + return; + } + int int_h_blocks = h_blocks * sizeof(int); - imlib_similatiry_line_op_state_t state; - state.sumBucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); - state.sumBucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); - state.sum2BucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); - state.sum2BucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); - state.sum2Buckets = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); + imlib_similarity_line_op_state_t state; + state.dssim = dssim; + state.sumBucketsOfX = fb_alloc0(int_h_blocks * 5, FB_ALLOC_NO_HINT); + state.sumBucketsOfY = state.sumBucketsOfX + int_h_blocks; + state.sum2BucketsOfX = state.sumBucketsOfY + int_h_blocks; + state.sum2BucketsOfY = state.sum2BucketsOfX + int_h_blocks; + state.sum2Buckets = state.sum2BucketsOfY + int_h_blocks; state.similarity_sum = 0.0f; state.similarity_sum_2 = 0.0f; state.similarity_min = FLT_MAX; state.similarity_max = -FLT_MAX; state.lines_processed = 0; + state.lines = p1.y - p0.y; + + void *dst_row_override = fb_alloc0(image_line_size(img), FB_ALLOC_CACHE_ALIGN); + imlib_draw_image(img, other, x_start, y_start, x_scale, y_scale, roi, + rgb_channel, alpha, color_palette, alpha_palette, + hint, imlib_similarity_line_op, &state, dst_row_override); - imlib_image_operation(img, path, other, scalar, imlib_similarity_line_op, &state); *avg = state.similarity_sum / blocks; *std = fast_sqrtf((state.similarity_sum_2 / blocks) - ((*avg) * (*avg))); *min = state.similarity_min; *max = state.similarity_max; - fb_free(); - fb_free(); - fb_free(); - fb_free(); - fb_free(); + fb_free(); // dst_row_override + fb_free(); // sumBucketsOfX } -#endif //IMLIB_ENABLE_GET_SIMILARITY +#endif // IMLIB_ENABLE_GET_SIMILARITY void imlib_get_histogram(histogram_t *out, image_t *ptr, rectangle_t *roi, list_t *thresholds, bool invert, image_t *other) { switch (ptr->pixfmt) { diff --git a/src/omv/modules/py_image.c b/src/omv/modules/py_image.c index 7fdd07a64..5db1d1157 100644 --- a/src/omv/modules/py_image.c +++ b/src/omv/modules/py_image.c @@ -2882,26 +2882,57 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE( locals_dict, &py_similarity_locals_dict ); -static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) { - image_t *arg_img = py_helper_arg_to_image(img_obj, ARG_IMAGE_MUTABLE); - float avg, std, min, max; +static mp_obj_t py_image_get_similarity(uint n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { + ARG_image, ARG_x, ARG_y, ARG_x_scale, ARG_y_scale, ARG_roi, + ARG_channel, ARG_alpha, ARG_color_palette, ARG_alpha_palette, ARG_hint, ARG_dssim + }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_image, MP_ARG_OBJ | MP_ARG_REQUIRED, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_x, MP_ARG_INT, {.u_int = 0 } }, + { MP_QSTR_y, MP_ARG_INT, {.u_int = 0 } }, + { MP_QSTR_x_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_y_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_roi, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_rgb_channel, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = -1 } }, + { MP_QSTR_alpha, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 256 } }, + { MP_QSTR_color_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_alpha_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_hint, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 0 } }, + { MP_QSTR_dssim, MP_ARG_BOOL | MP_ARG_KW_ONLY, {.u_bool = false } }, + }; + + // Parse args. + image_t *image = py_helper_arg_to_image(pos_args[0], ARG_IMAGE_MUTABLE); + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); fb_alloc_mark(); + image_t *other = py_helper_arg_to_image(args[ARG_image].u_obj, ARG_IMAGE_ANY | ARG_IMAGE_ALLOC); + rectangle_t roi = py_helper_arg_to_roi(args[ARG_roi].u_obj, other); - if (MP_OBJ_IS_STR(other_obj)) { - imlib_get_similarity(arg_img, mp_obj_str_get_str(other_obj), NULL, 0, &avg, &std, &min, &max); - } else if (MP_OBJ_IS_TYPE(other_obj, &py_image_type)) { - imlib_get_similarity(arg_img, NULL, - py_helper_arg_to_image(other_obj, ARG_IMAGE_MUTABLE), - 0, &avg, &std, &min, &max); - } else { - imlib_get_similarity(arg_img, NULL, NULL, - py_helper_keyword_color(arg_img, 1, &other_obj, 0, NULL, 0), - &avg, &std, &min, &max); + if (args[ARG_channel].u_int < -1 || args[ARG_channel].u_int > 2) { + mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("RGB channel can be 0, 1, or 2")); } - fb_alloc_free_till_mark(); + if (args[ARG_alpha].u_int < 0 || args[ARG_alpha].u_int > 256) { + mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("Alpha ranges between 0 and 256")); + } + float x_scale = 1.0f; + float y_scale = 1.0f; + py_helper_arg_to_scale(args[ARG_x_scale].u_obj, args[ARG_y_scale].u_obj, &x_scale, &y_scale); + + const uint16_t *color_palette = py_helper_arg_to_palette(args[ARG_color_palette].u_obj, PIXFORMAT_RGB565); + const uint8_t *alpha_palette = py_helper_arg_to_palette(args[ARG_alpha_palette].u_obj, PIXFORMAT_GRAYSCALE); + + float avg = 0.0f, std = 0.0f, min = 0.0f, max = 0.0f; + imlib_get_similarity(image, other, args[ARG_x].u_int, args[ARG_y].u_int, x_scale, y_scale, &roi, + args[ARG_channel].u_int, args[ARG_alpha].u_int, color_palette, alpha_palette, + args[ARG_hint].u_int | IMAGE_HINT_BLACK_BACKGROUND, args[ARG_dssim].u_bool, + &avg, &std, &min, &max); + + fb_alloc_free_till_mark(); py_similarity_obj_t *o = m_new_obj(py_similarity_obj_t); o->base.type = &py_similarity_type; o->avg = mp_obj_new_float(avg); @@ -2910,7 +2941,7 @@ static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) { o->max = mp_obj_new_float(max); return o; } -STATIC MP_DEFINE_CONST_FUN_OBJ_2(py_image_get_similarity_obj, py_image_get_similarity); +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(py_image_get_similarity_obj, 1, py_image_get_similarity); #endif // IMLIB_ENABLE_GET_SIMILARITY // Statistics Object //