Merge pull request #2190 from kwagyeman/kwabena/refactor_get_similarity

modules/py_image: Refactor get_similarity() to use draw_image backend.
This commit is contained in:
Ibrahim Abdelkader 2024-06-09 14:34:52 +02:00 committed by GitHub
commit 8c8c7dbde1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 167 additions and 101 deletions

View File

@ -1440,9 +1440,18 @@ void imlib_rotation_corr(image_t *img, float x_rotation, float y_rotation,
float zoom, float fov, float *corners); float zoom, float fov, float *corners);
// Statistics // Statistics
void imlib_get_similarity(image_t *img, void imlib_get_similarity(image_t *img,
const char *path,
image_t *other, image_t *other,
int scalar, int x_start,
int y_start,
float x_scale,
float y_scale,
rectangle_t *roi,
int rgb_channel,
int alpha,
const uint16_t *color_palette,
const uint8_t *alpha_palette,
image_hint_t hint,
bool dssim,
float *avg, float *avg,
float *std, float *std,
float *min, float *min,

View File

@ -11,69 +11,68 @@
#include "imlib.h" #include "imlib.h"
#ifdef IMLIB_ENABLE_GET_SIMILARITY #ifdef IMLIB_ENABLE_GET_SIMILARITY
typedef struct imlib_similatiry_line_op_state { typedef struct imlib_similarity_line_op_state {
bool dssim;
int *sumBucketsOfX, *sumBucketsOfY, *sum2BucketsOfX, *sum2BucketsOfY, *sum2Buckets; int *sumBucketsOfX, *sumBucketsOfY, *sum2BucketsOfX, *sum2BucketsOfY, *sum2Buckets;
float similarity_sum, similarity_sum_2, similarity_min, similarity_max; float similarity_sum, similarity_sum_2, similarity_min, similarity_max;
int lines_processed; int lines_processed, lines;
} imlib_similatiry_line_op_state_t; } imlib_similarity_line_op_state_t;
void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, bool vflipped) { static void imlib_similarity_line_op(int x, int x_end, int y_row, imlib_draw_row_data_t *data) {
imlib_similatiry_line_op_state_t *state = (imlib_similatiry_line_op_state_t *) data; vflipped = vflipped; imlib_similarity_line_op_state_t *state = data->callback_arg;
float c1 = 0, c2 = 0; float c1 = 0, c2 = 0;
int x_start = x;
switch (img->pixfmt) { switch (data->dst_img->pixfmt) {
case PIXFORMAT_BINARY: { case PIXFORMAT_BINARY: {
uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(img, line); uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(data->dst_img, y_row);
uint32_t *other_row_ptr = (uint32_t *) other; uint32_t *other_row_ptr = (uint32_t *) data->dst_row_override;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { for (; x < x_end; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x);
int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x + i); int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x);
int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x + i); int bucket = (x - x_start) / 8;
state->sumBucketsOfX[x] += pixel; state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[x] += other_pixel; state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel; state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel; state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel; state->sum2Buckets[bucket] += pixel * other_pixel;
}
} }
c1 = COLOR_BINARY_MAX * 0.01f; c1 = (COLOR_BINARY_MAX * 0.01f) * (COLOR_BINARY_MAX * 0.01f);
c2 = COLOR_BINARY_MAX * 0.03f; c2 = (COLOR_BINARY_MAX * 0.03f) * (COLOR_BINARY_MAX * 0.03f);
break; break;
} }
case PIXFORMAT_GRAYSCALE: { case PIXFORMAT_GRAYSCALE: {
uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(img, line); uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(data->dst_img, y_row);
uint8_t *other_row_ptr = (uint8_t *) other; uint8_t *other_row_ptr = (uint8_t *) data->dst_row_override;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { for (; x < x_end; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x);
int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x + i); int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x);
int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x + i); int bucket = (x - x_start) / 8;
state->sumBucketsOfX[x] += pixel; state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[x] += other_pixel; state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel; state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel; state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel; state->sum2Buckets[bucket] += pixel * other_pixel;
}
} }
c1 = COLOR_GRAYSCALE_MAX * 0.01f; c1 = (COLOR_GRAYSCALE_MAX * 0.01f) * (COLOR_GRAYSCALE_MAX * 0.01f);
c2 = COLOR_GRAYSCALE_MAX * 0.03f; c2 = (COLOR_GRAYSCALE_MAX * 0.03f) * (COLOR_GRAYSCALE_MAX * 0.03f);
break; break;
} }
case PIXFORMAT_RGB565: { case PIXFORMAT_RGB565: {
uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(img, line); uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(data->dst_img, y_row);
uint16_t *other_row_ptr = (uint16_t *) other; uint16_t *other_row_ptr = (uint16_t *) data->dst_row_override;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { for (; x < x_end; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) { int pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x));
int pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x + i)); int other_pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x));
int other_pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x + i)); int bucket = (x - x_start) / 8;
state->sumBucketsOfX[x] += pixel; state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[x] += other_pixel; state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel; state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel; state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel; state->sum2Buckets[bucket] += pixel * other_pixel;
}
} }
c1 = COLOR_L_MAX * 0.01f; c1 = (COLOR_Y_MAX * 0.01f) * (COLOR_Y_MAX * 0.01f);
c2 = COLOR_L_MAX * 0.03f; c2 = (COLOR_Y_MAX * 0.03f) * (COLOR_Y_MAX * 0.03f);
break; break;
} }
default: { default: {
@ -82,33 +81,41 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b
} }
// https://en.wikipedia.org/wiki/Structural_similarity // https://en.wikipedia.org/wiki/Structural_similarity
if (((state->lines_processed + 1) == img->h) || (!((state->lines_processed + 1) % 8))) { if ((!((state->lines_processed + 1) % 8)) || ((state->lines_processed + 1) == state->lines)) {
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) { for (x = x_start; x < x_end; x += 8) {
int w = IM_MIN((img->w - (x * 8)), 8); int bucket = (x - x_start) / 8;
int h = IM_MIN((img->h - ((state->lines_processed / 8) * 8)), 8); int w = IM_MIN((x_end - x), 8);
int size = w * h; int h = IM_MIN((state->lines - state->lines_processed), 8);
float size = w * h;
int mx = state->sumBucketsOfX[x] / size; // Dividng the sum squared buckets by size causes a loss of accuracy which results in
int my = state->sumBucketsOfY[x] / size; // the single pass standard deviation formula giving the wrong answer. To bypass this
int vx = state->sum2BucketsOfX[x] - ((mx * state->sumBucketsOfX[x]) + (mx * state->sumBucketsOfX[x])) + // vx, vy, vxy have been multiplied by size which will be divided back out in the final
(size * mx * mx); // ssim calculation (given c1/c2 ~= 0).
int vy = state->sum2BucketsOfY[x] - ((my * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfY[x])) +
(size * my * my);
int vxy = state->sum2Buckets[x] - ((mx * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfX[x])) +
(size * mx * my);
float ssim = ( ((2 * mx * my) + c1) * ((2 * vxy) + c2) ) / ( ((mx * mx) + (my * my) + c1) * (vx + vy + c2) ); float mx = state->sumBucketsOfX[bucket] / size;
float my = state->sumBucketsOfY[bucket] / size;
float vx = state->sum2BucketsOfX[bucket] - (size * mx * mx);
float vy = state->sum2BucketsOfY[bucket] - (size * my * my);
float vxy = state->sum2Buckets[bucket] - (size * mx * my);
float ssim = (((2 * mx * my) + c1) * ((2 * vxy) + c2)) /
(((mx * mx) + (my * my) + c1) * (vx + vy + c2));
if (state->dssim) {
ssim = (1.0f - ssim) / 2.0f;
}
state->similarity_sum += ssim; state->similarity_sum += ssim;
state->similarity_sum_2 += ssim * ssim; state->similarity_sum_2 += ssim * ssim;
state->similarity_min = IM_MIN(state->similarity_min, ssim); state->similarity_min = IM_MIN(state->similarity_min, ssim);
state->similarity_max = IM_MAX(state->similarity_max, ssim); state->similarity_max = IM_MAX(state->similarity_max, ssim);
state->sumBucketsOfX[x] = 0; state->sumBucketsOfX[bucket] = 0;
state->sumBucketsOfY[x] = 0; state->sumBucketsOfY[bucket] = 0;
state->sum2BucketsOfX[x] = 0; state->sum2BucketsOfX[bucket] = 0;
state->sum2BucketsOfY[x] = 0; state->sum2BucketsOfY[bucket] = 0;
state->sum2Buckets[x] = 0; state->sum2Buckets[bucket] = 0;
} }
} }
@ -116,43 +123,62 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b
} }
void imlib_get_similarity(image_t *img, void imlib_get_similarity(image_t *img,
const char *path,
image_t *other, image_t *other,
int scalar, int x_start,
int y_start,
float x_scale,
float y_scale,
rectangle_t *roi,
int rgb_channel,
int alpha,
const uint16_t *color_palette,
const uint8_t *alpha_palette,
image_hint_t hint,
bool dssim,
float *avg, float *avg,
float *std, float *std,
float *min, float *min,
float *max) { float *max) {
int h_blocks = (img->w + 7) / 8; point_t p0, p1;
int v_blocks = (img->h + 7) / 8; imlib_draw_image_get_bounds(img, other, x_start, y_start, x_scale, y_scale, roi,
alpha, alpha_palette, hint, &p0, &p1);
int h_blocks = ((p1.x - p0.x) + 7) / 8;
int v_blocks = ((p1.y - p0.y) + 7) / 8;
int blocks = h_blocks * v_blocks; int blocks = h_blocks * v_blocks;
if (!blocks) {
return;
}
int int_h_blocks = h_blocks * sizeof(int); int int_h_blocks = h_blocks * sizeof(int);
imlib_similatiry_line_op_state_t state; imlib_similarity_line_op_state_t state;
state.sumBucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); state.dssim = dssim;
state.sumBucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); state.sumBucketsOfX = fb_alloc0(int_h_blocks * 5, FB_ALLOC_NO_HINT);
state.sum2BucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); state.sumBucketsOfY = state.sumBucketsOfX + int_h_blocks;
state.sum2BucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); state.sum2BucketsOfX = state.sumBucketsOfY + int_h_blocks;
state.sum2Buckets = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT); state.sum2BucketsOfY = state.sum2BucketsOfX + int_h_blocks;
state.sum2Buckets = state.sum2BucketsOfY + int_h_blocks;
state.similarity_sum = 0.0f; state.similarity_sum = 0.0f;
state.similarity_sum_2 = 0.0f; state.similarity_sum_2 = 0.0f;
state.similarity_min = FLT_MAX; state.similarity_min = FLT_MAX;
state.similarity_max = -FLT_MAX; state.similarity_max = -FLT_MAX;
state.lines_processed = 0; state.lines_processed = 0;
state.lines = p1.y - p0.y;
void *dst_row_override = fb_alloc0(image_line_size(img), FB_ALLOC_CACHE_ALIGN);
imlib_draw_image(img, other, x_start, y_start, x_scale, y_scale, roi,
rgb_channel, alpha, color_palette, alpha_palette,
hint, imlib_similarity_line_op, &state, dst_row_override);
imlib_image_operation(img, path, other, scalar, imlib_similarity_line_op, &state);
*avg = state.similarity_sum / blocks; *avg = state.similarity_sum / blocks;
*std = fast_sqrtf((state.similarity_sum_2 / blocks) - ((*avg) * (*avg))); *std = fast_sqrtf((state.similarity_sum_2 / blocks) - ((*avg) * (*avg)));
*min = state.similarity_min; *min = state.similarity_min;
*max = state.similarity_max; *max = state.similarity_max;
fb_free(); fb_free(); // dst_row_override
fb_free(); fb_free(); // sumBucketsOfX
fb_free();
fb_free();
fb_free();
} }
#endif //IMLIB_ENABLE_GET_SIMILARITY #endif // IMLIB_ENABLE_GET_SIMILARITY
void imlib_get_histogram(histogram_t *out, image_t *ptr, rectangle_t *roi, list_t *thresholds, bool invert, image_t *other) { void imlib_get_histogram(histogram_t *out, image_t *ptr, rectangle_t *roi, list_t *thresholds, bool invert, image_t *other) {
switch (ptr->pixfmt) { switch (ptr->pixfmt) {

View File

@ -2712,26 +2712,57 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE(
locals_dict, &py_similarity_locals_dict locals_dict, &py_similarity_locals_dict
); );
static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) { static mp_obj_t py_image_get_similarity(uint n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
image_t *arg_img = py_helper_arg_to_image(img_obj, ARG_IMAGE_MUTABLE); enum {
float avg, std, min, max; ARG_image, ARG_x, ARG_y, ARG_x_scale, ARG_y_scale, ARG_roi,
ARG_channel, ARG_alpha, ARG_color_palette, ARG_alpha_palette, ARG_hint, ARG_dssim
};
static const mp_arg_t allowed_args[] = {
{ MP_QSTR_image, MP_ARG_OBJ | MP_ARG_REQUIRED, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_x, MP_ARG_INT, {.u_int = 0 } },
{ MP_QSTR_y, MP_ARG_INT, {.u_int = 0 } },
{ MP_QSTR_x_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_y_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_roi, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_rgb_channel, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = -1 } },
{ MP_QSTR_alpha, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 256 } },
{ MP_QSTR_color_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_alpha_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_hint, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 0 } },
{ MP_QSTR_dssim, MP_ARG_BOOL | MP_ARG_KW_ONLY, {.u_bool = false } },
};
// Parse args.
image_t *image = py_helper_arg_to_image(pos_args[0], ARG_IMAGE_MUTABLE);
mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
fb_alloc_mark(); fb_alloc_mark();
image_t *other = py_helper_arg_to_image(args[ARG_image].u_obj, ARG_IMAGE_ANY | ARG_IMAGE_ALLOC);
rectangle_t roi = py_helper_arg_to_roi(args[ARG_roi].u_obj, other);
if (MP_OBJ_IS_STR(other_obj)) { if (args[ARG_channel].u_int < -1 || args[ARG_channel].u_int > 2) {
imlib_get_similarity(arg_img, mp_obj_str_get_str(other_obj), NULL, 0, &avg, &std, &min, &max); mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("RGB channel can be 0, 1, or 2"));
} else if (MP_OBJ_IS_TYPE(other_obj, &py_image_type)) {
imlib_get_similarity(arg_img, NULL,
py_helper_arg_to_image(other_obj, ARG_IMAGE_MUTABLE),
0, &avg, &std, &min, &max);
} else {
imlib_get_similarity(arg_img, NULL, NULL,
py_helper_keyword_color(arg_img, 1, &other_obj, 0, NULL, 0),
&avg, &std, &min, &max);
} }
fb_alloc_free_till_mark(); if (args[ARG_alpha].u_int < 0 || args[ARG_alpha].u_int > 256) {
mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("Alpha ranges between 0 and 256"));
}
float x_scale = 1.0f;
float y_scale = 1.0f;
py_helper_arg_to_scale(args[ARG_x_scale].u_obj, args[ARG_y_scale].u_obj, &x_scale, &y_scale);
const uint16_t *color_palette = py_helper_arg_to_palette(args[ARG_color_palette].u_obj, PIXFORMAT_RGB565);
const uint8_t *alpha_palette = py_helper_arg_to_palette(args[ARG_alpha_palette].u_obj, PIXFORMAT_GRAYSCALE);
float avg = 0.0f, std = 0.0f, min = 0.0f, max = 0.0f;
imlib_get_similarity(image, other, args[ARG_x].u_int, args[ARG_y].u_int, x_scale, y_scale, &roi,
args[ARG_channel].u_int, args[ARG_alpha].u_int, color_palette, alpha_palette,
args[ARG_hint].u_int | IMAGE_HINT_BLACK_BACKGROUND, args[ARG_dssim].u_bool,
&avg, &std, &min, &max);
fb_alloc_free_till_mark();
py_similarity_obj_t *o = m_new_obj(py_similarity_obj_t); py_similarity_obj_t *o = m_new_obj(py_similarity_obj_t);
o->base.type = &py_similarity_type; o->base.type = &py_similarity_type;
o->avg = mp_obj_new_float(avg); o->avg = mp_obj_new_float(avg);
@ -2740,7 +2771,7 @@ static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) {
o->max = mp_obj_new_float(max); o->max = mp_obj_new_float(max);
return o; return o;
} }
STATIC MP_DEFINE_CONST_FUN_OBJ_2(py_image_get_similarity_obj, py_image_get_similarity); STATIC MP_DEFINE_CONST_FUN_OBJ_KW(py_image_get_similarity_obj, 1, py_image_get_similarity);
#endif // IMLIB_ENABLE_GET_SIMILARITY #endif // IMLIB_ENABLE_GET_SIMILARITY
// Statistics Object // // Statistics Object //