Merge pull request #2190 from kwagyeman/kwabena/refactor_get_similarity

modules/py_image: Refactor get_similarity() to use draw_image backend.
This commit is contained in:
Ibrahim Abdelkader 2024-06-09 14:34:52 +02:00 committed by GitHub
commit 8c8c7dbde1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 167 additions and 101 deletions

View File

@ -1440,9 +1440,18 @@ void imlib_rotation_corr(image_t *img, float x_rotation, float y_rotation,
float zoom, float fov, float *corners);
// Statistics
void imlib_get_similarity(image_t *img,
const char *path,
image_t *other,
int scalar,
int x_start,
int y_start,
float x_scale,
float y_scale,
rectangle_t *roi,
int rgb_channel,
int alpha,
const uint16_t *color_palette,
const uint8_t *alpha_palette,
image_hint_t hint,
bool dssim,
float *avg,
float *std,
float *min,

View File

@ -11,69 +11,68 @@
#include "imlib.h"
#ifdef IMLIB_ENABLE_GET_SIMILARITY
typedef struct imlib_similatiry_line_op_state {
typedef struct imlib_similarity_line_op_state {
bool dssim;
int *sumBucketsOfX, *sumBucketsOfY, *sum2BucketsOfX, *sum2BucketsOfY, *sum2Buckets;
float similarity_sum, similarity_sum_2, similarity_min, similarity_max;
int lines_processed;
} imlib_similatiry_line_op_state_t;
int lines_processed, lines;
} imlib_similarity_line_op_state_t;
void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, bool vflipped) {
imlib_similatiry_line_op_state_t *state = (imlib_similatiry_line_op_state_t *) data; vflipped = vflipped;
static void imlib_similarity_line_op(int x, int x_end, int y_row, imlib_draw_row_data_t *data) {
imlib_similarity_line_op_state_t *state = data->callback_arg;
float c1 = 0, c2 = 0;
int x_start = x;
switch (img->pixfmt) {
switch (data->dst_img->pixfmt) {
case PIXFORMAT_BINARY: {
uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(img, line);
uint32_t *other_row_ptr = (uint32_t *) other;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) {
int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x + i);
int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x + i);
state->sumBucketsOfX[x] += pixel;
state->sumBucketsOfY[x] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel;
uint32_t *row_ptr = IMAGE_COMPUTE_BINARY_PIXEL_ROW_PTR(data->dst_img, y_row);
uint32_t *other_row_ptr = (uint32_t *) data->dst_row_override;
for (; x < x_end; x++) {
int pixel = IMAGE_GET_BINARY_PIXEL_FAST(row_ptr, x);
int other_pixel = IMAGE_GET_BINARY_PIXEL_FAST(other_row_ptr, x);
int bucket = (x - x_start) / 8;
state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[bucket] += pixel * other_pixel;
}
}
c1 = COLOR_BINARY_MAX * 0.01f;
c2 = COLOR_BINARY_MAX * 0.03f;
c1 = (COLOR_BINARY_MAX * 0.01f) * (COLOR_BINARY_MAX * 0.01f);
c2 = (COLOR_BINARY_MAX * 0.03f) * (COLOR_BINARY_MAX * 0.03f);
break;
}
case PIXFORMAT_GRAYSCALE: {
uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(img, line);
uint8_t *other_row_ptr = (uint8_t *) other;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) {
int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x + i);
int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x + i);
state->sumBucketsOfX[x] += pixel;
state->sumBucketsOfY[x] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel;
uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(data->dst_img, y_row);
uint8_t *other_row_ptr = (uint8_t *) data->dst_row_override;
for (; x < x_end; x++) {
int pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(row_ptr, x);
int other_pixel = IMAGE_GET_GRAYSCALE_PIXEL_FAST(other_row_ptr, x);
int bucket = (x - x_start) / 8;
state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[bucket] += pixel * other_pixel;
}
}
c1 = COLOR_GRAYSCALE_MAX * 0.01f;
c2 = COLOR_GRAYSCALE_MAX * 0.03f;
c1 = (COLOR_GRAYSCALE_MAX * 0.01f) * (COLOR_GRAYSCALE_MAX * 0.01f);
c2 = (COLOR_GRAYSCALE_MAX * 0.03f) * (COLOR_GRAYSCALE_MAX * 0.03f);
break;
}
case PIXFORMAT_RGB565: {
uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(img, line);
uint16_t *other_row_ptr = (uint16_t *) other;
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) {
for (int i = 0, ii = IM_MIN((img->w - (x * 8)), 8); i < ii; i++) {
int pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x + i));
int other_pixel = COLOR_RGB565_TO_L(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x + i));
state->sumBucketsOfX[x] += pixel;
state->sumBucketsOfY[x] += other_pixel;
state->sum2BucketsOfX[x] += pixel * pixel;
state->sum2BucketsOfY[x] += other_pixel * other_pixel;
state->sum2Buckets[x] += pixel * other_pixel;
uint16_t *row_ptr = IMAGE_COMPUTE_RGB565_PIXEL_ROW_PTR(data->dst_img, y_row);
uint16_t *other_row_ptr = (uint16_t *) data->dst_row_override;
for (; x < x_end; x++) {
int pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(row_ptr, x));
int other_pixel = COLOR_RGB565_TO_Y(IMAGE_GET_RGB565_PIXEL_FAST(other_row_ptr, x));
int bucket = (x - x_start) / 8;
state->sumBucketsOfX[bucket] += pixel;
state->sumBucketsOfY[bucket] += other_pixel;
state->sum2BucketsOfX[bucket] += pixel * pixel;
state->sum2BucketsOfY[bucket] += other_pixel * other_pixel;
state->sum2Buckets[bucket] += pixel * other_pixel;
}
}
c1 = COLOR_L_MAX * 0.01f;
c2 = COLOR_L_MAX * 0.03f;
c1 = (COLOR_Y_MAX * 0.01f) * (COLOR_Y_MAX * 0.01f);
c2 = (COLOR_Y_MAX * 0.03f) * (COLOR_Y_MAX * 0.03f);
break;
}
default: {
@ -82,33 +81,41 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b
}
// https://en.wikipedia.org/wiki/Structural_similarity
if (((state->lines_processed + 1) == img->h) || (!((state->lines_processed + 1) % 8))) {
for (int x = 0, xx = (img->w + 7) / 8; x < xx; x++) {
int w = IM_MIN((img->w - (x * 8)), 8);
int h = IM_MIN((img->h - ((state->lines_processed / 8) * 8)), 8);
int size = w * h;
if ((!((state->lines_processed + 1) % 8)) || ((state->lines_processed + 1) == state->lines)) {
for (x = x_start; x < x_end; x += 8) {
int bucket = (x - x_start) / 8;
int w = IM_MIN((x_end - x), 8);
int h = IM_MIN((state->lines - state->lines_processed), 8);
float size = w * h;
int mx = state->sumBucketsOfX[x] / size;
int my = state->sumBucketsOfY[x] / size;
int vx = state->sum2BucketsOfX[x] - ((mx * state->sumBucketsOfX[x]) + (mx * state->sumBucketsOfX[x])) +
(size * mx * mx);
int vy = state->sum2BucketsOfY[x] - ((my * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfY[x])) +
(size * my * my);
int vxy = state->sum2Buckets[x] - ((mx * state->sumBucketsOfY[x]) + (my * state->sumBucketsOfX[x])) +
(size * mx * my);
// Dividng the sum squared buckets by size causes a loss of accuracy which results in
// the single pass standard deviation formula giving the wrong answer. To bypass this
// vx, vy, vxy have been multiplied by size which will be divided back out in the final
// ssim calculation (given c1/c2 ~= 0).
float ssim = ( ((2 * mx * my) + c1) * ((2 * vxy) + c2) ) / ( ((mx * mx) + (my * my) + c1) * (vx + vy + c2) );
float mx = state->sumBucketsOfX[bucket] / size;
float my = state->sumBucketsOfY[bucket] / size;
float vx = state->sum2BucketsOfX[bucket] - (size * mx * mx);
float vy = state->sum2BucketsOfY[bucket] - (size * my * my);
float vxy = state->sum2Buckets[bucket] - (size * mx * my);
float ssim = (((2 * mx * my) + c1) * ((2 * vxy) + c2)) /
(((mx * mx) + (my * my) + c1) * (vx + vy + c2));
if (state->dssim) {
ssim = (1.0f - ssim) / 2.0f;
}
state->similarity_sum += ssim;
state->similarity_sum_2 += ssim * ssim;
state->similarity_min = IM_MIN(state->similarity_min, ssim);
state->similarity_max = IM_MAX(state->similarity_max, ssim);
state->sumBucketsOfX[x] = 0;
state->sumBucketsOfY[x] = 0;
state->sum2BucketsOfX[x] = 0;
state->sum2BucketsOfY[x] = 0;
state->sum2Buckets[x] = 0;
state->sumBucketsOfX[bucket] = 0;
state->sumBucketsOfY[bucket] = 0;
state->sum2BucketsOfX[bucket] = 0;
state->sum2BucketsOfY[bucket] = 0;
state->sum2Buckets[bucket] = 0;
}
}
@ -116,41 +123,60 @@ void imlib_similarity_line_op(image_t *img, int line, void *other, void *data, b
}
void imlib_get_similarity(image_t *img,
const char *path,
image_t *other,
int scalar,
int x_start,
int y_start,
float x_scale,
float y_scale,
rectangle_t *roi,
int rgb_channel,
int alpha,
const uint16_t *color_palette,
const uint8_t *alpha_palette,
image_hint_t hint,
bool dssim,
float *avg,
float *std,
float *min,
float *max) {
int h_blocks = (img->w + 7) / 8;
int v_blocks = (img->h + 7) / 8;
point_t p0, p1;
imlib_draw_image_get_bounds(img, other, x_start, y_start, x_scale, y_scale, roi,
alpha, alpha_palette, hint, &p0, &p1);
int h_blocks = ((p1.x - p0.x) + 7) / 8;
int v_blocks = ((p1.y - p0.y) + 7) / 8;
int blocks = h_blocks * v_blocks;
if (!blocks) {
return;
}
int int_h_blocks = h_blocks * sizeof(int);
imlib_similatiry_line_op_state_t state;
state.sumBucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT);
state.sumBucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT);
state.sum2BucketsOfX = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT);
state.sum2BucketsOfY = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT);
state.sum2Buckets = fb_alloc0(int_h_blocks, FB_ALLOC_NO_HINT);
imlib_similarity_line_op_state_t state;
state.dssim = dssim;
state.sumBucketsOfX = fb_alloc0(int_h_blocks * 5, FB_ALLOC_NO_HINT);
state.sumBucketsOfY = state.sumBucketsOfX + int_h_blocks;
state.sum2BucketsOfX = state.sumBucketsOfY + int_h_blocks;
state.sum2BucketsOfY = state.sum2BucketsOfX + int_h_blocks;
state.sum2Buckets = state.sum2BucketsOfY + int_h_blocks;
state.similarity_sum = 0.0f;
state.similarity_sum_2 = 0.0f;
state.similarity_min = FLT_MAX;
state.similarity_max = -FLT_MAX;
state.lines_processed = 0;
state.lines = p1.y - p0.y;
void *dst_row_override = fb_alloc0(image_line_size(img), FB_ALLOC_CACHE_ALIGN);
imlib_draw_image(img, other, x_start, y_start, x_scale, y_scale, roi,
rgb_channel, alpha, color_palette, alpha_palette,
hint, imlib_similarity_line_op, &state, dst_row_override);
imlib_image_operation(img, path, other, scalar, imlib_similarity_line_op, &state);
*avg = state.similarity_sum / blocks;
*std = fast_sqrtf((state.similarity_sum_2 / blocks) - ((*avg) * (*avg)));
*min = state.similarity_min;
*max = state.similarity_max;
fb_free();
fb_free();
fb_free();
fb_free();
fb_free();
fb_free(); // dst_row_override
fb_free(); // sumBucketsOfX
}
#endif // IMLIB_ENABLE_GET_SIMILARITY

View File

@ -2712,26 +2712,57 @@ STATIC MP_DEFINE_CONST_OBJ_TYPE(
locals_dict, &py_similarity_locals_dict
);
static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) {
image_t *arg_img = py_helper_arg_to_image(img_obj, ARG_IMAGE_MUTABLE);
float avg, std, min, max;
static mp_obj_t py_image_get_similarity(uint n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
enum {
ARG_image, ARG_x, ARG_y, ARG_x_scale, ARG_y_scale, ARG_roi,
ARG_channel, ARG_alpha, ARG_color_palette, ARG_alpha_palette, ARG_hint, ARG_dssim
};
static const mp_arg_t allowed_args[] = {
{ MP_QSTR_image, MP_ARG_OBJ | MP_ARG_REQUIRED, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_x, MP_ARG_INT, {.u_int = 0 } },
{ MP_QSTR_y, MP_ARG_INT, {.u_int = 0 } },
{ MP_QSTR_x_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_y_scale, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_roi, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_rgb_channel, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = -1 } },
{ MP_QSTR_alpha, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 256 } },
{ MP_QSTR_color_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_alpha_palette, MP_ARG_OBJ | MP_ARG_KW_ONLY, {.u_rom_obj = MP_ROM_NONE} },
{ MP_QSTR_hint, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 0 } },
{ MP_QSTR_dssim, MP_ARG_BOOL | MP_ARG_KW_ONLY, {.u_bool = false } },
};
// Parse args.
image_t *image = py_helper_arg_to_image(pos_args[0], ARG_IMAGE_MUTABLE);
mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
fb_alloc_mark();
image_t *other = py_helper_arg_to_image(args[ARG_image].u_obj, ARG_IMAGE_ANY | ARG_IMAGE_ALLOC);
rectangle_t roi = py_helper_arg_to_roi(args[ARG_roi].u_obj, other);
if (MP_OBJ_IS_STR(other_obj)) {
imlib_get_similarity(arg_img, mp_obj_str_get_str(other_obj), NULL, 0, &avg, &std, &min, &max);
} else if (MP_OBJ_IS_TYPE(other_obj, &py_image_type)) {
imlib_get_similarity(arg_img, NULL,
py_helper_arg_to_image(other_obj, ARG_IMAGE_MUTABLE),
0, &avg, &std, &min, &max);
} else {
imlib_get_similarity(arg_img, NULL, NULL,
py_helper_keyword_color(arg_img, 1, &other_obj, 0, NULL, 0),
&avg, &std, &min, &max);
if (args[ARG_channel].u_int < -1 || args[ARG_channel].u_int > 2) {
mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("RGB channel can be 0, 1, or 2"));
}
fb_alloc_free_till_mark();
if (args[ARG_alpha].u_int < 0 || args[ARG_alpha].u_int > 256) {
mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("Alpha ranges between 0 and 256"));
}
float x_scale = 1.0f;
float y_scale = 1.0f;
py_helper_arg_to_scale(args[ARG_x_scale].u_obj, args[ARG_y_scale].u_obj, &x_scale, &y_scale);
const uint16_t *color_palette = py_helper_arg_to_palette(args[ARG_color_palette].u_obj, PIXFORMAT_RGB565);
const uint8_t *alpha_palette = py_helper_arg_to_palette(args[ARG_alpha_palette].u_obj, PIXFORMAT_GRAYSCALE);
float avg = 0.0f, std = 0.0f, min = 0.0f, max = 0.0f;
imlib_get_similarity(image, other, args[ARG_x].u_int, args[ARG_y].u_int, x_scale, y_scale, &roi,
args[ARG_channel].u_int, args[ARG_alpha].u_int, color_palette, alpha_palette,
args[ARG_hint].u_int | IMAGE_HINT_BLACK_BACKGROUND, args[ARG_dssim].u_bool,
&avg, &std, &min, &max);
fb_alloc_free_till_mark();
py_similarity_obj_t *o = m_new_obj(py_similarity_obj_t);
o->base.type = &py_similarity_type;
o->avg = mp_obj_new_float(avg);
@ -2740,7 +2771,7 @@ static mp_obj_t py_image_get_similarity(mp_obj_t img_obj, mp_obj_t other_obj) {
o->max = mp_obj_new_float(max);
return o;
}
STATIC MP_DEFINE_CONST_FUN_OBJ_2(py_image_get_similarity_obj, py_image_get_similarity);
STATIC MP_DEFINE_CONST_FUN_OBJ_KW(py_image_get_similarity_obj, 1, py_image_get_similarity);
#endif // IMLIB_ENABLE_GET_SIMILARITY
// Statistics Object //