mirror of
https://github.com/openmv/openmv.git
synced 2025-09-26 23:09:13 +08:00

Originally meant to abstract gc_collect but we could just use m_alloc and friends. Also was meant to provide functions like alloc0, alloc_maybe etc.. which are all available in MP anyway. Signed-off-by: iabdalkader <i.abdalkader@gmail.com>
466 lines
15 KiB
C
466 lines
15 KiB
C
/*
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
* Copyright (C) 2013-2024 OpenMV, LLC.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
*
|
|
* Selective search.
|
|
*/
|
|
#include <stdio.h>
|
|
#include <math.h>
|
|
#include <string.h>
|
|
#include <stdint.h>
|
|
#include "imlib.h"
|
|
#include "fb_alloc.h"
|
|
#ifdef IMLIB_ENABLE_SELECTIVE_SEARCH
|
|
|
|
#define THRESHOLD(size, c) (c / size)
|
|
typedef struct {
|
|
uint16_t y;
|
|
uint16_t h;
|
|
uint16_t x;
|
|
uint16_t w;
|
|
} region;
|
|
|
|
typedef struct {
|
|
uint16_t p;
|
|
uint16_t rank;
|
|
uint16_t size;
|
|
} uni_elt;
|
|
|
|
typedef struct {
|
|
int num;
|
|
uni_elt *elts;
|
|
} universe;
|
|
|
|
typedef struct {
|
|
float w;
|
|
uint16_t a;
|
|
uint16_t b;
|
|
} edge;
|
|
|
|
static inline int min(int a, int b) {
|
|
return (a < b) ? a : b;
|
|
}
|
|
static inline int max(int a, int b) {
|
|
return (a > b) ? a : b;
|
|
}
|
|
static inline float minf(float a, float b) {
|
|
return (a < b) ? a : b;
|
|
}
|
|
static inline float maxf(float a, float b) {
|
|
return (a > b) ? a : b;
|
|
}
|
|
extern uint32_t rng_randint(uint32_t min, uint32_t max);
|
|
|
|
static universe *universe_create(int elements) {
|
|
universe *uni = (universe *) fb_alloc(sizeof(universe), FB_ALLOC_NO_HINT);
|
|
uni->elts = (uni_elt *) fb_alloc(sizeof(uni_elt) * elements, FB_ALLOC_NO_HINT);
|
|
uni->num = elements;
|
|
for (int i = 0; i < elements; ++i) {
|
|
uni->elts[i].p = i;
|
|
uni->elts[i].rank = 0;
|
|
uni->elts[i].size = 1;
|
|
}
|
|
return uni;
|
|
}
|
|
|
|
static int universe_size(universe *uni, int x) {
|
|
return uni->elts[x].size;
|
|
}
|
|
|
|
static int universe_num_sets(universe *uni) {
|
|
return uni->num;
|
|
}
|
|
|
|
static int universe_find(universe *uni, int x) {
|
|
int y = x;
|
|
while (y != uni->elts[y].p) {
|
|
y = uni->elts[y].p;
|
|
}
|
|
// Path compression
|
|
uni->elts[x].p = y;
|
|
return y;
|
|
}
|
|
|
|
static void universe_join(universe *uni, int x, int y) {
|
|
if (uni->elts[x].rank > uni->elts[y].rank) {
|
|
uni->elts[y].p = x;
|
|
uni->elts[x].size += uni->elts[y].size;
|
|
} else {
|
|
uni->elts[x].p = y;
|
|
uni->elts[y].size += uni->elts[x].size;
|
|
if (uni->elts[x].rank == uni->elts[y].rank) {
|
|
uni->elts[y].rank++;
|
|
}
|
|
}
|
|
uni->num--;
|
|
}
|
|
|
|
static int universe_get_id(universe *this, int x) {
|
|
return this->elts[x].rank;
|
|
}
|
|
|
|
static void universe_set_id(universe *this, int x, int id) {
|
|
this->elts[x].rank = id;
|
|
}
|
|
|
|
static inline float color_similarity(float *hist1, float *hist2) {
|
|
float sim = 0;
|
|
for (int i = 0; i < 75; ++i) {
|
|
sim += minf(hist1[i], hist2[i]);
|
|
}
|
|
return sim;
|
|
}
|
|
|
|
static inline float size_similarity(int a, int b, int size) {
|
|
return 1.0f - (a + b) / size;
|
|
}
|
|
|
|
static inline float fill_similarity(region *ra, region *rb, int a, int b, int size) {
|
|
int width = max(ra->w, rb->w) - min(ra->x, rb->x);
|
|
int height = max(ra->h, rb->h) - min(ra->y, rb->y);
|
|
return 1.0f - (width * height - a - b) / size;
|
|
}
|
|
|
|
static inline float square(float x) {
|
|
return x * x;
|
|
};
|
|
|
|
static inline float diff(image_t *img, int x1, int y1, int x2, int y2) {
|
|
uint16_t p1 = IMAGE_GET_RGB565_PIXEL(img, x1, y1);
|
|
uint16_t p2 = IMAGE_GET_RGB565_PIXEL(img, x2, y2);
|
|
uint8_t r1 = COLOR_RGB565_TO_R8(p1);
|
|
uint8_t r2 = COLOR_RGB565_TO_R8(p2);
|
|
|
|
uint8_t g1 = COLOR_RGB565_TO_G8(p1);
|
|
uint8_t g2 = COLOR_RGB565_TO_G8(p2);
|
|
|
|
uint8_t b1 = COLOR_RGB565_TO_B8(p1);
|
|
uint8_t b2 = COLOR_RGB565_TO_B8(p2);
|
|
// dissimilarity measure between pixels
|
|
return sqrtf((r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2) + (b1 - b2) * (b1 - b2));
|
|
}
|
|
|
|
int comp(const void *elem1, const void *elem2) {
|
|
edge *f = (edge *) elem1;
|
|
edge *s = (edge *) elem2;
|
|
if (f->w > s->w) {
|
|
return 1;
|
|
}
|
|
if (f->w < s->w) {
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void segment_graph(universe *u, int num_vertices, int num_edges, edge *edges, float c) {
|
|
qsort(edges, num_edges, sizeof(edge), comp);
|
|
|
|
float *threshold = fb_alloc(num_vertices * sizeof(float), FB_ALLOC_NO_HINT);
|
|
for (int i = 0; i < num_vertices; i++) {
|
|
threshold[i] = THRESHOLD(1, c);
|
|
}
|
|
|
|
for (int i = 0; i < num_edges; i++) {
|
|
edge *pedge = edges + i;
|
|
int a = universe_find(u, pedge->a);
|
|
int b = universe_find(u, pedge->b);
|
|
if (a != b) {
|
|
if ((pedge->w <= threshold[a]) && (pedge->w <= threshold[b])) {
|
|
universe_join(u, a, b);
|
|
a = universe_find(u, a);
|
|
threshold[a] = pedge->w + THRESHOLD(universe_size(u, a), c);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Free thresholds.
|
|
fb_free();
|
|
}
|
|
|
|
static void image_scale(image_t *src, image_t *dst) {
|
|
int x_ratio = (int) ((src->w << 16) / dst->w) + 1;
|
|
int y_ratio = (int) ((src->h << 16) / dst->h) + 1;
|
|
|
|
for (int y = 0; y < dst->h; y++) {
|
|
int sy = (y * y_ratio) >> 16;
|
|
for (int x = 0; x < dst->w; x++) {
|
|
int sx = (x * x_ratio) >> 16;
|
|
((uint16_t *) dst->pixels)[y * dst->w + x] = ((uint16_t *) src->pixels)[sy * src->w + sx];
|
|
}
|
|
}
|
|
}
|
|
|
|
array_t *imlib_selective_search(image_t *src, float t, int min_size, float a1, float a2, float a3) {
|
|
int i, j;
|
|
int num = 0;
|
|
int width = 0, height = 0;
|
|
image_t *img = NULL;
|
|
|
|
fb_alloc_mark();
|
|
|
|
if ((src->w * src->h) <= (80 * 60)) {
|
|
img = src;
|
|
width = src->w;
|
|
height = src->h;
|
|
} else {
|
|
// Down scale image
|
|
width = src->w / 4;
|
|
height = src->h / 4;
|
|
img = fb_alloc(sizeof(image_t), FB_ALLOC_NO_HINT);
|
|
img->w = width;
|
|
img->h = height;
|
|
img->pixels = fb_alloc(width * height * 2, FB_ALLOC_NO_HINT);
|
|
image_scale(src, img);
|
|
}
|
|
|
|
// Region proposals array
|
|
array_t *proposals;
|
|
array_alloc(&proposals, m_free);
|
|
|
|
universe *u = universe_create(width * height);
|
|
edge *edges = (edge *) fb_alloc(width * height * sizeof(edge) * 4, FB_ALLOC_NO_HINT);
|
|
|
|
for (int y = 0; y < height; y++) {
|
|
for (int x = 0; x < width; x++) {
|
|
if (x < width - 1) {
|
|
edges[num].a = y * width + x;
|
|
edges[num].b = y * width + (x + 1);
|
|
edges[num].w = diff(img, x, y, x + 1, y);
|
|
num++;
|
|
}
|
|
|
|
if (y < height - 1) {
|
|
edges[num].a = y * width + x;
|
|
edges[num].b = (y + 1) * width + x;
|
|
edges[num].w = diff(img, x, y, x, y + 1);
|
|
num++;
|
|
}
|
|
|
|
if ((x < width - 1) && (y < height - 1)) {
|
|
edges[num].a = y * width + x;
|
|
edges[num].b = (y + 1) * width + (x + 1);
|
|
edges[num].w = diff(img, x, y, x + 1, y + 1);
|
|
num++;
|
|
}
|
|
|
|
if ((x < width - 1) && (y > 0)) {
|
|
edges[num].a = y * width + x;
|
|
edges[num].b = (y - 1) * width + (x + 1);
|
|
edges[num].w = diff(img, x, y, x + 1, y - 1);
|
|
num++;
|
|
}
|
|
}
|
|
}
|
|
|
|
segment_graph(u, width * height, num, edges, t);
|
|
|
|
for (i = 0; i < num; i++) {
|
|
int a = universe_find(u, edges[i].a);
|
|
int b = universe_find(u, edges[i].b);
|
|
if ((a != b) && ((universe_size(u, a) < min_size) || (universe_size(u, b) < min_size))) {
|
|
universe_join(u, a, b);
|
|
}
|
|
}
|
|
|
|
// Free graph edges
|
|
fb_free();
|
|
|
|
int num_ccs = universe_num_sets(u);
|
|
region *regions = (region *) fb_alloc(num_ccs * sizeof(region), FB_ALLOC_NO_HINT);
|
|
for (i = 0; i < num_ccs; i++) {
|
|
regions[i].x = width;
|
|
regions[i].w = 0;
|
|
regions[i].y = height;
|
|
regions[i].h = 0;
|
|
}
|
|
|
|
int next_component = 0;
|
|
int *counts = (int *) fb_alloc0(num_ccs * sizeof(int), FB_ALLOC_NO_HINT);
|
|
int *components = (int *) fb_alloc(num_ccs * sizeof(int), FB_ALLOC_NO_HINT);
|
|
float *histogram = (float *) fb_alloc0(num_ccs * sizeof(float) * 75, FB_ALLOC_NO_HINT);
|
|
|
|
// Calc histograms
|
|
for (int y = 0; y < height; y++) {
|
|
for (int x = 0; x < width; x++) {
|
|
int component_id = -1;
|
|
int comp = universe_find(u, y * width + x);
|
|
for (i = 0; i < next_component; i++) {
|
|
if (components[i] == comp) {
|
|
component_id = i;
|
|
break;
|
|
}
|
|
}
|
|
if (i == next_component) {
|
|
components[next_component] = comp;
|
|
component_id = next_component;
|
|
++next_component;
|
|
}
|
|
universe_set_id(u, y * width + x, component_id);
|
|
region *r = regions + component_id;
|
|
r->y = min(r->y, y);
|
|
r->h = max(r->h, y);
|
|
r->x = min(r->x, x);
|
|
r->w = max(r->w, x);
|
|
|
|
uint16_t p = IMAGE_GET_RGB565_PIXEL(img, x, y);
|
|
int r_bin = min(COLOR_RGB565_TO_R8(p), 240) / 10;
|
|
int g_bin = min(COLOR_RGB565_TO_G8(p), 240) / 10;
|
|
int b_bin = min(COLOR_RGB565_TO_B8(p), 240) / 10;
|
|
|
|
histogram[75 * component_id + 0 + r_bin]++;
|
|
histogram[75 * component_id + 25 + g_bin]++;
|
|
histogram[75 * component_id + 50 + b_bin]++;
|
|
counts[component_id]++;
|
|
}
|
|
}
|
|
|
|
// Normalize histograms
|
|
for (i = 0; i < num_ccs; i++) {
|
|
float max_val = 0;
|
|
for (j = 0; j < 75; j++) {
|
|
max_val = max(max_val, histogram[75 * i + j]);
|
|
}
|
|
for (j = 0; j < 75; j++) {
|
|
histogram[75 * i + j] /= max_val;
|
|
}
|
|
}
|
|
|
|
uint8_t *adjacency = (uint8_t *) fb_alloc0(num_ccs * num_ccs * sizeof(uint8_t), FB_ALLOC_NO_HINT);
|
|
for (int y = 0; y < height - 1; ++y) {
|
|
for (int x = 0; x < width - 1; ++x) {
|
|
int component1 = universe_get_id(u, y * width + x);
|
|
int component2 = universe_get_id(u, y * width + x + 1);
|
|
int component3 = universe_get_id(u, y * width + x + width);
|
|
|
|
if (component1 != component2) {
|
|
adjacency[component1 * num_ccs + component2] = 1;
|
|
adjacency[component2 * num_ccs + component1] = 1;
|
|
}
|
|
|
|
if (component1 != component3) {
|
|
adjacency[component1 * num_ccs + component3] = 1;
|
|
adjacency[component3 * num_ccs + component1] = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
int size = height * width;
|
|
float *similarity_table = (float *) fb_alloc(num_ccs * num_ccs * sizeof(float), FB_ALLOC_NO_HINT);
|
|
for (i = 0; i < num_ccs; ++i) {
|
|
for (j = i + 1; j < num_ccs; ++j) {
|
|
float color_sim = a1 * color_similarity(histogram + 75 * i, histogram + 75 * j);
|
|
float size_sim = a2 * size_similarity(counts[i], counts[j], size);
|
|
float fill_sim = a3 * fill_similarity(regions + i, regions + j, counts[i], counts[j], size);
|
|
float similarity = color_sim + size_sim + fill_sim;
|
|
similarity_table[i * num_ccs + j] = similarity;
|
|
similarity_table[j * num_ccs + i] = similarity;
|
|
}
|
|
}
|
|
|
|
int remaining = num_ccs;
|
|
while (remaining > 1) {
|
|
int best_i = -1;
|
|
int best_j = -1;
|
|
float best_similarity = 0;
|
|
for (i = 0; i < num_ccs; i++) {
|
|
for (j = i + 1; j < num_ccs; j++) {
|
|
if (adjacency[i * num_ccs + j] == 0) {
|
|
continue;
|
|
}
|
|
float similarity = similarity_table[i * num_ccs + j];
|
|
if (similarity > best_similarity) {
|
|
best_similarity = similarity;
|
|
best_i = i;
|
|
best_j = j;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (best_i == -1) {
|
|
printf("failed to build tree\n");
|
|
break;
|
|
}
|
|
|
|
// update regions, histograms, counts, adjacency, similarity
|
|
regions[best_i].x = min(regions[best_i].x, regions[best_j].x);
|
|
regions[best_i].y = min(regions[best_i].y, regions[best_j].y);
|
|
regions[best_i].w = max(regions[best_i].w, regions[best_j].w);
|
|
regions[best_i].h = max(regions[best_i].h, regions[best_j].h);
|
|
|
|
bool add = true;
|
|
for (i = 0; i < array_length(proposals); i++) {
|
|
rectangle_t *r = array_at(proposals, i);
|
|
if (regions[best_i].x == r->x && regions[best_i].y == r->y &&
|
|
regions[best_i].w == r->w && regions[best_i].h == r->h) {
|
|
add = false;
|
|
break;
|
|
}
|
|
}
|
|
if (add) {
|
|
array_push_back(proposals, rectangle_alloc(regions[best_i].x,
|
|
regions[best_i].y, regions[best_i].w, regions[best_i].h));
|
|
}
|
|
|
|
|
|
for (i = 0; i < 75; i++) {
|
|
histogram[75 * best_i + i] = (counts[best_i] * histogram[75 * best_i + i]
|
|
+ counts[best_j] * histogram[75 * best_j + i]) / (counts[best_i] + counts[best_j]);
|
|
}
|
|
counts[best_i] += counts[best_j];
|
|
|
|
for (i = 0; i < num_ccs; i++) {
|
|
adjacency[best_i * num_ccs + i] |= adjacency[best_j * num_ccs + i];
|
|
adjacency[i * num_ccs + best_i] |= adjacency[i * num_ccs + best_j];
|
|
adjacency[best_j * num_ccs + i] = adjacency[i * num_ccs + best_j] = 0;
|
|
}
|
|
adjacency[best_i * num_ccs + best_i] = 0;
|
|
|
|
for (i = 0; i < num_ccs; i++) {
|
|
if (adjacency[best_i * num_ccs + i] == 0) {
|
|
continue;
|
|
}
|
|
float color_sim = a1 * color_similarity(histogram + 75 * i, histogram + 75 * best_i);
|
|
float size_sim = a2 * size_similarity(counts[i], counts[best_i], size);
|
|
float fill_sim = a3 * fill_similarity(regions + i, regions + best_i, counts[i], counts[best_i], size);
|
|
float similarity = color_sim + size_sim + fill_sim;
|
|
similarity_table[i * num_ccs + best_i] = similarity;
|
|
similarity_table[best_i * num_ccs + i] = similarity;
|
|
}
|
|
--remaining;
|
|
}
|
|
|
|
for (int i = 0; i < array_length(proposals); i++) {
|
|
rectangle_t *r = array_at(proposals, i);
|
|
r->w = r->w - r->x;
|
|
r->h = r->h - r->y;
|
|
if ((src->w * src->h) > (80 * 60)) {
|
|
r->x *= 4;
|
|
r->y *= 4;
|
|
r->w *= 4;
|
|
r->h *= 4;
|
|
}
|
|
}
|
|
fb_alloc_free_till_mark();
|
|
return proposals;
|
|
}
|
|
#endif //IMLIB_ENABLE_SELECTIVE_SEARCH
|