mirror of
https://github.com/openmv/openmv.git
synced 2025-09-26 23:09:13 +08:00
Merge pull request #2801 from kwagyeman/kwabena/post_processing_speed_up
Some checks failed
🔥 Firmware Build / build-firmware (0, ARDUINO_GIGA) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NANO_33_BLE_SENSE) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NANO_RP2040_CONNECT) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NICLA_VISION) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_PORTENTA_H7) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, DOCKER) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV2) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV3) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV4) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV4P) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMVPT) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_AE3) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_N6) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_RT1060) (push) Has been cancelled
🔥 Firmware Build / build-firmware (1, OPENMV_N6) (push) Has been cancelled
🔥 Firmware Build / code-size-report (push) Has been cancelled
🔥 Firmware Build / stable-release (push) Has been cancelled
🔥 Firmware Build / development-release (push) Has been cancelled
Some checks failed
🔥 Firmware Build / build-firmware (0, ARDUINO_GIGA) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NANO_33_BLE_SENSE) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NANO_RP2040_CONNECT) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_NICLA_VISION) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, ARDUINO_PORTENTA_H7) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, DOCKER) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV2) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV3) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV4) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV4P) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMVPT) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_AE3) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_N6) (push) Has been cancelled
🔥 Firmware Build / build-firmware (0, OPENMV_RT1060) (push) Has been cancelled
🔥 Firmware Build / build-firmware (1, OPENMV_N6) (push) Has been cancelled
🔥 Firmware Build / code-size-report (push) Has been cancelled
🔥 Firmware Build / stable-release (push) Has been cancelled
🔥 Firmware Build / development-release (push) Has been cancelled
scripts/libraries: Post-processing speedup.
This commit is contained in:
commit
bc39969581
@ -65,7 +65,7 @@ static size_t py_ml_tuple_sum(mp_obj_tuple_t *o) {
|
||||
return size;
|
||||
}
|
||||
|
||||
static size_t pl_ml_dtype_size(char dtype) {
|
||||
static size_t py_ml_dtype_size(char dtype) {
|
||||
switch (dtype) {
|
||||
case 'f':
|
||||
return 4;
|
||||
@ -92,7 +92,7 @@ static void py_ml_process_input(py_ml_model_obj_t *model, mp_obj_t arg) {
|
||||
if (mp_obj_is_callable(input_arg)) {
|
||||
// Input is a callable. Call the object and pass the tensor buffer and dtype.
|
||||
mp_obj_t fargs[3] = {
|
||||
mp_obj_new_bytearray_by_ref(input_size * pl_ml_dtype_size(input_dtype), input_buffer),
|
||||
mp_obj_new_bytearray_by_ref(input_size * py_ml_dtype_size(input_dtype), input_buffer),
|
||||
MP_OBJ_FROM_PTR(input_shape),
|
||||
mp_obj_new_int(input_dtype)
|
||||
};
|
||||
@ -151,7 +151,7 @@ static void py_ml_process_input(py_ml_model_obj_t *model, mp_obj_t arg) {
|
||||
}
|
||||
}
|
||||
|
||||
static mp_obj_t py_ml_process_output(py_ml_model_obj_t *model) {
|
||||
static mp_obj_t py_ml_process_output(py_ml_model_obj_t *model, bool deep_copy) {
|
||||
mp_obj_list_t *output_list = MP_OBJ_TO_PTR(mp_obj_new_list(model->outputs_size, NULL));
|
||||
for (size_t i = 0; i < model->outputs_size; i++) {
|
||||
void *model_output = ml_backend_get_output(model, i);
|
||||
@ -172,31 +172,38 @@ static mp_obj_t py_ml_process_output(py_ml_model_obj_t *model) {
|
||||
shape[ulab_offset + j] = mp_obj_get_int(output_shape->items[j]);
|
||||
}
|
||||
|
||||
ndarray_obj_t *ndarray = ndarray_new_dense_ndarray(output_shape->len, shape, NDARRAY_FLOAT);
|
||||
ndarray_obj_t *ndarray;
|
||||
|
||||
if (output_dtype == 'f') {
|
||||
memcpy(ndarray->array, model_output, size * sizeof(float));
|
||||
} else if (output_dtype == 'b') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((int8_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'B') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((uint8_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'h') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((int16_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'H') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((uint16_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
if (deep_copy) {
|
||||
ndarray = ndarray_new_dense_ndarray(output_shape->len, shape, NDARRAY_FLOAT);
|
||||
|
||||
if (output_dtype == 'f') {
|
||||
memcpy(ndarray->array, model_output, size * sizeof(float));
|
||||
} else if (output_dtype == 'b') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((int8_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'B') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((uint8_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'h') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((int16_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
} else if (output_dtype == 'H') {
|
||||
for (size_t j = 0; j < size; j++) {
|
||||
float v = (((uint16_t *) model_output)[j] - output_zero_point);
|
||||
((float *) ndarray->array)[j] = v * output_scale;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ndarray = ndarray_new_ndarray(output_shape->len, shape, NULL, output_dtype, model_output);
|
||||
}
|
||||
|
||||
output_list->items[i] = MP_OBJ_FROM_PTR(ndarray);
|
||||
}
|
||||
|
||||
@ -254,11 +261,14 @@ static mp_obj_t py_ml_model_predict(size_t n_args, const mp_obj_t *pos_args, mp_
|
||||
mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("Unsupported input type. Expected a list"));
|
||||
}
|
||||
|
||||
bool callback = args[ARG_callback].u_obj != mp_const_none;
|
||||
|
||||
py_ml_process_input(model, pos_args[1]);
|
||||
ml_backend_run_inference(model);
|
||||
mp_obj_t output = py_ml_process_output(model);
|
||||
|
||||
if (args[ARG_callback].u_obj != mp_const_none) {
|
||||
mp_obj_t output = py_ml_process_output(model, !callback);
|
||||
|
||||
if (callback) {
|
||||
// Pass model, inputs, outputs to the post-processing callback.
|
||||
mp_obj_t fargs[3] = { MP_OBJ_FROM_PTR(model), pos_args[1], output };
|
||||
output = mp_call_function_n_kw(args[ARG_callback].u_obj, 3, 0, fargs);
|
||||
|
@ -9,9 +9,8 @@
|
||||
import sensor
|
||||
import time
|
||||
import ml
|
||||
from ml.utils import NMS
|
||||
from ml.postprocessing import fomo_postprocess
|
||||
import math
|
||||
import image
|
||||
|
||||
sensor.reset() # Reset and initialize the sensor.
|
||||
sensor.set_pixformat(sensor.RGB565) # Set pixel format to RGB565 (or GRAYSCALE)
|
||||
@ -19,9 +18,6 @@ sensor.set_framesize(sensor.QVGA) # Set frame size to QVGA (320x240)
|
||||
sensor.set_windowing((240, 240)) # Set 240x240 window.
|
||||
sensor.skip_frames(time=2000) # Let the camera adjust.
|
||||
|
||||
min_confidence = 0.4
|
||||
threshold_list = [(math.ceil(min_confidence * 255), 255)]
|
||||
|
||||
# Load built-in FOMO face detection model
|
||||
model = ml.Model("/rom/fomo_face_detection.tflite")
|
||||
print(model)
|
||||
@ -40,39 +36,13 @@ colors = [ # Add more colors if you are detecting more than 7 types of classes
|
||||
(255, 255, 255),
|
||||
]
|
||||
|
||||
|
||||
# FOMO outputs an image per class where each pixel in the image is the centroid of the trained
|
||||
# object. So, we will get those output images and then run find_blobs() on them to extract the
|
||||
# centroids. We will also run get_stats() on the detected blobs to determine their score.
|
||||
# The Non-Max-Supression (NMS) object then filters out overlapping detections and maps their
|
||||
# position in the output image back to the original input image. The function then returns a
|
||||
# list per class which each contain a list of (rect, score) tuples representing the detected
|
||||
# objects.
|
||||
def fomo_post_process(model, inputs, outputs):
|
||||
n, oh, ow, oc = model.output_shape[0]
|
||||
nms = NMS(ow, oh, inputs[0].roi)
|
||||
for i in range(oc):
|
||||
img = image.Image(outputs[0][0, :, :, i] * 255)
|
||||
blobs = img.find_blobs(
|
||||
threshold_list, x_stride=1, area_threshold=1, pixels_threshold=1
|
||||
)
|
||||
for b in blobs:
|
||||
rect = b.rect()
|
||||
x, y, w, h = rect
|
||||
score = (
|
||||
img.get_statistics(thresholds=threshold_list, roi=rect).l_mean() / 255.0
|
||||
)
|
||||
nms.add_bounding_box(x, y, x + w, y + h, score, i)
|
||||
return nms.get_bounding_boxes()
|
||||
|
||||
|
||||
clock = time.clock()
|
||||
while True:
|
||||
clock.tick()
|
||||
|
||||
img = sensor.snapshot()
|
||||
|
||||
for i, detection_list in enumerate(model.predict([img], callback=fomo_post_process)):
|
||||
for i, detection_list in enumerate(model.predict([img], callback=fomo_postprocess())):
|
||||
if i == 0:
|
||||
continue # background class
|
||||
if len(detection_list) == 0:
|
||||
|
@ -26,8 +26,6 @@
|
||||
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
import math
|
||||
import image
|
||||
from ml.utils import NMS
|
||||
from micropython import const
|
||||
from ulab import numpy as np
|
||||
@ -36,32 +34,86 @@ from ulab import numpy as np
|
||||
_NO_DETECTION = const(())
|
||||
|
||||
|
||||
# FOMO generates an image per class, where each pixel represents the centroid
|
||||
# of the trained object. These images are processed with `find_blobs()` to
|
||||
# extract centroids, and `get_stats()` is used to get their scores. Overlapping
|
||||
# detections are then filtered with NMS and positions are mapped back to the
|
||||
# original image, and a list of (rect, score) tuples is returned for each class,
|
||||
# representing detected objects.
|
||||
def mod(a, b):
|
||||
return a - (b * (a // b))
|
||||
|
||||
|
||||
def threshold(scores, threshold, scale, find_max=False, find_max_axis=1):
|
||||
if scale > 0:
|
||||
if find_max:
|
||||
scores = np.max(scores, axis=find_max_axis)
|
||||
return np.nonzero(scores > threshold)[0]
|
||||
else:
|
||||
if find_max:
|
||||
scores = np.min(scores, axis=find_max_axis)
|
||||
return np.nonzero(scores < threshold)[0]
|
||||
|
||||
|
||||
def quantize(model, value):
|
||||
if model.output_dtype[0] == 'f':
|
||||
return value
|
||||
return (value / model.output_scale[0]) + model.output_zero_point[0]
|
||||
|
||||
|
||||
def dequantize(model, value):
|
||||
if model.output_dtype[0] == 'f':
|
||||
return value
|
||||
return (value - model.output_zero_point[0]) * model.output_scale[0]
|
||||
|
||||
|
||||
class fomo_postprocess:
|
||||
def __init__(self, threshold=0.4):
|
||||
self.threshold_list = [(math.ceil(threshold * 255), 255)]
|
||||
_FOMO_CLASSES = const(1)
|
||||
|
||||
def __init__(self, threshold=0.4, w_scale=1.414214, h_scale=1.414214,
|
||||
nms_threshold=0.1, nms_sigma=0.001):
|
||||
self.threshold = threshold
|
||||
self.w_scale = w_scale
|
||||
self.h_scale = h_scale
|
||||
self.nms_threshold = nms_threshold
|
||||
self.nms_sigma = nms_sigma
|
||||
|
||||
def __call__(self, model, inputs, outputs):
|
||||
n, oh, ow, oc = model.output_shape[0]
|
||||
nms = NMS(ow, oh, inputs[0].roi)
|
||||
for i in range(oc):
|
||||
img = image.Image(outputs[0][0, :, :, i] * 255)
|
||||
blobs = img.find_blobs(
|
||||
self.threshold_list, x_stride=1, area_threshold=1, pixels_threshold=1
|
||||
)
|
||||
for b in blobs:
|
||||
rect = b.rect()
|
||||
x, y, w, h = rect
|
||||
score = (
|
||||
img.get_statistics(thresholds=self.threshold_list, roi=rect).l_mean() / 255.0
|
||||
)
|
||||
nms.add_bounding_box(x, y, x + w, y + h, score, i)
|
||||
return nms.get_bounding_boxes()
|
||||
ob, oh, ow, oc = model.output_shape[0]
|
||||
scale = model.output_scale[0]
|
||||
t = quantize(model, self.threshold)
|
||||
|
||||
# Reshape the output to a 2D array
|
||||
row_outputs = outputs[0].reshape((oh * ow, oc))
|
||||
|
||||
# Threshold all the scores
|
||||
score_indices = row_outputs[:, _FOMO_CLASSES:]
|
||||
score_indices = threshold(score_indices, t, scale, find_max=True, find_max_axis=1)
|
||||
if not len(score_indices):
|
||||
return _NO_DETECTION
|
||||
|
||||
# Get the bounding boxes that have a valid score
|
||||
bb = dequantize(model, np.take(row_outputs, score_indices, axis=0))
|
||||
|
||||
# Extract rows and columns
|
||||
bb_rows = score_indices // ow
|
||||
bb_cols = mod(score_indices, ow)
|
||||
|
||||
# Get the score information
|
||||
bb_scores = np.max(bb[:, _FOMO_CLASSES:], axis=1)
|
||||
|
||||
# Get the class information
|
||||
bb_classes = np.argmax(bb[:, _FOMO_CLASSES:], axis=1) + _FOMO_CLASSES
|
||||
|
||||
# Scale the bounding boxes to have enough integer precision for NMS
|
||||
ib, ih, iw, ic = model.input_shape[0]
|
||||
x_center = ((bb_cols + 0.5) / ow) * iw
|
||||
y_center = ((bb_rows + 0.5) / oh) * ih
|
||||
w_rel = np.full(len(bb_cols), self.w_scale / ow) * iw
|
||||
h_rel = np.full(len(bb_rows), self.h_scale / oh) * ih
|
||||
|
||||
nms = NMS(iw, ih, inputs[0].roi)
|
||||
for i in range(bb.shape[0]):
|
||||
nms.add_bounding_box(x_center[i] - (w_rel[i] / 2),
|
||||
y_center[i] - (h_rel[i] / 2),
|
||||
x_center[i] + (w_rel[i] / 2),
|
||||
y_center[i] + (h_rel[i] / 2),
|
||||
bb_scores[i], bb_classes[i])
|
||||
return nms.get_bounding_boxes(threshold=self.nms_threshold, sigma=self.nms_sigma)
|
||||
|
||||
|
||||
# This is a lightweight version of the tiny yolo v2 object detection algorithm.
|
||||
@ -88,31 +140,34 @@ class yolo_v2_postprocess:
|
||||
self.nms_sigma = nms_sigma
|
||||
|
||||
def __call__(self, model, inputs, outputs):
|
||||
ob, oh, ow, oc = model.output_shape[0]
|
||||
class_count = (oc // self.anchors_len) - _YOLO_V2_CLASSES
|
||||
|
||||
def logit(x):
|
||||
return np.log(x / (1.0 - x))
|
||||
|
||||
def sigmoid(x):
|
||||
return 1.0 / (1.0 + np.exp(-x))
|
||||
|
||||
def mod(a, b):
|
||||
return a - (b * (a // b))
|
||||
|
||||
def softmax(x):
|
||||
e_x = np.exp(x - np.max(x, axis=1, keepdims=True))
|
||||
return e_x / np.sum(e_x, axis=1, keepdims=True)
|
||||
|
||||
ob, oh, ow, oc = model.output_shape[0]
|
||||
scale = model.output_scale[0]
|
||||
t = quantize(model, logit(self.threshold))
|
||||
class_count = (oc // self.anchors_len) - _YOLO_V2_CLASSES
|
||||
|
||||
# Reshape the output to a 2D array
|
||||
row_outputs = outputs[0].reshape((oh * ow * self.anchors_len,
|
||||
_YOLO_V2_CLASSES + class_count))
|
||||
|
||||
# Threshold all the scores
|
||||
score_indices = sigmoid(row_outputs[:, _YOLO_V2_SCORE])
|
||||
score_indices = np.nonzero(score_indices > self.threshold)[0]
|
||||
score_indices = row_outputs[:, _YOLO_V2_SCORE]
|
||||
score_indices = threshold(score_indices, t, scale)
|
||||
if not len(score_indices):
|
||||
return _NO_DETECTION
|
||||
|
||||
# Get the bounding boxes that have a valid score
|
||||
bb = np.take(row_outputs, score_indices, axis=0)
|
||||
bb = dequantize(model, np.take(row_outputs, score_indices, axis=0))
|
||||
|
||||
# Extract rows, columns, and anchor indices
|
||||
bb_rows = score_indices // (ow * self.anchors_len)
|
||||
@ -179,6 +234,8 @@ class yolo_v5_postprocess:
|
||||
|
||||
def __call__(self, model, inputs, outputs):
|
||||
oh, ow, oc = model.output_shape[0]
|
||||
scale = model.output_scale[0]
|
||||
t = quantize(model, self.threshold)
|
||||
class_count = oc - _YOLO_V5_CLASSES
|
||||
|
||||
# Reshape the output to a 2D array
|
||||
@ -186,12 +243,12 @@ class yolo_v5_postprocess:
|
||||
|
||||
# Threshold all the scores
|
||||
score_indices = row_outputs[:, _YOLO_V5_SCORE]
|
||||
score_indices = np.nonzero(score_indices > self.threshold)[0]
|
||||
score_indices = threshold(score_indices, t, scale)
|
||||
if not len(score_indices):
|
||||
return _NO_DETECTION
|
||||
|
||||
# Get the bounding boxes that have a valid score
|
||||
bb = np.take(row_outputs, score_indices, axis=0)
|
||||
bb = dequantize(model, np.take(row_outputs, score_indices, axis=0))
|
||||
|
||||
# Get the score information
|
||||
bb_scores = bb[:, _YOLO_V5_SCORE]
|
||||
@ -233,19 +290,21 @@ class yolo_v8_postprocess:
|
||||
|
||||
def __call__(self, model, inputs, outputs):
|
||||
oh, ow, oc = model.output_shape[0]
|
||||
scale = model.output_scale[0]
|
||||
t = quantize(model, self.threshold)
|
||||
class_count = ow - _YOLO_V8_CLASSES
|
||||
|
||||
# Reshape the output to a 2D array
|
||||
column_outputs = outputs[0].reshape((oh * (_YOLO_V8_CLASSES + class_count), oc))
|
||||
|
||||
# Threshold all the scores
|
||||
score_indices = np.max(column_outputs[_YOLO_V8_CLASSES:, :], axis=0)
|
||||
score_indices = np.nonzero(score_indices > self.threshold)[0]
|
||||
score_indices = column_outputs[_YOLO_V8_CLASSES:, :]
|
||||
score_indices = threshold(score_indices, t, scale, find_max=True, find_max_axis=0)
|
||||
if not len(score_indices):
|
||||
return _NO_DETECTION
|
||||
|
||||
# Get the bounding boxes that have a valid score
|
||||
bb = np.take(column_outputs, score_indices, axis=1)
|
||||
bb = dequantize(model, np.take(column_outputs, score_indices, axis=1))
|
||||
|
||||
# Get the score information
|
||||
bb_scores = np.max(bb[_YOLO_V8_CLASSES:, :], axis=0)
|
||||
|
Loading…
Reference in New Issue
Block a user