diff --git a/scripts/libraries/ml/manifest.py b/scripts/libraries/ml/manifest.py new file mode 100644 index 000000000..8cab052ea --- /dev/null +++ b/scripts/libraries/ml/manifest.py @@ -0,0 +1,5 @@ +metadata( + description="Machine Learning Extension Package.", + version="0.0.1", +) +package("ml") diff --git a/scripts/libraries/ml/ml/__init__.py b/scripts/libraries/ml/ml/__init__.py new file mode 100644 index 000000000..e7ccae809 --- /dev/null +++ b/scripts/libraries/ml/ml/__init__.py @@ -0,0 +1,11 @@ +# This file is part of the OpenMV project. +# +# Copyright (c) 2024 Ibrahim Abdelkader +# Copyright (c) 2024 Kwabena W. Agyeman +# +# This work is licensed under the MIT license, see the file LICENSE for details. +# +# This is an extension package to the ml C user-module. + +from uml import NMS # noqa +from .model import * # noqa diff --git a/scripts/libraries/ml.py b/scripts/libraries/ml/ml/apps.py similarity index 74% rename from scripts/libraries/ml.py rename to scripts/libraries/ml/ml/apps.py index 47e24723a..770f7198b 100644 --- a/scripts/libraries/ml.py +++ b/scripts/libraries/ml/ml/apps.py @@ -1,46 +1,21 @@ # This file is part of the OpenMV project. # -# Copyright (c) 2023 Ibrahim Abdelkader -# Copyright (c) 2023 Kwabena W. Agyeman +# Copyright (c) 2024 Ibrahim Abdelkader +# Copyright (c) 2024 Kwabena W. Agyeman # # This work is licensed under the MIT license, see the file LICENSE for details. -# -# This is an extension to the display C user-module. Add or import any display-related -# drivers here, and freeze this module in the board's manifest, and those drivers will -# be importable from display. import time -from uml import * # noqa +from ml import Model from micropython import const from ulab import numpy as np + try: import audio except (ImportError, AttributeError): pass -def draw_predictions(img, boxes, labels, colors, format="pascal_voc", text_color=(255, 255, 255)): - CHAR_W = 8 - CHAR_H = 10 - img_w = img.width() - img_h = img.height() - for i, (x, y, w, h) in enumerate(boxes): - label = labels[i] - box_color = colors[i] - - if format == "pascal_voc": - x = int(x * img_w) - y = int(y * img_h) - w = int(w * img_w) - x - h = int(h * img_h) - y - - img.draw_rectangle(x, y, w, h, color=box_color) - img.draw_rectangle( - x, y - CHAR_H, len(label) * CHAR_W, CHAR_H, fill=True, color=box_color - ) - img.draw_string(x, y - CHAR_H, label.upper(), text_color) - - class MicroSpeech: _SLICE_SIZE = const(40) _SLICE_COUNT = const(49) @@ -71,13 +46,11 @@ class MicroSpeech: # Roll the spectrogram to the left and add the new slice. self.spectrogram = np.roll(self.spectrogram, -_SLICE_SIZE, axis=1) - self.spectrogram[0, -_SLICE_SIZE:] = self.preprocessor.predict( - self.audio_buffer - ) + self.spectrogram[0, -_SLICE_SIZE:] = self.preprocessor.predict([self.audio_buffer]) # Roll the prediction history and add the new prediction. self.pred_history = np.roll(self.pred_history, -1, axis=0) - self.pred_history[-1] = self.micro_speech.predict(self.spectrogram)[0] + self.pred_history[-1] = self.micro_speech.predict([self.spectrogram])[0] def start_audio_streaming(self): if self.audio_started is False: diff --git a/scripts/libraries/ml/ml/model.py b/scripts/libraries/ml/ml/model.py new file mode 100644 index 000000000..b015d3295 --- /dev/null +++ b/scripts/libraries/ml/ml/model.py @@ -0,0 +1,26 @@ +# This file is part of the OpenMV project. +# +# Copyright (c) 2024 Ibrahim Abdelkader +# Copyright (c) 2024 Kwabena W. Agyeman +# +# This work is licensed under the MIT license, see the file LICENSE for details. +import uml +import image +from ml.preprocessing import Normalization + + +class Model: + def __new__(cls, *args, **kwargs): + self = super().__new__(cls) + retobj = uml.Model(*args, **kwargs) + if isinstance(retobj, tuple): + labels, self.model = retobj + return labels, self + return self + + def __str__(self): + return str(self.model) + + def predict(self, args, **kwargs): + args = [Normalization()(x) if isinstance(x, image.Image) else x for x in args] + return self.model.predict(args, **kwargs) diff --git a/scripts/libraries/ml/ml/preprocessing.py b/scripts/libraries/ml/ml/preprocessing.py new file mode 100644 index 000000000..7cd37fb3b --- /dev/null +++ b/scripts/libraries/ml/ml/preprocessing.py @@ -0,0 +1,41 @@ +# This file is part of the OpenMV project. +# +# Copyright (c) 2024 Ibrahim Abdelkader +# Copyright (c) 2024 Kwabena W. Agyeman +# +# This work is licensed under the MIT license, see the file LICENSE for details. + +import image + + +class Normalization: + def __init__( + self, + image=None, + scale=(0.0, 1.0), + mean=(0.0, 0.0, 0.0), + stdev=(1.0, 1.0, 1.0), + roi=None, + ): + self.image = image + self.scale = scale + self.mean = mean + self.stdev = stdev + self.roi = roi + + def __call__(self, *args): + if len(args) == 1: + img = args[0] + if not isinstance(img, image.Image): + raise ValueError("Expected an image input") + if self.roi is None: + self.roi = (0, 0, img.width(), img.height()) + return Normalization(img, self.scale, self.mean, self.stdev, self.roi) + buffer, shape, dtype = args + # Create an image using the input tensor as buffer. + img = image.Image(shape[2], shape[1], self.image.format(), buffer=buffer) + # Copy and scale (if needed) the input image to the input buffer. + hints = image.BILINEAR | image.CENTER | image.SCALE_ASPECT_EXPAND | image.BLACK_BACKGROUND + img.draw_image(self.image, 0, 0, roi=self.roi, hint=hints) + # Scale and convert the image to input tensor data. + img.unpack(buffer, dtype, scale=self.scale, mean=self.mean, stdev=self.stdev) diff --git a/scripts/libraries/ml/ml/utils.py b/scripts/libraries/ml/ml/utils.py new file mode 100644 index 000000000..370c1378a --- /dev/null +++ b/scripts/libraries/ml/ml/utils.py @@ -0,0 +1,39 @@ +# This file is part of the OpenMV project. +# +# Copyright (c) 2024 Ibrahim Abdelkader +# Copyright (c) 2024 Kwabena W. Agyeman +# +# This work is licensed under the MIT license, see the file LICENSE for details. + +def draw_predictions( + image, + boxes, + labels, + colors, + format="pascal_voc", + font_width=8, + font_height=10, + text_color=(255, 255, 255), +): + image_w = image.width() + image_h = image.height() + for i, (x, y, w, h) in enumerate(boxes): + label = labels[i] + box_color = colors[i] + + if format == "pascal_voc": + x = int(x * image_w) + y = int(y * image_h) + w = int(w * image_w) - x + h = int(h * image_h) - y + + image.draw_rectangle(x, y, w, h, color=box_color) + image.draw_rectangle( + x, + y - font_height, + len(label) * font_width, + font_height, + fill=True, + color=box_color, + ) + image.draw_string(x, y - font_height, label.upper(), text_color) diff --git a/src/omv/boards/ARDUINO_GIGA/manifest.py b/src/omv/boards/ARDUINO_GIGA/manifest.py index 83a4c1aec..528561800 100644 --- a/src/omv/boards/ARDUINO_GIGA/manifest.py +++ b/src/omv/boards/ARDUINO_GIGA/manifest.py @@ -10,7 +10,7 @@ freeze ("$(OMV_LIB_DIR)/", "gt911.py") freeze ("$(OMV_LIB_DIR)/", "st7701.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/ARDUINO_NICLA_VISION/manifest.py b/src/omv/boards/ARDUINO_NICLA_VISION/manifest.py index 460ba0dd8..d7b084a0b 100644 --- a/src/omv/boards/ARDUINO_NICLA_VISION/manifest.py +++ b/src/omv/boards/ARDUINO_NICLA_VISION/manifest.py @@ -10,7 +10,7 @@ freeze ("$(OMV_LIB_DIR)/", "modbus.py") freeze ("$(OMV_LIB_DIR)/", "pid.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/ARDUINO_PORTENTA_H7/manifest.py b/src/omv/boards/ARDUINO_PORTENTA_H7/manifest.py index 3d8aba416..efb5c5162 100644 --- a/src/omv/boards/ARDUINO_PORTENTA_H7/manifest.py +++ b/src/omv/boards/ARDUINO_PORTENTA_H7/manifest.py @@ -14,7 +14,7 @@ freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "bno055.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/OPENMV4/manifest.py b/src/omv/boards/OPENMV4/manifest.py index c6c9e4d9d..9dcf56ed8 100644 --- a/src/omv/boards/OPENMV4/manifest.py +++ b/src/omv/boards/OPENMV4/manifest.py @@ -13,7 +13,7 @@ freeze ("$(OMV_LIB_DIR)/", "tb6612.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/OPENMV4P/manifest.py b/src/omv/boards/OPENMV4P/manifest.py index c6c9e4d9d..9dcf56ed8 100644 --- a/src/omv/boards/OPENMV4P/manifest.py +++ b/src/omv/boards/OPENMV4P/manifest.py @@ -13,7 +13,7 @@ freeze ("$(OMV_LIB_DIR)/", "tb6612.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/OPENMV4_PRO/manifest.py b/src/omv/boards/OPENMV4_PRO/manifest.py index b9fdd2856..9dcf56ed8 100644 --- a/src/omv/boards/OPENMV4_PRO/manifest.py +++ b/src/omv/boards/OPENMV4_PRO/manifest.py @@ -13,6 +13,7 @@ freeze ("$(OMV_LIB_DIR)/", "tb6612.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/OPENMVPT/manifest.py b/src/omv/boards/OPENMVPT/manifest.py index c6c9e4d9d..9dcf56ed8 100644 --- a/src/omv/boards/OPENMVPT/manifest.py +++ b/src/omv/boards/OPENMVPT/manifest.py @@ -13,7 +13,7 @@ freeze ("$(OMV_LIB_DIR)/", "tb6612.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl") diff --git a/src/omv/boards/OPENMV_RT1060/manifest.py b/src/omv/boards/OPENMV_RT1060/manifest.py index 78af2d849..0cb4ca59f 100644 --- a/src/omv/boards/OPENMV_RT1060/manifest.py +++ b/src/omv/boards/OPENMV_RT1060/manifest.py @@ -13,7 +13,7 @@ freeze ("$(OMV_LIB_DIR)/", "tb6612.py") freeze ("$(OMV_LIB_DIR)/", "vl53l1x.py") freeze ("$(OMV_LIB_DIR)/", "machine.py") freeze ("$(OMV_LIB_DIR)/", "display.py") -freeze ("$(OMV_LIB_DIR)/", "ml.py") +freeze ("$(OMV_LIB_DIR)/ml") # Networking require("ssl")