This commit is contained in:
Kwabena W Agyeman 2025-11-02 20:42:18 +00:00 committed by GitHub
commit b335b43724
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 218 additions and 67 deletions

View File

@ -44,6 +44,12 @@
"alignment": 16,
"optimize": "Performance"
},
{
"type": "tflite",
"path": "{TOP}/lib/models/face_landmarks_192.tflite",
"alignment": 16,
"optimize": "Performance"
},
{
"type": "tflite",
"path": "{TOP}/lib/models/palm_detection_full_192.tflite",

View File

@ -26,6 +26,12 @@
"alignment": 32,
"profile": "default"
},
{
"type": "tflite",
"path": "{TOP}/lib/models/face_landmarks_192.tflite",
"alignment": 32,
"profile": "default"
},
{
"type": "tflite",
"path": "{TOP}/lib/models/palm_detection_full_192.tflite",

Binary file not shown.

View File

@ -26,11 +26,7 @@ while True:
img = csi0.snapshot()
# faces is a list of ((x, y, w, h), score, keypoints) tuples
faces = model.predict([img])
# Draw bounding boxes around the detected faces and keypoints.
if faces:
for r, score, keypoints in faces[0]:
for r, score, keypoints in model.predict([img]):
ml.utils.draw_predictions(img, [r], ("face",), ((0, 0, 255),), format=None)
# keypoints is a ndarray of shape (6, 2)

View File

@ -31,11 +31,7 @@ while True:
img = csi0.snapshot()
# palms is a list of ((x, y, w, h), score, keypoints) tuples
palms = model.predict([img])
# Draw bounding boxes around the detected palms and keypoints.
if palms:
for r, score, keypoints in palms[0]:
for r, score, keypoints in model.predict([img]):
ml.utils.draw_predictions(img, [r], ("palm",), ((0, 0, 255),), format=None)
# keypoints is a ndarray of shape (7, 2)

View File

@ -0,0 +1,50 @@
# This work is licensed under the MIT license.
# Copyright (c) 2013-2025 OpenMV LLC. All rights reserved.
# https://github.com/openmv/openmv/blob/master/LICENSE
#
# This example shows off Google's MediaPipe Face Landmark Detection model for multiple faces.
#
# NOTE: This exaxmple requires an OpenMV Cam with an NPU like the AE3 or N6 to run real-time.
import csi
import time
import ml
from ml.preprocessing import Normalization
from ml.postprocessing.mediapipe import BlazeFace
from ml.postprocessing.mediapipe import FaceLandmarks
# Initialize the sensor.
csi0 = csi.CSI()
csi0.reset()
csi0.pixformat(csi.RGB565)
csi0.framesize(csi.VGA)
csi0.window((400, 400))
# Load built-in face detection model
face_detection = ml.Model("/rom/blazeface_front_128.tflite", postprocess=BlazeFace(threshold=0.4))
print(face_detection)
# Load built-in face landmark model
face_landmarks = ml.Model("/rom/face_landmarks_192.tflite", postprocess=FaceLandmarks(threshold=0.4))
print(face_landmarks)
clock = time.clock()
while True:
clock.tick()
img = csi0.snapshot()
# faces is a list of ((x, y, w, h), score, keypoints) tuples
for r, score, keypoints in face_detection.predict([img]):
# rect is (x, y, w, h) - enlarge by 2x for face landmarks model
wider_rect = (r[0] - r[2] // 2, r[1] - r[3] // 2, r[2] * 2, r[3] * 2)
# Operate on just the ROI of the detected face
n = Normalization(roi=wider_rect)
# marks is a list of ((x, y, w, h), score, keypoints) tuples
for r, score, keypoints in face_landmarks.predict([n(img)]):
ml.utils.draw_predictions(img, [r], ("face",), ((0, 0, 255),), format=None)
# keypoints is a ndarray of shape (468, 3) where each keypoint is (x, y, z)
ml.utils.draw_keypoints(img, keypoints, radius=0, color=(255, 0, 0))
print(clock.fps(), "fps")

View File

@ -0,0 +1,71 @@
# This work is licensed under the MIT license.
# Copyright (c) 2013-2025 OpenMV LLC. All rights reserved.
# https://github.com/openmv/openmv/blob/master/LICENSE
#
# This example shows off Google's MediaPipe Face Landmark Detection model for a single face.
#
# NOTE: This exaxmple requires an OpenMV Cam with an NPU like the AE3 or N6 to run real-time.
import csi
import time
import ml
from ml.preprocessing import Normalization
from ml.postprocessing.mediapipe import BlazeFace
from ml.postprocessing.mediapipe import FaceLandmarks
# Initialize the sensor.
csi0 = csi.CSI()
csi0.reset()
csi0.pixformat(csi.RGB565)
csi0.framesize(csi.VGA)
csi0.window((400, 400))
# Load built-in face detection model
face_detection = ml.Model("/rom/blazeface_front_128.tflite", postprocess=BlazeFace(threshold=0.4))
print(face_detection)
# Load built-in face landmark model
face_landmarks = ml.Model("/rom/face_landmarks_192.tflite", postprocess=FaceLandmarks(threshold=0.4))
print(face_landmarks)
# Tracking vars.
n = None
clock = time.clock()
while True:
clock.tick()
img = csi0.snapshot()
if n is None:
# faces is a list of ((x, y, w, h), score, keypoints) tuples
for r, score, keypoints in face_detection.predict([img]):
# rect is (x, y, w, h) - enlarge by 2x for face landmarks model
wider_rect = (r[0] - r[2] // 2, r[1] - r[3] // 2, r[2] * 2, r[3] * 2)
# Operate on just the ROI of the detected face
n = Normalization(roi=wider_rect)
else:
# marks is a list of ((x, y, w, h), score, keypoints) tuples
marks = face_landmarks.predict([n(img)])
# No faces detected, reset the tracker.
if not marks:
n = None
continue
# Draw bounding boxes around the detected faces and keypoints.
for r, score, keypoints in marks:
ml.utils.draw_predictions(img, [r], ("face",), ((0, 0, 255),), format=None)
# keypoints is a ndarray of shape (468, 3) where each keypoint is (x, y, z)
ml.utils.draw_keypoints(img, keypoints, radius=0, color=(255, 0, 0))
# Center new_wider_rect on face for tracking
new_wider_rect = (r[0] + (r[2] // 2) - (wider_rect[2] // 2),
r[1] + (r[3] // 2) - (wider_rect[3] // 2),
wider_rect[2],
wider_rect[3])
# Operate on just the ROI of the detected face
n = Normalization(roi=new_wider_rect)
print(clock.fps(), "fps")

View File

@ -39,10 +39,7 @@ while True:
img = csi0.snapshot()
# palms is a list of ((x, y, w, h), score, keypoints) tuples
palms = palm_detection.predict([img])
if palms:
for r, score, keypoints in palms[0]:
for r, score, keypoints in palm_detection.predict([img]):
# rect is (x, y, w, h) - enlarge by 3x for hand landmarks model
wider_rect = (r[0] - r[2], r[1] - r[3], r[2] * 3, r[3] * 3)
# Operate on just the ROI of the detected palm

View File

@ -43,10 +43,7 @@ while True:
if n is None:
# palms is a list of ((x, y, w, h), score, keypoints) tuples
palms = palm_detection.predict([img])
if palms:
for r, score, keypoints in palms[0]:
for r, score, keypoints in palm_detection.predict([img]):
# rect is (x, y, w, h) - enlarge by 3x for hand landmarks model
wider_rect = (r[0] - r[2], r[1] - r[3], r[2] * 3, r[3] * 3)
# Operate on just the ROI of the detected palm

View File

@ -78,7 +78,7 @@ class mediapipe_detection_postprocess:
self.detection_post_process(ih, iw, nms, model, inputs, outputs, self.scores[1], self.cords[1],
self.threshold, self.anchors[output_len:])
return nms.get_bounding_boxes(threshold=self.nms_threshold, sigma=self.nms_sigma)
return nms.get_bounding_boxes(threshold=self.nms_threshold, sigma=self.nms_sigma)[0]
def detection_post_process(self, ih, iw, nms, model, inputs, outputs, score_idx, cords_idx, t, anchors):
s_oh, s_ow, s_oc = model.output_shape[score_idx]
@ -173,3 +173,35 @@ class HandLandmarks:
nms.add_bounding_box(xmin, ymin, xmax, ymax, score, left_right, keypoints=keypoints)
return nms.get_bounding_boxes(threshold=self.nms_threshold, sigma=self.nms_sigma)
class FaceLandmarks:
def __init__(self, threshold=0.6, nms_threshold=0.1, nms_sigma=0.1):
self.threshold = threshold
self.nms_threshold = nms_threshold
self.nms_sigma = nms_sigma
def __call__(self, model, inputs, outputs):
ib, ih, iw, ic = model.input_shape[0]
nms = NMS(iw, ih, inputs[0].roi)
score = sigmoid(outputs[1][0, 0, 0, 0])
if score < self.threshold:
return _NO_DETECTION
cords = outputs[0][0, 0, 0, :]
# Get the keypoint information
keypoints = np.empty((len(cords) // 3, 3))
keypoints[:, 0] = cords[0::3]
keypoints[:, 1] = cords[1::3]
keypoints[:, 2] = cords[2::3]
# Get bounding box information
xmin = np.min(keypoints[:, 0])
ymin = np.min(keypoints[:, 1])
xmax = np.max(keypoints[:, 0])
ymax = np.max(keypoints[:, 1])
nms.add_bounding_box(xmin, ymin, xmax, ymax, score, 0, keypoints=keypoints)
return nms.get_bounding_boxes(threshold=self.nms_threshold, sigma=self.nms_sigma)[0]