fix: add new leap simplified model

This commit is contained in:
Prohurtz 2024-09-19 19:12:06 -07:00
parent 9798fb13a6
commit 71ddd8ceba
3 changed files with 54 additions and 76 deletions

View File

@ -350,9 +350,9 @@ class AHSF:
init_rect_down = self.rect_scale(init_rect, params["ratio_downsample"], False)
init_rect_down = self.intersect_rect(init_rect_down, imgboundary)
img_blur = img_gray[
init_rect_down[1] : init_rect_down[1] + init_rect_down[3],
init_rect_down[0] : init_rect_down[0] + init_rect_down[2],
]
init_rect_down[1]: init_rect_down[1] + init_rect_down[3],
init_rect_down[0]: init_rect_down[0] + init_rect_down[2],
]
(
frame_int,
@ -375,53 +375,36 @@ class AHSF:
) = self.get_empty_array(img_blur.shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer)
cv2.integral(
img_blur, sum=frame_int, sdepth=cv2.CV_32S
) # memo: It becomes slower when using float64, probably because the increase in bits from 32 to 64 causes the arrays to be larger.
)
# memo: If axis=1 is too slow, just transpose and "take" with axis=0.
# memo: This URL gave me an idea. https://numpy.org/doc/1.25/dev/internals.html#multidimensional-array-indexing-order-issues
out_p_temp = frame_int.take(y_out_n, axis=0, mode="clip") # , out=out_p_temp)
out_p_temp = frame_int.take(y_out_n, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p00 = out_p_temp.take(x_out_n, axis=0, mode="clip") # , out=out_p00)
# p01 calc
out_p01 = out_p_temp.take(x_out_w, axis=0, mode="clip") # , out=out_p01)
# p11 calc
out_p_temp = frame_int.take(y_out_h, axis=0, mode="clip") # , out=out_p_temp)
out_p00 = out_p_temp.take(x_out_n, axis=0, mode="clip")
out_p01 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p_temp = frame_int.take(y_out_h, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p11 = out_p_temp.take(x_out_w, axis=0, mode="clip") # , out=out_p11)
# p10 calc
out_p10 = out_p_temp.take(x_out_n, axis=0, mode="clip") # , out=out_p10)
out_p11 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p10 = out_p_temp.take(x_out_n, axis=0, mode="clip")
# outer_sum[:, :] = out_p00 + out_p11 - out_p01 - out_p10
outer_sum = cv2.add(out_p00, out_p11) # , dst=outer_sum)
outer_sum = cv2.add(out_p00, out_p11)
cv2.subtract(outer_sum, out_p01, dst=outer_sum)
cv2.subtract(outer_sum, out_p10, dst=outer_sum)
# outer_sum=outer_sum.astype(np.float64)
# outer_sum = cv2.transpose(outer_sum)
in_p_temp = frame_int.take(y_in_n, axis=0, mode="clip") # , out=in_p_temp)
in_p_temp = frame_int.take(y_in_n, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p00 = in_p_temp.take(x_in_n, axis=0, mode="clip") # , out=in_p00)
# p01 calc
in_p01 = in_p_temp.take(x_in_w, axis=0, mode="clip") # , out=in_p01)
# p11 calc
in_p_temp = frame_int.take(y_in_h, axis=0, mode="clip") # , out=in_p_temp)
in_p00 = in_p_temp.take(x_in_n, axis=0, mode="clip")
in_p01 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p_temp = frame_int.take(y_in_h, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p11 = in_p_temp.take(x_in_w, axis=0, mode="clip") # , out=in_p11)
# p10 calc
in_p10 = in_p_temp.take(x_in_n, axis=0, mode="clip") # , out=in_p10)
in_p11 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p10 = in_p_temp.take(x_in_n, axis=0, mode="clip")
# inner_sum[:, :] = in_p00 + in_p11 - in_p01 - in_p10
inner_sum = cv2.add(in_p00, in_p11)
cv2.subtract(inner_sum, in_p01, dst=inner_sum)
cv2.subtract(inner_sum, in_p10, dst=inner_sum)
# memo: Multiplication, etc. can be faster by self-assignment, but care must be taken because array initialization is required.
# https://stackoverflow.com/questions/71204415/opencv-python-fastest-way-to-multiply-pixel-value
inner_sum_f = np.empty(inner_sum.shape, dtype=np.float64)
inner_sum_f[:, :] = inner_sum
outer_sum_f = np.empty(outer_sum.shape, dtype=np.float64)
outer_sum_f[:, :] = outer_sum
inner_sum_f = inner_sum.astype(np.float64)
outer_sum_f = outer_sum.astype(np.float64)
response_value = np.empty(outer_sum.shape, dtype=np.float64)
inout_rect_sum = mu_outer_rect2.copy()
@ -432,14 +415,10 @@ class AHSF:
cv2.add(inout_rect_mul, inout_rect_sum, dst=inout_rect_sum)
cv2.multiply(inner_sum_f, wh_in_arr, inner_sum_f, kf)
cv2.add(inout_rect_sum, inner_sum_f, dst=response_value)
# mu_outer_left+(kf*inner_sum*wh_in_arr)
# memo: The input image is transposed, so the coordinate output of this function has x and y swapped.
min_response, max_response, min_loc, max_loc = cv2.minMaxLoc(response_value)
# The sign is reversed from the original calculation result, so using min.
rec_o = (
x_out_n[min_loc[1]],
y_out_n[min_loc[0]],
@ -597,7 +576,6 @@ class AHSF:
)
def External_Run_AHSF(self, frame_gray):
average_color = np.mean(frame_gray)
height, width = frame_gray.shape
@ -614,19 +592,20 @@ class AHSF:
"use_init_rect": False,
"mu_outer": 200,
"mu_inner": 50,
"ratio_outer": 0.9,
"ratio_outer": 1,
"kf": 1,
"width_min": 16,
"width_min": 25,
"width_max": 50,
"wh_step": 5,
"xy_step": 10,
"wh_step": 1,
"xy_step": 5,
"roi": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
"init_rect_flag": False,
"init_rect": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
}
try:
pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer = self.coarse_detection(frame_gray, params)
ellipse_rect, center_fitting = self.fine_detection(frame_gray, pupil_rect_coarse)
# ellipse_rect, center_fitting = self.fine_detection(frame_gray, pupil_rect_coarse)
except TypeError:
return frame_gray, frame_gray, 0, 0, 0
@ -634,10 +613,27 @@ class AHSF:
y_center = outer_rect_coarse[1] + outer_rect_coarse[3] / 2
x, y, width, height = outer_rect_coarse
cv2.circle(frame_gray, (int(x_center), int(y_center)), 2, (255, 255, 255), -1)
thickness = 1
cv2.rectangle(frame_gray, (pupil_rect_coarse[0], pupil_rect_coarse[1]), (pupil_rect_coarse[0] + pupil_rect_coarse[2], pupil_rect_coarse[1] + pupil_rect_coarse[3]), (255, 255, 255), thickness)
cv2.rectangle(frame_gray, (outer_rect_coarse[0], outer_rect_coarse[1]), (outer_rect_coarse[0] + outer_rect_coarse[2], outer_rect_coarse[1] + outer_rect_coarse[3]), (255, 255, 255), thickness)
cv2.rectangle(frame_gray, (pupil_rect_coarse[0], pupil_rect_coarse[1]),
(pupil_rect_coarse[0] + pupil_rect_coarse[2], pupil_rect_coarse[1] + pupil_rect_coarse[3]),
(0, 255, 0), 2)
cv2.rectangle(frame_gray, (outer_rect_coarse[0], outer_rect_coarse[1]),
(outer_rect_coarse[0] + outer_rect_coarse[2], outer_rect_coarse[1] + outer_rect_coarse[3]),
(255, 0, 0), 2)
major_diameter = math.sqrt(width**2 + height**2)
minor_diameter = min(width, height)

Binary file not shown.

View File

@ -1,5 +1,4 @@
import os
os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime
import numpy as np
import cv2
@ -8,15 +7,15 @@ import math
from queue import Queue
import threading
from one_euro_filter import OneEuroFilter
import psutil, os
import sys
import psutil
from utils.misc_utils import resource_path
from pathlib import Path
os.environ["OMP_NUM_THREADS"] = "1"
frames = 0
models = Path("Models")
def run_model(input_queue, output_queue, session):
while True:
frame = input_queue.get()
@ -26,7 +25,6 @@ def run_model(input_queue, output_queue, session):
img_np = np.array(frame, dtype=np.float32) / 255.0
gray_img = 0.299 * img_np[:, :, 0] + 0.587 * img_np[:, :, 1] + 0.114 * img_np[:, :, 2]
# Add the channel and batch dimensions
gray_img = np.expand_dims(np.expand_dims(gray_img, axis=0), axis=0)
ort_inputs = {session.get_inputs()[0].name: gray_img}
@ -34,44 +32,35 @@ def run_model(input_queue, output_queue, session):
pre_landmark = np.reshape(pre_landmark, (-1, 2))
output_queue.put((frame, pre_landmark))
def run_onnx_model(queues, session, frame):
for queue in queues:
if not queue.full():
queue.put(frame)
break
def to_numpy(tensor):
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
class LEAP_C:
def __init__(self):
self.last_lid = None
self.current_image_gray = None
self.current_image_gray_clean = None
onnxruntime.disable_telemetry_events()
self.num_threads = 2
self.num_threads = 1
self.queue_max_size = 1
self.model_path = resource_path(models / "pfld-sim.onnx")
self.print_fps = False
self.frames = 0
self.queues = []
self.queues = [Queue(maxsize=self.queue_max_size) for _ in range(self.num_threads)]
self.threads = []
self.model_output = np.zeros((12, 2))
self.output_queue = Queue(maxsize=self.queue_max_size)
self.start_time = time.time()
for _ in range(self.num_threads):
queue = Queue(maxsize=self.queue_max_size)
self.queues.append(queue)
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1 # fps hit
opts.intra_op_num_threads = 1
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
opts.enable_mem_pattern = False
self.one_euro_filter_float = OneEuroFilter(np.random.rand(1, 2), min_cutoff=0.0004, beta=0.9)
self.dmax = 0
@ -85,7 +74,8 @@ class LEAP_C:
self.total_velocity_old = 0
self.old_per = 0.0
self.delta_per_neg = 0.0
self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=["CPUExecutionProvider"])
self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
for i in range(self.num_threads):
thread = threading.Thread(
@ -129,13 +119,6 @@ class LEAP_C:
if len(self.openlist) < 2500:
self.openlist.append(d)
else:
pass
# print("full")
#print(len(self.openlist))
# self.openlist.pop(0)
# self.openlist.append(d)
try:
if len(self.openlist) > 0:
@ -162,7 +145,6 @@ class LEAP_C:
imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
return imgvis, 0, 0, 0
class External_Run_LEAP:
def __init__(self):
self.algo = LEAP_C()
@ -171,4 +153,4 @@ class External_Run_LEAP:
self.algo.current_image_gray = current_image_gray
self.algo.current_image_gray_clean = current_image_gray_clean
img, x, y, per = self.algo.leap_run()
return img, x, y, per
return img, x, y, per