fix: revert LEAP bug fix

This commit is contained in:
Prohurtz 2024-07-04 19:39:17 -05:00
parent c220521172
commit cda06b830b
2 changed files with 147 additions and 83 deletions

View File

@ -907,6 +907,8 @@ if __name__ == "__main__":
def External_Run_AHSF(frame_gray): def External_Run_AHSF(frame_gray):
average_color = np.mean(frame_gray) average_color = np.mean(frame_gray)
orig_height, orig_width = frame_gray.shape orig_height, orig_width = frame_gray.shape
frame_clear_resize = frame_gray.copy()
org_frame_gray = frame_gray.copy() org_frame_gray = frame_gray.copy()
frame_gray = cv2.resize(frame_gray, (130, 130)) # TODO TEST FIXED RESIZE frame_gray = cv2.resize(frame_gray, (130, 130)) # TODO TEST FIXED RESIZE
@ -928,7 +930,7 @@ def External_Run_AHSF(frame_gray):
square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray
frame_gray = square_background frame_gray = square_background
frame_clear_resize = frame_gray.copy()
wh_step = max((int(max_dimension / 80)),1) # TODO: FINETUNE VALUES wh_step = max((int(max_dimension / 80)),1) # TODO: FINETUNE VALUES
xy_step = max(int(max_dimension / 24), 1) # TODO: FINETUNE VALUES xy_step = max(int(max_dimension / 24), 1) # TODO: FINETUNE VALUES
@ -1025,7 +1027,7 @@ def External_Run_AHSF(frame_gray):
minor_diameter = min(width, height) minor_diameter = min(width, height)
average_diameter = (major_diameter + minor_diameter) / 2 average_diameter = (major_diameter + minor_diameter) / 2
# print(orig_width, orig_height)
return org_frame_gray, frame_clear_resize, x_center, y_center, abs(width - height) return org_frame_gray, frame_clear_resize, x_center, y_center, abs(width - height)

View File

@ -50,9 +50,37 @@ DATASET CONTRIBUTIONS:
""" """
"""
------------------------------------------------------------------------------------------------------
,@@@@@@
@@@@@@@@@@@ @@@
@@@@@@@@@@@@ @@@@@@@@@@@
@@@@@@@@@@@@@ @@@@@@@@@@@@@@
@@@@@@@/ ,@@@@@@@@@@@@@
/@@@@@@@@@@@@@@@ @@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@
@@@@@@@@ @@@@@
,@@@ @@@@&
@@@@@@. @@@@
@@@ @@@@@@@@@/ @@@@@
,@@@. @@@@@@((@ @@@@(
//@@@ ,, @@@@ @@@@@
@@@( @@@@@@@
@@@ @ @@@@@@@@#
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
LEAP by: Prohurtz
Algorithm App Implementation By: Prohurtz
Copyright (c) 2023 EyeTrackVR <3
LICENSE: GNU GPLv3
------------------------------------------------------------------------------------------------------
"""
# LEAP = Lightweight Eyelid And Pupil # LEAP = Lightweight Eyelid And Pupil
import os import os
os.environ["OMP_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime import onnxruntime
import numpy as np import numpy as np
@ -67,14 +95,40 @@ import sys
from utils.misc_utils import resource_path from utils.misc_utils import resource_path
from pathlib import Path from pathlib import Path
frames = 0 frames = 0
models = Path("Models") models = Path("Models")
def run_model(input_queue, output_queue, session):
while True:
frame = input_queue.get()
if frame is None:
break
img_np = np.array(frame)
img_np = img_np.astype(np.float32) / 255.0
gray_img = 0.299 * img_np[:, :, 0] + 0.587 * img_np[:, :, 1] + 0.114 * img_np[:, :, 2]
# Add the channel and batch dimensions
gray_img = np.expand_dims(gray_img, axis=0) # Add channel dimension
img_np = np.expand_dims(gray_img, axis=0) # Add batch dimension
# img_np = np.transpose(img_np, (2, 0, 1))
# img_np = np.expand_dims(img_np, axis=0)
ort_inputs = {session.get_inputs()[0].name: img_np}
pre_landmark = session.run(None, ort_inputs)
# pre_landmark = pre_landmark[1]
# pre_landmark = np.reshape(pre_landmark, (12, 2))
pre_landmark = np.reshape(pre_landmark, (-1, 2))
output_queue.put((frame, pre_landmark))
class LEAP_C(object): class LEAP_C(object):
def __init__(self): def __init__(self):
onnxruntime.disable_telemetry_events() onnxruntime.disable_telemetry_events()
# Config variables # Config variables
self.num_threads = 1 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage) self.num_threads = 4 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak. self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
self.model_path = resource_path(models / 'LEAP062120246epoch.onnx') self.model_path = resource_path(models / 'LEAP062120246epoch.onnx')
@ -142,28 +196,25 @@ class LEAP_C(object):
self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=["CPUExecutionProvider"]) self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=["CPUExecutionProvider"])
def run_model(output_queue, session, frame): threads = []
for i in range(self.num_threads):
thread = threading.Thread(
target=run_model,
args=(self.queues[i], self.output_queue, self.ort_session1),
name=f"Thread {i}",
)
threads.append(thread)
thread.start()
img_np = np.array(frame)
img_np = img_np.astype(np.float32) / 255.0
gray_img = 0.299 * img_np[:, :, 0] + 0.587 * img_np[:, :, 1] + 0.114 * img_np[:, :, 2]
# Add the channel and batch dimensions
gray_img = np.expand_dims(gray_img, axis=0) # Add channel dimension
img_np = np.expand_dims(gray_img, axis=0) # Add batch dimension
# img_np = np.transpose(img_np, (2, 0, 1))
# img_np = np.expand_dims(img_np, axis=0)
ort_inputs = {session.get_inputs()[0].name: img_np}
pre_landmark = session.run(None, ort_inputs)
# pre_landmark = pre_landmark[1]
# pre_landmark = np.reshape(pre_landmark, (12, 2))
pre_landmark = np.reshape(pre_landmark, (-1, 2))
# output_queue.put((frame, pre_landmark))
return frame, pre_landmark
def to_numpy(self, tensor): def to_numpy(self, tensor):
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
def run_onnx_model(self, queues, session, frame):
for i in range(len(queues)):
if not queues[i].full():
queues[i].put(frame)
break
def leap_run(self): def leap_run(self):
img = self.current_image_gray_clean.copy() img = self.current_image_gray_clean.copy()
@ -173,90 +224,103 @@ class LEAP_C(object):
frame = cv2.resize(img, (112, 112)) frame = cv2.resize(img, (112, 112))
imgvis = self.current_image_gray.copy() imgvis = self.current_image_gray.copy()
self.run_onnx_model(self.queues, self.ort_session1, frame)
frame, pre_landmark = self.run_model(self.ort_session1, frame) if not self.output_queue.empty():
for point in pre_landmark: frame, pre_landmark = self.output_queue.get()
# x, y = (point*112).astype(int) # pre_landmark = np.reshape(pre_landmark, (-1, 2))
x, y = point # Assuming point is a tuple (x, y) # pre_landmark = self.one_euro_filter(pre_landmark)
# Scale the coordinates to image width and height for point in pre_landmark:
x = int(x * img_width) # x, y = (point*112).astype(int)
y = int(y * img_height)
# x, y = int(x), int(y) # Ensure x and y are integers
cv2.circle(imgvis, (int(x), int(y)), 2, (255, 255, 0), -1) x, y = point # Assuming point is a tuple (x, y)
# Scale the coordinates to image width and height
x = int(x * img_width)
y = int(y * img_height)
# x, y = int(x), int(y) # Ensure x and y are integers
cv2.circle(imgvis, (int(x), int(y)), 2, (255, 255, 0), -1)
x1, y1 = pre_landmark[1]
x2, y2 = pre_landmark[3]
d1 = math.dist(pre_landmark[1], pre_landmark[3]) x3, y3 = pre_landmark[4]
# a more fancy method could be used taking into acount the relative size of the landmarks so that weirdness can be acounted for better x4, y4 = pre_landmark[2]
d2 = math.dist(pre_landmark[2], pre_landmark[4])
d = (d1 + d2) / 2
# by averaging both sets we can get less error? i think part of why 1 eye was better than the other is because we only considered one offset points.
# considering both should smooth things out between eyes
try: d1 = math.dist(pre_landmark[1], pre_landmark[3])
if d >= np.percentile( # a more fancy method could be used taking into acount the relative size of the landmarks so that weirdness can be acounted for better
self.openlist, 80 # do not go above 85, but this value can be tuned d2 = math.dist(pre_landmark[2], pre_landmark[4])
): # an aditional approach could be using the place where on average it is most stable, denoting what distance is the most stable "open" d = (d1 + d2) / 2
self.maxlist.append(d) # by averaging both sets we can get less error? i think part of why 1 eye was better than the other is because we only considered one offset points.
# considering both should smooth things out between eyes
if len(self.maxlist) > 2000: # i feel that this is very cpu intensive. think of a better method try:
self.maxlist.pop(0) if d >= np.percentile(
self.openlist, 80 # do not go above 85, but this value can be tuned
): # an aditional approach could be using the place where on average it is most stable, denoting what distance is the most stable "open"
self.maxlist.append(d)
# this should be the average most open value, the average of top 2000 values in rolling calibration if len(self.maxlist) > 2000: # i feel that this is very cpu intensive. think of a better method
# with this we can use it as the "openstate" (0.7, for expanded squeeze) self.maxlist.pop(0)
# weighted values to shift slightly to max value # this should be the average most open value, the average of top 2000 values in rolling calibration
normal_open = ((sum(self.maxlist) / len(self.maxlist)) * 0.90 + max(self.openlist) * 0.10) / ( # with this we can use it as the "openstate" (0.7, for expanded squeeze)
0.95 + 0.15
)
except: # weighted values to shift slightly to max value
normal_open = 0.8 normal_open = ((sum(self.maxlist) / len(self.maxlist)) * 0.90 + max(self.openlist) * 0.10) / (
0.95 + 0.15
)
if len(self.openlist) < 5000: # TODO expose as setting? except:
self.openlist.append(d) normal_open = 0.8
else:
self.openlist.pop(0)
self.openlist.append(d)
try: if len(self.openlist) < 5000: # TODO expose as setting?
per = (d - normal_open) / (min(self.openlist) - normal_open) self.openlist.append(d)
else:
self.openlist.pop(0)
self.openlist.append(d)
oldper = (d - max(self.openlist)) / ( try:
min(self.openlist) - max(self.openlist) per = (d - normal_open) / (min(self.openlist) - normal_open)
) # TODO: remove when testing is done
per = 1 - per oldper = (d - max(self.openlist)) / (
per = per - 0.2 # allow for eye widen? might require a more legit math way but this makes sense. min(self.openlist) - max(self.openlist)
per = min(per, 1.0) # clamp to 1.0 max ) # TODO: remove when testing is done
per = max(per, 0.0) # clamp to 1.0 min
# print("new: ", per, "vs old: ", oldper) per = 1 - per
per = per - 0.2 # allow for eye widen? might require a more legit math way but this makes sense.
per = min(per, 1.0) # clamp to 1.0 max
per = max(per, 0.0) # clamp to 1.0 min
except: # print("new: ", per, "vs old: ", oldper)
per = 0.8
pass
x = pre_landmark[6][0] except:
y = pre_landmark[6][1] per = 0.8
pass
self.last_lid = per x = pre_landmark[6][0]
calib_array = np.array([per, per]).reshape(1, 2) y = pre_landmark[6][1]
per = self.one_euro_filter_float(calib_array) self.last_lid = per
calib_array = np.array([per, per]).reshape(1, 2)
per = per[0][0] per = self.one_euro_filter_float(calib_array)
# print(per)
if per <= 0.2: # TODO: EXPOSE AS SETTING
per == 0.0
# this should be tuned, i could make this auto calib based on min from a list of per values.
return imgvis, float(x), float(y), per per = per[0][0]
# print(per)
if per <= 0.2: # TODO: EXPOSE AS SETTING
per == 0.0
# this should be tuned, i could make this auto calib based on min from a list of per values.
return imgvis, float(x), float(y), per
imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
return imgvis, 0, 0, 0
class External_Run_LEAP(object): class External_Run_LEAP(object):
@ -267,6 +331,4 @@ class External_Run_LEAP(object):
self.algo.current_image_gray = current_image_gray self.algo.current_image_gray = current_image_gray
self.algo.current_image_gray_clean = current_image_gray_clean self.algo.current_image_gray_clean = current_image_gray_clean
img, x, y, per = self.algo.leap_run() img, x, y, per = self.algo.leap_run()
return img, x, y, per return img, x, y, per