fix: leap and leap lid both enabled crash

This commit is contained in:
Prohurtz 2024-08-17 17:13:02 -05:00
parent bf4b77abe7
commit 8acaa1c643
2 changed files with 47 additions and 53 deletions

View File

@ -54,6 +54,7 @@ from ellipse_based_pupil_dilation import *
from AHSF import * from AHSF import *
from osc.OSCMessage import OSCMessageType, OSCMessage from osc.OSCMessage import OSCMessageType, OSCMessage
def run_once(f): def run_once(f):
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
if not wrapper.has_run: if not wrapper.has_run:
@ -168,7 +169,6 @@ class EyeProcessor:
self.avg_velocity = 0.0 self.avg_velocity = 0.0
self.angle = 621 self.angle = 621
try: try:
min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004 min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004
beta = float(self.settings.gui_speed_coefficient) # 0.9 beta = float(self.settings.gui_speed_coefficient) # 0.9
@ -214,26 +214,19 @@ class EyeProcessor:
# fill with avg color + 10. # fill with avg color + 10.
# fill with white (self.current_image_white) and average in-bounds color (self.current_image). # fill with white (self.current_image_white) and average in-bounds color (self.current_image).
crop_matrix = np.float32([[1, 0, -roi_x], crop_matrix = np.float32([[1, 0, -roi_x], [0, 1, -roi_y], [0, 0, 1]])
[0, 1, -roi_y],
[0, 0, 1]])
img_center = (roi_w / 2, roi_h / 2) img_center = (roi_w / 2, roi_h / 2)
rotation_matrix = cv2.getRotationMatrix2D( rotation_matrix = cv2.getRotationMatrix2D(img_center, self.config.rotation_angle, 1)
img_center, self.config.rotation_angle, 1
)
# rows, cols = self.current_image.shape[:2]
# rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), self.config.rotation_angle, 1)
#cos_theta = np.abs(rotation_matrix[0, 0])
# sin_theta = np.abs(rotation_matrix[0, 1])
# new_cols = int((cols * cos_theta) + (rows * sin_theta))
# new_rows = int((cols * sin_theta) + (rows * cos_theta))
# rotation_matrix[0, 2] += (new_cols - cols) / 2
# rotation_matrix[1, 2] += (new_rows - rows) / 2
# rows, cols = self.current_image.shape[:2]
# rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), self.config.rotation_angle, 1)
# cos_theta = np.abs(rotation_matrix[0, 0])
# sin_theta = np.abs(rotation_matrix[0, 1])
# new_cols = int((cols * cos_theta) + (rows * sin_theta))
# new_rows = int((cols * sin_theta) + (rows * cos_theta))
# rotation_matrix[0, 2] += (new_cols - cols) / 2
# rotation_matrix[1, 2] += (new_rows - rows) / 2
matrix = np.matmul(rotation_matrix, crop_matrix) matrix = np.matmul(rotation_matrix, crop_matrix)
self.current_image_white = cv2.warpAffine( self.current_image_white = cv2.warpAffine(
@ -254,13 +247,8 @@ class EyeProcessor:
inv_matrix = np.linalg.inv(np.vstack((matrix, [0, 0, 1])))[:-1] inv_matrix = np.linalg.inv(np.vstack((matrix, [0, 0, 1])))[:-1]
# calculate crop corner locations in original image space # calculate crop corner locations in original image space
corners = np.matmul([[0, 0, 1], corners = np.matmul([[0, 0, 1], [roi_w, 0, 1], [0, roi_h, 1], [roi_w, roi_h, 1]], np.transpose(inv_matrix))
[roi_w, 0, 1], fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h for (x, y) in corners)
[0, roi_h, 1],
[roi_w, roi_h, 1]],
np.transpose(inv_matrix))
fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h
for (x, y) in corners)
if fits_in_bounds: if fits_in_bounds:
# crop is entirely within original image bounds so average color and white are identical # crop is entirely within original image bounds so average color and white are identical
@ -290,11 +278,8 @@ class EyeProcessor:
rgb_ch = self.current_image[:, :, :3] rgb_ch = self.current_image[:, :, :3]
inv_alpha_ch = 255 - self.current_image[:, :, 3] inv_alpha_ch = 255 - self.current_image[:, :, 3]
self.current_image = rgb_ch + np.stack( self.current_image = rgb_ch + np.stack(
np.uint8([inv_alpha_ch * ar, np.uint8([inv_alpha_ch * ar, inv_alpha_ch * ag, inv_alpha_ch * ab]), axis=-1
inv_alpha_ch * ag, )
inv_alpha_ch * ab]),
axis=-1)
return True return True
except: except:
@ -332,7 +317,7 @@ class EyeProcessor:
self.settings.ibo_average_output_samples, self.settings.ibo_average_output_samples,
) )
if self.settings.gui_LEAP_lid and self.eyeopen != 0.0: if self.settings.gui_LEAP_lid and self.eyeopen != 0.0 and not self.settings.gui_LEAP:
( (
self.current_image_gray, self.current_image_gray,
self.rawx, self.rawx,
@ -340,14 +325,12 @@ class EyeProcessor:
self.eyeopen, self.eyeopen,
) = self.er_leap.run(self.current_image_gray, self.current_image_gray_clean) ) = self.er_leap.run(self.current_image_gray, self.current_image_gray_clean)
if len(self.prev_y_list) >= 100: # "lock" eye when close/blink IN TESTING, kinda broke if len(self.prev_y_list) >= 100: # "lock" eye when close/blink IN TESTING, kinda broke
self.prev_y_list.pop(0) self.prev_y_list.pop(0)
self.prev_y_list.append(self.out_y) self.prev_y_list.append(self.out_y)
else: else:
self.prev_y_list.append(self.out_y) self.prev_y_list.append(self.out_y)
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1 blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
if blink_vec >= 0.18: if blink_vec >= 0.18:
@ -384,25 +367,27 @@ class EyeProcessor:
), ),
) )
# if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0: why is this here # if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0: why is this here
# pass # pass
# else: # else:
# self.eyeopen = 0.81 # self.eyeopen = 0.81
osc_message = OSCMessage( osc_message = OSCMessage(
type=OSCMessageType.EYE_INFO, type=OSCMessageType.EYE_INFO,
data=(self.eye_id, EyeInfo( data=(
self.current_algo, self.eye_id,
self.out_x, EyeInfo(
self.out_y, self.current_algo,
self.pupil_dilation, self.out_x,
self.eyeopen, self.out_y,
self.avg_velocity, self.pupil_dilation,
)), self.eyeopen,
self.avg_velocity,
),
),
) )
self.osc_queue.put(osc_message) self.osc_queue.put(osc_message)
self.eyeopen = 0.8 # TODO: remove this by fixing checks if is 0.0 self.eyeopen = 0.8 # TODO: remove this by fixing checks if is 0.0
def BLINKM(self): def BLINKM(self):
self.eyeopen = BLINK(self) self.eyeopen = BLINK(self)

View File

@ -1,4 +1,5 @@
import os import os
os.environ["OMP_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime import onnxruntime
import numpy as np import numpy as np
@ -16,6 +17,7 @@ from pathlib import Path
frames = 0 frames = 0
models = Path("Models") models = Path("Models")
def run_model(input_queue, output_queue, session): def run_model(input_queue, output_queue, session):
while True: while True:
frame = input_queue.get() frame = input_queue.get()
@ -33,15 +35,18 @@ def run_model(input_queue, output_queue, session):
pre_landmark = np.reshape(pre_landmark, (-1, 2)) pre_landmark = np.reshape(pre_landmark, (-1, 2))
output_queue.put((frame, pre_landmark)) output_queue.put((frame, pre_landmark))
def run_onnx_model(queues, session, frame): def run_onnx_model(queues, session, frame):
for queue in queues: for queue in queues:
if not queue.full(): if not queue.full():
queue.put(frame) queue.put(frame)
break break
def to_numpy(tensor): def to_numpy(tensor):
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
class LEAP_C: class LEAP_C:
def __init__(self): def __init__(self):
self.last_lid = None self.last_lid = None
@ -50,7 +55,7 @@ class LEAP_C:
onnxruntime.disable_telemetry_events() onnxruntime.disable_telemetry_events()
self.num_threads = 2 self.num_threads = 2
self.queue_max_size = 1 self.queue_max_size = 1
self.model_path = resource_path(models / 'LEAP071024_E16.onnx') self.model_path = resource_path(models / "LEAP071024_E16.onnx")
self.print_fps = False self.print_fps = False
self.frames = 0 self.frames = 0
@ -66,7 +71,7 @@ class LEAP_C:
opts = onnxruntime.SessionOptions() opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1 opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1 #fps hit opts.intra_op_num_threads = 1 # fps hit
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
self.one_euro_filter_float = OneEuroFilter(np.random.rand(1, 2), min_cutoff=0.0004, beta=0.9) self.one_euro_filter_float = OneEuroFilter(np.random.rand(1, 2), min_cutoff=0.0004, beta=0.9)
@ -123,15 +128,18 @@ class LEAP_C:
normal_open = np.percentile(self.openlist, 70) if len(self.openlist) >= 500 else 0.8 normal_open = np.percentile(self.openlist, 70) if len(self.openlist) >= 500 else 0.8
if len(self.openlist) < 5000: if len(self.openlist) < 2500:
self.openlist.append(d) self.openlist.append(d)
else: else:
self.openlist.pop(0) print("full")
self.openlist.append(d)
print(len(self.openlist))
# self.openlist.pop(0)
# self.openlist.append(d)
try: try:
if len(self.openlist) > 0: if len(self.openlist) > 0:
per = (d - normal_open) / (np.percentile(self.openlist, 1.7) - normal_open) per = (d - normal_open) / (np.percentile(self.openlist, 1) - normal_open)
per = 1 - per per = 1 - per
per = np.clip(per - 0.2, 0.0, 1.0) per = np.clip(per - 0.2, 0.0, 1.0)
else: else:
@ -154,6 +162,7 @@ class LEAP_C:
imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
return imgvis, 0, 0, 0 return imgvis, 0, 0, 0
class External_Run_LEAP: class External_Run_LEAP:
def __init__(self): def __init__(self):
self.algo = LEAP_C() self.algo = LEAP_C()