mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix: leap and leap lid both enabled crash
This commit is contained in:
parent
bf4b77abe7
commit
8acaa1c643
@ -54,6 +54,7 @@ from ellipse_based_pupil_dilation import *
|
||||
from AHSF import *
|
||||
from osc.OSCMessage import OSCMessageType, OSCMessage
|
||||
|
||||
|
||||
def run_once(f):
|
||||
def wrapper(*args, **kwargs):
|
||||
if not wrapper.has_run:
|
||||
@ -168,7 +169,6 @@ class EyeProcessor:
|
||||
self.avg_velocity = 0.0
|
||||
self.angle = 621
|
||||
|
||||
|
||||
try:
|
||||
min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004
|
||||
beta = float(self.settings.gui_speed_coefficient) # 0.9
|
||||
@ -214,26 +214,19 @@ class EyeProcessor:
|
||||
# fill with avg color + 10.
|
||||
# fill with white (self.current_image_white) and average in-bounds color (self.current_image).
|
||||
|
||||
crop_matrix = np.float32([[1, 0, -roi_x],
|
||||
[0, 1, -roi_y],
|
||||
[0, 0, 1]])
|
||||
crop_matrix = np.float32([[1, 0, -roi_x], [0, 1, -roi_y], [0, 0, 1]])
|
||||
img_center = (roi_w / 2, roi_h / 2)
|
||||
|
||||
rotation_matrix = cv2.getRotationMatrix2D(
|
||||
img_center, self.config.rotation_angle, 1
|
||||
)
|
||||
|
||||
# rows, cols = self.current_image.shape[:2]
|
||||
# rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), self.config.rotation_angle, 1)
|
||||
#cos_theta = np.abs(rotation_matrix[0, 0])
|
||||
# sin_theta = np.abs(rotation_matrix[0, 1])
|
||||
# new_cols = int((cols * cos_theta) + (rows * sin_theta))
|
||||
# new_rows = int((cols * sin_theta) + (rows * cos_theta))
|
||||
# rotation_matrix[0, 2] += (new_cols - cols) / 2
|
||||
# rotation_matrix[1, 2] += (new_rows - rows) / 2
|
||||
|
||||
|
||||
rotation_matrix = cv2.getRotationMatrix2D(img_center, self.config.rotation_angle, 1)
|
||||
|
||||
# rows, cols = self.current_image.shape[:2]
|
||||
# rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), self.config.rotation_angle, 1)
|
||||
# cos_theta = np.abs(rotation_matrix[0, 0])
|
||||
# sin_theta = np.abs(rotation_matrix[0, 1])
|
||||
# new_cols = int((cols * cos_theta) + (rows * sin_theta))
|
||||
# new_rows = int((cols * sin_theta) + (rows * cos_theta))
|
||||
# rotation_matrix[0, 2] += (new_cols - cols) / 2
|
||||
# rotation_matrix[1, 2] += (new_rows - rows) / 2
|
||||
|
||||
matrix = np.matmul(rotation_matrix, crop_matrix)
|
||||
self.current_image_white = cv2.warpAffine(
|
||||
@ -254,13 +247,8 @@ class EyeProcessor:
|
||||
|
||||
inv_matrix = np.linalg.inv(np.vstack((matrix, [0, 0, 1])))[:-1]
|
||||
# calculate crop corner locations in original image space
|
||||
corners = np.matmul([[0, 0, 1],
|
||||
[roi_w, 0, 1],
|
||||
[0, roi_h, 1],
|
||||
[roi_w, roi_h, 1]],
|
||||
np.transpose(inv_matrix))
|
||||
fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h
|
||||
for (x, y) in corners)
|
||||
corners = np.matmul([[0, 0, 1], [roi_w, 0, 1], [0, roi_h, 1], [roi_w, roi_h, 1]], np.transpose(inv_matrix))
|
||||
fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h for (x, y) in corners)
|
||||
|
||||
if fits_in_bounds:
|
||||
# crop is entirely within original image bounds so average color and white are identical
|
||||
@ -290,11 +278,8 @@ class EyeProcessor:
|
||||
rgb_ch = self.current_image[:, :, :3]
|
||||
inv_alpha_ch = 255 - self.current_image[:, :, 3]
|
||||
self.current_image = rgb_ch + np.stack(
|
||||
np.uint8([inv_alpha_ch * ar,
|
||||
inv_alpha_ch * ag,
|
||||
inv_alpha_ch * ab]),
|
||||
axis=-1)
|
||||
|
||||
np.uint8([inv_alpha_ch * ar, inv_alpha_ch * ag, inv_alpha_ch * ab]), axis=-1
|
||||
)
|
||||
|
||||
return True
|
||||
except:
|
||||
@ -332,7 +317,7 @@ class EyeProcessor:
|
||||
self.settings.ibo_average_output_samples,
|
||||
)
|
||||
|
||||
if self.settings.gui_LEAP_lid and self.eyeopen != 0.0:
|
||||
if self.settings.gui_LEAP_lid and self.eyeopen != 0.0 and not self.settings.gui_LEAP:
|
||||
(
|
||||
self.current_image_gray,
|
||||
self.rawx,
|
||||
@ -340,14 +325,12 @@ class EyeProcessor:
|
||||
self.eyeopen,
|
||||
) = self.er_leap.run(self.current_image_gray, self.current_image_gray_clean)
|
||||
|
||||
|
||||
if len(self.prev_y_list) >= 100: # "lock" eye when close/blink IN TESTING, kinda broke
|
||||
self.prev_y_list.pop(0)
|
||||
self.prev_y_list.append(self.out_y)
|
||||
else:
|
||||
self.prev_y_list.append(self.out_y)
|
||||
|
||||
|
||||
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
|
||||
|
||||
if blink_vec >= 0.18:
|
||||
@ -384,25 +367,27 @@ class EyeProcessor:
|
||||
),
|
||||
)
|
||||
|
||||
# if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0: why is this here
|
||||
# pass
|
||||
# else:
|
||||
# self.eyeopen = 0.81
|
||||
|
||||
# if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0: why is this here
|
||||
# pass
|
||||
# else:
|
||||
# self.eyeopen = 0.81
|
||||
|
||||
osc_message = OSCMessage(
|
||||
type=OSCMessageType.EYE_INFO,
|
||||
data=(self.eye_id, EyeInfo(
|
||||
self.current_algo,
|
||||
self.out_x,
|
||||
self.out_y,
|
||||
self.pupil_dilation,
|
||||
self.eyeopen,
|
||||
self.avg_velocity,
|
||||
)),
|
||||
data=(
|
||||
self.eye_id,
|
||||
EyeInfo(
|
||||
self.current_algo,
|
||||
self.out_x,
|
||||
self.out_y,
|
||||
self.pupil_dilation,
|
||||
self.eyeopen,
|
||||
self.avg_velocity,
|
||||
),
|
||||
),
|
||||
)
|
||||
self.osc_queue.put(osc_message)
|
||||
self.eyeopen = 0.8 # TODO: remove this by fixing checks if is 0.0
|
||||
self.eyeopen = 0.8 # TODO: remove this by fixing checks if is 0.0
|
||||
|
||||
def BLINKM(self):
|
||||
self.eyeopen = BLINK(self)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import os
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
import onnxruntime
|
||||
import numpy as np
|
||||
@ -16,6 +17,7 @@ from pathlib import Path
|
||||
frames = 0
|
||||
models = Path("Models")
|
||||
|
||||
|
||||
def run_model(input_queue, output_queue, session):
|
||||
while True:
|
||||
frame = input_queue.get()
|
||||
@ -33,15 +35,18 @@ def run_model(input_queue, output_queue, session):
|
||||
pre_landmark = np.reshape(pre_landmark, (-1, 2))
|
||||
output_queue.put((frame, pre_landmark))
|
||||
|
||||
|
||||
def run_onnx_model(queues, session, frame):
|
||||
for queue in queues:
|
||||
if not queue.full():
|
||||
queue.put(frame)
|
||||
break
|
||||
|
||||
|
||||
def to_numpy(tensor):
|
||||
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
|
||||
|
||||
|
||||
class LEAP_C:
|
||||
def __init__(self):
|
||||
self.last_lid = None
|
||||
@ -50,7 +55,7 @@ class LEAP_C:
|
||||
onnxruntime.disable_telemetry_events()
|
||||
self.num_threads = 2
|
||||
self.queue_max_size = 1
|
||||
self.model_path = resource_path(models / 'LEAP071024_E16.onnx')
|
||||
self.model_path = resource_path(models / "LEAP071024_E16.onnx")
|
||||
|
||||
self.print_fps = False
|
||||
self.frames = 0
|
||||
@ -66,7 +71,7 @@ class LEAP_C:
|
||||
|
||||
opts = onnxruntime.SessionOptions()
|
||||
opts.inter_op_num_threads = 1
|
||||
opts.intra_op_num_threads = 1 #fps hit
|
||||
opts.intra_op_num_threads = 1 # fps hit
|
||||
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
|
||||
self.one_euro_filter_float = OneEuroFilter(np.random.rand(1, 2), min_cutoff=0.0004, beta=0.9)
|
||||
@ -123,15 +128,18 @@ class LEAP_C:
|
||||
|
||||
normal_open = np.percentile(self.openlist, 70) if len(self.openlist) >= 500 else 0.8
|
||||
|
||||
if len(self.openlist) < 5000:
|
||||
if len(self.openlist) < 2500:
|
||||
self.openlist.append(d)
|
||||
else:
|
||||
self.openlist.pop(0)
|
||||
self.openlist.append(d)
|
||||
print("full")
|
||||
|
||||
print(len(self.openlist))
|
||||
# self.openlist.pop(0)
|
||||
# self.openlist.append(d)
|
||||
|
||||
try:
|
||||
if len(self.openlist) > 0:
|
||||
per = (d - normal_open) / (np.percentile(self.openlist, 1.7) - normal_open)
|
||||
per = (d - normal_open) / (np.percentile(self.openlist, 1) - normal_open)
|
||||
per = 1 - per
|
||||
per = np.clip(per - 0.2, 0.0, 1.0)
|
||||
else:
|
||||
@ -154,6 +162,7 @@ class LEAP_C:
|
||||
imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
||||
return imgvis, 0, 0, 0
|
||||
|
||||
|
||||
class External_Run_LEAP:
|
||||
def __init__(self):
|
||||
self.algo = LEAP_C()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user