mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix: revert LEAP bug fix
This commit is contained in:
parent
c220521172
commit
cda06b830b
@ -907,6 +907,8 @@ if __name__ == "__main__":
|
|||||||
def External_Run_AHSF(frame_gray):
|
def External_Run_AHSF(frame_gray):
|
||||||
average_color = np.mean(frame_gray)
|
average_color = np.mean(frame_gray)
|
||||||
orig_height, orig_width = frame_gray.shape
|
orig_height, orig_width = frame_gray.shape
|
||||||
|
|
||||||
|
frame_clear_resize = frame_gray.copy()
|
||||||
org_frame_gray = frame_gray.copy()
|
org_frame_gray = frame_gray.copy()
|
||||||
|
|
||||||
frame_gray = cv2.resize(frame_gray, (130, 130)) # TODO TEST FIXED RESIZE
|
frame_gray = cv2.resize(frame_gray, (130, 130)) # TODO TEST FIXED RESIZE
|
||||||
@ -928,7 +930,7 @@ def External_Run_AHSF(frame_gray):
|
|||||||
square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray
|
square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray
|
||||||
|
|
||||||
frame_gray = square_background
|
frame_gray = square_background
|
||||||
frame_clear_resize = frame_gray.copy()
|
|
||||||
|
|
||||||
wh_step = max((int(max_dimension / 80)),1) # TODO: FINETUNE VALUES
|
wh_step = max((int(max_dimension / 80)),1) # TODO: FINETUNE VALUES
|
||||||
xy_step = max(int(max_dimension / 24), 1) # TODO: FINETUNE VALUES
|
xy_step = max(int(max_dimension / 24), 1) # TODO: FINETUNE VALUES
|
||||||
@ -1025,7 +1027,7 @@ def External_Run_AHSF(frame_gray):
|
|||||||
minor_diameter = min(width, height)
|
minor_diameter = min(width, height)
|
||||||
average_diameter = (major_diameter + minor_diameter) / 2
|
average_diameter = (major_diameter + minor_diameter) / 2
|
||||||
|
|
||||||
|
# print(orig_width, orig_height)
|
||||||
|
|
||||||
|
|
||||||
return org_frame_gray, frame_clear_resize, x_center, y_center, abs(width - height)
|
return org_frame_gray, frame_clear_resize, x_center, y_center, abs(width - height)
|
||||||
|
|||||||
@ -50,9 +50,37 @@ DATASET CONTRIBUTIONS:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
,@@@@@@
|
||||||
|
@@@@@@@@@@@ @@@
|
||||||
|
@@@@@@@@@@@@ @@@@@@@@@@@
|
||||||
|
@@@@@@@@@@@@@ @@@@@@@@@@@@@@
|
||||||
|
@@@@@@@/ ,@@@@@@@@@@@@@
|
||||||
|
/@@@@@@@@@@@@@@@ @@@@@@@@
|
||||||
|
@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@
|
||||||
|
@@@@@@@@ @@@@@
|
||||||
|
,@@@ @@@@&
|
||||||
|
@@@@@@. @@@@
|
||||||
|
@@@ @@@@@@@@@/ @@@@@
|
||||||
|
,@@@. @@@@@@((@ @@@@(
|
||||||
|
//@@@ ,, @@@@ @@@@@
|
||||||
|
@@@( @@@@@@@
|
||||||
|
@@@ @ @@@@@@@@#
|
||||||
|
@@@@@@@@@@@@@@@@@
|
||||||
|
@@@@@@@@@@@@@(
|
||||||
|
|
||||||
|
LEAP by: Prohurtz
|
||||||
|
Algorithm App Implementation By: Prohurtz
|
||||||
|
|
||||||
|
Copyright (c) 2023 EyeTrackVR <3
|
||||||
|
LICENSE: GNU GPLv3
|
||||||
|
------------------------------------------------------------------------------------------------------
|
||||||
|
"""
|
||||||
# LEAP = Lightweight Eyelid And Pupil
|
# LEAP = Lightweight Eyelid And Pupil
|
||||||
import os
|
import os
|
||||||
|
|
||||||
os.environ["OMP_NUM_THREADS"] = "1"
|
os.environ["OMP_NUM_THREADS"] = "1"
|
||||||
import onnxruntime
|
import onnxruntime
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@ -67,14 +95,40 @@ import sys
|
|||||||
from utils.misc_utils import resource_path
|
from utils.misc_utils import resource_path
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
frames = 0
|
frames = 0
|
||||||
models = Path("Models")
|
models = Path("Models")
|
||||||
|
|
||||||
|
|
||||||
|
def run_model(input_queue, output_queue, session):
|
||||||
|
while True:
|
||||||
|
frame = input_queue.get()
|
||||||
|
if frame is None:
|
||||||
|
break
|
||||||
|
|
||||||
|
img_np = np.array(frame)
|
||||||
|
img_np = img_np.astype(np.float32) / 255.0
|
||||||
|
gray_img = 0.299 * img_np[:, :, 0] + 0.587 * img_np[:, :, 1] + 0.114 * img_np[:, :, 2]
|
||||||
|
|
||||||
|
# Add the channel and batch dimensions
|
||||||
|
gray_img = np.expand_dims(gray_img, axis=0) # Add channel dimension
|
||||||
|
img_np = np.expand_dims(gray_img, axis=0) # Add batch dimension
|
||||||
|
# img_np = np.transpose(img_np, (2, 0, 1))
|
||||||
|
# img_np = np.expand_dims(img_np, axis=0)
|
||||||
|
ort_inputs = {session.get_inputs()[0].name: img_np}
|
||||||
|
pre_landmark = session.run(None, ort_inputs)
|
||||||
|
|
||||||
|
# pre_landmark = pre_landmark[1]
|
||||||
|
# pre_landmark = np.reshape(pre_landmark, (12, 2))
|
||||||
|
pre_landmark = np.reshape(pre_landmark, (-1, 2))
|
||||||
|
output_queue.put((frame, pre_landmark))
|
||||||
|
|
||||||
|
|
||||||
class LEAP_C(object):
|
class LEAP_C(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
onnxruntime.disable_telemetry_events()
|
onnxruntime.disable_telemetry_events()
|
||||||
# Config variables
|
# Config variables
|
||||||
self.num_threads = 1 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
|
self.num_threads = 4 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
|
||||||
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
|
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
|
||||||
self.model_path = resource_path(models / 'LEAP062120246epoch.onnx')
|
self.model_path = resource_path(models / 'LEAP062120246epoch.onnx')
|
||||||
|
|
||||||
@ -142,28 +196,25 @@ class LEAP_C(object):
|
|||||||
|
|
||||||
self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=["CPUExecutionProvider"])
|
self.ort_session1 = onnxruntime.InferenceSession(self.model_path, opts, providers=["CPUExecutionProvider"])
|
||||||
|
|
||||||
def run_model(output_queue, session, frame):
|
threads = []
|
||||||
|
for i in range(self.num_threads):
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=run_model,
|
||||||
|
args=(self.queues[i], self.output_queue, self.ort_session1),
|
||||||
|
name=f"Thread {i}",
|
||||||
|
)
|
||||||
|
threads.append(thread)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
img_np = np.array(frame)
|
|
||||||
img_np = img_np.astype(np.float32) / 255.0
|
|
||||||
gray_img = 0.299 * img_np[:, :, 0] + 0.587 * img_np[:, :, 1] + 0.114 * img_np[:, :, 2]
|
|
||||||
|
|
||||||
# Add the channel and batch dimensions
|
|
||||||
gray_img = np.expand_dims(gray_img, axis=0) # Add channel dimension
|
|
||||||
img_np = np.expand_dims(gray_img, axis=0) # Add batch dimension
|
|
||||||
# img_np = np.transpose(img_np, (2, 0, 1))
|
|
||||||
# img_np = np.expand_dims(img_np, axis=0)
|
|
||||||
ort_inputs = {session.get_inputs()[0].name: img_np}
|
|
||||||
pre_landmark = session.run(None, ort_inputs)
|
|
||||||
|
|
||||||
# pre_landmark = pre_landmark[1]
|
|
||||||
# pre_landmark = np.reshape(pre_landmark, (12, 2))
|
|
||||||
pre_landmark = np.reshape(pre_landmark, (-1, 2))
|
|
||||||
# output_queue.put((frame, pre_landmark))
|
|
||||||
return frame, pre_landmark
|
|
||||||
def to_numpy(self, tensor):
|
def to_numpy(self, tensor):
|
||||||
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
|
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
|
||||||
|
|
||||||
|
def run_onnx_model(self, queues, session, frame):
|
||||||
|
for i in range(len(queues)):
|
||||||
|
if not queues[i].full():
|
||||||
|
queues[i].put(frame)
|
||||||
|
break
|
||||||
|
|
||||||
def leap_run(self):
|
def leap_run(self):
|
||||||
|
|
||||||
img = self.current_image_gray_clean.copy()
|
img = self.current_image_gray_clean.copy()
|
||||||
@ -173,8 +224,14 @@ class LEAP_C(object):
|
|||||||
|
|
||||||
frame = cv2.resize(img, (112, 112))
|
frame = cv2.resize(img, (112, 112))
|
||||||
imgvis = self.current_image_gray.copy()
|
imgvis = self.current_image_gray.copy()
|
||||||
|
self.run_onnx_model(self.queues, self.ort_session1, frame)
|
||||||
|
|
||||||
frame, pre_landmark = self.run_model(self.ort_session1, frame)
|
if not self.output_queue.empty():
|
||||||
|
|
||||||
|
frame, pre_landmark = self.output_queue.get()
|
||||||
|
# pre_landmark = np.reshape(pre_landmark, (-1, 2))
|
||||||
|
|
||||||
|
# pre_landmark = self.one_euro_filter(pre_landmark)
|
||||||
|
|
||||||
for point in pre_landmark:
|
for point in pre_landmark:
|
||||||
# x, y = (point*112).astype(int)
|
# x, y = (point*112).astype(int)
|
||||||
@ -189,6 +246,11 @@ class LEAP_C(object):
|
|||||||
cv2.circle(imgvis, (int(x), int(y)), 2, (255, 255, 0), -1)
|
cv2.circle(imgvis, (int(x), int(y)), 2, (255, 255, 0), -1)
|
||||||
|
|
||||||
|
|
||||||
|
x1, y1 = pre_landmark[1]
|
||||||
|
x2, y2 = pre_landmark[3]
|
||||||
|
|
||||||
|
x3, y3 = pre_landmark[4]
|
||||||
|
x4, y4 = pre_landmark[2]
|
||||||
|
|
||||||
d1 = math.dist(pre_landmark[1], pre_landmark[3])
|
d1 = math.dist(pre_landmark[1], pre_landmark[3])
|
||||||
# a more fancy method could be used taking into acount the relative size of the landmarks so that weirdness can be acounted for better
|
# a more fancy method could be used taking into acount the relative size of the landmarks so that weirdness can be acounted for better
|
||||||
@ -257,6 +319,8 @@ class LEAP_C(object):
|
|||||||
|
|
||||||
return imgvis, float(x), float(y), per
|
return imgvis, float(x), float(y), per
|
||||||
|
|
||||||
|
imgvis = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
||||||
|
return imgvis, 0, 0, 0
|
||||||
|
|
||||||
|
|
||||||
class External_Run_LEAP(object):
|
class External_Run_LEAP(object):
|
||||||
@ -268,5 +332,3 @@ class External_Run_LEAP(object):
|
|||||||
self.algo.current_image_gray_clean = current_image_gray_clean
|
self.algo.current_image_gray_clean = current_image_gray_clean
|
||||||
img, x, y, per = self.algo.leap_run()
|
img, x, y, per = self.algo.leap_run()
|
||||||
return img, x, y, per
|
return img, x, y, per
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user