feat: LEAP Lid blink algo

This commit is contained in:
Prohurtz 2023-09-26 08:46:44 -05:00
parent 6297ce327c
commit ab631fad19
4 changed files with 317 additions and 248 deletions

View File

@ -7,7 +7,9 @@ from threading import Event
class AlgoSettingsWidget:
def __init__(self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue):
def __init__(
self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue
):
self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-"
self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-"
@ -55,91 +57,97 @@ class AlgoSettingsWidget:
self.gui_legacy_ransac = f"-LEGACYRANSACTHRESH{widget_id}-"
self.gui_legacy_ransac_thresh_right = f"-THRESHRIGHT{widget_id}-"
self.gui_legacy_ransac_thresh_left = f"-THRESHLEFT{widget_id}-"
self.gui_LEAP_lid = f"-LEAPLID{widget_id}-"
self.main_config = main_config
self.config = main_config.settings
self.osc_queue = osc_queue
# Define the window's contents
self.general_settings_layout = [
[sg.Checkbox(
"",
default=self.config.gui_HSRAC,
key=self.gui_HSRAC,
background_color='#424042',
tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_HSRACP,
key=self.gui_HSRACP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("HSRAC", background_color='#424042'),
[
sg.Checkbox(
"",
default=self.config.gui_HSRAC,
key=self.gui_HSRAC,
background_color="#424042",
tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.",
),
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_HSRACP,
key=self.gui_HSRACP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("HSRAC", background_color="#424042"),
# ],
# [
sg.Checkbox(
"",
default=self.config.gui_HSF,
key=self.gui_HSF,
background_color='#424042',
background_color="#424042",
tooltip="HSF Is a new, lower resolution tracking algorithim that provides excelent resilancy to lighting conditions and great speed.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_HSFP,
key=self.gui_HSFP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithims.",
),
sg.Text("Haar Surround Feature", background_color='#424042'),
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_HSFP,
key=self.gui_HSFP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithims.",
),
sg.Text("Haar Surround Feature", background_color="#424042"),
],
[sg.Checkbox(
"",
default=self.config.gui_DADDY,
key=self.gui_DADDY,
background_color='#424042',
tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_DADDYP,
key=self.gui_DADDYP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("DADDY", background_color='#424042'),
[
sg.Checkbox(
"",
default=self.config.gui_DADDY,
key=self.gui_DADDY,
background_color="#424042",
tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.",
),
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_DADDYP,
key=self.gui_DADDYP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("DADDY", background_color="#424042"),
# ],
# [
sg.Checkbox(
"",
default=self.config.gui_RANSAC3D,
key=self.gui_RANSAC3D,
background_color='#424042',
background_color="#424042",
tooltip="RANSAC3D provides good tracking quality, however does not do well in bad lighting conditions.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_RANSAC3DP,
key=self.gui_RANSAC3DP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("RANSAC 3D", background_color='#424042'),
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_RANSAC3DP,
key=self.gui_RANSAC3DP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("RANSAC 3D", background_color="#424042"),
sg.Checkbox(
"Legacy RANSAC Thresh",
default=self.config.gui_legacy_ransac,
key=self.gui_legacy_ransac,
background_color='#424042',
background_color="#424042",
),
],
[
@ -147,77 +155,80 @@ class AlgoSettingsWidget:
"",
default=self.config.gui_LEAP,
key=self.gui_LEAP,
background_color='#424042',
background_color="#424042",
tooltip="LEAP Uses a lightweight deep learning algorithm.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_LEAPP,
key=self.gui_LEAPP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("LEAP", background_color='#424042'),
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_LEAPP,
key=self.gui_LEAPP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("LEAP", background_color="#424042"),
sg.Checkbox(
"",
default=self.config.gui_BLOB,
key=self.gui_BLOB,
background_color='#424042',
background_color="#424042",
tooltip="Blob tracking is the oldest and worst tracking algorithm, it provides fast, though sometimes inaccurate tracking.",
),
sg.Combo(['1', '2', '3', '4', '5', '6'],
default_value=self.config.gui_BLOBP,
key=self.gui_BLOBP,
background_color='#424042',
text_color='white',
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("Blob", background_color='#424042'),
],
[
sg.Text("Blink Algo Settings:", background_color='#242224')
sg.Combo(
["1", "2", "3", "4", "5", "6"],
default_value=self.config.gui_BLOBP,
key=self.gui_BLOBP,
background_color="#424042",
text_color="white",
button_arrow_color="black",
button_background_color="#6f4ca1",
tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("Blob", background_color="#424042"),
],
[sg.Text("Blink Algo Settings:", background_color="#242224")],
[
sg.Checkbox(
"Intensity Based Openness",
default=self.config.gui_IBO,
key=self.gui_IBO,
background_color='#424042',
background_color="#424042",
),
sg.Checkbox(
"RANSAC Quick Blink Algo",
default=self.config.gui_RANSACBLINK,
key=self.gui_RANSACBLINK,
background_color='#424042',
background_color="#424042",
),
sg.Checkbox(
"Binary Blink Algo",
default=self.config.gui_BLINK,
key=self.gui_BLINK,
background_color='#424042',
background_color="#424042",
),
sg.Checkbox(
"LEAP Lid",
default=self.config.gui_LEAP_lid,
key=self.gui_LEAP_lid,
background_color="#424042",
),
],
[
sg.Text("IBO Filter Sample Size", background_color='#424042'),
sg.Text("IBO Filter Sample Size", background_color="#424042"),
sg.InputText(
self.config.ibo_filter_samples,
key=self.ibo_filter_samples,
size=(0, 10),
),
sg.Text("Calibration Samples", background_color='#424042'),
sg.Text("Calibration Samples", background_color="#424042"),
sg.InputText(
self.config.calibration_samples,
key=self.calibration_samples,
size=(0, 10),
),
sg.Text("IBO Close Threshold", background_color='#424042'),
sg.Text("IBO Close Threshold", background_color="#424042"),
sg.InputText(
self.config.ibo_fully_close_eye_threshold,
key=self.ibo_fully_close_eye_threshold,
@ -229,121 +240,130 @@ class AlgoSettingsWidget:
"Left Eye Circle crop",
default=self.config.gui_circular_crop_left,
key=self.gui_circular_crop_left,
background_color='#424042',
background_color="#424042",
),
sg.Checkbox(
"Right Eye Circle crop",
default=self.config.gui_circular_crop_right,
key=self.gui_circular_crop_right,
background_color='#424042',
background_color="#424042",
),
],
[
sg.Text("Advanced Tracking Algorithim Settings:", background_color='#242224')
],
[sg.Checkbox(
"HSF: Skip Auto Radius",
default=self.config.gui_skip_autoradius,
key=self.gui_skip_autoradius,
background_color='#424042',
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
),
sg.Text(
"Advanced Tracking Algorithim Settings:", background_color="#242224"
)
],
[
sg.Text("Left HSF Radius:", background_color='#424042'),
sg.Checkbox(
"HSF: Skip Auto Radius",
default=self.config.gui_skip_autoradius,
key=self.gui_skip_autoradius,
background_color="#424042",
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
),
],
[
sg.Text("Left HSF Radius:", background_color="#424042"),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_HSF_radius_left,
orientation="h",
key=self.gui_HSF_radius_left,
background_color='#424042',
background_color="#424042",
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
),
sg.Text("Right HSF Radius:", background_color='#424042'),
sg.Text("Right HSF Radius:", background_color="#424042"),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_HSF_radius_right,
orientation="h",
key=self.gui_HSF_radius_right,
background_color='#424042',
background_color="#424042",
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
),
],
[sg.Text("RANSAC Thresh Add", background_color='#424042'),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_thresh_add,
orientation="h",
key=self.gui_thresh_add,
background_color='#424042',
tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.",
),
sg.Text("Blob Threshold", background_color='#424042'),
# TODO make this for right and left eyes? I dont know how vital that is..
sg.Slider(
range=(0, 110),
default_value=self.config.gui_threshold,
orientation="h",
key=self.gui_threshold_slider,
background_color='#424042',
tooltip="Adjusts the threshold for blob tracking.",
),
],
[sg.Text("Min Blob Size:", background_color='#424042'),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_blob_minsize,
orientation="h",
key=self.gui_blob_minsize,
background_color='#424042',
tooltip="Minimum size a blob has to be for blob tracking.",
),
sg.Text("Max Blob Size:", background_color='#424042'),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_blob_maxsize,
orientation="h",
key=self.gui_blob_maxsize,
background_color='#424042',
tooltip="Maximum size a blob can be for blob tracking.",
),
],
[
sg.Text("Right Eye Thresh:", background_color='#424042'),
sg.Slider(
range=(1, 120),
default_value=self.config.gui_legacy_ransac_thresh_right,
orientation="h",
key=self.gui_legacy_ransac_thresh_right,
background_color='#424042',
tooltip="Threshold for right eye, legacy RANSAC only",
),
sg.Text("Left Eye Thresh:", background_color='#424042'),
sg.Slider(
range=(1, 120),
default_value=self.config.gui_legacy_ransac_thresh_left,
orientation="h",
key=self.gui_legacy_ransac_thresh_left,
background_color='#424042',
tooltip="Threshold for left eye, legacy RANSAC only",
),
[
sg.Text("RANSAC Thresh Add", background_color="#424042"),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_thresh_add,
orientation="h",
key=self.gui_thresh_add,
background_color="#424042",
tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.",
),
sg.Text("Blob Threshold", background_color="#424042"),
# TODO make this for right and left eyes? I dont know how vital that is..
sg.Slider(
range=(0, 110),
default_value=self.config.gui_threshold,
orientation="h",
key=self.gui_threshold_slider,
background_color="#424042",
tooltip="Adjusts the threshold for blob tracking.",
),
],
[
sg.Text("Min Blob Size:", background_color="#424042"),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_blob_minsize,
orientation="h",
key=self.gui_blob_minsize,
background_color="#424042",
tooltip="Minimum size a blob has to be for blob tracking.",
),
sg.Text("Max Blob Size:", background_color="#424042"),
sg.Slider(
range=(1, 50),
default_value=self.config.gui_blob_maxsize,
orientation="h",
key=self.gui_blob_maxsize,
background_color="#424042",
tooltip="Maximum size a blob can be for blob tracking.",
),
],
[
sg.Text("Right Eye Thresh:", background_color="#424042"),
sg.Slider(
range=(1, 120),
default_value=self.config.gui_legacy_ransac_thresh_right,
orientation="h",
key=self.gui_legacy_ransac_thresh_right,
background_color="#424042",
tooltip="Threshold for right eye, legacy RANSAC only",
),
sg.Text("Left Eye Thresh:", background_color="#424042"),
sg.Slider(
range=(1, 120),
default_value=self.config.gui_legacy_ransac_thresh_left,
orientation="h",
key=self.gui_legacy_ransac_thresh_left,
background_color="#424042",
tooltip="Threshold for left eye, legacy RANSAC only",
),
],
]
self.widget_layout = [
[
sg.Text("Tracking Algorithm Order Settings:", background_color='#242224'),
sg.Text(
"Tracking Algorithm Order Settings:", background_color="#242224"
),
],
[
sg.Column(self.general_settings_layout, key=self.gui_general_settings_layout,
background_color='#424042'),
sg.Column(
self.general_settings_layout,
key=self.gui_general_settings_layout,
background_color="#424042",
),
],
]
self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called.
self.cancellation_event = (
Event()
) # Set the event until start is called, otherwise we can block if shutdown is called.
self.cancellation_event.set()
self.image_queue = Queue()
@ -383,7 +403,8 @@ class AlgoSettingsWidget:
changed = True
if self.config.gui_RANSAC3DP != int(
values[self.gui_RANSAC3DP]): # TODO check that priority order is unique/auto fix it.
values[self.gui_RANSAC3DP]
): # TODO check that priority order is unique/auto fix it.
self.config.gui_RANSAC3DP = int(values[self.gui_RANSAC3DP])
changed = True
@ -427,6 +448,10 @@ class AlgoSettingsWidget:
self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK]
changed = True
if self.config.gui_LEAP_lid != values[self.gui_LEAP_lid]:
self.config.gui_LEAP_lid = values[self.gui_LEAP_lid]
changed = True
if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]:
self.config.gui_circular_crop_left = values[self.gui_circular_crop_left]
changed = True
@ -467,20 +492,32 @@ class AlgoSettingsWidget:
self.config.ibo_filter_samples = int(values[self.ibo_filter_samples])
changed = True
if self.config.ibo_fully_close_eye_threshold != float(values[self.ibo_fully_close_eye_threshold]):
self.config.ibo_fully_close_eye_threshold = float(values[self.ibo_fully_close_eye_threshold])
if self.config.ibo_fully_close_eye_threshold != float(
values[self.ibo_fully_close_eye_threshold]
):
self.config.ibo_fully_close_eye_threshold = float(
values[self.ibo_fully_close_eye_threshold]
)
changed = True
if self.config.calibration_samples != int(values[self.calibration_samples]):
self.config.calibration_samples = int(values[self.calibration_samples])
changed = True
if self.config.gui_legacy_ransac_thresh_left != int(values[self.gui_legacy_ransac_thresh_left]):
self.config.gui_legacy_ransac_thresh_left = int(values[self.gui_legacy_ransac_thresh_left])
if self.config.gui_legacy_ransac_thresh_left != int(
values[self.gui_legacy_ransac_thresh_left]
):
self.config.gui_legacy_ransac_thresh_left = int(
values[self.gui_legacy_ransac_thresh_left]
)
changed = True
if self.config.gui_legacy_ransac_thresh_right != int(values[self.gui_legacy_ransac_thresh_right]):
self.config.gui_legacy_ransac_thresh_right = int(values[self.gui_legacy_ransac_thresh_right])
if self.config.gui_legacy_ransac_thresh_right != int(
values[self.gui_legacy_ransac_thresh_right]
):
self.config.gui_legacy_ransac_thresh_right = int(
values[self.gui_legacy_ransac_thresh_right]
)
changed = True
if changed:

View File

@ -77,7 +77,7 @@ class EyeTrackSettingsConfig(BaseModel):
gui_legacy_ransac: bool = False
gui_legacy_ransac_thresh_right: int = 80
gui_legacy_ransac_thresh_left: int = 80
gui_LEAP_lid: bool = False
class EyeTrackConfig(BaseModel):
@ -85,7 +85,7 @@ class EyeTrackConfig(BaseModel):
right_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
left_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
# algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
# algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
eye_display_id: EyeId = EyeId.RIGHT
@staticmethod

View File

@ -34,6 +34,7 @@ from operator import truth
from dataclasses import dataclass
import sys
import asyncio
sys.path.append(".")
from config import EyeTrackCameraConfig
from config import EyeTrackSettingsConfig
@ -252,7 +253,9 @@ class EyeProcessor:
if self.settings.gui_BLINK:
self.eyeopen = BLINK(self)
if self.settings.gui_IBO and self.eyeopen != 0.0: #TODO make ransac blink it's pwn self var to rid of this non-sense
if (
self.settings.gui_IBO and self.eyeopen != 0.0
): # TODO make ransac blink it's pwn self var to rid of this non-sense
self.eyeopen = self.ibo.intense(
self.rawx,
self.rawy,
@ -268,8 +271,7 @@ class EyeProcessor:
if self.bd_blink == True:
pass
if self.settings.gui_IBO and self.settings.gui_BLINK and self.eyeopen != 0.0:
if self.settings.gui_IBO and self.eyeopen != 0.0:
ibo = self.ibo.intense(
self.rawx,
self.rawy,
@ -278,27 +280,30 @@ class EyeProcessor:
self.settings.ibo_average_output_samples,
)
blink = BLINK(self)
if blink == 0.0:
self.eyeopen = 0.0
else:
self.eyeopen = ibo
if self.settings.gui_LEAP_lid:
(
self.current_image_gray,
self.rawx,
self.rawy,
self.eyeopen,
) = self.er_leap.run(self.current_image_gray)
if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING, kinda broke
if (
len(self.prev_y_list) >= 200
): # "lock" eye when close/blink IN TESTING, kinda broke
self.prev_y_list.pop(0)
self.prev_y_list.append(self.out_y)
else:
self.prev_y_list.append(self.out_y)
# print(abs(self.eyeopen - self.past_blink))
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1
# print(abs(self.eyeopen - self.past_blink))
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
#if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
# if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
# print('AVG', self.out_y, len(self.prev_y_list))
# print('AVG', self.out_y, len(self.prev_y_list))
self.past_blink = self.eyeopen
self.prev_x = self.out_x
@ -313,24 +318,23 @@ class EyeProcessor:
else:
self.eyeopen = 0.9
def BLINKM(self):
self.eyeopen = BLINK(self)
def LEAPM(self):
self.thresh = self.current_image_gray.copy()
self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(self.current_image_gray) #TODO: make own self var and LEAP toggle
self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(
self.current_image_gray
) # TODO: make own self var and LEAP toggle
self.thresh = self.current_image_gray.copy()
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.LEAP
def DADDYM(self):
# todo: We should have a proper variable for drawing.
#self.thresh = self.current_image_gray.copy()
# self.thresh = self.current_image_gray.copy()
self.thresh = self.current_image_gray.copy()
self.rawx, self.rawy, self.radius = self.er_daddy.run(
self.current_image_gray
)
self.rawx, self.rawy, self.radius = self.er_daddy.run(self.current_image_gray)
# Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it.
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.DADDY
@ -355,7 +359,7 @@ class EyeProcessor:
self.current_image_gray
)
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
if self.settings.gui_RANSACBLINK: #might be redundant
if self.settings.gui_RANSACBLINK: # might be redundant
self.eyeopen = ranblink
# print(self.radius)
@ -379,7 +383,9 @@ class EyeProcessor:
else:
pass
# todo: add process to initialise er_hsf when resolution changes
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray)
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
self.current_image_gray
)
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.HSF
@ -526,7 +532,7 @@ class EyeProcessor:
if self.er_daddy is not None:
self.er_daddy = None
if self.settings.gui_LEAP:
if self.settings.gui_LEAP or self.settings.gui_LEAP_lid:
if self.er_leap is None:
self.er_leap = External_Run_LEAP()
algolist[self.settings.gui_LEAP] = self.LEAPM

View File

@ -27,6 +27,7 @@ Copyright (c) 2023 EyeTrackVR <3
"""
# LEAP = Lightweight Eyelid And Pupil
import os
os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime
import numpy as np
@ -37,21 +38,23 @@ from queue import Queue
import threading
from one_euro_filter import OneEuroFilter
import psutil, os
import sys
import sys
from utils.misc_utils import resource_path
import platform
frames = 0
def run_model(input_queue, output_queue, session):
while True:
frame = input_queue.get()
if frame is None:
break
# to_tensor = transforms.ToTensor()
# img_tensor = to_tensor(frame)
# img_tensor.unsqueeze_(0)
# img_np = img_tensor.numpy()
# to_tensor = transforms.ToTensor()
# img_tensor = to_tensor(frame)
# img_tensor.unsqueeze_(0)
# img_np = img_tensor.numpy()
img_np = np.array(frame)
# Normalize the pixel values to [0, 1] and convert the data type to float32
img_np = img_np.astype(np.float32) / 255.0
@ -69,8 +72,6 @@ def run_model(input_queue, output_queue, session):
output_queue.put((frame, pre_landmark))
class LEAP_C(object):
def __init__(self):
onnxruntime.disable_telemetry_events()
@ -78,9 +79,11 @@ class LEAP_C(object):
self.num_threads = 3 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
if platform.system() == "Darwin":
self.model_path = resource_path('EyeTrackApp/Models/mommy072623.onnx') # funny MacOS files issues :P
self.model_path = resource_path(
"EyeTrackApp/Models/mommy072623.onnx"
) # funny MacOS files issues :P
else:
self.model_path = resource_path('Models/mommy072623.onnx')
self.model_path = resource_path("Models/mommy072623.onnx")
self.interval = 1 # FPS print update rate
self.low_priority = True # set process priority to low
self.print_fps = True
@ -99,9 +102,13 @@ class LEAP_C(object):
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
opts.optimized_model_filepath = ''
self.ort_session = onnxruntime.InferenceSession(self.model_path, opts, providers=['CPUExecutionProvider'])
opts.graph_optimization_level = (
onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
)
opts.optimized_model_filepath = ""
self.ort_session = onnxruntime.InferenceSession(
self.model_path, opts, providers=["CPUExecutionProvider"]
)
if self.low_priority:
process = psutil.Process(os.getpid()) # set process priority to low
@ -119,9 +126,7 @@ class LEAP_C(object):
# print(np.random.rand(22, 2))
# noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
one_euro_filter = OneEuroFilter(
np.random.rand(7, 2),
min_cutoff=min_cutoff,
beta=beta
np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta
)
self.dmax = 0
self.dmin = 0
@ -129,37 +134,40 @@ class LEAP_C(object):
self.x = 0
self.y = 0
self.ort_session1 = onnxruntime.InferenceSession(
self.model_path, opts,
providers=['CPUExecutionProvider'])
self.model_path, opts, providers=["CPUExecutionProvider"]
)
# ort_session1 = onnxruntime.InferenceSession("C:/Users/beaul/PycharmProjects/EyeTrackVR/EyeTrackApp/Models/mommy062023.onnx", opts, providers=['DmlExecutionProvider'])
threads = []
for i in range(self.num_threads):
thread = threading.Thread(target=run_model, args=(self.queues[i], self.output_queue, self.ort_session1),
name=f"Thread {i}")
thread = threading.Thread(
target=run_model,
args=(self.queues[i], self.output_queue, self.ort_session1),
name=f"Thread {i}",
)
threads.append(thread)
thread.start()
def to_numpy(self, tensor):
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
return (
tensor.detach().cpu().numpy()
if tensor.requires_grad
else tensor.cpu().numpy()
)
def run_onnx_model(self, queues, session, frame):
for i in range(len(queues)):
if not queues[i].full():
queues[i].put(frame)
break
def leap_run(self):
def leap_run(self):
img = self.current_image_gray.copy()
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# img = imutils.rotate(img, angle=320)
img_height, img_width = img.shape[:2] # Move outside the loop
frame = cv2.resize(img, (112, 112))
self.run_onnx_model(self.queues, self.ort_session1, frame)
@ -170,12 +178,26 @@ class LEAP_C(object):
for point in pre_landmark:
x, y = point
cv2.circle(img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1)
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
# print(pre_landmark)
cv2.circle(
img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1
)
cv2.circle(
img,
tuple(int(x * img_width) for x in pre_landmark[2]),
1,
(255, 255, 0),
-1,
)
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
cv2.circle(
img,
tuple(int(x * img_width) for x in pre_landmark[4]),
1,
(255, 255, 255),
-1,
)
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
# print(pre_landmark)
x1, y1 = pre_landmark[0]
x2, y2 = pre_landmark[6]
@ -188,27 +210,31 @@ class LEAP_C(object):
x4, y4 = pre_landmark[2]
euclidean_dist_open = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
# d = area / euclidean_dist_width
# print(area)
# d = area / euclidean_dist_width
# print(area)
eyesize_dist = math.dist(pre_landmark[0], pre_landmark[6])
distance = math.dist(pre_landmark[1], pre_landmark[3])
# d = distance / eyesize_dist
# d = distance / eyesize_dist
d = math.dist(pre_landmark[1], pre_landmark[3])
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
# d = d + d2
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
# d = d + d2
if len(self.openlist) < 2000: # TODO expose as setting?
if len(self.openlist) < 2000: # TODO expose as setting?
self.openlist.append(d)
else:
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(self.openlist, 1):
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(
self.openlist, 1
):
pass
else:
self.openlist.pop(0)
self.openlist.append(d)
try:
per = ((d - max(self.openlist)) / (min(self.openlist) - max(self.openlist)))
per = (d - max(self.openlist)) / (
min(self.openlist) - max(self.openlist)
)
per = 1 - per
except:
per = 0.7
@ -221,13 +247,13 @@ class LEAP_C(object):
self.last_lid = per
if per <= 0.1:
per == 0.0
# print(per)
# print(per)
return frame, float(x), float(y), per
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
return frame, 0, 0, 0
class External_Run_LEAP(object):
def __init__(self):
self.algo = LEAP_C()
@ -235,4 +261,4 @@ class External_Run_LEAP(object):
def run(self, current_image_gray):
self.algo.current_image_gray = current_image_gray
img, x, y, per = self.algo.leap_run()
return img, x, y, per
return img, x, y, per