feat: LEAP Lid blink algo

This commit is contained in:
Prohurtz 2023-09-26 08:46:44 -05:00
parent 6297ce327c
commit ab631fad19
4 changed files with 317 additions and 248 deletions

View File

@ -7,7 +7,9 @@ from threading import Event
class AlgoSettingsWidget: class AlgoSettingsWidget:
def __init__(self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue): def __init__(
self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue
):
self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-" self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-"
self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-" self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-"
@ -55,91 +57,97 @@ class AlgoSettingsWidget:
self.gui_legacy_ransac = f"-LEGACYRANSACTHRESH{widget_id}-" self.gui_legacy_ransac = f"-LEGACYRANSACTHRESH{widget_id}-"
self.gui_legacy_ransac_thresh_right = f"-THRESHRIGHT{widget_id}-" self.gui_legacy_ransac_thresh_right = f"-THRESHRIGHT{widget_id}-"
self.gui_legacy_ransac_thresh_left = f"-THRESHLEFT{widget_id}-" self.gui_legacy_ransac_thresh_left = f"-THRESHLEFT{widget_id}-"
self.gui_LEAP_lid = f"-LEAPLID{widget_id}-"
self.main_config = main_config self.main_config = main_config
self.config = main_config.settings self.config = main_config.settings
self.osc_queue = osc_queue self.osc_queue = osc_queue
# Define the window's contents # Define the window's contents
self.general_settings_layout = [ self.general_settings_layout = [
[
[sg.Checkbox( sg.Checkbox(
"", "",
default=self.config.gui_HSRAC, default=self.config.gui_HSRAC,
key=self.gui_HSRAC, key=self.gui_HSRAC,
background_color='#424042', background_color="#424042",
tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.", tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.",
), ),
sg.Combo(['1', '2', '3', '4', '5', '6'], sg.Combo(
default_value=self.config.gui_HSRACP, ["1", "2", "3", "4", "5", "6"],
key=self.gui_HSRACP, default_value=self.config.gui_HSRACP,
background_color='#424042', key=self.gui_HSRACP,
text_color='white', background_color="#424042",
button_arrow_color="black", text_color="white",
button_background_color="#6f4ca1", button_arrow_color="black",
tooltip="Select the priority of eyetracking algorithms.", button_background_color="#6f4ca1",
), tooltip="Select the priority of eyetracking algorithms.",
sg.Text("HSRAC", background_color='#424042'), ),
sg.Text("HSRAC", background_color="#424042"),
# ], # ],
# [ # [
sg.Checkbox( sg.Checkbox(
"", "",
default=self.config.gui_HSF, default=self.config.gui_HSF,
key=self.gui_HSF, key=self.gui_HSF,
background_color='#424042', background_color="#424042",
tooltip="HSF Is a new, lower resolution tracking algorithim that provides excelent resilancy to lighting conditions and great speed.", tooltip="HSF Is a new, lower resolution tracking algorithim that provides excelent resilancy to lighting conditions and great speed.",
), ),
sg.Combo(['1', '2', '3', '4', '5', '6'], sg.Combo(
default_value=self.config.gui_HSFP, ["1", "2", "3", "4", "5", "6"],
key=self.gui_HSFP, default_value=self.config.gui_HSFP,
background_color='#424042', key=self.gui_HSFP,
text_color='white', background_color="#424042",
button_arrow_color="black", text_color="white",
button_background_color="#6f4ca1", button_arrow_color="black",
tooltip="Select the priority of eyetracking algorithims.", button_background_color="#6f4ca1",
), tooltip="Select the priority of eyetracking algorithims.",
sg.Text("Haar Surround Feature", background_color='#424042'), ),
sg.Text("Haar Surround Feature", background_color="#424042"),
], ],
[sg.Checkbox( [
"", sg.Checkbox(
default=self.config.gui_DADDY, "",
key=self.gui_DADDY, default=self.config.gui_DADDY,
background_color='#424042', key=self.gui_DADDY,
tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.", background_color="#424042",
), tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.",
sg.Combo(['1', '2', '3', '4', '5', '6'], ),
default_value=self.config.gui_DADDYP, sg.Combo(
key=self.gui_DADDYP, ["1", "2", "3", "4", "5", "6"],
background_color='#424042', default_value=self.config.gui_DADDYP,
text_color='white', key=self.gui_DADDYP,
button_arrow_color="black", background_color="#424042",
button_background_color="#6f4ca1", text_color="white",
tooltip="Select the priority of eyetracking algorithms.", button_arrow_color="black",
), button_background_color="#6f4ca1",
sg.Text("DADDY", background_color='#424042'), tooltip="Select the priority of eyetracking algorithms.",
),
sg.Text("DADDY", background_color="#424042"),
# ], # ],
# [ # [
sg.Checkbox( sg.Checkbox(
"", "",
default=self.config.gui_RANSAC3D, default=self.config.gui_RANSAC3D,
key=self.gui_RANSAC3D, key=self.gui_RANSAC3D,
background_color='#424042', background_color="#424042",
tooltip="RANSAC3D provides good tracking quality, however does not do well in bad lighting conditions.", tooltip="RANSAC3D provides good tracking quality, however does not do well in bad lighting conditions.",
), ),
sg.Combo(['1', '2', '3', '4', '5', '6'], sg.Combo(
default_value=self.config.gui_RANSAC3DP, ["1", "2", "3", "4", "5", "6"],
key=self.gui_RANSAC3DP, default_value=self.config.gui_RANSAC3DP,
background_color='#424042', key=self.gui_RANSAC3DP,
text_color='white', background_color="#424042",
button_arrow_color="black", text_color="white",
button_background_color="#6f4ca1", button_arrow_color="black",
tooltip="Select the priority of eyetracking algorithms.", button_background_color="#6f4ca1",
), tooltip="Select the priority of eyetracking algorithms.",
sg.Text("RANSAC 3D", background_color='#424042'), ),
sg.Text("RANSAC 3D", background_color="#424042"),
sg.Checkbox( sg.Checkbox(
"Legacy RANSAC Thresh", "Legacy RANSAC Thresh",
default=self.config.gui_legacy_ransac, default=self.config.gui_legacy_ransac,
key=self.gui_legacy_ransac, key=self.gui_legacy_ransac,
background_color='#424042', background_color="#424042",
), ),
], ],
[ [
@ -147,77 +155,80 @@ class AlgoSettingsWidget:
"", "",
default=self.config.gui_LEAP, default=self.config.gui_LEAP,
key=self.gui_LEAP, key=self.gui_LEAP,
background_color='#424042', background_color="#424042",
tooltip="LEAP Uses a lightweight deep learning algorithm.", tooltip="LEAP Uses a lightweight deep learning algorithm.",
), ),
sg.Combo(['1', '2', '3', '4', '5', '6'], sg.Combo(
default_value=self.config.gui_LEAPP, ["1", "2", "3", "4", "5", "6"],
key=self.gui_LEAPP, default_value=self.config.gui_LEAPP,
background_color='#424042', key=self.gui_LEAPP,
text_color='white', background_color="#424042",
button_arrow_color="black", text_color="white",
button_background_color="#6f4ca1", button_arrow_color="black",
tooltip="Select the priority of eyetracking algorithms.", button_background_color="#6f4ca1",
), tooltip="Select the priority of eyetracking algorithms.",
sg.Text("LEAP", background_color='#424042'), ),
sg.Text("LEAP", background_color="#424042"),
sg.Checkbox( sg.Checkbox(
"", "",
default=self.config.gui_BLOB, default=self.config.gui_BLOB,
key=self.gui_BLOB, key=self.gui_BLOB,
background_color='#424042', background_color="#424042",
tooltip="Blob tracking is the oldest and worst tracking algorithm, it provides fast, though sometimes inaccurate tracking.", tooltip="Blob tracking is the oldest and worst tracking algorithm, it provides fast, though sometimes inaccurate tracking.",
), ),
sg.Combo(['1', '2', '3', '4', '5', '6'], sg.Combo(
default_value=self.config.gui_BLOBP, ["1", "2", "3", "4", "5", "6"],
key=self.gui_BLOBP, default_value=self.config.gui_BLOBP,
background_color='#424042', key=self.gui_BLOBP,
text_color='white', background_color="#424042",
button_arrow_color="black", text_color="white",
button_background_color="#6f4ca1", button_arrow_color="black",
tooltip="Select the priority of eyetracking algorithms.", button_background_color="#6f4ca1",
), tooltip="Select the priority of eyetracking algorithms.",
sg.Text("Blob", background_color='#424042'), ),
], sg.Text("Blob", background_color="#424042"),
[
sg.Text("Blink Algo Settings:", background_color='#242224')
], ],
[sg.Text("Blink Algo Settings:", background_color="#242224")],
[ [
sg.Checkbox( sg.Checkbox(
"Intensity Based Openness", "Intensity Based Openness",
default=self.config.gui_IBO, default=self.config.gui_IBO,
key=self.gui_IBO, key=self.gui_IBO,
background_color='#424042', background_color="#424042",
), ),
sg.Checkbox( sg.Checkbox(
"RANSAC Quick Blink Algo", "RANSAC Quick Blink Algo",
default=self.config.gui_RANSACBLINK, default=self.config.gui_RANSACBLINK,
key=self.gui_RANSACBLINK, key=self.gui_RANSACBLINK,
background_color='#424042', background_color="#424042",
), ),
sg.Checkbox( sg.Checkbox(
"Binary Blink Algo", "Binary Blink Algo",
default=self.config.gui_BLINK, default=self.config.gui_BLINK,
key=self.gui_BLINK, key=self.gui_BLINK,
background_color='#424042', background_color="#424042",
),
sg.Checkbox(
"LEAP Lid",
default=self.config.gui_LEAP_lid,
key=self.gui_LEAP_lid,
background_color="#424042",
), ),
], ],
[ [
sg.Text("IBO Filter Sample Size", background_color='#424042'), sg.Text("IBO Filter Sample Size", background_color="#424042"),
sg.InputText( sg.InputText(
self.config.ibo_filter_samples, self.config.ibo_filter_samples,
key=self.ibo_filter_samples, key=self.ibo_filter_samples,
size=(0, 10), size=(0, 10),
), ),
sg.Text("Calibration Samples", background_color='#424042'), sg.Text("Calibration Samples", background_color="#424042"),
sg.InputText( sg.InputText(
self.config.calibration_samples, self.config.calibration_samples,
key=self.calibration_samples, key=self.calibration_samples,
size=(0, 10), size=(0, 10),
), ),
sg.Text("IBO Close Threshold", background_color="#424042"),
sg.Text("IBO Close Threshold", background_color='#424042'),
sg.InputText( sg.InputText(
self.config.ibo_fully_close_eye_threshold, self.config.ibo_fully_close_eye_threshold,
key=self.ibo_fully_close_eye_threshold, key=self.ibo_fully_close_eye_threshold,
@ -229,121 +240,130 @@ class AlgoSettingsWidget:
"Left Eye Circle crop", "Left Eye Circle crop",
default=self.config.gui_circular_crop_left, default=self.config.gui_circular_crop_left,
key=self.gui_circular_crop_left, key=self.gui_circular_crop_left,
background_color='#424042', background_color="#424042",
), ),
sg.Checkbox( sg.Checkbox(
"Right Eye Circle crop", "Right Eye Circle crop",
default=self.config.gui_circular_crop_right, default=self.config.gui_circular_crop_right,
key=self.gui_circular_crop_right, key=self.gui_circular_crop_right,
background_color='#424042', background_color="#424042",
), ),
], ],
[ [
sg.Text("Advanced Tracking Algorithim Settings:", background_color='#242224') sg.Text(
], "Advanced Tracking Algorithim Settings:", background_color="#242224"
[sg.Checkbox( )
"HSF: Skip Auto Radius",
default=self.config.gui_skip_autoradius,
key=self.gui_skip_autoradius,
background_color='#424042',
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
),
], ],
[ [
sg.Text("Left HSF Radius:", background_color='#424042'), sg.Checkbox(
"HSF: Skip Auto Radius",
default=self.config.gui_skip_autoradius,
key=self.gui_skip_autoradius,
background_color="#424042",
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
),
],
[
sg.Text("Left HSF Radius:", background_color="#424042"),
sg.Slider( sg.Slider(
range=(1, 50), range=(1, 50),
default_value=self.config.gui_HSF_radius_left, default_value=self.config.gui_HSF_radius_left,
orientation="h", orientation="h",
key=self.gui_HSF_radius_left, key=self.gui_HSF_radius_left,
background_color='#424042', background_color="#424042",
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.", tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
), ),
sg.Text("Right HSF Radius:", background_color='#424042'), sg.Text("Right HSF Radius:", background_color="#424042"),
sg.Slider( sg.Slider(
range=(1, 50), range=(1, 50),
default_value=self.config.gui_HSF_radius_right, default_value=self.config.gui_HSF_radius_right,
orientation="h", orientation="h",
key=self.gui_HSF_radius_right, key=self.gui_HSF_radius_right,
background_color='#424042', background_color="#424042",
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.", tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
), ),
], ],
[sg.Text("RANSAC Thresh Add", background_color='#424042'), [
sg.Slider( sg.Text("RANSAC Thresh Add", background_color="#424042"),
range=(1, 50), sg.Slider(
default_value=self.config.gui_thresh_add, range=(1, 50),
orientation="h", default_value=self.config.gui_thresh_add,
key=self.gui_thresh_add, orientation="h",
background_color='#424042', key=self.gui_thresh_add,
tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.", background_color="#424042",
), tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.",
sg.Text("Blob Threshold", background_color='#424042'), ),
# TODO make this for right and left eyes? I dont know how vital that is.. sg.Text("Blob Threshold", background_color="#424042"),
sg.Slider( # TODO make this for right and left eyes? I dont know how vital that is..
range=(0, 110), sg.Slider(
default_value=self.config.gui_threshold, range=(0, 110),
orientation="h", default_value=self.config.gui_threshold,
key=self.gui_threshold_slider, orientation="h",
background_color='#424042', key=self.gui_threshold_slider,
tooltip="Adjusts the threshold for blob tracking.", background_color="#424042",
), tooltip="Adjusts the threshold for blob tracking.",
], ),
[sg.Text("Min Blob Size:", background_color='#424042'), ],
sg.Slider( [
range=(1, 50), sg.Text("Min Blob Size:", background_color="#424042"),
default_value=self.config.gui_blob_minsize, sg.Slider(
orientation="h", range=(1, 50),
key=self.gui_blob_minsize, default_value=self.config.gui_blob_minsize,
background_color='#424042', orientation="h",
tooltip="Minimum size a blob has to be for blob tracking.", key=self.gui_blob_minsize,
), background_color="#424042",
tooltip="Minimum size a blob has to be for blob tracking.",
sg.Text("Max Blob Size:", background_color='#424042'), ),
sg.Slider( sg.Text("Max Blob Size:", background_color="#424042"),
range=(1, 50), sg.Slider(
default_value=self.config.gui_blob_maxsize, range=(1, 50),
orientation="h", default_value=self.config.gui_blob_maxsize,
key=self.gui_blob_maxsize, orientation="h",
background_color='#424042', key=self.gui_blob_maxsize,
tooltip="Maximum size a blob can be for blob tracking.", background_color="#424042",
), tooltip="Maximum size a blob can be for blob tracking.",
], ),
[ ],
sg.Text("Right Eye Thresh:", background_color='#424042'), [
sg.Slider( sg.Text("Right Eye Thresh:", background_color="#424042"),
range=(1, 120), sg.Slider(
default_value=self.config.gui_legacy_ransac_thresh_right, range=(1, 120),
orientation="h", default_value=self.config.gui_legacy_ransac_thresh_right,
key=self.gui_legacy_ransac_thresh_right, orientation="h",
background_color='#424042', key=self.gui_legacy_ransac_thresh_right,
tooltip="Threshold for right eye, legacy RANSAC only", background_color="#424042",
), tooltip="Threshold for right eye, legacy RANSAC only",
sg.Text("Left Eye Thresh:", background_color='#424042'), ),
sg.Slider( sg.Text("Left Eye Thresh:", background_color="#424042"),
range=(1, 120), sg.Slider(
default_value=self.config.gui_legacy_ransac_thresh_left, range=(1, 120),
orientation="h", default_value=self.config.gui_legacy_ransac_thresh_left,
key=self.gui_legacy_ransac_thresh_left, orientation="h",
background_color='#424042', key=self.gui_legacy_ransac_thresh_left,
tooltip="Threshold for left eye, legacy RANSAC only", background_color="#424042",
), tooltip="Threshold for left eye, legacy RANSAC only",
),
], ],
] ]
self.widget_layout = [ self.widget_layout = [
[ [
sg.Text("Tracking Algorithm Order Settings:", background_color='#242224'), sg.Text(
"Tracking Algorithm Order Settings:", background_color="#242224"
),
], ],
[ [
sg.Column(self.general_settings_layout, key=self.gui_general_settings_layout, sg.Column(
background_color='#424042'), self.general_settings_layout,
key=self.gui_general_settings_layout,
background_color="#424042",
),
], ],
] ]
self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called. self.cancellation_event = (
Event()
) # Set the event until start is called, otherwise we can block if shutdown is called.
self.cancellation_event.set() self.cancellation_event.set()
self.image_queue = Queue() self.image_queue = Queue()
@ -383,7 +403,8 @@ class AlgoSettingsWidget:
changed = True changed = True
if self.config.gui_RANSAC3DP != int( if self.config.gui_RANSAC3DP != int(
values[self.gui_RANSAC3DP]): # TODO check that priority order is unique/auto fix it. values[self.gui_RANSAC3DP]
): # TODO check that priority order is unique/auto fix it.
self.config.gui_RANSAC3DP = int(values[self.gui_RANSAC3DP]) self.config.gui_RANSAC3DP = int(values[self.gui_RANSAC3DP])
changed = True changed = True
@ -427,6 +448,10 @@ class AlgoSettingsWidget:
self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK] self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK]
changed = True changed = True
if self.config.gui_LEAP_lid != values[self.gui_LEAP_lid]:
self.config.gui_LEAP_lid = values[self.gui_LEAP_lid]
changed = True
if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]: if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]:
self.config.gui_circular_crop_left = values[self.gui_circular_crop_left] self.config.gui_circular_crop_left = values[self.gui_circular_crop_left]
changed = True changed = True
@ -467,20 +492,32 @@ class AlgoSettingsWidget:
self.config.ibo_filter_samples = int(values[self.ibo_filter_samples]) self.config.ibo_filter_samples = int(values[self.ibo_filter_samples])
changed = True changed = True
if self.config.ibo_fully_close_eye_threshold != float(values[self.ibo_fully_close_eye_threshold]): if self.config.ibo_fully_close_eye_threshold != float(
self.config.ibo_fully_close_eye_threshold = float(values[self.ibo_fully_close_eye_threshold]) values[self.ibo_fully_close_eye_threshold]
):
self.config.ibo_fully_close_eye_threshold = float(
values[self.ibo_fully_close_eye_threshold]
)
changed = True changed = True
if self.config.calibration_samples != int(values[self.calibration_samples]): if self.config.calibration_samples != int(values[self.calibration_samples]):
self.config.calibration_samples = int(values[self.calibration_samples]) self.config.calibration_samples = int(values[self.calibration_samples])
changed = True changed = True
if self.config.gui_legacy_ransac_thresh_left != int(values[self.gui_legacy_ransac_thresh_left]): if self.config.gui_legacy_ransac_thresh_left != int(
self.config.gui_legacy_ransac_thresh_left = int(values[self.gui_legacy_ransac_thresh_left]) values[self.gui_legacy_ransac_thresh_left]
):
self.config.gui_legacy_ransac_thresh_left = int(
values[self.gui_legacy_ransac_thresh_left]
)
changed = True changed = True
if self.config.gui_legacy_ransac_thresh_right != int(values[self.gui_legacy_ransac_thresh_right]): if self.config.gui_legacy_ransac_thresh_right != int(
self.config.gui_legacy_ransac_thresh_right = int(values[self.gui_legacy_ransac_thresh_right]) values[self.gui_legacy_ransac_thresh_right]
):
self.config.gui_legacy_ransac_thresh_right = int(
values[self.gui_legacy_ransac_thresh_right]
)
changed = True changed = True
if changed: if changed:

View File

@ -77,7 +77,7 @@ class EyeTrackSettingsConfig(BaseModel):
gui_legacy_ransac: bool = False gui_legacy_ransac: bool = False
gui_legacy_ransac_thresh_right: int = 80 gui_legacy_ransac_thresh_right: int = 80
gui_legacy_ransac_thresh_left: int = 80 gui_legacy_ransac_thresh_left: int = 80
gui_LEAP_lid: bool = False
class EyeTrackConfig(BaseModel): class EyeTrackConfig(BaseModel):
@ -85,7 +85,7 @@ class EyeTrackConfig(BaseModel):
right_eye: EyeTrackCameraConfig = EyeTrackCameraConfig() right_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
left_eye: EyeTrackCameraConfig = EyeTrackCameraConfig() left_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig() settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
# algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig() # algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
eye_display_id: EyeId = EyeId.RIGHT eye_display_id: EyeId = EyeId.RIGHT
@staticmethod @staticmethod

View File

@ -34,6 +34,7 @@ from operator import truth
from dataclasses import dataclass from dataclasses import dataclass
import sys import sys
import asyncio import asyncio
sys.path.append(".") sys.path.append(".")
from config import EyeTrackCameraConfig from config import EyeTrackCameraConfig
from config import EyeTrackSettingsConfig from config import EyeTrackSettingsConfig
@ -252,7 +253,9 @@ class EyeProcessor:
if self.settings.gui_BLINK: if self.settings.gui_BLINK:
self.eyeopen = BLINK(self) self.eyeopen = BLINK(self)
if self.settings.gui_IBO and self.eyeopen != 0.0: #TODO make ransac blink it's pwn self var to rid of this non-sense if (
self.settings.gui_IBO and self.eyeopen != 0.0
): # TODO make ransac blink it's pwn self var to rid of this non-sense
self.eyeopen = self.ibo.intense( self.eyeopen = self.ibo.intense(
self.rawx, self.rawx,
self.rawy, self.rawy,
@ -268,8 +271,7 @@ class EyeProcessor:
if self.bd_blink == True: if self.bd_blink == True:
pass pass
if self.settings.gui_IBO and self.eyeopen != 0.0:
if self.settings.gui_IBO and self.settings.gui_BLINK and self.eyeopen != 0.0:
ibo = self.ibo.intense( ibo = self.ibo.intense(
self.rawx, self.rawx,
self.rawy, self.rawy,
@ -278,27 +280,30 @@ class EyeProcessor:
self.settings.ibo_average_output_samples, self.settings.ibo_average_output_samples,
) )
blink = BLINK(self) if self.settings.gui_LEAP_lid:
if blink == 0.0: (
self.eyeopen = 0.0 self.current_image_gray,
else: self.rawx,
self.eyeopen = ibo self.rawy,
self.eyeopen,
) = self.er_leap.run(self.current_image_gray)
if (
if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING, kinda broke len(self.prev_y_list) >= 200
): # "lock" eye when close/blink IN TESTING, kinda broke
self.prev_y_list.pop(0) self.prev_y_list.pop(0)
self.prev_y_list.append(self.out_y) self.prev_y_list.append(self.out_y)
else: else:
self.prev_y_list.append(self.out_y) self.prev_y_list.append(self.out_y)
# print(abs(self.eyeopen - self.past_blink)) # print(abs(self.eyeopen - self.past_blink))
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1 blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0: if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
#if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0: # if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list) # self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list) self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
# print('AVG', self.out_y, len(self.prev_y_list)) # print('AVG', self.out_y, len(self.prev_y_list))
self.past_blink = self.eyeopen self.past_blink = self.eyeopen
self.prev_x = self.out_x self.prev_x = self.out_x
@ -313,24 +318,23 @@ class EyeProcessor:
else: else:
self.eyeopen = 0.9 self.eyeopen = 0.9
def BLINKM(self): def BLINKM(self):
self.eyeopen = BLINK(self) self.eyeopen = BLINK(self)
def LEAPM(self): def LEAPM(self):
self.thresh = self.current_image_gray.copy() self.thresh = self.current_image_gray.copy()
self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(self.current_image_gray) #TODO: make own self var and LEAP toggle self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(
self.current_image_gray
) # TODO: make own self var and LEAP toggle
self.thresh = self.current_image_gray.copy() self.thresh = self.current_image_gray.copy()
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.LEAP self.current_algorithm = EyeInfoOrigin.LEAP
def DADDYM(self): def DADDYM(self):
# todo: We should have a proper variable for drawing. # todo: We should have a proper variable for drawing.
#self.thresh = self.current_image_gray.copy() # self.thresh = self.current_image_gray.copy()
self.thresh = self.current_image_gray.copy() self.thresh = self.current_image_gray.copy()
self.rawx, self.rawy, self.radius = self.er_daddy.run( self.rawx, self.rawy, self.radius = self.er_daddy.run(self.current_image_gray)
self.current_image_gray
)
# Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it. # Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it.
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.DADDY self.current_algorithm = EyeInfoOrigin.DADDY
@ -355,7 +359,7 @@ class EyeProcessor:
self.current_image_gray self.current_image_gray
) )
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True) self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
if self.settings.gui_RANSACBLINK: #might be redundant if self.settings.gui_RANSACBLINK: # might be redundant
self.eyeopen = ranblink self.eyeopen = ranblink
# print(self.radius) # print(self.radius)
@ -379,7 +383,9 @@ class EyeProcessor:
else: else:
pass pass
# todo: add process to initialise er_hsf when resolution changes # todo: add process to initialise er_hsf when resolution changes
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray) self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
self.current_image_gray
)
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.HSF self.current_algorithm = EyeInfoOrigin.HSF
@ -526,7 +532,7 @@ class EyeProcessor:
if self.er_daddy is not None: if self.er_daddy is not None:
self.er_daddy = None self.er_daddy = None
if self.settings.gui_LEAP: if self.settings.gui_LEAP or self.settings.gui_LEAP_lid:
if self.er_leap is None: if self.er_leap is None:
self.er_leap = External_Run_LEAP() self.er_leap = External_Run_LEAP()
algolist[self.settings.gui_LEAP] = self.LEAPM algolist[self.settings.gui_LEAP] = self.LEAPM

View File

@ -27,6 +27,7 @@ Copyright (c) 2023 EyeTrackVR <3
""" """
# LEAP = Lightweight Eyelid And Pupil # LEAP = Lightweight Eyelid And Pupil
import os import os
os.environ["OMP_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime import onnxruntime
import numpy as np import numpy as np
@ -37,21 +38,23 @@ from queue import Queue
import threading import threading
from one_euro_filter import OneEuroFilter from one_euro_filter import OneEuroFilter
import psutil, os import psutil, os
import sys import sys
from utils.misc_utils import resource_path from utils.misc_utils import resource_path
import platform import platform
frames = 0 frames = 0
def run_model(input_queue, output_queue, session): def run_model(input_queue, output_queue, session):
while True: while True:
frame = input_queue.get() frame = input_queue.get()
if frame is None: if frame is None:
break break
# to_tensor = transforms.ToTensor() # to_tensor = transforms.ToTensor()
# img_tensor = to_tensor(frame) # img_tensor = to_tensor(frame)
# img_tensor.unsqueeze_(0) # img_tensor.unsqueeze_(0)
# img_np = img_tensor.numpy() # img_np = img_tensor.numpy()
img_np = np.array(frame) img_np = np.array(frame)
# Normalize the pixel values to [0, 1] and convert the data type to float32 # Normalize the pixel values to [0, 1] and convert the data type to float32
img_np = img_np.astype(np.float32) / 255.0 img_np = img_np.astype(np.float32) / 255.0
@ -69,8 +72,6 @@ def run_model(input_queue, output_queue, session):
output_queue.put((frame, pre_landmark)) output_queue.put((frame, pre_landmark))
class LEAP_C(object): class LEAP_C(object):
def __init__(self): def __init__(self):
onnxruntime.disable_telemetry_events() onnxruntime.disable_telemetry_events()
@ -78,9 +79,11 @@ class LEAP_C(object):
self.num_threads = 3 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage) self.num_threads = 3 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak. self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
if platform.system() == "Darwin": if platform.system() == "Darwin":
self.model_path = resource_path('EyeTrackApp/Models/mommy072623.onnx') # funny MacOS files issues :P self.model_path = resource_path(
"EyeTrackApp/Models/mommy072623.onnx"
) # funny MacOS files issues :P
else: else:
self.model_path = resource_path('Models/mommy072623.onnx') self.model_path = resource_path("Models/mommy072623.onnx")
self.interval = 1 # FPS print update rate self.interval = 1 # FPS print update rate
self.low_priority = True # set process priority to low self.low_priority = True # set process priority to low
self.print_fps = True self.print_fps = True
@ -99,9 +102,13 @@ class LEAP_C(object):
opts = onnxruntime.SessionOptions() opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1 opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1 opts.intra_op_num_threads = 1
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL opts.graph_optimization_level = (
opts.optimized_model_filepath = '' onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
self.ort_session = onnxruntime.InferenceSession(self.model_path, opts, providers=['CPUExecutionProvider']) )
opts.optimized_model_filepath = ""
self.ort_session = onnxruntime.InferenceSession(
self.model_path, opts, providers=["CPUExecutionProvider"]
)
if self.low_priority: if self.low_priority:
process = psutil.Process(os.getpid()) # set process priority to low process = psutil.Process(os.getpid()) # set process priority to low
@ -119,9 +126,7 @@ class LEAP_C(object):
# print(np.random.rand(22, 2)) # print(np.random.rand(22, 2))
# noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]) # noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
one_euro_filter = OneEuroFilter( one_euro_filter = OneEuroFilter(
np.random.rand(7, 2), np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta
min_cutoff=min_cutoff,
beta=beta
) )
self.dmax = 0 self.dmax = 0
self.dmin = 0 self.dmin = 0
@ -129,37 +134,40 @@ class LEAP_C(object):
self.x = 0 self.x = 0
self.y = 0 self.y = 0
self.ort_session1 = onnxruntime.InferenceSession( self.ort_session1 = onnxruntime.InferenceSession(
self.model_path, opts, self.model_path, opts, providers=["CPUExecutionProvider"]
providers=['CPUExecutionProvider']) )
# ort_session1 = onnxruntime.InferenceSession("C:/Users/beaul/PycharmProjects/EyeTrackVR/EyeTrackApp/Models/mommy062023.onnx", opts, providers=['DmlExecutionProvider']) # ort_session1 = onnxruntime.InferenceSession("C:/Users/beaul/PycharmProjects/EyeTrackVR/EyeTrackApp/Models/mommy062023.onnx", opts, providers=['DmlExecutionProvider'])
threads = [] threads = []
for i in range(self.num_threads): for i in range(self.num_threads):
thread = threading.Thread(target=run_model, args=(self.queues[i], self.output_queue, self.ort_session1), thread = threading.Thread(
name=f"Thread {i}") target=run_model,
args=(self.queues[i], self.output_queue, self.ort_session1),
name=f"Thread {i}",
)
threads.append(thread) threads.append(thread)
thread.start() thread.start()
def to_numpy(self, tensor): def to_numpy(self, tensor):
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() return (
tensor.detach().cpu().numpy()
if tensor.requires_grad
else tensor.cpu().numpy()
)
def run_onnx_model(self, queues, session, frame): def run_onnx_model(self, queues, session, frame):
for i in range(len(queues)): for i in range(len(queues)):
if not queues[i].full(): if not queues[i].full():
queues[i].put(frame) queues[i].put(frame)
break break
def leap_run(self):
def leap_run(self):
img = self.current_image_gray.copy() img = self.current_image_gray.copy()
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# img = imutils.rotate(img, angle=320) # img = imutils.rotate(img, angle=320)
img_height, img_width = img.shape[:2] # Move outside the loop img_height, img_width = img.shape[:2] # Move outside the loop
frame = cv2.resize(img, (112, 112)) frame = cv2.resize(img, (112, 112))
self.run_onnx_model(self.queues, self.ort_session1, frame) self.run_onnx_model(self.queues, self.ort_session1, frame)
@ -170,12 +178,26 @@ class LEAP_C(object):
for point in pre_landmark: for point in pre_landmark:
x, y = point x, y = point
cv2.circle(img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1) cv2.circle(
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[2]), 1, (255, 255, 0), -1) img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1) )
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[4]), 1, (255, 255, 255), -1) cv2.circle(
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1) img,
# print(pre_landmark) tuple(int(x * img_width) for x in pre_landmark[2]),
1,
(255, 255, 0),
-1,
)
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
cv2.circle(
img,
tuple(int(x * img_width) for x in pre_landmark[4]),
1,
(255, 255, 255),
-1,
)
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
# print(pre_landmark)
x1, y1 = pre_landmark[0] x1, y1 = pre_landmark[0]
x2, y2 = pre_landmark[6] x2, y2 = pre_landmark[6]
@ -188,27 +210,31 @@ class LEAP_C(object):
x4, y4 = pre_landmark[2] x4, y4 = pre_landmark[2]
euclidean_dist_open = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) euclidean_dist_open = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
# d = area / euclidean_dist_width # d = area / euclidean_dist_width
# print(area) # print(area)
eyesize_dist = math.dist(pre_landmark[0], pre_landmark[6]) eyesize_dist = math.dist(pre_landmark[0], pre_landmark[6])
distance = math.dist(pre_landmark[1], pre_landmark[3]) distance = math.dist(pre_landmark[1], pre_landmark[3])
# d = distance / eyesize_dist # d = distance / eyesize_dist
d = math.dist(pre_landmark[1], pre_landmark[3]) d = math.dist(pre_landmark[1], pre_landmark[3])
# d2 = math.dist(pre_landmark[2], pre_landmark[4]) # d2 = math.dist(pre_landmark[2], pre_landmark[4])
# d = d + d2 # d = d + d2
if len(self.openlist) < 2000: # TODO expose as setting? if len(self.openlist) < 2000: # TODO expose as setting?
self.openlist.append(d) self.openlist.append(d)
else: else:
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(self.openlist, 1): if d >= np.percentile(self.openlist, 99) or d <= np.percentile(
self.openlist, 1
):
pass pass
else: else:
self.openlist.pop(0) self.openlist.pop(0)
self.openlist.append(d) self.openlist.append(d)
try: try:
per = ((d - max(self.openlist)) / (min(self.openlist) - max(self.openlist))) per = (d - max(self.openlist)) / (
min(self.openlist) - max(self.openlist)
)
per = 1 - per per = 1 - per
except: except:
per = 0.7 per = 0.7
@ -221,13 +247,13 @@ class LEAP_C(object):
self.last_lid = per self.last_lid = per
if per <= 0.1: if per <= 0.1:
per == 0.0 per == 0.0
# print(per) # print(per)
return frame, float(x), float(y), per return frame, float(x), float(y), per
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
return frame, 0, 0, 0 return frame, 0, 0, 0
class External_Run_LEAP(object): class External_Run_LEAP(object):
def __init__(self): def __init__(self):
self.algo = LEAP_C() self.algo = LEAP_C()
@ -235,4 +261,4 @@ class External_Run_LEAP(object):
def run(self, current_image_gray): def run(self, current_image_gray):
self.algo.current_image_gray = current_image_gray self.algo.current_image_gray = current_image_gray
img, x, y, per = self.algo.leap_run() img, x, y, per = self.algo.leap_run()
return img, x, y, per return img, x, y, per