mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
feat: LEAP Lid blink algo
This commit is contained in:
parent
6297ce327c
commit
ab631fad19
@ -7,7 +7,9 @@ from threading import Event
|
||||
|
||||
|
||||
class AlgoSettingsWidget:
|
||||
def __init__(self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue):
|
||||
def __init__(
|
||||
self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue
|
||||
):
|
||||
|
||||
self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-"
|
||||
self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-"
|
||||
@ -55,91 +57,97 @@ class AlgoSettingsWidget:
|
||||
self.gui_legacy_ransac = f"-LEGACYRANSACTHRESH{widget_id}-"
|
||||
self.gui_legacy_ransac_thresh_right = f"-THRESHRIGHT{widget_id}-"
|
||||
self.gui_legacy_ransac_thresh_left = f"-THRESHLEFT{widget_id}-"
|
||||
self.gui_LEAP_lid = f"-LEAPLID{widget_id}-"
|
||||
self.main_config = main_config
|
||||
self.config = main_config.settings
|
||||
self.osc_queue = osc_queue
|
||||
|
||||
# Define the window's contents
|
||||
self.general_settings_layout = [
|
||||
|
||||
[sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_HSRAC,
|
||||
key=self.gui_HSRAC,
|
||||
background_color='#424042',
|
||||
tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_HSRACP,
|
||||
key=self.gui_HSRACP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("HSRAC", background_color='#424042'),
|
||||
[
|
||||
sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_HSRAC,
|
||||
key=self.gui_HSRAC,
|
||||
background_color="#424042",
|
||||
tooltip="Our flagship algorithm, utilizing both HSF and RANSAC for best tracking quality and lighting resistance.",
|
||||
),
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_HSRACP,
|
||||
key=self.gui_HSRACP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("HSRAC", background_color="#424042"),
|
||||
# ],
|
||||
# [
|
||||
sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_HSF,
|
||||
key=self.gui_HSF,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="HSF Is a new, lower resolution tracking algorithim that provides excelent resilancy to lighting conditions and great speed.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_HSFP,
|
||||
key=self.gui_HSFP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithims.",
|
||||
),
|
||||
sg.Text("Haar Surround Feature", background_color='#424042'),
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_HSFP,
|
||||
key=self.gui_HSFP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithims.",
|
||||
),
|
||||
sg.Text("Haar Surround Feature", background_color="#424042"),
|
||||
],
|
||||
[sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_DADDY,
|
||||
key=self.gui_DADDY,
|
||||
background_color='#424042',
|
||||
tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_DADDYP,
|
||||
key=self.gui_DADDYP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("DADDY", background_color='#424042'),
|
||||
[
|
||||
sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_DADDY,
|
||||
key=self.gui_DADDY,
|
||||
background_color="#424042",
|
||||
tooltip="DADDY Uses a Deep learning algorithm. This has a big CPU usage impact.",
|
||||
),
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_DADDYP,
|
||||
key=self.gui_DADDYP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("DADDY", background_color="#424042"),
|
||||
# ],
|
||||
# [
|
||||
sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_RANSAC3D,
|
||||
key=self.gui_RANSAC3D,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="RANSAC3D provides good tracking quality, however does not do well in bad lighting conditions.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_RANSAC3DP,
|
||||
key=self.gui_RANSAC3DP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("RANSAC 3D", background_color='#424042'),
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_RANSAC3DP,
|
||||
key=self.gui_RANSAC3DP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("RANSAC 3D", background_color="#424042"),
|
||||
sg.Checkbox(
|
||||
"Legacy RANSAC Thresh",
|
||||
default=self.config.gui_legacy_ransac,
|
||||
key=self.gui_legacy_ransac,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
],
|
||||
[
|
||||
@ -147,77 +155,80 @@ class AlgoSettingsWidget:
|
||||
"",
|
||||
default=self.config.gui_LEAP,
|
||||
key=self.gui_LEAP,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="LEAP Uses a lightweight deep learning algorithm.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_LEAPP,
|
||||
key=self.gui_LEAPP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("LEAP", background_color='#424042'),
|
||||
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_LEAPP,
|
||||
key=self.gui_LEAPP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("LEAP", background_color="#424042"),
|
||||
sg.Checkbox(
|
||||
"",
|
||||
default=self.config.gui_BLOB,
|
||||
key=self.gui_BLOB,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="Blob tracking is the oldest and worst tracking algorithm, it provides fast, though sometimes inaccurate tracking.",
|
||||
),
|
||||
sg.Combo(['1', '2', '3', '4', '5', '6'],
|
||||
default_value=self.config.gui_BLOBP,
|
||||
key=self.gui_BLOBP,
|
||||
background_color='#424042',
|
||||
text_color='white',
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("Blob", background_color='#424042'),
|
||||
],
|
||||
[
|
||||
sg.Text("Blink Algo Settings:", background_color='#242224')
|
||||
sg.Combo(
|
||||
["1", "2", "3", "4", "5", "6"],
|
||||
default_value=self.config.gui_BLOBP,
|
||||
key=self.gui_BLOBP,
|
||||
background_color="#424042",
|
||||
text_color="white",
|
||||
button_arrow_color="black",
|
||||
button_background_color="#6f4ca1",
|
||||
tooltip="Select the priority of eyetracking algorithms.",
|
||||
),
|
||||
sg.Text("Blob", background_color="#424042"),
|
||||
],
|
||||
[sg.Text("Blink Algo Settings:", background_color="#242224")],
|
||||
[
|
||||
sg.Checkbox(
|
||||
"Intensity Based Openness",
|
||||
default=self.config.gui_IBO,
|
||||
key=self.gui_IBO,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
sg.Checkbox(
|
||||
"RANSAC Quick Blink Algo",
|
||||
default=self.config.gui_RANSACBLINK,
|
||||
key=self.gui_RANSACBLINK,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
sg.Checkbox(
|
||||
"Binary Blink Algo",
|
||||
default=self.config.gui_BLINK,
|
||||
key=self.gui_BLINK,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
sg.Checkbox(
|
||||
"LEAP Lid",
|
||||
default=self.config.gui_LEAP_lid,
|
||||
key=self.gui_LEAP_lid,
|
||||
background_color="#424042",
|
||||
),
|
||||
|
||||
],
|
||||
[
|
||||
sg.Text("IBO Filter Sample Size", background_color='#424042'),
|
||||
sg.Text("IBO Filter Sample Size", background_color="#424042"),
|
||||
sg.InputText(
|
||||
self.config.ibo_filter_samples,
|
||||
key=self.ibo_filter_samples,
|
||||
size=(0, 10),
|
||||
),
|
||||
sg.Text("Calibration Samples", background_color='#424042'),
|
||||
sg.Text("Calibration Samples", background_color="#424042"),
|
||||
sg.InputText(
|
||||
self.config.calibration_samples,
|
||||
key=self.calibration_samples,
|
||||
size=(0, 10),
|
||||
),
|
||||
|
||||
sg.Text("IBO Close Threshold", background_color='#424042'),
|
||||
sg.Text("IBO Close Threshold", background_color="#424042"),
|
||||
sg.InputText(
|
||||
self.config.ibo_fully_close_eye_threshold,
|
||||
key=self.ibo_fully_close_eye_threshold,
|
||||
@ -229,121 +240,130 @@ class AlgoSettingsWidget:
|
||||
"Left Eye Circle crop",
|
||||
default=self.config.gui_circular_crop_left,
|
||||
key=self.gui_circular_crop_left,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
sg.Checkbox(
|
||||
"Right Eye Circle crop",
|
||||
default=self.config.gui_circular_crop_right,
|
||||
key=self.gui_circular_crop_right,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Advanced Tracking Algorithim Settings:", background_color='#242224')
|
||||
],
|
||||
[sg.Checkbox(
|
||||
"HSF: Skip Auto Radius",
|
||||
default=self.config.gui_skip_autoradius,
|
||||
key=self.gui_skip_autoradius,
|
||||
background_color='#424042',
|
||||
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
|
||||
),
|
||||
sg.Text(
|
||||
"Advanced Tracking Algorithim Settings:", background_color="#242224"
|
||||
)
|
||||
],
|
||||
[
|
||||
sg.Text("Left HSF Radius:", background_color='#424042'),
|
||||
sg.Checkbox(
|
||||
"HSF: Skip Auto Radius",
|
||||
default=self.config.gui_skip_autoradius,
|
||||
key=self.gui_skip_autoradius,
|
||||
background_color="#424042",
|
||||
tooltip="To gain more control and possibly better tracking quality of HSF, please disable auto radius to enable manual adjustment.",
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Left HSF Radius:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_HSF_radius_left,
|
||||
orientation="h",
|
||||
key=self.gui_HSF_radius_left,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
|
||||
),
|
||||
sg.Text("Right HSF Radius:", background_color='#424042'),
|
||||
sg.Text("Right HSF Radius:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_HSF_radius_right,
|
||||
orientation="h",
|
||||
key=self.gui_HSF_radius_right,
|
||||
background_color='#424042',
|
||||
background_color="#424042",
|
||||
tooltip="Adjusts the radius parameter for HSF. Only adjust if you are having tracking issues.",
|
||||
),
|
||||
|
||||
],
|
||||
[sg.Text("RANSAC Thresh Add", background_color='#424042'),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_thresh_add,
|
||||
orientation="h",
|
||||
key=self.gui_thresh_add,
|
||||
background_color='#424042',
|
||||
tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.",
|
||||
),
|
||||
sg.Text("Blob Threshold", background_color='#424042'),
|
||||
# TODO make this for right and left eyes? I dont know how vital that is..
|
||||
sg.Slider(
|
||||
range=(0, 110),
|
||||
default_value=self.config.gui_threshold,
|
||||
orientation="h",
|
||||
key=self.gui_threshold_slider,
|
||||
background_color='#424042',
|
||||
tooltip="Adjusts the threshold for blob tracking.",
|
||||
),
|
||||
],
|
||||
[sg.Text("Min Blob Size:", background_color='#424042'),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_blob_minsize,
|
||||
orientation="h",
|
||||
key=self.gui_blob_minsize,
|
||||
background_color='#424042',
|
||||
tooltip="Minimum size a blob has to be for blob tracking.",
|
||||
),
|
||||
|
||||
sg.Text("Max Blob Size:", background_color='#424042'),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_blob_maxsize,
|
||||
orientation="h",
|
||||
key=self.gui_blob_maxsize,
|
||||
background_color='#424042',
|
||||
tooltip="Maximum size a blob can be for blob tracking.",
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Right Eye Thresh:", background_color='#424042'),
|
||||
sg.Slider(
|
||||
range=(1, 120),
|
||||
default_value=self.config.gui_legacy_ransac_thresh_right,
|
||||
orientation="h",
|
||||
key=self.gui_legacy_ransac_thresh_right,
|
||||
background_color='#424042',
|
||||
tooltip="Threshold for right eye, legacy RANSAC only",
|
||||
),
|
||||
sg.Text("Left Eye Thresh:", background_color='#424042'),
|
||||
sg.Slider(
|
||||
range=(1, 120),
|
||||
default_value=self.config.gui_legacy_ransac_thresh_left,
|
||||
orientation="h",
|
||||
key=self.gui_legacy_ransac_thresh_left,
|
||||
background_color='#424042',
|
||||
tooltip="Threshold for left eye, legacy RANSAC only",
|
||||
),
|
||||
[
|
||||
sg.Text("RANSAC Thresh Add", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_thresh_add,
|
||||
orientation="h",
|
||||
key=self.gui_thresh_add,
|
||||
background_color="#424042",
|
||||
tooltip="Adjusts the amount of threshold to add to RANSAC. Useful for fine tuning your setup.",
|
||||
),
|
||||
sg.Text("Blob Threshold", background_color="#424042"),
|
||||
# TODO make this for right and left eyes? I dont know how vital that is..
|
||||
sg.Slider(
|
||||
range=(0, 110),
|
||||
default_value=self.config.gui_threshold,
|
||||
orientation="h",
|
||||
key=self.gui_threshold_slider,
|
||||
background_color="#424042",
|
||||
tooltip="Adjusts the threshold for blob tracking.",
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Min Blob Size:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_blob_minsize,
|
||||
orientation="h",
|
||||
key=self.gui_blob_minsize,
|
||||
background_color="#424042",
|
||||
tooltip="Minimum size a blob has to be for blob tracking.",
|
||||
),
|
||||
sg.Text("Max Blob Size:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 50),
|
||||
default_value=self.config.gui_blob_maxsize,
|
||||
orientation="h",
|
||||
key=self.gui_blob_maxsize,
|
||||
background_color="#424042",
|
||||
tooltip="Maximum size a blob can be for blob tracking.",
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Right Eye Thresh:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 120),
|
||||
default_value=self.config.gui_legacy_ransac_thresh_right,
|
||||
orientation="h",
|
||||
key=self.gui_legacy_ransac_thresh_right,
|
||||
background_color="#424042",
|
||||
tooltip="Threshold for right eye, legacy RANSAC only",
|
||||
),
|
||||
sg.Text("Left Eye Thresh:", background_color="#424042"),
|
||||
sg.Slider(
|
||||
range=(1, 120),
|
||||
default_value=self.config.gui_legacy_ransac_thresh_left,
|
||||
orientation="h",
|
||||
key=self.gui_legacy_ransac_thresh_left,
|
||||
background_color="#424042",
|
||||
tooltip="Threshold for left eye, legacy RANSAC only",
|
||||
),
|
||||
],
|
||||
|
||||
]
|
||||
|
||||
self.widget_layout = [
|
||||
[
|
||||
sg.Text("Tracking Algorithm Order Settings:", background_color='#242224'),
|
||||
sg.Text(
|
||||
"Tracking Algorithm Order Settings:", background_color="#242224"
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Column(self.general_settings_layout, key=self.gui_general_settings_layout,
|
||||
background_color='#424042'),
|
||||
sg.Column(
|
||||
self.general_settings_layout,
|
||||
key=self.gui_general_settings_layout,
|
||||
background_color="#424042",
|
||||
),
|
||||
],
|
||||
]
|
||||
|
||||
self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called.
|
||||
self.cancellation_event = (
|
||||
Event()
|
||||
) # Set the event until start is called, otherwise we can block if shutdown is called.
|
||||
self.cancellation_event.set()
|
||||
self.image_queue = Queue()
|
||||
|
||||
@ -383,7 +403,8 @@ class AlgoSettingsWidget:
|
||||
changed = True
|
||||
|
||||
if self.config.gui_RANSAC3DP != int(
|
||||
values[self.gui_RANSAC3DP]): # TODO check that priority order is unique/auto fix it.
|
||||
values[self.gui_RANSAC3DP]
|
||||
): # TODO check that priority order is unique/auto fix it.
|
||||
self.config.gui_RANSAC3DP = int(values[self.gui_RANSAC3DP])
|
||||
changed = True
|
||||
|
||||
@ -427,6 +448,10 @@ class AlgoSettingsWidget:
|
||||
self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_LEAP_lid != values[self.gui_LEAP_lid]:
|
||||
self.config.gui_LEAP_lid = values[self.gui_LEAP_lid]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]:
|
||||
self.config.gui_circular_crop_left = values[self.gui_circular_crop_left]
|
||||
changed = True
|
||||
@ -467,20 +492,32 @@ class AlgoSettingsWidget:
|
||||
self.config.ibo_filter_samples = int(values[self.ibo_filter_samples])
|
||||
changed = True
|
||||
|
||||
if self.config.ibo_fully_close_eye_threshold != float(values[self.ibo_fully_close_eye_threshold]):
|
||||
self.config.ibo_fully_close_eye_threshold = float(values[self.ibo_fully_close_eye_threshold])
|
||||
if self.config.ibo_fully_close_eye_threshold != float(
|
||||
values[self.ibo_fully_close_eye_threshold]
|
||||
):
|
||||
self.config.ibo_fully_close_eye_threshold = float(
|
||||
values[self.ibo_fully_close_eye_threshold]
|
||||
)
|
||||
changed = True
|
||||
|
||||
if self.config.calibration_samples != int(values[self.calibration_samples]):
|
||||
self.config.calibration_samples = int(values[self.calibration_samples])
|
||||
changed = True
|
||||
|
||||
if self.config.gui_legacy_ransac_thresh_left != int(values[self.gui_legacy_ransac_thresh_left]):
|
||||
self.config.gui_legacy_ransac_thresh_left = int(values[self.gui_legacy_ransac_thresh_left])
|
||||
if self.config.gui_legacy_ransac_thresh_left != int(
|
||||
values[self.gui_legacy_ransac_thresh_left]
|
||||
):
|
||||
self.config.gui_legacy_ransac_thresh_left = int(
|
||||
values[self.gui_legacy_ransac_thresh_left]
|
||||
)
|
||||
changed = True
|
||||
|
||||
if self.config.gui_legacy_ransac_thresh_right != int(values[self.gui_legacy_ransac_thresh_right]):
|
||||
self.config.gui_legacy_ransac_thresh_right = int(values[self.gui_legacy_ransac_thresh_right])
|
||||
if self.config.gui_legacy_ransac_thresh_right != int(
|
||||
values[self.gui_legacy_ransac_thresh_right]
|
||||
):
|
||||
self.config.gui_legacy_ransac_thresh_right = int(
|
||||
values[self.gui_legacy_ransac_thresh_right]
|
||||
)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
|
||||
@ -77,7 +77,7 @@ class EyeTrackSettingsConfig(BaseModel):
|
||||
gui_legacy_ransac: bool = False
|
||||
gui_legacy_ransac_thresh_right: int = 80
|
||||
gui_legacy_ransac_thresh_left: int = 80
|
||||
|
||||
gui_LEAP_lid: bool = False
|
||||
|
||||
|
||||
class EyeTrackConfig(BaseModel):
|
||||
@ -85,7 +85,7 @@ class EyeTrackConfig(BaseModel):
|
||||
right_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
|
||||
left_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
|
||||
settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
|
||||
# algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
|
||||
# algo_settings: EyeTrackSettingsConfig = EyeTrackSettingsConfig()
|
||||
eye_display_id: EyeId = EyeId.RIGHT
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -34,6 +34,7 @@ from operator import truth
|
||||
from dataclasses import dataclass
|
||||
import sys
|
||||
import asyncio
|
||||
|
||||
sys.path.append(".")
|
||||
from config import EyeTrackCameraConfig
|
||||
from config import EyeTrackSettingsConfig
|
||||
@ -252,7 +253,9 @@ class EyeProcessor:
|
||||
if self.settings.gui_BLINK:
|
||||
self.eyeopen = BLINK(self)
|
||||
|
||||
if self.settings.gui_IBO and self.eyeopen != 0.0: #TODO make ransac blink it's pwn self var to rid of this non-sense
|
||||
if (
|
||||
self.settings.gui_IBO and self.eyeopen != 0.0
|
||||
): # TODO make ransac blink it's pwn self var to rid of this non-sense
|
||||
self.eyeopen = self.ibo.intense(
|
||||
self.rawx,
|
||||
self.rawy,
|
||||
@ -268,8 +271,7 @@ class EyeProcessor:
|
||||
if self.bd_blink == True:
|
||||
pass
|
||||
|
||||
|
||||
if self.settings.gui_IBO and self.settings.gui_BLINK and self.eyeopen != 0.0:
|
||||
if self.settings.gui_IBO and self.eyeopen != 0.0:
|
||||
ibo = self.ibo.intense(
|
||||
self.rawx,
|
||||
self.rawy,
|
||||
@ -278,27 +280,30 @@ class EyeProcessor:
|
||||
self.settings.ibo_average_output_samples,
|
||||
)
|
||||
|
||||
blink = BLINK(self)
|
||||
if blink == 0.0:
|
||||
self.eyeopen = 0.0
|
||||
else:
|
||||
self.eyeopen = ibo
|
||||
if self.settings.gui_LEAP_lid:
|
||||
(
|
||||
self.current_image_gray,
|
||||
self.rawx,
|
||||
self.rawy,
|
||||
self.eyeopen,
|
||||
) = self.er_leap.run(self.current_image_gray)
|
||||
|
||||
|
||||
if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING, kinda broke
|
||||
if (
|
||||
len(self.prev_y_list) >= 200
|
||||
): # "lock" eye when close/blink IN TESTING, kinda broke
|
||||
self.prev_y_list.pop(0)
|
||||
self.prev_y_list.append(self.out_y)
|
||||
else:
|
||||
self.prev_y_list.append(self.out_y)
|
||||
|
||||
# print(abs(self.eyeopen - self.past_blink))
|
||||
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1
|
||||
# print(abs(self.eyeopen - self.past_blink))
|
||||
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
|
||||
|
||||
if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
||||
#if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
||||
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
|
||||
# if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
||||
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
|
||||
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
|
||||
# print('AVG', self.out_y, len(self.prev_y_list))
|
||||
# print('AVG', self.out_y, len(self.prev_y_list))
|
||||
|
||||
self.past_blink = self.eyeopen
|
||||
self.prev_x = self.out_x
|
||||
@ -313,24 +318,23 @@ class EyeProcessor:
|
||||
else:
|
||||
self.eyeopen = 0.9
|
||||
|
||||
|
||||
def BLINKM(self):
|
||||
self.eyeopen = BLINK(self)
|
||||
|
||||
def LEAPM(self):
|
||||
self.thresh = self.current_image_gray.copy()
|
||||
self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(self.current_image_gray) #TODO: make own self var and LEAP toggle
|
||||
self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(
|
||||
self.current_image_gray
|
||||
) # TODO: make own self var and LEAP toggle
|
||||
self.thresh = self.current_image_gray.copy()
|
||||
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
||||
self.current_algorithm = EyeInfoOrigin.LEAP
|
||||
|
||||
def DADDYM(self):
|
||||
# todo: We should have a proper variable for drawing.
|
||||
#self.thresh = self.current_image_gray.copy()
|
||||
# self.thresh = self.current_image_gray.copy()
|
||||
self.thresh = self.current_image_gray.copy()
|
||||
self.rawx, self.rawy, self.radius = self.er_daddy.run(
|
||||
self.current_image_gray
|
||||
)
|
||||
self.rawx, self.rawy, self.radius = self.er_daddy.run(self.current_image_gray)
|
||||
# Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it.
|
||||
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
||||
self.current_algorithm = EyeInfoOrigin.DADDY
|
||||
@ -355,7 +359,7 @@ class EyeProcessor:
|
||||
self.current_image_gray
|
||||
)
|
||||
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
|
||||
if self.settings.gui_RANSACBLINK: #might be redundant
|
||||
if self.settings.gui_RANSACBLINK: # might be redundant
|
||||
self.eyeopen = ranblink
|
||||
|
||||
# print(self.radius)
|
||||
@ -379,7 +383,9 @@ class EyeProcessor:
|
||||
else:
|
||||
pass
|
||||
# todo: add process to initialise er_hsf when resolution changes
|
||||
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray)
|
||||
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
|
||||
self.current_image_gray
|
||||
)
|
||||
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
||||
self.current_algorithm = EyeInfoOrigin.HSF
|
||||
|
||||
@ -526,7 +532,7 @@ class EyeProcessor:
|
||||
if self.er_daddy is not None:
|
||||
self.er_daddy = None
|
||||
|
||||
if self.settings.gui_LEAP:
|
||||
if self.settings.gui_LEAP or self.settings.gui_LEAP_lid:
|
||||
if self.er_leap is None:
|
||||
self.er_leap = External_Run_LEAP()
|
||||
algolist[self.settings.gui_LEAP] = self.LEAPM
|
||||
|
||||
@ -27,6 +27,7 @@ Copyright (c) 2023 EyeTrackVR <3
|
||||
"""
|
||||
# LEAP = Lightweight Eyelid And Pupil
|
||||
import os
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
import onnxruntime
|
||||
import numpy as np
|
||||
@ -37,21 +38,23 @@ from queue import Queue
|
||||
import threading
|
||||
from one_euro_filter import OneEuroFilter
|
||||
import psutil, os
|
||||
import sys
|
||||
import sys
|
||||
from utils.misc_utils import resource_path
|
||||
import platform
|
||||
|
||||
frames = 0
|
||||
|
||||
|
||||
def run_model(input_queue, output_queue, session):
|
||||
while True:
|
||||
frame = input_queue.get()
|
||||
if frame is None:
|
||||
break
|
||||
|
||||
# to_tensor = transforms.ToTensor()
|
||||
# img_tensor = to_tensor(frame)
|
||||
# img_tensor.unsqueeze_(0)
|
||||
# img_np = img_tensor.numpy()
|
||||
# to_tensor = transforms.ToTensor()
|
||||
# img_tensor = to_tensor(frame)
|
||||
# img_tensor.unsqueeze_(0)
|
||||
# img_np = img_tensor.numpy()
|
||||
img_np = np.array(frame)
|
||||
# Normalize the pixel values to [0, 1] and convert the data type to float32
|
||||
img_np = img_np.astype(np.float32) / 255.0
|
||||
@ -69,8 +72,6 @@ def run_model(input_queue, output_queue, session):
|
||||
output_queue.put((frame, pre_landmark))
|
||||
|
||||
|
||||
|
||||
|
||||
class LEAP_C(object):
|
||||
def __init__(self):
|
||||
onnxruntime.disable_telemetry_events()
|
||||
@ -78,9 +79,11 @@ class LEAP_C(object):
|
||||
self.num_threads = 3 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
|
||||
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
|
||||
if platform.system() == "Darwin":
|
||||
self.model_path = resource_path('EyeTrackApp/Models/mommy072623.onnx') # funny MacOS files issues :P
|
||||
self.model_path = resource_path(
|
||||
"EyeTrackApp/Models/mommy072623.onnx"
|
||||
) # funny MacOS files issues :P
|
||||
else:
|
||||
self.model_path = resource_path('Models/mommy072623.onnx')
|
||||
self.model_path = resource_path("Models/mommy072623.onnx")
|
||||
self.interval = 1 # FPS print update rate
|
||||
self.low_priority = True # set process priority to low
|
||||
self.print_fps = True
|
||||
@ -99,9 +102,13 @@ class LEAP_C(object):
|
||||
opts = onnxruntime.SessionOptions()
|
||||
opts.inter_op_num_threads = 1
|
||||
opts.intra_op_num_threads = 1
|
||||
opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
opts.optimized_model_filepath = ''
|
||||
self.ort_session = onnxruntime.InferenceSession(self.model_path, opts, providers=['CPUExecutionProvider'])
|
||||
opts.graph_optimization_level = (
|
||||
onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
)
|
||||
opts.optimized_model_filepath = ""
|
||||
self.ort_session = onnxruntime.InferenceSession(
|
||||
self.model_path, opts, providers=["CPUExecutionProvider"]
|
||||
)
|
||||
|
||||
if self.low_priority:
|
||||
process = psutil.Process(os.getpid()) # set process priority to low
|
||||
@ -119,9 +126,7 @@ class LEAP_C(object):
|
||||
# print(np.random.rand(22, 2))
|
||||
# noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
|
||||
one_euro_filter = OneEuroFilter(
|
||||
np.random.rand(7, 2),
|
||||
min_cutoff=min_cutoff,
|
||||
beta=beta
|
||||
np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta
|
||||
)
|
||||
self.dmax = 0
|
||||
self.dmin = 0
|
||||
@ -129,37 +134,40 @@ class LEAP_C(object):
|
||||
self.x = 0
|
||||
self.y = 0
|
||||
|
||||
|
||||
self.ort_session1 = onnxruntime.InferenceSession(
|
||||
self.model_path, opts,
|
||||
providers=['CPUExecutionProvider'])
|
||||
self.model_path, opts, providers=["CPUExecutionProvider"]
|
||||
)
|
||||
# ort_session1 = onnxruntime.InferenceSession("C:/Users/beaul/PycharmProjects/EyeTrackVR/EyeTrackApp/Models/mommy062023.onnx", opts, providers=['DmlExecutionProvider'])
|
||||
threads = []
|
||||
for i in range(self.num_threads):
|
||||
thread = threading.Thread(target=run_model, args=(self.queues[i], self.output_queue, self.ort_session1),
|
||||
name=f"Thread {i}")
|
||||
thread = threading.Thread(
|
||||
target=run_model,
|
||||
args=(self.queues[i], self.output_queue, self.ort_session1),
|
||||
name=f"Thread {i}",
|
||||
)
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
def to_numpy(self, tensor):
|
||||
return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
|
||||
|
||||
|
||||
return (
|
||||
tensor.detach().cpu().numpy()
|
||||
if tensor.requires_grad
|
||||
else tensor.cpu().numpy()
|
||||
)
|
||||
|
||||
def run_onnx_model(self, queues, session, frame):
|
||||
for i in range(len(queues)):
|
||||
if not queues[i].full():
|
||||
queues[i].put(frame)
|
||||
break
|
||||
def leap_run(self):
|
||||
|
||||
def leap_run(self):
|
||||
|
||||
img = self.current_image_gray.copy()
|
||||
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
|
||||
# img = imutils.rotate(img, angle=320)
|
||||
img_height, img_width = img.shape[:2] # Move outside the loop
|
||||
|
||||
|
||||
frame = cv2.resize(img, (112, 112))
|
||||
self.run_onnx_model(self.queues, self.ort_session1, frame)
|
||||
|
||||
@ -170,12 +178,26 @@ class LEAP_C(object):
|
||||
|
||||
for point in pre_landmark:
|
||||
x, y = point
|
||||
cv2.circle(img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1)
|
||||
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
|
||||
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
|
||||
cv2.circle(img, tuple(int(x*img_width) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
|
||||
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
|
||||
# print(pre_landmark)
|
||||
cv2.circle(
|
||||
img, (int(x * img_width), int(y * img_height)), 2, (0, 0, 50), -1
|
||||
)
|
||||
cv2.circle(
|
||||
img,
|
||||
tuple(int(x * img_width) for x in pre_landmark[2]),
|
||||
1,
|
||||
(255, 255, 0),
|
||||
-1,
|
||||
)
|
||||
# cv2.circle(img, tuple(int(x*112) for x in pre_landmark[2]), 1, (255, 255, 0), -1)
|
||||
cv2.circle(
|
||||
img,
|
||||
tuple(int(x * img_width) for x in pre_landmark[4]),
|
||||
1,
|
||||
(255, 255, 255),
|
||||
-1,
|
||||
)
|
||||
# cv2.circle(img, tuple(int(x * 112) for x in pre_landmark[4]), 1, (255, 255, 255), -1)
|
||||
# print(pre_landmark)
|
||||
|
||||
x1, y1 = pre_landmark[0]
|
||||
x2, y2 = pre_landmark[6]
|
||||
@ -188,27 +210,31 @@ class LEAP_C(object):
|
||||
x4, y4 = pre_landmark[2]
|
||||
euclidean_dist_open = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
|
||||
|
||||
# d = area / euclidean_dist_width
|
||||
# print(area)
|
||||
# d = area / euclidean_dist_width
|
||||
# print(area)
|
||||
eyesize_dist = math.dist(pre_landmark[0], pre_landmark[6])
|
||||
distance = math.dist(pre_landmark[1], pre_landmark[3])
|
||||
# d = distance / eyesize_dist
|
||||
# d = distance / eyesize_dist
|
||||
|
||||
d = math.dist(pre_landmark[1], pre_landmark[3])
|
||||
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
|
||||
# d = d + d2
|
||||
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
|
||||
# d = d + d2
|
||||
|
||||
if len(self.openlist) < 2000: # TODO expose as setting?
|
||||
if len(self.openlist) < 2000: # TODO expose as setting?
|
||||
self.openlist.append(d)
|
||||
else:
|
||||
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(self.openlist, 1):
|
||||
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(
|
||||
self.openlist, 1
|
||||
):
|
||||
pass
|
||||
else:
|
||||
self.openlist.pop(0)
|
||||
self.openlist.append(d)
|
||||
|
||||
try:
|
||||
per = ((d - max(self.openlist)) / (min(self.openlist) - max(self.openlist)))
|
||||
per = (d - max(self.openlist)) / (
|
||||
min(self.openlist) - max(self.openlist)
|
||||
)
|
||||
per = 1 - per
|
||||
except:
|
||||
per = 0.7
|
||||
@ -221,13 +247,13 @@ class LEAP_C(object):
|
||||
self.last_lid = per
|
||||
if per <= 0.1:
|
||||
per == 0.0
|
||||
# print(per)
|
||||
# print(per)
|
||||
return frame, float(x), float(y), per
|
||||
|
||||
|
||||
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
||||
return frame, 0, 0, 0
|
||||
|
||||
|
||||
class External_Run_LEAP(object):
|
||||
def __init__(self):
|
||||
self.algo = LEAP_C()
|
||||
@ -235,4 +261,4 @@ class External_Run_LEAP(object):
|
||||
def run(self, current_image_gray):
|
||||
self.algo.current_image_gray = current_image_gray
|
||||
img, x, y, per = self.algo.leap_run()
|
||||
return img, x, y, per
|
||||
return img, x, y, per
|
||||
|
||||
Loading…
Reference in New Issue
Block a user