This commit is contained in:
Blabz 2025-04-03 21:13:02 +02:00 committed by GitHub
commit 3390b0ef69
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 512 additions and 1 deletions

2
.gitignore vendored
View File

@ -23,3 +23,5 @@ EyeTrackApp/IBO_RIGHT.png
/IBO_LEFT.png
/eyetrack_settings.backup
/eyetrack_settings.json
zBuild.bat
EyeTrackApp/z_CompileBuild.bat

View File

@ -217,6 +217,16 @@ class EyeTrackSettingsConfig(BaseModel):
gui_OutputMultiplier: float = 1
gui_use_module: bool = False
#mirrortrackTracking
gui_mirrortrack_enabled: bool = False
gui_mirrortrack_enable_inv: bool = True
gui_mirrortrack_enable_smooth: bool = True
gui_mirrortrack_select_right: bool = False
gui_mirrortrack_cycle_count_inv: int = 15
gui_mirrortrack_cycle_count_stare: int = 10
gui_mirrortrack_smooth_rate: float = 0.2
gui_mirrortrack_minthresh: float = 0.125
gui_mirrortrack_rotation_clamp: float = 0.3
class EyeTrackConfig(BaseModel):
version: int = 1

View File

@ -38,6 +38,7 @@ from settings.algo_settings_widget import AlgoSettingsWidget
from osc.osc import OSCManager
from osc.OSCMessage import OSCMessage
from utils.misc_utils import is_nt, resource_path
from utils.mirrortrack import MirrorTrack
import cv2
import numpy as np
import uuid
@ -283,6 +284,7 @@ def main():
config.register_listener_callback(osc_manager.update)
config.register_listener_callback(eyes[0].on_config_update)
config.register_listener_callback(eyes[1].on_config_update)
config.register_listener_callback(MirrorTrack.config_update)
osc_manager.register_listeners(
config.settings.gui_osc_recenter_address,
@ -300,6 +302,7 @@ def main():
)
osc_manager.start()
MirrorTrack.init_config(config)
while True:
tint = 33

View File

@ -30,6 +30,7 @@ import time
from enum import IntEnum
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path
from utils.eye_falloff import velocity_falloff
from utils.mirrortrack import MirrorTrack
import socket
import struct
import threading
@ -39,6 +40,7 @@ import math
from utils.calibration_3d import receive_calibration_data, converge_3d
from utils.misc_utils import resource_path
from pathlib import Path
from utils.misc_utils import clamp
tool = Path("Tools")
class TimeoutError(RuntimeError):
@ -336,7 +338,10 @@ class cal:
var.past_x = out_x_mult
var.past_y = out_y_mult
out_x, out_y = velocity_falloff(self, var, out_x, out_y)
if(self.settings.gui_mirrortrack_enabled):
out_x, out_y = MirrorTrack.process(self.eye_id, out_x, out_y)
else:
out_x, out_y = velocity_falloff(self, var, out_x, out_y)
try:
noisy_point = np.array([float(out_x), float(out_y)]) # fliter our values with a One Euro Filter

View File

@ -31,6 +31,7 @@ from settings.BaseSettings import BaseSettingsWidget
from settings.modules.GeneralSettingsModule import GeneralSettingsModule
from settings.modules.OneEuroSettingsModule import OneEuroSettingsModule
from settings.modules.OSCSettingsModule import OSCSettingsModule
from settings.modules.MirrorTrackSettingsModule import MirrorTrackSettingsModule
class SettingsWidget(BaseSettingsWidget):
@ -38,6 +39,7 @@ class SettingsWidget(BaseSettingsWidget):
settings_modules = [
GeneralSettingsModule,
OneEuroSettingsModule,
MirrorTrackSettingsModule,
OSCSettingsModule,
]
super().__init__(widget_id, main_config, settings_modules)

View File

@ -32,3 +32,12 @@ def check_is_ip_address(v: str):
return v
except ValueError:
raise ValueError("Please provide a valid IP Address")
def try_convert_to_int(v: str):
""""
Checks if value provided can be converted to an integer and returns the converted result
"""
try:
return int(v)
except ValueError:
raise ValueError("Please provide a number with no decimal points")

View File

@ -0,0 +1,144 @@
from pydantic import AfterValidator
from typing_extensions import Annotated
from settings.modules.BaseModule import BaseSettingsModule, BaseValidationModel
from settings.constants import BACKGROUND_COLOR
import PySimpleGUI as sg
from settings.modules.CommonFieldValidators import try_convert_to_float
from settings.modules.CommonFieldValidators import try_convert_to_int
class MirrorTrackValidationModule(BaseValidationModel):
gui_mirrortrack_enabled: bool
gui_mirrortrack_enable_inv: bool
#gui_mirrortrack_enable_smooth: bool
gui_mirrortrack_select_right: bool
gui_mirrortrack_cycle_count_inv: Annotated[int, AfterValidator(try_convert_to_int)]
gui_mirrortrack_cycle_count_stare: Annotated[int, AfterValidator(try_convert_to_int)]
gui_mirrortrack_minthresh: Annotated[float, AfterValidator(try_convert_to_float)]
gui_mirrortrack_rotation_clamp: Annotated[float, AfterValidator(try_convert_to_float)]
gui_mirrortrack_smooth_rate: Annotated[float, AfterValidator(try_convert_to_float)]
class MirrorTrackSettingsModule(BaseSettingsModule):
def __init__(self, config, widget_id, **kwargs):
super().__init__(config=config, widget_id=widget_id, **kwargs)
self.validation_model = MirrorTrackValidationModule
self.gui_mirrortrack_enabled = f"-gui_mirrortrack_enabled{widget_id}-"
self.gui_mirrortrack_enable_inv =f"-gui_mirrortrack_enable_inv{widget_id}-"
#self.gui_mirrortrack_enable_smooth =f"-gui_mirrortrack_enable_smooth{widget_id}-"
self.gui_mirrortrack_select_right = f"-gui_mirrortrack_select_right{widget_id}-"
self.gui_mirrortrack_cycle_count_inv =f"-gui_mirrortrack_cycle_count_inv{widget_id}-"
self.gui_mirrortrack_cycle_count_stare =f"-gui_mirrortrack_cycle_count_stare{widget_id}-"
self.gui_mirrortrack_minthresh =f"-gui_mirrortrack_minthresh{widget_id}-"
self.gui_mirrortrack_rotation_clamp =f"-gui_mirrortrack_rotation_clamp{widget_id}-"
self.gui_mirrortrack_smooth_rate =f"-gui_mirrortrack_smooth_rate{widget_id}-"
def get_layout(self):
return [
[
sg.Text("MirrorTrack System:", background_color='#242224'),
],
[
sg.Checkbox(
"Enable MirrorTrack",
default=self.config.gui_mirrortrack_enabled,
key=self.gui_mirrortrack_enabled,
background_color="#424042",
tooltip="Enables MirrorTrack System",
),
],
[
sg.Radio(
"Use Left Eye",
"mirrortrack_selectedeye",
default = not self.config.gui_mirrortrack_select_right,
background_color="#424042",
tooltip="Uses the left eye as the tracked eye.",
),
sg.Radio(
"Use Right Eye",
"mirrortrack_selectedeye",
default=self.config.gui_mirrortrack_select_right,
key=self.gui_mirrortrack_select_right,
background_color="#424042",
tooltip="Uses the right eye as the tracked eye.",
)
],
[
sg.Text("Stare Ahead Detection Duration", background_color=BACKGROUND_COLOR,tooltip=
"How long it takes to detect you are staring ahead, or no longer staring ahead."
"\n Higher number means longer duration before changing in or out of being in stare ahead state."
"\n Exit conditions are half the duration of entry conditions."
),
sg.InputText(
self.config.gui_mirrortrack_cycle_count_stare,
key=self.gui_mirrortrack_cycle_count_stare,
size=(0, 10),
),
],
[
sg.Checkbox(
"Enable Cross-Eye Detection",
default=self.config.gui_mirrortrack_enable_inv,
key=self.gui_mirrortrack_enable_inv,
background_color="#424042",
tooltip="Enables cross-eye functionality",
),
],
[
sg.Text("Detection Threshold", background_color=BACKGROUND_COLOR,tooltip=
"Sets the minimum distance of looking in that's required before state will changed to cross-eyed."
"\n Lower value will make cross-eye detection more sensitive."
),
sg.InputText(
self.config.gui_mirrortrack_minthresh,
key=self.gui_mirrortrack_minthresh,
size=(0, 10),
),
],
[
sg.Text("Detection Duration", background_color=BACKGROUND_COLOR,tooltip=
"How long it takes to detect you are cross-eyed, or no longer cross-eyed."
"\n Higher number means longer duration before changing in or out of being cross-eyed state."
"\n Exit conditions are half the duration of entry conditions."
),
sg.InputText(
self.config.gui_mirrortrack_cycle_count_inv,
key=self.gui_mirrortrack_cycle_count_inv,
size=(0, 10),
),
],
[
sg.Text("Rotation Limit", background_color=BACKGROUND_COLOR,tooltip=
"Defines the maximum inwards rotation that is output when cross-eyed."
"\n0 = will only look straight ahead \n0.5 = will go a little bit cross-eyed \n1 = maximum hurr durr "
),
sg.InputText(
self.config.gui_mirrortrack_rotation_clamp,
key=self.gui_mirrortrack_rotation_clamp,
size=(0, 10),
),
],
[
#sg.Checkbox(
# "Allow cross-eye smoothing",
# default=self.config.gui_mirrortrack_enable_smooth,
# key=self.gui_mirrortrack_enable_smooth,
# background_color="#424042",
# tooltip="Enables smoothing when transitioning to cross-eye",
#),
sg.Text("Transition Smoothing Rate", background_color=BACKGROUND_COLOR,tooltip=
"How quickly smoothing decays when you enter or leave the cross-eyed state."
"\nHigher number = shorter smoothing duration, snappier transition."
"\nLower number = longer smoothing duration, smoother transition"
),
sg.InputText(
self.config.gui_mirrortrack_smooth_rate,
key=self.gui_mirrortrack_smooth_rate,
size=(0, 10),
),
],
]

View File

@ -0,0 +1,39 @@
#Counts how many cycles a condition has been true
class CycleCounter:
def __init__ (self, max_count):
self.count = 0
self.max_count = max_count
def increase(self):
self.count += 1
self.count = min(self.count,self.max_count)
def decrease(self):
self.count -= 1
self.count = max(self.count,0)
def reset(self):
self.count = 0
def is_complete(self):
if self.count >= self.max_count:
return True
return False
def active(self):
if self.count == 0:
return False
else:
return True
def get_count(self):
return self.count
def update(self,max_count):
self.max_count = max_count
def force_complete(self):
self.count = self.max_count
def less_than_percentage(self,mult):
return self.count <= self.max_count * mult

View File

@ -0,0 +1,286 @@
from eye import EyeId
from enum import Enum
from config import EyeTrackConfig
from utils.misc_utils import clamp
from utils.CycleCounter import *
import threading
class MirrorTrack:
#Defines our possible tracking states
class States(Enum):
TRACKING = 0
STARE = 1
INVERTED = 2
state = States.TRACKING
#Defines our tracked positions
rx_left_x = 0.0
rx_left_y = 0.0
rx_right_x = 0.0
rx_right_y = 0.0
rx_dom_eye_x = 0.0
rx_dom_eye_y = 0.0
#Defines our processed positions
tx_left_x = 0.0
tx_right_x = 0.0
tx_right_y = 0.0
tx_dom_eye_x = 0.0
tx_dom_eye_y = 0.0
#Defines other global variables
inv_x_thresh = 0.3
is_r_dom = True
bypass_stare = False
smoothing_trigger = False
#Defines variables that require settings
@classmethod
def init_config(cls,config: EyeTrackConfig):
if config.settings.gui_mirrortrack_select_right:
cls.dom_eye = EyeId.RIGHT
cls.rec_eye = EyeId.LEFT
cls.is_r_dom = True
else:
cls.dom_eye = EyeId.LEFT
cls.rec_eye = EyeId.RIGHT
cls.is_r_dom = False
#Inversion related
cls.is_inv_enabled = config.settings.gui_mirrortrack_enable_inv
cls.inv_x_thresh = config.settings.gui_mirrortrack_minthresh
cls.inv_clamp = config.settings.gui_mirrortrack_rotation_clamp
cls.cyc_counts_inv = config.settings.gui_mirrortrack_cycle_count_inv
cls.cyc_counts_stare = config.settings.gui_mirrortrack_cycle_count_stare
cls.cyc_counter_inv = CycleCounter(cls.cyc_counts_inv)
cls.cyc_counter_stare = CycleCounter(cls.cyc_counts_stare)
cls.is_smooth_enabled = config.settings.gui_mirrortrack_enable_smooth
cls.smoothing_rate = config.settings.gui_mirrortrack_smooth_rate
cls.thread_lock = threading.Lock()
#Receives changes in configuration and applies them accordingly
@classmethod
def config_update(cls,data):
if "gui_mirrortrack_select_right" in data:
cls.dom_eye = EyeId.RIGHT if data["gui_mirrortrack_select_right"] else EyeId.LEFT
cls.rec_eye = EyeId.LEFT if data["gui_mirrortrack_select_right"] else EyeId.RIGHT
cls.is_r_dom = True if data["gui_mirrortrack_select_right"] else False
#print(f"Dominant eye changed to {cls.dom_eye.name}")
if "gui_mirrortrack_minthresh" in data:
cls.inv_x_thresh = data["gui_mirrortrack_minthresh"]
#print(f"MirrorTrack transition threshold changed to {cls.inv_x_thresh}")
if "gui_mirrortrack_cycle_count_inv" in data:
cls.cyc_counts_inv = data["gui_mirrortrack_cycle_count_inv"]
cls.cyc_counter_inv.update(cls.cyc_counts_inv)
#print(f"MirrorTrack inversion transition condition required cycle count changed to {cls.cyc_counts_inv}")
if "gui_mirrortrack_cycle_count_stare" in data:
cls.cyc_counts_stare = data["gui_mirrortrack_cycle_count_stare"]
cls.cyc_counter_stare.update(cls.cyc_counts_stare)
#print(f"MirrorTrack stare transition condition required cycle count changed to {cls.cyc_counts_stare}")
if "gui_mirrortrack_rotation_clamp" in data:
cls.inv_clamp = data["gui_mirrortrack_rotation_clamp"]
#print(f"MirrorTrack maximum allowed cross-eye changed to {cls.inv_clamp}")
if "gui_mirrortrack_enable_inv" in data:
cls.is_inv_enabled = data["gui_mirrortrack_enable_inv"]
if not cls.is_inv_enabled and cls.is_inverted_mode():
cls.set_state("STARE")
cls.bypass_stare = False
#print(f"MirrorTrack allow cross-eye is set to {cls.is_inv_enabled}")
if "gui_mirrortrack_enable_smooth" in data:
cls.is_smooth_enabled = data["gui_mirrortrack_enable_smooth"]
#print(f"MirrorTrack cross-eye smoothing is set to {cls.is_smooth_enabled}")
if "gui_mirrortrack_smooth_rate" in data:
cls.smoothing_rate = data["gui_mirrortrack_smooth_rate"]
#print(f"MirrorTrack cross-eye smoothing rate is set to {cls.smoothing_rate}")
#Main processing function
@classmethod
def process(cls,eye_id,out_x,out_y):
with cls.thread_lock:
cls.store_tracked_positions(eye_id,out_x,out_y)
cls.check_for_stare()
cls.check_for_inversion()
out_x, out_y = cls.update_new_position(eye_id,out_x,out_y)
out_x = cls.process_smoothing(eye_id,out_x)
cls.store_processed_positions(eye_id,out_x,out_y)
return out_x, out_y
#Main methods that are called during processing
@classmethod
def store_tracked_positions(cls, eye_id, out_x,out_y):
if cls.is_processing_right_eye(eye_id):
cls.rx_right_x = out_x
cls.rx_right_y = out_y
if cls.is_dominant_eye(eye_id):
cls.rx_dom_eye_x = cls.rx_right_x
cls.rx_dom_eye_y = cls.rx_right_y
else:
cls.rx_left_x = out_x
cls.rx_left_y = out_y
if cls.is_dominant_eye(eye_id):
cls.rx_dom_eye_x = cls.rx_left_x
cls.rx_dom_eye_y = cls.rx_left_y
@classmethod
def store_processed_positions(cls, eye_id, out_x,out_y):
if cls.is_processing_right_eye(eye_id):
cls.tx_right_x = out_x
cls.tx_right_y = out_y
if cls.is_dominant_eye(eye_id):
cls.tx_dom_eye_x = cls.tx_right_x
cls.tx_dom_eye_y = cls.tx_right_y
else:
cls.tx_left_x = out_x
cls.tx_left_y = out_y
if cls.is_dominant_eye(eye_id):
cls.tx_dom_eye_x = cls.tx_left_x
cls.tx_dom_eye_y = cls.tx_left_y
@classmethod
def check_for_stare(cls,inv_call=False):
if cls.dom_is_inward() and cls.rec_is_inward():
#If inversion exits to stare, force complete the counter to force stare mode.
if inv_call:
cls.cyc_counter_stare.force_complete()
#Updates the counter for stare activation
if not cls.is_stare_mode() and not cls.cyc_counter_stare.is_complete():
cls.cyc_counter_stare.increase()
#Sets the state to stare if count is_completes
elif not cls.is_stare_mode() and cls.cyc_counter_stare.is_complete() and not cls.bypass_stare:
cls.set_state("STARE")
#If inversion exits and doesn't meet stare, reset stare counter to force tracking.
elif inv_call:
cls.cyc_counter_stare.reset()
elif cls.cyc_counter_stare.active():
cls.cyc_counter_stare.decrease()
if cls.cyc_counter_stare.less_than_percentage(0.5):
if cls.bypass_stare:
cls.bypass_stare = False
if not cls.is_tracking_mode():
cls.set_state("TRACKING")
@classmethod
def check_for_inversion(cls):
if cls.is_inv_enabled:
if cls.dom_meets_thresh() and cls.rec_meets_thresh() and (cls.is_stare_mode() or cls.bypass_stare):
#Updates the counter for activation
if not cls.is_inverted_mode() and not cls.cyc_counter_inv.is_complete():
cls.cyc_counter_inv.increase()
#print(f"Inversion activation counter is increasing: {cls.cyc_counter_inv.get_count()}")
#Sets the state to inverted if enough cycles have is_completed
elif not cls.is_inverted_mode() and cls.cyc_counter_inv.is_complete():
cls.set_state("INVERTED")
cls.bypass_stare = True
cls.smoothing_trigger = True
#print(f"Smoothing trigger is {cls.smoothing_trigger}")
#Begins the counter for deactivation if conditions are not met
elif cls.cyc_counter_inv.active():
cls.cyc_counter_inv.decrease()
if cls.cyc_counter_inv.less_than_percentage(0.5):
if cls.bypass_stare:
cls.bypass_stare = False
if cls.is_inverted_mode():
cls.smoothing_trigger = True
#print(f"Smoothing trigger is {cls.smoothing_trigger}")
cls.check_for_stare(True)
else:
return
@classmethod
def update_new_position(cls,eye_id,out_x,out_y):
if cls.is_stare_mode():
out_x, out_y = 0, cls.rx_dom_eye_y
elif not cls.is_dominant_eye(eye_id) and cls.is_inverted_mode():
out_x, out_y = -cls.rx_dom_eye_x, cls.rx_dom_eye_y
else:
out_x, out_y = cls.rx_dom_eye_x, cls.rx_dom_eye_y
if cls.is_inverted_mode():
if cls.is_processing_right_eye(eye_id):
out_x = clamp(out_x,-cls.inv_clamp,0)
else:
out_x = clamp(out_x,0,cls.inv_clamp)
return out_x, out_y
@classmethod
def process_smoothing(cls,eye_id,out_x):
if cls.is_smooth_enabled and cls.smoothing_trigger:
smoothing_out_x = cls.tx_right_x if cls.is_processing_right_eye(eye_id) else cls.tx_left_x
smoothing_out_x += (out_x - smoothing_out_x) * cls.smoothing_rate
if abs(out_x - smoothing_out_x) < 0.1:
cls.smoothing_trigger = False
#print(f"Smoothing trigger is {cls.smoothing_trigger}")
return smoothing_out_x
return out_x
#Helper Methods
@classmethod
def is_tracking_mode(cls):
return cls.state == cls.States.TRACKING
@classmethod
def is_stare_mode(cls):
return cls.state == cls.States.STARE
@classmethod
def is_inverted_mode(cls):
return cls.state == cls.States.INVERTED
@classmethod
def set_state(cls,new_state: str):
cls.state = cls.States[new_state]
#print(f"State set to {cls.state.name}")
@classmethod
def is_dominant_eye(cls,eye_id):
return eye_id == cls.dom_eye
@classmethod
def is_processing_right_eye(cls,eye_id):
return eye_id == EyeId.RIGHT
@classmethod
def dom_is_inward(cls):
return (cls.is_r_dom and cls.rx_right_x < 0) or (not cls.is_r_dom and cls.rx_left_x > 0)
@classmethod
def rec_is_inward(cls):
return (cls.is_r_dom and cls.rx_left_x > 0) or (not cls.is_r_dom and cls.rx_right_x < 0)
@classmethod
def dom_meets_thresh(cls):
return (cls.is_r_dom and cls.rx_right_x < -cls.inv_x_thresh) or (not cls.is_r_dom and cls.rx_left_x > cls.inv_x_thresh)
@classmethod
def rec_meets_thresh(cls):
return (cls.is_r_dom and cls.rx_left_x > cls.inv_x_thresh) or (not cls.is_r_dom and cls.rx_right_x < -cls.inv_x_thresh)
@classmethod
def x_diff(cls):
return abs(cls.rx_left_x - cls.rx_right_x)

View File

@ -74,6 +74,17 @@ def eyetrack_settings_config():
gui_osc_vrcft_v2=False,
gui_vrc_native=False,
gui_pupil_dilation=True,
#MirrorTrack
gui_mirrortrack_enabled=False,
gui_mirrortrack_enable_inv=False
gui_mirrortrack_enable_smooth=True
gui_mirrortrack_select_right=False,
gui_mirrortrack_cycle_count_inv=15,
gui_mirrortrack_cycle_count_stare=10,
gui_mirrortrack_smooth_rate=0.2,
gui_mirrortrack_minthresh=0.125,
gui_mirrortrack_rotation_clamp=0.3,
)