From 8e2809260180f7e091ceba97c19eb52f883a3b91 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Tue, 3 Oct 2023 14:51:36 -0500 Subject: [PATCH] feat: stage one of new output formats --- EyeTrackApp/config.py | 4 +- EyeTrackApp/general_settings_widget.py | 222 ++++++++------ EyeTrackApp/intensity_based_openness.py | 21 +- EyeTrackApp/leap.py | 29 +- EyeTrackApp/osc.py | 391 ++++++++++++++++-------- 5 files changed, 417 insertions(+), 250 deletions(-) diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index 37e4a72..30db8bd 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -63,7 +63,6 @@ class EyeTrackSettingsConfig(BaseModel): gui_thresh_add: int = 11 gui_update_check: bool = False gui_ROSC: bool = False - gui_vrc_native: bool = True gui_circular_crop_right: bool = False gui_circular_crop_left: bool = False ibo_filter_samples: int = 400 @@ -81,6 +80,9 @@ class EyeTrackSettingsConfig(BaseModel): gui_legacy_ransac_thresh_right: int = 80 gui_legacy_ransac_thresh_left: int = 80 gui_LEAP_lid: bool = False + gui_osc_vrcft_v1: bool = False + gui_osc_vrcft_v2: bool = False + gui_vrc_native: bool = True class EyeTrackConfig(BaseModel): diff --git a/EyeTrackApp/general_settings_widget.py b/EyeTrackApp/general_settings_widget.py index e667e0a..c2c35d5 100644 --- a/EyeTrackApp/general_settings_widget.py +++ b/EyeTrackApp/general_settings_widget.py @@ -5,8 +5,11 @@ from osc import EyeId from queue import Queue from threading import Event + class SettingsWidget: - def __init__(self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue): + def __init__( + self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue + ): self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-" self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-" @@ -43,160 +46,179 @@ class SettingsWidget: self.gui_circular_crop_right = f"-CIRCLECROPRIGHT{widget_id}-" self.gui_update_check = f"-UPDATECHECK{widget_id}-" self.gui_threshold_slider = f"-BLOBTHRESHOLD{widget_id}-" + self.gui_osc_vrcft_v1 = f"-OSCVRCFTV1{widget_id}-" + self.gui_osc_vrcft_v2 = f"-OSCVRCFTV2{widget_id}-" self.main_config = main_config self.config = main_config.settings self.osc_queue = osc_queue # Define the window's contents self.general_settings_layout = [ - [ sg.Checkbox( "Flip Left Eye X Axis", default=self.config.gui_flip_x_axis_left, key=self.gui_flip_x_axis_left, - background_color='#424042', - tooltip = "Flips the left eye's X axis.", + background_color="#424042", + tooltip="Flips the left eye's X axis.", ), sg.Checkbox( "Flip Right Eye X Axis", default=self.config.gui_flip_x_axis_right, key=self.gui_flip_x_axis_right, - background_color='#424042', - tooltip = "Flips the right eye's X axis.", + background_color="#424042", + tooltip="Flips the right eye's X axis.", ), - - # ], - #[ - sg.Checkbox( + # ], + # [ + sg.Checkbox( "Flip Y Axis", default=self.config.gui_flip_y_axis, key=self.gui_flip_y_axis, - background_color='#424042', - tooltip = "Flips the eye's Y axis.", + background_color="#424042", + tooltip="Flips the eye's Y axis.", ), ], - [sg.Checkbox( - "VRC Native Eyetracking", - default=self.config.gui_vrc_native, - key=self.gui_vrc_native, - background_color='#424042', - tooltip = "Toggle VRCFT output or VRC native", - ), + [ sg.Checkbox( "Dual Eye Falloff", default=self.config.gui_eye_falloff, key=self.gui_eye_falloff, - background_color='#424042', - tooltip = "If one eye stops tracking, we send tracking data from your other eye.", + background_color="#424042", + tooltip="If one eye stops tracking, we send tracking data from your other eye.", ), ], - [sg.Checkbox( + [ + sg.Checkbox( "Check For Updates", default=self.config.gui_update_check, key=self.gui_update_check, - background_color='#424042', - tooltip = "Toggle update check on launch.", + background_color="#424042", + tooltip="Toggle update check on launch.", ), ], [ - sg.Text("One Euro Filter Paramaters:", background_color='#242224'), + sg.Text("One Euro Filter Paramaters:", background_color="#242224"), ], [ - - sg.Text("Min Frequency Cutoff", background_color='#424042'), + sg.Text("Min Frequency Cutoff", background_color="#424042"), sg.InputText( self.config.gui_min_cutoff, key=self.gui_min_cutoff, - size=(0,10), + size=(0, 10), ), - #], - #[ - sg.Text("Speed Coefficient", background_color='#424042'), + # ], + # [ + sg.Text("Speed Coefficient", background_color="#424042"), sg.InputText( - self.config.gui_speed_coefficient, + self.config.gui_speed_coefficient, key=self.gui_speed_coefficient, - size=(0,10), + size=(0, 10), ), ], - [ - sg.Text("OSC Settings:", background_color='#242224'), + [ + sg.Text("OSC Settings:", background_color="#242224"), ], [ - sg.Text("Address:", background_color='#424042'), + sg.Checkbox( + "VRC Native", + default=self.config.gui_vrc_native, + key=self.gui_vrc_native, + background_color="#424042", + tooltip="Toggle VRC's Native Eyetracking format.", + ), + sg.Checkbox( + "VRCFT v1", + default=self.config.gui_osc_vrcft_v1, + key=self.gui_osc_vrcft_v1, + background_color="#424042", + tooltip="Toggle VRCFT's v1 Eyetracking format.", + ), + sg.Checkbox( + "VRCFT v2 (UE)", + default=self.config.gui_osc_vrcft_v2, + key=self.gui_osc_vrcft_v2, + background_color="#424042", + tooltip="Toggle VRCFT's v2 (UE) Eyetracking format.", + ), + ], + [ + sg.Text("Address:", background_color="#424042"), sg.InputText( - self.config.gui_osc_address, + self.config.gui_osc_address, key=self.gui_osc_address, - size=(0,20), - tooltip = "IP address we send OSC data to.", + size=(0, 20), + tooltip="IP address we send OSC data to.", ), - - # ], - # [ - sg.Text("Port:", background_color='#424042'), + # ], + # [ + sg.Text("Port:", background_color="#424042"), sg.InputText( - self.config.gui_osc_port, + self.config.gui_osc_port, key=self.gui_osc_port, - size=(0,10), - tooltip = "OSC port we send data to.", + size=(0, 10), + tooltip="OSC port we send data to.", ), ], [ - sg.Text("Receive functions", background_color='#424042'), + sg.Text("Receive functions", background_color="#424042"), sg.Checkbox( "", default=self.config.gui_ROSC, key=self.gui_ROSC, - background_color='#424042', - size=(0,10), - tooltip = "Toggle OSC receive functions.", + background_color="#424042", + size=(0, 10), + tooltip="Toggle OSC receive functions.", ), ], [ - sg.Text("Receiver Port:", background_color='#424042'), + sg.Text("Receiver Port:", background_color="#424042"), sg.InputText( - self.config.gui_osc_receiver_port, + self.config.gui_osc_receiver_port, key=self.gui_osc_receiver_port, - size=(0,10), - tooltip = "Port we receive OSC data from (used to recalibrate or recenter app from within VRChat.", + size=(0, 10), + tooltip="Port we receive OSC data from (used to recalibrate or recenter app from within VRChat.", ), - #], - # [ - sg.Text("Recenter Address:", background_color='#424042'), + # ], + # [ + sg.Text("Recenter Address:", background_color="#424042"), sg.InputText( - self.config.gui_osc_recenter_address, + self.config.gui_osc_recenter_address, key=self.gui_osc_recenter_address, - size=(0,10), - tooltip = "OSC Address used for recentering your eye.", - ), + size=(0, 10), + tooltip="OSC Address used for recentering your eye.", + ), ], [ - sg.Text("Recalibrate Address:", background_color='#424042'), + sg.Text("Recalibrate Address:", background_color="#424042"), sg.InputText( - self.config.gui_osc_recalibrate_address, + self.config.gui_osc_recalibrate_address, key=self.gui_osc_recalibrate_address, - size=(0,10), - tooltip = "OSC address we use for recalibrating your eye", - ), - ] - + size=(0, 10), + tooltip="OSC address we use for recalibrating your eye", + ), + ], ] - self.widget_layout = [ - [ - sg.Text("General Settings:", background_color='#242224'), + [ + sg.Text("General Settings:", background_color="#242224"), ], [ - sg.Column(self.general_settings_layout, key=self.gui_general_settings_layout, background_color='#424042' ), + sg.Column( + self.general_settings_layout, + key=self.gui_general_settings_layout, + background_color="#424042", + ), ], ] - self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called. + self.cancellation_event = ( + Event() + ) # Set the event until start is called, otherwise we can block if shutdown is called. self.cancellation_event.set() self.image_queue = Queue() - def started(self): return not self.cancellation_event.is_set() @@ -218,43 +240,61 @@ class SettingsWidget: if self.config.gui_osc_port != int(values[self.gui_osc_port]): print(self.config.gui_osc_port, values[self.gui_osc_port]) - try: + try: int(values[self.gui_osc_port]) if len(values[self.gui_osc_port]) <= 5: self.config.gui_osc_port = int(values[self.gui_osc_port]) changed = True else: - print("\033[91m[ERROR] OSC port value must be an integer 0-65535\033[0m") + print( + "\033[91m[ERROR] OSC port value must be an integer 0-65535\033[0m" + ) except: - print("\033[91m[ERROR] OSC port value must be an integer 0-65535\033[0m") + print( + "\033[91m[ERROR] OSC port value must be an integer 0-65535\033[0m" + ) if self.config.gui_osc_receiver_port != int(values[self.gui_osc_receiver_port]): - try: + try: int(values[self.gui_osc_receiver_port]) if len(values[self.gui_osc_receiver_port]) <= 5: - self.config.gui_osc_receiver_port = int(values[self.gui_osc_receiver_port]) + self.config.gui_osc_receiver_port = int( + values[self.gui_osc_receiver_port] + ) changed = True else: - print("\033[91m[ERROR] OSC receive port value must be an integer 0-65535\033[0m") + print( + "\033[91m[ERROR] OSC receive port value must be an integer 0-65535\033[0m" + ) except: - print("\033[91m[ERROR] OSC receive port value must be an integer 0-65535\033[0m") + print( + "\033[91m[ERROR] OSC receive port value must be an integer 0-65535\033[0m" + ) if self.config.gui_osc_address != values[self.gui_osc_address]: self.config.gui_osc_address = values[self.gui_osc_address] changed = True - if self.config.gui_osc_recenter_address != values[self.gui_osc_recenter_address]: + if ( + self.config.gui_osc_recenter_address + != values[self.gui_osc_recenter_address] + ): self.config.gui_osc_recenter_address = values[self.gui_osc_recenter_address] changed = True - if self.config.gui_osc_recalibrate_address != values[self.gui_osc_recalibrate_address]: - self.config.gui_osc_recalibrate_address = values[self.gui_osc_recalibrate_address] + if ( + self.config.gui_osc_recalibrate_address + != values[self.gui_osc_recalibrate_address] + ): + self.config.gui_osc_recalibrate_address = values[ + self.gui_osc_recalibrate_address + ] changed = True if self.config.gui_min_cutoff != values[self.gui_min_cutoff]: self.config.gui_min_cutoff = values[self.gui_min_cutoff] changed = True - + if self.config.gui_speed_coefficient != values[self.gui_speed_coefficient]: self.config.gui_speed_coefficient = values[self.gui_speed_coefficient] changed = True @@ -267,10 +307,18 @@ class SettingsWidget: self.config.gui_flip_x_axis_left = values[self.gui_flip_x_axis_left] changed = True + if self.config.gui_osc_vrcft_v1 != values[self.gui_osc_vrcft_v1]: + self.config.gui_osc_vrcft_v1 = values[self.gui_osc_vrcft_v1] + changed = True + + if self.config.gui_osc_vrcft_v2 != values[self.gui_osc_vrcft_v2]: + self.config.gui_osc_vrcft_v2 = values[self.gui_osc_vrcft_v2] + changed = True + if self.config.gui_vrc_native != values[self.gui_vrc_native]: self.config.gui_vrc_native = values[self.gui_vrc_native] changed = True - + if self.config.gui_update_check != values[self.gui_update_check]: self.config.gui_update_check = values[self.gui_update_check] changed = True @@ -278,7 +326,7 @@ class SettingsWidget: if self.config.gui_flip_y_axis != values[self.gui_flip_y_axis]: self.config.gui_flip_y_axis = values[self.gui_flip_y_axis] changed = True - + if self.config.gui_eye_falloff != values[self.gui_eye_falloff]: self.config.gui_eye_falloff = values[self.gui_eye_falloff] changed = True diff --git a/EyeTrackApp/intensity_based_openness.py b/EyeTrackApp/intensity_based_openness.py index 997c2c4..ab151b0 100644 --- a/EyeTrackApp/intensity_based_openness.py +++ b/EyeTrackApp/intensity_based_openness.py @@ -37,7 +37,7 @@ import psutil import sys process = psutil.Process(os.getpid()) # set process priority to low -try: # medium chance this does absolutely nothing but eh +try: # medium chance this does absolutely nothing but eh sys.getwindowsversion() except AttributeError: process.nice(0) # UNIX: 0 low 10 high @@ -46,6 +46,7 @@ else: process.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) # Windows process.nice() + class EyeId(IntEnum): RIGHT = 0 LEFT = 1 @@ -247,7 +248,7 @@ class IntensityBasedOpeness: upper_y = min(int_y + 25, frame.shape[0] - 1) lower_y = max(int_y - 25, 0) - # frame_crop = frame[lower_y:upper_y, lower_x:upper_x] + # frame_crop = frame[lower_y:upper_y, lower_x:upper_x] # frame = safe_crop(frame, lower_x, lower_y, upper_x, upper_y, False) # ret_, th = cv2.threshold(frame_crop, 80, 1.0, cv2.THRESH_BINARY_INV, dst=frame_crop) frame_crop = frame @@ -274,10 +275,10 @@ class IntensityBasedOpeness: intensity = self.maxval # if intensity <= np.percentile( # TODO test this - # self.filterlist, 0.3 - # ): # filter abnormally low values - # print('filter, assume blink') - # intensity = self.data[int_y, int_x] + # self.filterlist, 0.3 + # ): # filter abnormally low values + # print('filter, assume blink') + # intensity = self.data[int_y, int_x] except: pass # self.tri_filter.append(intensity) @@ -341,10 +342,9 @@ class IntensityBasedOpeness: intensitya = max( data_val + 5000, 1 ) # if current intensity value is not less use this is an agressive adjust, test - self.data[int_y, int_x] = intensitya # set value + self.data[int_y, int_x] = intensitya # set value changed = True - # min pupil global if self.maxval == 0: # that value is not yet saved self.maxval = intensity # set value at 0 index @@ -371,7 +371,6 @@ class IntensityBasedOpeness: ) # for whatever reason when input and maxp are too close it outputs high eyeopen = 1 - eyeopen - if outputSamples > 0: if len(self.averageList) < outputSamples: self.averageList.append(eyeopen) @@ -386,8 +385,6 @@ class IntensityBasedOpeness: if eyeopen < 0: eyeopen = 0.0 - - if changed and ( (time.time() - self.lct) > 5 ): # save every 5 seconds if something changed to save disk usage @@ -412,4 +409,4 @@ class IntensityBasedOpeness: # if eyevec > 0.4: # print("BLINK LCOK") - return eyeopen \ No newline at end of file + return eyeopen diff --git a/EyeTrackApp/leap.py b/EyeTrackApp/leap.py index 449e2b9..315fa99 100644 --- a/EyeTrackApp/leap.py +++ b/EyeTrackApp/leap.py @@ -85,7 +85,7 @@ class LEAP_C(object): else: self.model_path = resource_path("Models/mommy072623.onnx") self.interval = 1 # FPS print update rate - self.low_priority = True # set process priority to low + self.low_priority = True # set process priority to low may cause issues self.print_fps = True # Init variables self.frames = 0 @@ -106,9 +106,6 @@ class LEAP_C(object): onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL ) opts.optimized_model_filepath = "" - self.ort_session = onnxruntime.InferenceSession( - self.model_path, opts, providers=["CPUExecutionProvider"] - ) if self.low_priority: process = psutil.Process(os.getpid()) # set process priority to low @@ -128,9 +125,9 @@ class LEAP_C(object): self.one_euro_filter = OneEuroFilter( np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta ) - #self.one_euro_filter_open = OneEuroFilter( - # np.random.rand(1, 2), min_cutoff=0.01, beta=0.04 - #) + # self.one_euro_filter_open = OneEuroFilter( + # np.random.rand(1, 2), min_cutoff=0.01, beta=0.04 + # ) self.dmax = 0 self.dmin = 0 self.openlist = [] @@ -227,11 +224,11 @@ class LEAP_C(object): if len(self.openlist) < 5000: # TODO expose as setting? self.openlist.append(d) else: - # if d >= np.percentile(self.openlist, 99) or d <= np.percentile( + # if d >= np.percentile(self.openlist, 99) or d <= np.percentile( # self.openlist, 1 - # ): + # ): # pass - #else: + # else: self.openlist.pop(0) self.openlist.append(d) @@ -243,18 +240,18 @@ class LEAP_C(object): except: per = 0.7 pass - # print(d, per) + # print(d, per) x = pre_landmark[6][0] y = pre_landmark[6][1] frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) - # per = d - 0.1 + # per = d - 0.1 self.last_lid = per - # pera = np.array([per, per]) - #self.one_euro_filter_open(pera) - if per <= 0.2: #TODO: EXPOSE AS SETTING + # pera = np.array([per, per]) + # self.one_euro_filter_open(pera) + if per <= 0.2: # TODO: EXPOSE AS SETTING per == 0.0 - # print(per) + # print(per) return frame, float(x), float(y), per frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) diff --git a/EyeTrackApp/osc.py b/EyeTrackApp/osc.py index db0a6f5..aead0ed 100644 --- a/EyeTrackApp/osc.py +++ b/EyeTrackApp/osc.py @@ -1,22 +1,25 @@ - from pythonosc import udp_client from pythonosc import osc_server from pythonosc import dispatcher -from utils.misc_utils import PlaySound,SND_FILENAME,SND_ASYNC +from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC import queue import threading from enum import IntEnum import time + class EyeId(IntEnum): RIGHT = 0 LEFT = 1 BOTH = 2 SETTINGS = 3 ALGOSETTINGS = 4 + + from config import EyeTrackConfig -def eyelid_transformer(self,eye_blink): + +def eyelid_transformer(self, eye_blink): if self.config.osc_invert_eye_close: return float(1 - eye_blink) else: @@ -24,158 +27,260 @@ def eyelid_transformer(self,eye_blink): se = False + + def output_osc(eye_x, eye_y, eye_blink, last_blink, self): - global se + global se + # self.config.gui_osc_vrcft_v2 + # self.config.gui_osc_vrcft_v1 + # self.config.gui_vrc_native - if not self.config.gui_vrc_native: + if self.config.gui_osc_vrcft_v1: - if self.main_config.eye_display_id in [EyeId.RIGHT, EyeId.LEFT]: #we are in single eye mode - se = True + if self.main_config.eye_display_id in [ + EyeId.RIGHT, + EyeId.LEFT, + ]: # we are in single eye mode + se = True - self.client.send_message(self.config.osc_left_eye_x_address, eye_x) - self.client.send_message(self.config.osc_right_eye_x_address, eye_x) - self.client.send_message(self.config.osc_eyes_y_address, eye_y) - - self.config.osc_left_eye_close_address - - self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,eye_blink)) - self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,eye_blink)) - else: - se = False + self.client.send_message(self.config.osc_left_eye_x_address, eye_x) + self.client.send_message(self.config.osc_right_eye_x_address, eye_x) + self.client.send_message(self.config.osc_eyes_y_address, eye_y) - if self.eye_id in [EyeId.LEFT] and not se: #left eye, send data to left - self.l_eye_x = eye_x - self.l_eye_blink = eye_blink + self.config.osc_left_eye_close_address - if self.l_eye_blink == 0.0: - if last_blink > 0.15: #when binary blink is on, blinks may be too fast for OSC so we repeat them. - for i in range(4): - self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink)) - last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: - if self.r_eye_blink == 0.0: #if both eyes closed and DEF is enables, blink - self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink)) - self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.l_eye_blink)) - self.l_eye_x = self.r_eye_x + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, eye_blink), + ) + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, eye_blink), + ) + else: + se = False - self.client.send_message(self.config.osc_left_eye_x_address, self.l_eye_x) - self.left_y = eye_y + if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left + self.l_eye_x = eye_x + self.l_eye_blink = eye_blink - self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink)) + if self.l_eye_blink == 0.0: + if ( + last_blink > 0.15 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(4): + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, self.l_eye_blink), + ) + last_blink = time.time() - last_blink + if self.config.gui_eye_falloff: + if ( + self.r_eye_blink == 0.0 + ): # if both eyes closed and DEF is enables, blink + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, self.l_eye_blink), + ) + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, self.l_eye_blink), + ) + self.l_eye_x = self.r_eye_x + self.client.send_message(self.config.osc_left_eye_x_address, self.l_eye_x) + self.left_y = eye_y - elif self.eye_id in [EyeId.RIGHT] and not se: #Right eye, send data to right - self.r_eye_x = eye_x - self.r_eye_blink = eye_blink + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, self.l_eye_blink), + ) - if self.r_eye_blink == 0.0: - if last_blink > 0.15: #when binary blink is on, blinks may be too fast for OSC so we repeat them. - print("REPEATING R BLINK") - for i in range(4): - self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink)) - last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: - if self.l_eye_blink == 0.0: #if both eyes closed and DEF is enables, blink - self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.r_eye_blink)) - self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink)) + elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right + self.r_eye_x = eye_x + self.r_eye_blink = eye_blink - self.r_eye_x = self.l_eye_x - - self.client.send_message(self.config.osc_right_eye_x_address, eye_x) - self.right_y = eye_y + if self.r_eye_blink == 0.0: + if ( + last_blink > 0.15 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + print("REPEATING R BLINK") + for i in range(4): + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, self.r_eye_blink), + ) + last_blink = time.time() - last_blink + if self.config.gui_eye_falloff: + if ( + self.l_eye_blink == 0.0 + ): # if both eyes closed and DEF is enables, blink + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, self.r_eye_blink), + ) + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, self.r_eye_blink), + ) - self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink)) + self.r_eye_x = self.l_eye_x - if self.main_config.eye_display_id in [EyeId.BOTH] and self.right_y != 621 and self.left_y != 621: - y = (self.right_y + self.left_y) / 2 - self.client.send_message(self.config.osc_eyes_y_address, y) + self.client.send_message(self.config.osc_right_eye_x_address, eye_x) + self.right_y = eye_y + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, self.r_eye_blink), + ) + if ( + self.main_config.eye_display_id in [EyeId.BOTH] + and self.right_y != 621 + and self.left_y != 621 + ): + y = (self.right_y + self.left_y) / 2 + self.client.send_message(self.config.osc_eyes_y_address, y) - else: # VRC NATIVE + if self.config.gui_vrc_native: # VRC NATIVE - if self.main_config.eye_display_id in [EyeId.RIGHT, EyeId.LEFT]: # we are in single eye mode - se = True - if eye_blink == 0.0: - if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. - for i in range(5): - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1 - eye_blink)) - eye_blink += 0.02 #TODO finish tuning value - last_blink = time.time() - last_blink - - else: - self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink)) - self.client.send_message("/tracking/eye/LeftRightVec", [float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0]) # vrc native ET + if self.main_config.eye_display_id in [ + EyeId.RIGHT, + EyeId.LEFT, + ]: # we are in single eye mode + se = True + if eye_blink == 0.0: + if ( + last_blink > 0.2 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(5): + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) + eye_blink += 0.02 # TODO finish tuning value + last_blink = time.time() - last_blink else: - se = False + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) + self.client.send_message( + "/tracking/eye/LeftRightVec", + [float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0], + ) # vrc native ET - if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left - self.l_eye_x = eye_x - self.l_eye_blink = eye_blink - self.left_y = eye_y - self.client.send_message(self.config.osc_left_eye_close_address,eyelid_transformer(self,eye_blink)) + else: + se = False - if self.l_eye_blink == 0.0: - if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. - for i in range(5): - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1 - eye_blink)) - last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: - if self.r_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1 - eye_blink)) - self.l_eye_x = self.r_eye_x + if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left + self.l_eye_x = eye_x + self.l_eye_blink = eye_blink + self.left_y = eye_y + self.client.send_message( + self.config.osc_left_eye_close_address, + eyelid_transformer(self, eye_blink), + ) + if self.l_eye_blink == 0.0: + if ( + last_blink > 0.2 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(5): + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) + last_blink = time.time() - last_blink + if self.config.gui_eye_falloff: + if ( + self.r_eye_blink == 0.0 + ): # if both eyes closed and DEF is enables, blink + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) + self.l_eye_x = self.r_eye_x - elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right - self.r_eye_x = eye_x - self.r_eye_blink = eye_blink - self.right_y = eye_y - self.client.send_message(self.config.osc_right_eye_close_address,eyelid_transformer(self,eye_blink)) + elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right + self.r_eye_x = eye_x + self.r_eye_blink = eye_blink + self.right_y = eye_y + self.client.send_message( + self.config.osc_right_eye_close_address, + eyelid_transformer(self, eye_blink), + ) - if self.r_eye_blink == 0.0: - if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. - for i in range(5): - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1 - eye_blink)) - last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: - if self.l_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(0)) + if self.r_eye_blink == 0.0: + if ( + last_blink > 0.2 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(5): + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) + last_blink = time.time() - last_blink + if self.config.gui_eye_falloff: + if ( + self.l_eye_blink == 0.0 + ): # if both eyes closed and DEF is enables, blink + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(0) + ) - self.r_eye_x = self.l_eye_x + self.r_eye_x = self.l_eye_x + if ( + self.main_config.eye_display_id in [EyeId.BOTH] + and self.r_eye_blink != 621 + and self.r_eye_blink != 621 + ): + if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0: + if ( + last_blink > 0.2 + ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(5): + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1) + ) + last_blink = time.time() - last_blink + eye_blink = (self.r_eye_blink + self.l_eye_blink) / 2 + self.client.send_message( + "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) + ) - if self.main_config.eye_display_id in [EyeId.BOTH] and self.r_eye_blink != 621 and self.r_eye_blink != 621: - if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0: - if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. - for i in range(5): - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1)) - last_blink = time.time() - last_blink - eye_blink = (self.r_eye_blink + self.l_eye_blink) / 2 - self.client.send_message("/tracking/eye/EyesClosedAmount", - float(1 - eye_blink)) + if ( + self.main_config.eye_display_id in [EyeId.BOTH] + and self.right_y != 621 + and self.left_y != 621 + ): + eye_y = (self.right_y + self.left_y) / 2 - - if self.main_config.eye_display_id in [EyeId.BOTH] and self.right_y != 621 and self.left_y != 621: - eye_y = (self.right_y + self.left_y) / 2 - - if not se: - # vrc native ET (z values may need tweaking, they act like a scalar) - self.client.send_message("/tracking/eye/LeftRightVec",[float(self.l_eye_x), float(self.left_y), 1.0, float(self.r_eye_x), float(self.right_y), 1.0]) + if not se: + # vrc native ET (z values may need tweaking, they act like a scalar) + self.client.send_message( + "/tracking/eye/LeftRightVec", + [ + float(self.l_eye_x), + float(self.left_y), + 1.0, + float(self.r_eye_x), + float(self.right_y), + 1.0, + ], + ) class VRChatOSC: - # Use a tuple of blink (true, blinking, false, not), x, y for now. - def __init__(self, cancellation_event: threading.Event, msg_queue: queue.Queue[tuple[bool, int, int]], main_config: EyeTrackConfig,): + # Use a tuple of blink (true, blinking, false, not), x, y for now. + def __init__( + self, + cancellation_event: threading.Event, + msg_queue: queue.Queue[tuple[bool, int, int]], + main_config: EyeTrackConfig, + ): self.main_config = main_config self.config = main_config.settings - self.client = udp_client.SimpleUDPClient(self.config.gui_osc_address, int(self.config.gui_osc_port)) # use OSC port and address that was set in the config + self.client = udp_client.SimpleUDPClient( + self.config.gui_osc_address, int(self.config.gui_osc_port) + ) # use OSC port and address that was set in the config self.cancellation_event = cancellation_event self.msg_queue = msg_queue self.eye_id = EyeId.RIGHT @@ -186,7 +291,6 @@ class VRChatOSC: self.r_eye_blink = 0.7 self.l_eye_blink = 0.7 - def run(self): start = time.time() last_blink = time.time() @@ -203,15 +307,22 @@ class VRChatOSC: class VRChatOSCReceiver: - def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []): + def __init__( + self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: [] + ): self.config = main_config.settings self.cancellation_event = cancellation_event self.dispatcher = dispatcher.Dispatcher() self.eyes = eyes # we cant import CameraWidget so any type it is try: - self.server = osc_server.OSCUDPServer((self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)), self.dispatcher) + self.server = osc_server.OSCUDPServer( + (self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)), + self.dispatcher, + ) except: - print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m") + print( + f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m" + ) def shutdown(self): print("\033[94m[INFO] Exiting OSC Receiver\033[0m") @@ -221,28 +332,40 @@ class VRChatOSCReceiver: pass def recenter_eyes(self, address, osc_value): - if type(osc_value) != bool: return # just incase we get anything other than bool + if type(osc_value) != bool: + return # just incase we get anything other than bool if osc_value: for eye in self.eyes: eye.settings.gui_recenter_eyes = True def recalibrate_eyes(self, address, osc_value): - if type(osc_value) != bool: return # just incase we get anything other than bool + if type(osc_value) != bool: + return # just incase we get anything other than bool if osc_value: for eye in self.eyes: eye.ransac.ibo.clear_filter() eye.ransac.calibration_frame_counter = self.config.calibration_samples - PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) + PlaySound("Audio/start.wav", SND_FILENAME | SND_ASYNC) def run(self): - + # bind what function to run when specified OSC message is received try: - self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes) - self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes) + self.dispatcher.map( + self.config.gui_osc_recalibrate_address, self.recalibrate_eyes + ) + self.dispatcher.map( + self.config.gui_osc_recenter_address, self.recenter_eyes + ) # start the server - print("\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format(self.server.server_address)) + print( + "\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format( + self.server.server_address + ) + ) self.server.serve_forever() - + except: - print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m") \ No newline at end of file + print( + f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m" + )