diff --git a/EyeTrackApp/osc.py b/EyeTrackApp/osc.py index 68bfc10..7a84598 100644 --- a/EyeTrackApp/osc.py +++ b/EyeTrackApp/osc.py @@ -7,6 +7,7 @@ import queue import threading from enum import IntEnum import time +import numpy as np class EyeId(IntEnum): @@ -25,15 +26,12 @@ def eyelid_transformer(self, eye_blink): se = False +falloff = False def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity, self): # print(pupil_dilation) - global se - - if self.eye_id in [EyeId.LEFT]: - # avg_velocity - pass + global se, falloff # self.config.gui_osc_vrcft_v2 # self.config.gui_osc_vrcft_v1 @@ -67,6 +65,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left self.l_eye_x = eye_x self.l_eye_blink = eye_blink + self.l_eye_velocity = avg_velocity if self.l_eye_blink == 0.0: if ( @@ -92,19 +91,20 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right self.r_eye_x = eye_x self.r_eye_blink = eye_blink + self.l_eye_velocity = avg_velocity if self.r_eye_blink == 0.0: if ( last_blink > 0.15 ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. - print("REPEATING R BLINK") + # print("REPEATING R BLINK") for i in range(4): self.client.send_message( self.config.osc_right_eye_close_address, eyelid_transformer(self, self.r_eye_blink), ) last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: + if self.config.gui_outer_side_falloff: if ( self.l_eye_blink == 0.0 ): # if both eyes closed and DEF is enables, blink @@ -159,6 +159,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left self.l_eye_x = eye_x self.l_eye_blink = eye_blink + self.r_eye_velocity = avg_velocity if self.l_eye_blink == 0.0: if ( @@ -170,7 +171,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity eyelid_transformer(self, self.l_eye_blink), ) last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: + if self.config.gui_outer_side_falloff: if ( self.r_eye_blink == 0.0 ): # if both eyes closed and DEF is enables, blink @@ -200,19 +201,20 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right self.r_eye_x = eye_x self.r_eye_blink = eye_blink + self.r_eye_velocity = avg_velocity if self.r_eye_blink == 0.0: if ( last_blink > 0.15 ): # when binary blink is on, blinks may be too fast for OSC so we repeat them. - print("REPEATING R BLINK") + # print("REPEATING R BLINK") for i in range(4): self.client.send_message( "/avatar/parameters/v2/EyeLidRight", eyelid_transformer(self, self.r_eye_blink), ) last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: + if self.config.gui_outer_side_falloff: if ( self.l_eye_blink == 0.0 ): # if both eyes closed and DEF is enables, blink @@ -282,6 +284,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity self.l_eye_x = eye_x self.l_eye_blink = eye_blink self.left_y = eye_y + self.l_eye_velocity = avg_velocity self.client.send_message( self.config.osc_left_eye_close_address, eyelid_transformer(self, eye_blink), @@ -296,7 +299,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) ) last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: + if self.config.gui_outer_side_falloff: if ( self.r_eye_blink == 0.0 ): # if both eyes closed and DEF is enables, blink @@ -309,6 +312,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity self.r_eye_x = eye_x self.r_eye_blink = eye_blink self.right_y = eye_y + self.r_eye_velocity = avg_velocity self.client.send_message( self.config.osc_right_eye_close_address, eyelid_transformer(self, eye_blink), @@ -323,7 +327,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity "/tracking/eye/EyesClosedAmount", float(1 - eye_blink) ) last_blink = time.time() - last_blink - if self.config.gui_eye_falloff: + if self.config.gui_outer_side_falloff: if ( self.l_eye_blink == 0.0 ): # if both eyes closed and DEF is enables, blink @@ -396,6 +400,8 @@ class VRChatOSC: self.l_eye_x = 0 self.r_eye_blink = 0.7 self.l_eye_blink = 0.7 + self.l_eye_velocity = 0 + self.r_eye_velocity = 1 def run(self): start = time.time() diff --git a/EyeTrackApp/osc_calibrate_filter.py b/EyeTrackApp/osc_calibrate_filter.py index 13631b6..ba73566 100644 --- a/EyeTrackApp/osc_calibrate_filter.py +++ b/EyeTrackApp/osc_calibrate_filter.py @@ -2,6 +2,7 @@ import numpy as np import time from enum import IntEnum from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path +from utils.eye_falloff import velocity_falloff class EyeId(IntEnum): @@ -17,6 +18,12 @@ class var: past_x = 0 past_y = 0 start_time = time.time() + r_eye_x = 0.0 + l_eye_x = 0.0 + left_y = 0.0 + right_y = 0.0 + l_eye_velocity = 0.0 + r_eye_velocity = 0.0 class cal: @@ -31,6 +38,7 @@ class cal: flipx = self.settings.gui_flip_x_axis_right else: flipx = self.settings.gui_flip_x_axis_left + if self.calibration_frame_counter == 0: self.calibration_frame_counter = None self.config.calib_XOFF = cx @@ -134,7 +142,7 @@ class cal: ) / ((var.start_time - run_time) * 10) ) - if len(var.velocity_rolling_list) < 30: + if len(var.velocity_rolling_list) < 15: var.velocity_rolling_list.append(float(velocity)) else: var.velocity_rolling_list.pop(0) @@ -155,6 +163,8 @@ class cal: except: pass + out_x, out_y = velocity_falloff(self, var, out_x, out_y) + return out_x, out_y, var.average_velocity else: if self.printcal: diff --git a/EyeTrackApp/utils/eye_falloff.py b/EyeTrackApp/utils/eye_falloff.py new file mode 100644 index 0000000..4738f06 --- /dev/null +++ b/EyeTrackApp/utils/eye_falloff.py @@ -0,0 +1,75 @@ +import numpy as np +from enum import IntEnum + + +class EyeId(IntEnum): + RIGHT = 0 + LEFT = 1 + BOTH = 2 + SETTINGS = 3 + + +def velocity_falloff(self, var, out_x, out_y): + + if self.eye_id in [EyeId.LEFT]: + var.l_eye_velocity = var.average_velocity + if self.settings.gui_outer_side_falloff: + dist = abs( + np.sqrt( + abs(np.square(out_x - var.r_eye_x) - np.square(out_y - var.right_y)) + ) + ) + # print(dist) # TODO remove once testing is done + if dist > self.settings.gui_eye_dominant_diff_thresh: + falloff = True + if ( + not self.settings.gui_left_eye_dominant + and not self.settings.gui_right_eye_dominant + ): + if var.l_eye_velocity < var.r_eye_velocity: + var.r_eye_x = out_x + var.right_y = out_y + else: + eye_x = var.l_eye_x + eye_y = var.left_y + else: + var.l_eye_x = out_x + var.left_y = out_y + falloff = False + + if self.eye_id == EyeId.RIGHT: + var.r_eye_velocity = var.average_velocity + if self.settings.gui_outer_side_falloff: + dist = abs( + np.sqrt( + abs(np.square(var.l_eye_x - out_x) - np.square(var.left_y - out_y)) + ) + ) + # print(dist, "r") # TODO remove once testing is done + if dist > self.settings.gui_eye_dominant_diff_thresh: + falloff = True + if ( + not self.settings.gui_left_eye_dominant + and not self.settings.gui_right_eye_dominant + ): + if var.l_eye_velocity < var.r_eye_velocity: + var.r_eye_x = out_x + var.right_y = out_y + else: + var.l_eye_x = out_x + var.left_y = out_y # need to make sure we send these values... might re think the whole file + else: + falloff = False + var.r_eye_x = out_x + var.right_y = out_y + + if self.eye_id == EyeId.LEFT and falloff: + out_x = var.r_eye_x + out_y = var.right_y + if self.eye_id == EyeId.RIGHT and falloff: + out_x = var.l_eye_x + out_y = ( + var.left_y + ) # needs to not be in this file lol... well if we want proper visualization.. + + return out_x, out_y