From b1472949c715737f00987271117ba82558ed6e69 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:49:23 -0800 Subject: [PATCH] fix bugs, add basic blink sync, adjust default values, bump ver, fix pyinstaller, osc --- EyeTrackApp/config.py | 7 ++-- EyeTrackApp/eye_processor.py | 33 ++++------------ EyeTrackApp/eyetrackapp.py | 2 +- EyeTrackApp/eyetrackapp.spec | 2 +- EyeTrackApp/osc.py | 71 ++++++++++++++++++++++++---------- EyeTrackApp/settings_widget.py | 13 +++++++ 6 files changed, 76 insertions(+), 52 deletions(-) diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index 3e5ea01..c70deef 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -7,8 +7,8 @@ CONFIG_FILE_NAME: str = "eyetrack_settings.json" class EyeTrackCameraConfig(BaseModel): - threshold: int = 0 - rotation_angle: int = 50 + threshold: int = 50 + rotation_angle: int = 0 roi_window_x: int = 0 roi_window_y: int = 0 roi_window_w: int = 0 @@ -34,7 +34,8 @@ class EyeTrackSettingsConfig(BaseModel): gui_blob_minsize: float = 10 gui_recenter_eyes: bool = False gui_eye_falloff: bool = False - tracker_single_eye: float = 0 + tracker_single_eye: int = 0 + gui_blink_sync: bool = False class EyeTrackConfig(BaseModel): diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 88eaa5a..8c433fd 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -268,16 +268,12 @@ class EyeProcessor: # draw filled circle in white on black background as mask mask = np.zeros((ht, wd), dtype=np.uint8) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) - # create white colored background color = np.full_like(self.current_image_gray, (255)) - # apply mask to image masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) - # apply inverse mask to colored image masked_color = cv2.bitwise_and(color, color, mask=255 - mask) - # combine the two masked images self.current_image_gray = cv2.add(masked_img, masked_color) except: @@ -285,15 +281,7 @@ class EyeProcessor: else: self.cct = self.cct - 1 - # Increase our threshold value slightly, in order to have a better possibility of getting back - # something to do blob tracking on. - hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) - histr = hist.ravel() - peaks, properties = sp.find_peaks(histr, distance=5) - minpeak = np.min(peaks) - thresholdoptics = np.array(minpeak + int(self.config.threshold + 12)) - larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold - larger_threshold = cv2.bitwise_not(larger_threshold) + _, larger_threshold = cv2.threshold(self.current_image_gray, int(self.config.threshold + 12), 255, cv2.THRESH_BINARY) # Blob tracking requires that we have a vague idea of where the eye may be at the moment. This # means we need to have had at least one successful runthrough of the Pupil Labs algorithm in # order to have a projected sphere. @@ -507,23 +495,18 @@ class EyeProcessor: if self.cct == 0: try: ht, wd = self.current_image_gray.shape[:2] - radius = int(float(self.lkg_projected_sphere["axes"][0])) self.xc = int(float(self.lkg_projected_sphere["center"][0])) self.yc = int(float(self.lkg_projected_sphere["center"][1])) # draw filled circle in white on black background as mask mask = np.zeros((ht, wd), dtype=np.uint8) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) - # create white colored background color = np.full_like(self.current_image_gray, (255)) - # apply mask to image masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) - # apply inverse mask to colored image masked_color = cv2.bitwise_and(color, color, mask=255 - mask) - # combine the two masked images self.current_image_gray = cv2.add(masked_img, masked_color) except: @@ -533,14 +516,12 @@ class EyeProcessor: else: self.cct = 300 - # Using Histogram based thresholding. Improves robustness insanely - hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) - histr = hist.ravel() - peaks, properties = sp.find_peaks(histr, distance=5) - minpeak = np.min(peaks) - thresholdoptics = np.array(minpeak + int(self.config.threshold)) - thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold - thresh = cv2.bitwise_not(thresh) + _, thresh = cv2.threshold( + self.current_image_gray, + int(self.config.threshold), + 255, + cv2.THRESH_BINARY, + ) # Set up morphological transforms, for smoothing and clearing the image we get out of the # thresholding operation. After this, we'd really like to just have a black blob in the middle diff --git a/EyeTrackApp/eyetrackapp.py b/EyeTrackApp/eyetrackapp.py index 3983cbf..297ca86 100644 --- a/EyeTrackApp/eyetrackapp.py +++ b/EyeTrackApp/eyetrackapp.py @@ -119,7 +119,7 @@ def main(): osc_receiver_thread.start() # Create the window - window = sg.Window("EyeTrackVR v0.1.7", layout, icon='Images/logo.ico', background_color='#292929') + window = sg.Window("EyeTrackVR v0.1.7.1", layout, icon='Images/logo.ico', background_color='#292929') # GUI Render loop while True: diff --git a/EyeTrackApp/eyetrackapp.spec b/EyeTrackApp/eyetrackapp.spec index 6d4cb04..1e4f3e9 100644 --- a/EyeTrackApp/eyetrackapp.spec +++ b/EyeTrackApp/eyetrackapp.spec @@ -6,7 +6,7 @@ block_cipher = None a = Analysis(['eyetrackapp.py'], pathex=[], binaries=[], - datas=[("Audio/*", "Audio"), ("Images/*", "Images/")], + datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("pye3dcustom/refraction_models/*", "pye3dcustom/refraction_models")], hiddenimports=['cv2', 'numpy', 'PySimpleGui'], hookspath=[], hooksconfig={}, diff --git a/EyeTrackApp/osc.py b/EyeTrackApp/osc.py index 51c7862..cb397b5 100644 --- a/EyeTrackApp/osc.py +++ b/EyeTrackApp/osc.py @@ -54,15 +54,22 @@ class VRChatOSC: self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye + if self.config.gui_blink_sync and not rb and not lb: + self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r + self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left + self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye + else: if eye_id in [EyeId.RIGHT]: yr = eye_info.y sx = eye_info.x sy = eye_info.y rb = False - self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x) - self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right - self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye + self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x) + if not self.config.gui_blink_sync or self.config.gui_blink_sync and not lb: + self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye if eye_id in [EyeId.LEFT]: yl = eye_info.y @@ -70,27 +77,21 @@ class VRChatOSC: sy = eye_info.y lb = False self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x) - self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left - self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye + if not self.config.gui_blink_sync or self.config.gui_blink_sync and not rb: + self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left + self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye if (yr != 621 and yl != 621) and (lb == False and rb == False): y = (yr + yl) / 2 self.client.send_message("/avatar/parameters/EyesY", y) else: - if self.config.gui_eye_falloff: + + if self.config.gui_blink_sync: if eye_id in [EyeId.LEFT]: lb = True if eye_id in [EyeId.RIGHT]: rb = True - if rb == True or lb == True: # If one eye closed and fall off is enabled, mirror data - self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes. - self.client.send_message("/avatar/parameters/RightEyeX", sx) - self.client.send_message("/avatar/parameters/EyesY", sy) - self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right - self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r - self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left - self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye - if rb == True and lb == True: # If both eyes are closed, blink + if rb == True and lb == True : # If both eyes are closed, blink if last_blink > 0.5: for i in range(4): self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye @@ -107,7 +108,8 @@ class VRChatOSC: self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0)) last_blink = time.time() - else: + + if not self.config.gui_eye_falloff: if eye_id in [EyeId.LEFT]: lb = True if last_blink > 0.5: @@ -124,6 +126,28 @@ class VRChatOSC: self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye last_blink = time.time() + else: + if eye_id in [EyeId.LEFT]: + lb = True + if eye_id in [EyeId.RIGHT]: + rb = True + if rb or lb: # If one eye closed and fall off is enabled, mirror data + self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes. + self.client.send_message("/avatar/parameters/RightEyeX", sx) + self.client.send_message("/avatar/parameters/EyesY", sy) + self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r + self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left + self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye + if rb and lb: # If both eyes are closed, blink + if last_blink > 0.5: + for i in range(4): + self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye + self.client.send_message("/avatar/parameters/LeftEyeLid", float(1)) + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye + self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0)) + last_blink = time.time() + class VRChatOSCReceiver: def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []): @@ -151,9 +175,14 @@ class VRChatOSCReceiver: PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) def run(self): - print("VRChatOSCReceiver serving on {}".format(self.server.server_address)) + # bind what function to run when specified OSC message is received - self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes) - self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes) - # start the server - self.server.serve_forever() + try: + self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes) + self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes) + # start the server + print("VRChatOSCReceiver serving on {}".format(self.server.server_address)) + self.server.serve_forever() + + except: + print("[WARN] OSC Receive port taken! Please use another OSC port.") diff --git a/EyeTrackApp/settings_widget.py b/EyeTrackApp/settings_widget.py index e7a21cc..a7b208b 100644 --- a/EyeTrackApp/settings_widget.py +++ b/EyeTrackApp/settings_widget.py @@ -26,6 +26,7 @@ class SettingsWidget: self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-" self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-" self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-" + self.gui_blink_sync = f"-BLINKSYNC{widget_id}-" self.main_config = main_config self.config = main_config.settings self.osc_queue = osc_queue @@ -62,6 +63,13 @@ class SettingsWidget: background_color='#424042', ), ], + [sg.Checkbox( + "Sync Blinks (disables winking)", + default=self.config.gui_blink_sync, + key=self.gui_blink_sync, + background_color='#424042', + ), + ], [ sg.Text("Tracking Algorithim Settings:", background_color='#242224'), @@ -221,6 +229,11 @@ class SettingsWidget: self.config.gui_eye_falloff = values[self.gui_eye_falloff] changed = True + if self.config.gui_blink_sync != values[self.gui_blink_sync]: + self.config.gui_blink_sync = values[self.gui_blink_sync] + changed = True + + if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]: self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]