From 03c9ad6b89b66e616b6a4e36424a4d6aa89d607f Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Mon, 12 Sep 2022 17:20:31 -0700 Subject: [PATCH] initial commit --- EyeTrackApp/config.py | 2 +- EyeTrackApp/eyetrackapp.py | 47 +++++- EyeTrackApp/settings_widget.py | 284 +++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 4 deletions(-) create mode 100644 EyeTrackApp/settings_widget.py diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index a34eebc..584ce85 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -20,7 +20,7 @@ class EyeTrackCameraConfig: focal_length: "int" = 30 capture_source: "Union[int, str, None]" = None vrc_eye_position_scalar: "int" = 3000 - #show_color_image: "bool" = False + show_color_image: "bool" = False CONFIG_FILE_NAME = "eyetrack_settings.json" diff --git a/EyeTrackApp/eyetrackapp.py b/EyeTrackApp/eyetrackapp.py index 5659149..c0eac57 100644 --- a/EyeTrackApp/eyetrackapp.py +++ b/EyeTrackApp/eyetrackapp.py @@ -6,6 +6,7 @@ os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0" from osc import VRChatOSC, EyeId from config import EyeTrackConfig from camera_widget import CameraWidget +from settings_widget import SettingsWidget import queue import threading import PySimpleGUI as sg @@ -13,11 +14,12 @@ import PySimpleGUI as sg WINDOW_NAME = "EyeTrackApp" RIGHT_EYE_NAME = "-RIGHTEYEWIDGET-" LEFT_EYE_NAME = "-LEFTEYEWIDGET-" +SETTINGS_NAME = "-SETTINGSWIDGET-" LEFT_EYE_RADIO_NAME = "-LEFTEYERADIO-" RIGHT_EYE_RADIO_NAME = "-RIGHTEYERADIO-" BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-" - +SETTINGS_RADIO_NAME = '-SETTINGSRADIO-' def main(): # Get Configuration @@ -47,6 +49,9 @@ def main(): CameraWidget(EyeId.RIGHT, config, osc_queue), CameraWidget(EyeId.LEFT, config, osc_queue), ] + Settings = [ + SettingsWidget(EyeId.SETTINGS, config, osc_queue), + ] layout = [ [ @@ -71,6 +76,13 @@ def main(): default=(config.eye_display_id == EyeId.BOTH), key=BOTH_EYE_RADIO_NAME, ), + sg.Radio( + "Settings", + "SETTINGSRADIO", + background_color='#292929', + default=(config.eye_display_id == EyeId.SETTINGS), + key=SETTINGS_RADIO_NAME, + ), ], [ sg.Column( @@ -87,6 +99,13 @@ def main(): visible=(config.eye_display_id in [EyeId.LEFT, EyeId.BOTH]), background_color='#424042', ), + sg.Column( + Settings[0].widget_layout, + vertical_alignment="top", + key=SETTINGS_NAME, + visible=(config.eye_display_id in [EyeId.SETTINGS]), + background_color='#424042', + ), ], ] @@ -94,9 +113,11 @@ def main(): eyes[0].start() if config.eye_display_id in [EyeId.LEFT, EyeId.BOTH]: eyes[1].start() + if config.eye_display_id in [EyeId.SETTINGS]: + Settings[0].start() # Create the window - window = sg.Window("EyeTrackVR v0.1.3", layout, icon='Images/logo.ico', background_color='#292929') + window = sg.Window("EyeTrackVR v0.1.3 nightly", layout, icon='Images/logo.ico', background_color='#292929') # GUI Render loop while True: @@ -105,6 +126,7 @@ def main(): # If we're in either mode and someone hits q, quit immediately if event == "Exit" or event == sg.WIN_CLOSED: + Settings[0].stop() for eye in eyes: eye.stop() cancellation_event.set() @@ -114,29 +136,48 @@ def main(): # t2s_thread.join() print("Exiting EyeTrackApp") return - + print(config.eye_display_id) if values[RIGHT_EYE_RADIO_NAME] and config.eye_display_id != EyeId.RIGHT: + Settings[0].stop() eyes[0].start() eyes[1].stop() + Settings[0].stop() window[RIGHT_EYE_NAME].update(visible=True) window[LEFT_EYE_NAME].update(visible=False) + window[SETTINGS_NAME].update(visible=False) config.eye_display_id = EyeId.RIGHT config.save() elif values[LEFT_EYE_RADIO_NAME] and config.eye_display_id != EyeId.LEFT: + Settings[0].stop() eyes[0].stop() eyes[1].start() window[RIGHT_EYE_NAME].update(visible=False) window[LEFT_EYE_NAME].update(visible=True) + window[SETTINGS_NAME].update(visible=False) config.eye_display_id = EyeId.LEFT config.save() elif values[BOTH_EYE_RADIO_NAME] and config.eye_display_id != EyeId.BOTH: + Settings[0].stop() eyes[0].start() eyes[1].start() window[RIGHT_EYE_NAME].update(visible=True) window[LEFT_EYE_NAME].update(visible=True) + window[SETTINGS_NAME].update(visible=False) config.eye_display_id = EyeId.BOTH config.save() + elif values[SETTINGS_RADIO_NAME] and config.eye_display_id != EyeId.SETTINGS: + eyes[0].stop() + eyes[1].stop() + Settings[0].start() + window[RIGHT_EYE_NAME].update(visible=False) + window[LEFT_EYE_NAME].update(visible=False) + window.Element(LEFT_EYE_NAME).Update(value=False) + window[SETTINGS_NAME].update(visible=True) + + config.eye_display_id = EyeId.SETTINGS + config.save() + # Otherwise, render all of our cameras for eye in eyes: if eye.started(): diff --git a/EyeTrackApp/settings_widget.py b/EyeTrackApp/settings_widget.py new file mode 100644 index 0000000..35178e6 --- /dev/null +++ b/EyeTrackApp/settings_widget.py @@ -0,0 +1,284 @@ +import PySimpleGUI as sg +from config import EyeTrackConfig +from threading import Event, Thread +from eye_processor import EyeProcessor, InformationOrigin +from enum import Enum +from queue import Queue, Empty +from camera import Camera, CameraState +import cv2 +from osc import EyeId + +class SettingsWidget: + def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue): + + self.gui_show_color_image = f"-SETTING1{widget_id}-" + self.gui_camera_addr = f"-CAMERAADDR{widget_id}-" + self.gui_threshold_slider = f"-THREADHOLDSLIDER{widget_id}-" + self.gui_rotation_slider = f"-ROTATIONSLIDER{widget_id}-" + self.gui_roi_button = f"-ROIMODE{widget_id}-" + self.gui_roi_layout = f"-ROILAYOUT{widget_id}-" + self.gui_roi_selection = f"-GRAPH{widget_id}-" + self.gui_tracking_button = f"-TRACKINGMODE{widget_id}-" + self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-" + self.gui_tracking_layout = f"-TRACKINGLAYOUT{widget_id}-" + self.gui_tracking_image = f"-IMAGE{widget_id}-" + self.gui_output_graph = f"-OUTPUTGRAPH{widget_id}-" + self.gui_restart_calibration = f"-RESTARTCALIBRATION{widget_id}-" + self.gui_recenter_eye = f"-RECENTEREYE{widget_id}-" + self.gui_mode_readout = f"-APPMODE{widget_id}-" + self.gui_show_color_image = f"-SHOWCOLORIMAGE{widget_id}-" + self.gui_roi_message = f"-ROIMESSAGE{widget_id}-" + + self.main_config = main_config + self.config = main_config.left_eye + + + # Define the window's contents + self.setting_layout = [ + [ + sg.Text("Rotation", background_color='#424042'), + sg.Slider( + range=(0, 360), + default_value=5, + orientation="h", + key=self.gui_rotation_slider, + background_color='#424042' + ), + ], + [ + sg.Button("Recenter Eye", key=self.gui_recenter_eye, button_color = '#6f4ca1'), + sg.Checkbox( + "Show Color Image:", + default=self.config.show_color_image, + key=self.gui_show_color_image, + ), + ], + [sg.Image(filename="", key=self.gui_tracking_image)], + [ + sg.Graph( + (200, 200), + (-100, 100), + (100, -100), + background_color="white", + key=self.gui_output_graph, + drag_submits=True, + enable_events=True, + ), + sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', visible=False), + ], + ] + + self.widget_layout = [ + [ + sg.Text("Camera Address ass", background_color='#424042'), + sg.InputText(self.config.capture_source, key=self.gui_camera_addr), + ], + [ + sg.Button( + "Save and Restart Tracking", key=self.gui_save_tracking_button, button_color = '#6f4ca1' + ), + ], + [ + sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color = '#6f4ca1'), + sg.Button("Cropping Mode", key=self.gui_roi_button, button_color = '#6f4ca1'), + ], + # [ + # sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042' ), + # sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False), + # ], + ] + + self.cancellation_event = Event() + # Set the event until start is called, otherwise we can block if shutdown is called. + self.cancellation_event.set() + self.capture_event = Event() + self.capture_queue = Queue() + self.roi_queue = Queue() + + self.image_queue = Queue() + + self.ransac = EyeProcessor( + self.config, + self.cancellation_event, + self.capture_event, + self.capture_queue, + self.image_queue, + ) + + self.camera_status_queue = Queue() + self.camera = Camera( + self.config, + 0, + self.cancellation_event, + self.capture_event, + self.camera_status_queue, + self.capture_queue, + ) + + + def started(self): + return not self.cancellation_event.is_set() + + def start(self): + # If we're already running, bail + if not self.cancellation_event.is_set(): + return + self.cancellation_event.clear() + self.ransac_thread = Thread(target=self.ransac.run) + self.ransac_thread.start() + self.camera_thread = Thread(target=self.camera.run) + self.camera_thread.start() + + def stop(self): + # If we're not running yet, bail + if self.cancellation_event.is_set(): + return + self.cancellation_event.set() + self.ransac_thread.join() + self.camera_thread.join() + + def render(self, window, event, values): + changed = False + # If anything has changed in our configuration settings, change/update those. + if ( + event == self.gui_save_tracking_button + and values[self.gui_camera_addr] != self.config.capture_source + ): + print("New value: {}".format(values[self.gui_camera_addr])) + try: + # Try storing ints as ints, for those using wired cameras. + self.config.capture_source = int(values[self.gui_camera_addr]) + except ValueError: + if values[self.gui_camera_addr] == "": + self.config.capture_source = None + else: + self.config.capture_source = values[self.gui_camera_addr] + changed = True + + if self.config.threshold != values[self.gui_threshold_slider]: + self.config.threshold = int(values[self.gui_threshold_slider]) + changed = True + + if self.config.rotation_angle != values[self.gui_rotation_slider]: + self.config.rotation_angle = int(values[self.gui_rotation_slider]) + changed = True + + if self.config.show_color_image != values[self.gui_show_color_image]: + self.config.show_color_image = values[self.gui_show_color_image] + changed = True + + if changed: + self.main_config.save() + + if event == self.gui_tracking_button: + print("Moving to tracking mode") + + elif event == self.gui_roi_button: + print("Move to roi mode") + self.in_roi_mode = True + self.camera.set_output_queue(self.roi_queue) + window[self.gui_roi_layout].update(visible=True) + window[self.gui_tracking_layout].update(visible=False) + elif event == "{}+UP".format(self.gui_roi_selection): + # Event for mouse button up in ROI mode + self.is_mouse_up = True + if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: + self.config.roi_window_x = min([self.x0, self.x1]) + self.config.roi_window_y = min([self.y0, self.y1]) + self.config.roi_window_w = abs(self.x0 - self.x1) + self.config.roi_window_h = abs(self.y0 - self.y1) + self.main_config.save() + elif event == self.gui_roi_selection: + # Event for mouse button down or mouse drag in ROI mode + if self.is_mouse_up: + self.is_mouse_up = False + self.x0, self.y0 = values[self.gui_roi_selection] + self.x1, self.y1 = values[self.gui_roi_selection] + elif event == self.gui_restart_calibration: + self.ransac.calibration_frame_counter = 300 + elif event == self.gui_recenter_eye: + self.ransac.recenter_eye = True + + needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 + + if self.config.capture_source is None or self.config.capture_source == "": + window[self.gui_mode_readout].update("Waiting for camera address") + window[self.gui_roi_message].update(visible=False) + window[self.gui_output_graph].update(visible=False) + elif self.camera.camera_status == CameraState.CONNECTING: + window[self.gui_mode_readout].update("Camera Connecting") + elif self.camera.camera_status == CameraState.DISCONNECTED: + window[self.gui_mode_readout].update("CAMERA DISCONNECTED") + elif needs_roi_set: + window[self.gui_mode_readout].update("Awaiting Eye Cropping Setting") + elif self.ransac.calibration_frame_counter != None: + window[self.gui_mode_readout].update("Calibration") + else: + window[self.gui_mode_readout].update("Tracking") + + if self.in_roi_mode: + try: + if self.roi_queue.empty(): + self.capture_event.set() + maybe_image = self.roi_queue.get(block=False) + imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes() + graph = window[self.gui_roi_selection] + if self.figure: + graph.delete_figure(self.figure) + # INCREDIBLY IMPORTANT ERASE. Drawing images does NOT overwrite the buffer, the fucking + # graph keeps every image fed in until you call this. Therefore we have to make sure we + # erase before we redraw, otherwise we'll leak memory *very* quickly. + graph.erase() + graph.draw_image(data=imgbytes, location=(0, 0)) + if None not in (self.x0, self.y0, self.x1, self.y1): + self.figure = graph.draw_rectangle( + (self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1" + ) + except Empty: + pass + else: + if needs_roi_set: + window[self.gui_roi_message].update(visible=True) + window[self.gui_output_graph].update(visible=False) + return + try: + window[self.gui_roi_message].update(visible=False) + window[self.gui_output_graph].update(visible=True) + (maybe_image, eye_info) = self.image_queue.get(block=False) + imgbytes = cv2.imencode(".ppm", maybe_image)[1].tobytes() + window[self.gui_tracking_image].update(data=imgbytes) + + # Update the GUI + graph = window[self.gui_output_graph] + graph.erase() + + if ( + eye_info.info_type != InformationOrigin.FAILURE + and not eye_info.blink + ): + graph.update(background_color="white") + + # if eye_info.y < 0: # flip visualzation to be correct + # visy = abs(eye_info.y) + # elif eye_info.y >= 0: + # visy = -abs(eye_info.y) + try: + + graph.draw_circle( + (eye_info.x * -100, eye_info.y * -100), + 23, + fill_color="black", + line_color="white", + ) + except: + pass + elif eye_info.blink: + graph.update(background_color="#6f4ca1") + elif eye_info.info_type == InformationOrigin.FAILURE: + graph.update(background_color="red") + + # Relay information to OSC + if eye_info.info_type != InformationOrigin.FAILURE: + self.osc_queue.put((EyeId.SETTINGS, eye_info)) + except Empty: + return