mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
commit
c6b48a196c
1
EyeTrackApp/.json
Normal file
1
EyeTrackApp/.json
Normal file
@ -0,0 +1 @@
|
||||
{"version": 1, "right_eye": {"threshold": 60, "rotation_angle": 334, "roi_window_x": 134, "roi_window_y": 104, "roi_window_w": 132, "roi_window_h": 77, "focal_length": 30, "capture_source": "http://192.168.0.238/", "vrc_eye_position_scalar": 3000, "show_color_image": false, "gui_flip_x_axis": false}, "left_eye": {"threshold": 82, "rotation_angle": 37, "roi_window_x": 146, "roi_window_y": 108, "roi_window_w": 115, "roi_window_h": 110, "focal_length": 30, "capture_source": "http://192.168.0.62", "vrc_eye_position_scalar": 3000, "show_color_image": false, "gui_flip_x_axis": false}, "settings": {"threshold": 0, "rotation_angle": 50, "roi_window_x": 0, "roi_window_y": 0, "roi_window_w": 0, "roi_window_h": 0, "focal_length": 30, "capture_source": null, "vrc_eye_position_scalar": 3000, "show_color_image": false, "gui_flip_x_axis": false}, "eye_display_id": 0}
|
||||
BIN
EyeTrackApp/Audio/compleated.wav
Normal file
BIN
EyeTrackApp/Audio/compleated.wav
Normal file
Binary file not shown.
BIN
EyeTrackApp/Audio/start.wav
Normal file
BIN
EyeTrackApp/Audio/start.wav
Normal file
Binary file not shown.
@ -7,7 +7,7 @@ from queue import Queue, Empty
|
||||
from camera import Camera, CameraState
|
||||
from osc import EyeId
|
||||
import cv2
|
||||
|
||||
from winsound import PlaySound, SND_FILENAME, SND_ASYNC
|
||||
|
||||
class CameraWidget:
|
||||
def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue):
|
||||
@ -35,6 +35,8 @@ class CameraWidget:
|
||||
self.config = main_config.right_eye
|
||||
elif self.eye_id == EyeId.LEFT:
|
||||
self.config = main_config.left_eye
|
||||
elif self.eye_id == EyeId.SETTINGS:
|
||||
self.config = main_config.settings
|
||||
else:
|
||||
raise RuntimeError("Cannot have a camera widget represent both eyes!")
|
||||
|
||||
@ -134,6 +136,7 @@ class CameraWidget:
|
||||
self.capture_event,
|
||||
self.capture_queue,
|
||||
self.image_queue,
|
||||
self.eye_id,
|
||||
)
|
||||
|
||||
self.camera_status_queue = Queue()
|
||||
@ -235,6 +238,8 @@ class CameraWidget:
|
||||
self.x1, self.y1 = values[self.gui_roi_selection]
|
||||
elif event == self.gui_restart_calibration:
|
||||
self.ransac.calibration_frame_counter = 300
|
||||
PlaySound('Audio/start.wav', SND_FILENAME|SND_ASYNC)
|
||||
|
||||
elif event == self.gui_recenter_eye:
|
||||
self.ransac.recenter_eye = True
|
||||
|
||||
@ -297,15 +302,11 @@ class CameraWidget:
|
||||
):
|
||||
graph.update(background_color="white")
|
||||
|
||||
# if eye_info.y < 0: # flip visualzation to be correct
|
||||
# visy = abs(eye_info.y)
|
||||
# elif eye_info.y >= 0:
|
||||
# visy = -abs(eye_info.y)
|
||||
try:
|
||||
|
||||
graph.draw_circle(
|
||||
(eye_info.x * -100, eye_info.y * -100),
|
||||
23,
|
||||
25,
|
||||
fill_color="black",
|
||||
line_color="white",
|
||||
)
|
||||
@ -315,9 +316,9 @@ class CameraWidget:
|
||||
graph.update(background_color="#6f4ca1")
|
||||
elif eye_info.info_type == InformationOrigin.FAILURE:
|
||||
graph.update(background_color="red")
|
||||
|
||||
# Relay information to OSC
|
||||
if eye_info.info_type != InformationOrigin.FAILURE:
|
||||
|
||||
self.osc_queue.put((self.eye_id, eye_info))
|
||||
except Empty:
|
||||
return
|
||||
|
||||
@ -19,10 +19,15 @@ class EyeTrackCameraConfig:
|
||||
roi_window_h: "int" = 0
|
||||
focal_length: "int" = 30
|
||||
capture_source: "Union[int, str, None]" = None
|
||||
vrc_eye_position_scalar: "int" = 3000
|
||||
#show_color_image: "bool" = False
|
||||
|
||||
|
||||
gui_flip_x_axis_left: "bool" = False
|
||||
gui_flip_x_axis_right: "bool" = False
|
||||
gui_flip_y_axis: "bool" = False
|
||||
gui_blob_fallback: "bool" = True
|
||||
gui_min_cutoff: "str" = "0.0004"
|
||||
gui_speed_coefficient: "str" = "0.9"
|
||||
gui_osc_address: "str" = "127.0.0.1"
|
||||
gui_osc_port: "str" = "9000"
|
||||
|
||||
CONFIG_FILE_NAME = "eyetrack_settings.json"
|
||||
|
||||
|
||||
@ -31,6 +36,7 @@ class EyeTrackConfig:
|
||||
version: "int" = 1
|
||||
right_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
|
||||
left_eye: EyeTrackCameraConfig = EyeTrackCameraConfig()
|
||||
settings: EyeTrackCameraConfig = EyeTrackCameraConfig()
|
||||
eye_display_id: "EyeId" = EyeId.RIGHT
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -13,6 +13,7 @@ import time
|
||||
import statistics
|
||||
from one_euro_filter import OneEuroFilter
|
||||
from sympy import symbols, Eq, solve
|
||||
from winsound import PlaySound, SND_FILENAME, SND_ASYNC
|
||||
class InformationOrigin(Enum):
|
||||
RANSAC = 1
|
||||
BLOB = 2
|
||||
@ -138,16 +139,20 @@ class EyeProcessor:
|
||||
capture_event: "threading.Event",
|
||||
capture_queue_incoming: "queue.Queue",
|
||||
image_queue_outgoing: "queue.Queue",
|
||||
eye_id,
|
||||
):
|
||||
self.config = config
|
||||
|
||||
|
||||
|
||||
# Cross-thread communication management
|
||||
self.capture_queue_incoming = capture_queue_incoming
|
||||
self.image_queue_outgoing = image_queue_outgoing
|
||||
self.cancellation_event = cancellation_event
|
||||
self.capture_event = capture_event
|
||||
self.eye_id = eye_id
|
||||
|
||||
# Cross algo state
|
||||
|
||||
self.lkg_projected_sphere = None
|
||||
self.xc = None
|
||||
self.yc = None
|
||||
@ -175,15 +180,20 @@ class EyeProcessor:
|
||||
self.recenter_eye = False
|
||||
self.calibration_frame_counter
|
||||
|
||||
|
||||
min_cutoff = 0.0004
|
||||
beta = 0.9
|
||||
try:
|
||||
min_cutoff = int(self.config.gui_min_cutoff) #0.0004
|
||||
beta = int(self.config.gui_speed_coefficient) #0.9
|
||||
except:
|
||||
print('[WARN] OneEuroFilter values must be a legal number.')
|
||||
min_cutoff = 0.0004
|
||||
beta = 0.9
|
||||
noisy_point = np.array([1, 1])
|
||||
self.one_euro_filter = OneEuroFilter(
|
||||
noisy_point,
|
||||
min_cutoff=min_cutoff,
|
||||
beta=beta
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@ -260,9 +270,9 @@ class EyeProcessor:
|
||||
# define circle
|
||||
try:
|
||||
ht, wd = self.current_image_gray.shape[:2]
|
||||
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
||||
|
||||
|
||||
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
||||
|
||||
# draw filled circle in white on black background as mask
|
||||
mask = np.zeros((ht,wd), dtype=np.uint8)
|
||||
mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1)
|
||||
@ -279,7 +289,7 @@ class EyeProcessor:
|
||||
# combine the two masked images
|
||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||
except:
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
# Increase our threshold value slightly, in order to have a better possibility of getting back
|
||||
@ -364,7 +374,8 @@ class EyeProcessor:
|
||||
self.calibration_frame_counter = None
|
||||
self.recenter_eye = False
|
||||
self.xoff = cx
|
||||
self.yoff = cy
|
||||
self.yoff = cy
|
||||
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC)
|
||||
elif self.calibration_frame_counter != None:
|
||||
if cx > self.xmax:
|
||||
self.xmax = cx
|
||||
@ -395,14 +406,27 @@ class EyeProcessor:
|
||||
# print(self.)
|
||||
out_x = 0
|
||||
out_y = 0
|
||||
if xr > 0:
|
||||
out_x = max(0.0, min(1.0, xr))
|
||||
if xl > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xl)))
|
||||
if yd > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yd)))
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
if self.config.gui_flip_y_axis == True: #check config on flipped values settings and apply accordingly
|
||||
if yd > 0:
|
||||
out_y = max(0.0, min(1.0, yd))
|
||||
if yu > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yu)))
|
||||
else:
|
||||
if yd > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yd)))
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
|
||||
if self.config.gui_flip_x_axis_right == True:
|
||||
if xr > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xr)))
|
||||
if xl > 0:
|
||||
out_x = max(0.0, min(1.0, xl))
|
||||
else:
|
||||
if xr > 0:
|
||||
out_x = max(0.0, min(1.0, xr))
|
||||
if xl > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xl)))
|
||||
|
||||
try:
|
||||
noisy_point = np.array([out_x, out_y]) #fliter our values with a One Euro Filter
|
||||
@ -432,10 +456,19 @@ class EyeProcessor:
|
||||
yf = []
|
||||
pd = []
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
out_pupil_dialation = 1
|
||||
|
||||
|
||||
|
||||
if self.eye_id == "EyeId.RIGHT":
|
||||
flipx = self.config.gui_flip_x_axis_right
|
||||
#elif self.eye_id == "EyeId.LEFT":
|
||||
# flipx = self.config.gui_flip_x_axis_left
|
||||
else:
|
||||
flipx = self.config.gui_flip_x_axis_left
|
||||
while True:
|
||||
# oef = init_filter()
|
||||
|
||||
@ -496,27 +529,16 @@ class EyeProcessor:
|
||||
self.current_image_gray = cv2.cvtColor(
|
||||
self.current_image, cv2.COLOR_BGR2GRAY
|
||||
)
|
||||
_, thresh = cv2.threshold(
|
||||
self.current_image_gray,
|
||||
int(self.config.threshold),
|
||||
255,
|
||||
cv2.THRESH_BINARY,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# define circle for "cropping"
|
||||
|
||||
try:
|
||||
ht, wd = self.current_image_gray.shape
|
||||
ht, wd = self.current_image_gray.shape[:2]
|
||||
|
||||
|
||||
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
||||
|
||||
self.xc = int(self.lkg_projected_sphere["center"][0])
|
||||
self.yc = int(self.lkg_projected_sphere["center"][1])
|
||||
|
||||
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
|
||||
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
|
||||
# draw filled circle in white on black background as mask
|
||||
mask = np.zeros((ht,wd), dtype=np.uint8)
|
||||
mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1)
|
||||
@ -532,9 +554,21 @@ class EyeProcessor:
|
||||
|
||||
# combine the two masked images
|
||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||
|
||||
except:
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
_, thresh = cv2.threshold(
|
||||
self.current_image_gray,
|
||||
int(self.config.threshold),
|
||||
255,
|
||||
cv2.THRESH_BINARY,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -561,7 +595,11 @@ class EyeProcessor:
|
||||
# using blob tracking.
|
||||
#
|
||||
if len(convex_hulls) == 0:
|
||||
self.blob_tracking_fallback()
|
||||
if self.config.gui_blob_fallback == True:
|
||||
self.blob_tracking_fallback()
|
||||
else:
|
||||
print("[INFO] Blob fallback disabled. Assuming blink.")
|
||||
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, 0, 0, 0, False))
|
||||
continue
|
||||
|
||||
# Find our largest hull, which we expect will probably be the ellipse that represents the 2d
|
||||
@ -576,7 +614,11 @@ class EyeProcessor:
|
||||
largest_hull.reshape(-1, 2)
|
||||
)
|
||||
except:
|
||||
self.blob_tracking_fallback()
|
||||
if self.config.gui_blob_fallback == True:
|
||||
self.blob_tracking_fallback()
|
||||
else:
|
||||
print("[INFO] Blob fallback disabled. Assuming blink.")
|
||||
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, 0, 0, 0, False))
|
||||
continue
|
||||
|
||||
# Get axis and angle of the ellipse, using pupil labs 2d algos. The next bit of code ranges
|
||||
@ -618,6 +660,7 @@ class EyeProcessor:
|
||||
self.recenter_eye = False
|
||||
self.xoff = exm
|
||||
self.yoff = eym
|
||||
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC)
|
||||
elif self.calibration_frame_counter != None: # TODO reset calibration values on button press
|
||||
if exm > self.xmax:
|
||||
self.xmax = exm
|
||||
@ -655,15 +698,28 @@ class EyeProcessor:
|
||||
|
||||
out_x = 0
|
||||
out_y = 0
|
||||
if xr > 0:
|
||||
out_x = max(0.0, min(1.0, xr))
|
||||
if xl > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xl)))
|
||||
if yd > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yd)))
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
|
||||
if self.config.gui_flip_y_axis == True:
|
||||
if yd > 0:
|
||||
out_y = max(0.0, min(1.0, yd))
|
||||
if yu > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yu)))
|
||||
else:
|
||||
if yd > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yd)))
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
|
||||
if flipx == True:
|
||||
if xr > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xr)))
|
||||
if xl > 0:
|
||||
out_x = max(0.0, min(1.0, xl))
|
||||
else:
|
||||
if xr > 0:
|
||||
out_x = max(0.0, min(1.0, xr))
|
||||
if xl > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xl)))
|
||||
|
||||
|
||||
try:
|
||||
|
||||
@ -6,6 +6,7 @@ os.environ["OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS"] = "0"
|
||||
from osc import VRChatOSC, EyeId
|
||||
from config import EyeTrackConfig
|
||||
from camera_widget import CameraWidget
|
||||
from settings_widget import SettingsWidget
|
||||
import queue
|
||||
import threading
|
||||
import PySimpleGUI as sg
|
||||
@ -13,11 +14,12 @@ import PySimpleGUI as sg
|
||||
WINDOW_NAME = "EyeTrackApp"
|
||||
RIGHT_EYE_NAME = "-RIGHTEYEWIDGET-"
|
||||
LEFT_EYE_NAME = "-LEFTEYEWIDGET-"
|
||||
SETTINGS_NAME = "-SETTINGSWIDGET-"
|
||||
|
||||
LEFT_EYE_RADIO_NAME = "-LEFTEYERADIO-"
|
||||
RIGHT_EYE_RADIO_NAME = "-RIGHTEYERADIO-"
|
||||
BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-"
|
||||
|
||||
SETTINGS_RADIO_NAME = '-SETTINGSRADIO-'
|
||||
|
||||
def main():
|
||||
# Get Configuration
|
||||
@ -33,7 +35,7 @@ def main():
|
||||
|
||||
# Spawn worker threads
|
||||
osc_queue: "queue.Queue[tuple[bool, int, int]]" = queue.Queue()
|
||||
osc = VRChatOSC(cancellation_event, osc_queue)
|
||||
osc = VRChatOSC(cancellation_event, osc_queue, config)
|
||||
osc_thread = threading.Thread(target=osc.run)
|
||||
osc_thread.start()
|
||||
|
||||
@ -46,6 +48,11 @@ def main():
|
||||
eyes = [
|
||||
CameraWidget(EyeId.RIGHT, config, osc_queue),
|
||||
CameraWidget(EyeId.LEFT, config, osc_queue),
|
||||
# CameraWidget(EyeId.SETTINGS, config, osc_queue),
|
||||
|
||||
]
|
||||
settings = [
|
||||
SettingsWidget(EyeId.SETTINGS, config, osc_queue),
|
||||
]
|
||||
|
||||
layout = [
|
||||
@ -71,6 +78,13 @@ def main():
|
||||
default=(config.eye_display_id == EyeId.BOTH),
|
||||
key=BOTH_EYE_RADIO_NAME,
|
||||
),
|
||||
sg.Radio(
|
||||
"Settings",
|
||||
"EYESELECTRADIO",
|
||||
background_color='#292929',
|
||||
default=(config.eye_display_id == EyeId.SETTINGS),
|
||||
key=SETTINGS_RADIO_NAME,
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Column(
|
||||
@ -87,6 +101,13 @@ def main():
|
||||
visible=(config.eye_display_id in [EyeId.LEFT, EyeId.BOTH]),
|
||||
background_color='#424042',
|
||||
),
|
||||
sg.Column(
|
||||
settings[0].widget_layout,
|
||||
vertical_alignment="top",
|
||||
key=SETTINGS_NAME,
|
||||
visible=(config.eye_display_id in [EyeId.SETTINGS]),
|
||||
background_color='#424042',
|
||||
),
|
||||
],
|
||||
]
|
||||
|
||||
@ -94,9 +115,11 @@ def main():
|
||||
eyes[0].start()
|
||||
if config.eye_display_id in [EyeId.LEFT, EyeId.BOTH]:
|
||||
eyes[1].start()
|
||||
if config.eye_display_id in [EyeId.SETTINGS, EyeId.BOTH]:
|
||||
settings[0].start()
|
||||
|
||||
# Create the window
|
||||
window = sg.Window("EyeTrackVR v0.1.3", layout, icon='Images/logo.ico', background_color='#292929')
|
||||
window = sg.Window("EyeTrackVR v0.1.5 dev", layout, icon='Images/logo.ico', background_color='#292929')
|
||||
|
||||
# GUI Render loop
|
||||
while True:
|
||||
@ -105,6 +128,7 @@ def main():
|
||||
|
||||
# If we're in either mode and someone hits q, quit immediately
|
||||
if event == "Exit" or event == sg.WIN_CLOSED:
|
||||
# eyes[2].stop()
|
||||
for eye in eyes:
|
||||
eye.stop()
|
||||
cancellation_event.set()
|
||||
@ -118,29 +142,46 @@ def main():
|
||||
if values[RIGHT_EYE_RADIO_NAME] and config.eye_display_id != EyeId.RIGHT:
|
||||
eyes[0].start()
|
||||
eyes[1].stop()
|
||||
settings[0].stop()
|
||||
window[RIGHT_EYE_NAME].update(visible=True)
|
||||
window[LEFT_EYE_NAME].update(visible=False)
|
||||
window[SETTINGS_NAME].update(visible=False)
|
||||
config.eye_display_id = EyeId.RIGHT
|
||||
config.save()
|
||||
elif values[LEFT_EYE_RADIO_NAME] and config.eye_display_id != EyeId.LEFT:
|
||||
settings[0].stop()
|
||||
eyes[0].stop()
|
||||
eyes[1].start()
|
||||
window[RIGHT_EYE_NAME].update(visible=False)
|
||||
window[LEFT_EYE_NAME].update(visible=True)
|
||||
window[SETTINGS_NAME].update(visible=False)
|
||||
config.eye_display_id = EyeId.LEFT
|
||||
config.save()
|
||||
elif values[BOTH_EYE_RADIO_NAME] and config.eye_display_id != EyeId.BOTH:
|
||||
eyes[0].start()
|
||||
settings[0].stop()
|
||||
eyes[1].start()
|
||||
eyes[0].start()
|
||||
window[RIGHT_EYE_NAME].update(visible=True)
|
||||
window[LEFT_EYE_NAME].update(visible=True)
|
||||
window[SETTINGS_NAME].update(visible=False)
|
||||
config.eye_display_id = EyeId.BOTH
|
||||
config.save()
|
||||
|
||||
elif values[SETTINGS_RADIO_NAME] and config.eye_display_id != EyeId.SETTINGS:
|
||||
eyes[0].stop()
|
||||
eyes[1].stop()
|
||||
settings[0].start()
|
||||
window[RIGHT_EYE_NAME].update(visible=False)
|
||||
window[LEFT_EYE_NAME].update(visible=False)
|
||||
window[SETTINGS_NAME].update(visible=True)
|
||||
config.eye_display_id = EyeId.SETTINGS
|
||||
config.save()
|
||||
|
||||
# Otherwise, render all of our cameras
|
||||
for eye in eyes:
|
||||
if eye.started():
|
||||
eye.render(window, event, values)
|
||||
settings[0].render(window, event, values)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -2,53 +2,49 @@ from pythonosc import udp_client
|
||||
import queue
|
||||
import threading
|
||||
from enum import IntEnum
|
||||
|
||||
import time
|
||||
|
||||
class EyeId(IntEnum):
|
||||
RIGHT = 0
|
||||
LEFT = 1
|
||||
BOTH = 2
|
||||
SETTINGS = 3
|
||||
|
||||
from config import EyeTrackConfig
|
||||
|
||||
class VRChatOSC:
|
||||
# VRChat OSC Networking Info. For now, we'll assume it's always local.
|
||||
OSC_IP = "127.0.0.1"
|
||||
OSC_PORT = 9000 # VR Chat OSC port
|
||||
|
||||
# Use a tuple of blink (true, blinking, false, not), x, y for now. Probably clearer as a class but
|
||||
# we're stuck in python 3.6 so still no dataclasses. God I hate python.
|
||||
def __init__(
|
||||
self,
|
||||
cancellation_event: "threading.Event",
|
||||
msg_queue: "queue.Queue[tuple[bool, int, int, int]]",
|
||||
):
|
||||
self.client = udp_client.SimpleUDPClient(VRChatOSC.OSC_IP, VRChatOSC.OSC_PORT)
|
||||
def __init__(self, cancellation_event: "threading.Event", msg_queue: "queue.Queue[tuple[bool, int, int, int]]", main_config: EyeTrackConfig,):
|
||||
|
||||
self.main_config = main_config
|
||||
self.config = main_config.settings
|
||||
self.client = udp_client.SimpleUDPClient(self.config.gui_osc_address, int(self.config.gui_osc_port)) # use OSC port and address that was set in the config
|
||||
self.cancellation_event = cancellation_event
|
||||
self.msg_queue = msg_queue
|
||||
|
||||
|
||||
def run(self):
|
||||
|
||||
start = time.time()
|
||||
last_blink = 0
|
||||
yl = 621
|
||||
yr = 621
|
||||
sx = 0
|
||||
sy = 0
|
||||
se = 0
|
||||
lec = 1
|
||||
rec = 1
|
||||
lec = 0
|
||||
rec = 0
|
||||
while True:
|
||||
if self.cancellation_event.is_set():
|
||||
print("Exiting OSC Queue")
|
||||
return
|
||||
try:
|
||||
(eye_id, eye_info) = self.msg_queue.get(block=True, timeout=0.1)
|
||||
except queue.Empty:
|
||||
except:
|
||||
continue
|
||||
|
||||
|
||||
|
||||
if not eye_info.blink:
|
||||
|
||||
if eye_id in [EyeId.RIGHT, EyeId.BOTH]:
|
||||
if eye_id in [EyeId.RIGHT]:
|
||||
sx = eye_info.x
|
||||
yr = eye_info.y
|
||||
sy = eye_info.y
|
||||
@ -61,7 +57,7 @@ class VRChatOSC:
|
||||
# self.client.send_message(
|
||||
# "/avatar/parameters/EyesDilation", eye_info.pupil_dialation
|
||||
#)
|
||||
if eye_id in [EyeId.LEFT, EyeId.BOTH]:
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
sx = eye_info.x
|
||||
yl = eye_info.y
|
||||
sy = eye_info.y
|
||||
@ -75,28 +71,42 @@ class VRChatOSC:
|
||||
self.client.send_message("/avatar/parameters/EyesY", y)
|
||||
se = 0
|
||||
|
||||
if rec == 1 and lec == 1:
|
||||
if rec == 0 or lec == 0:
|
||||
print("here")
|
||||
se = 1
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", sx) # only one eye is detected or there is an error. Send mirrored data to both eyes.
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", sx)
|
||||
self.client.send_message("/avatar/parameters/EyesY", sy)
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
|
||||
|
||||
|
||||
|
||||
else:
|
||||
|
||||
if eye_id in [EyeId.LEFT, EyeId.BOTH]:
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
|
||||
if eye_id in [EyeId.RIGHT, EyeId.BOTH]:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
if last_blink > 0.5:
|
||||
for i in range(4):
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
last_blink = time.time()
|
||||
if eye_id in [EyeId.RIGHT]:
|
||||
if last_blink > 0.5:
|
||||
for i in range(4):
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
last_blink = time.time()
|
||||
if se == 1:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
if last_blink > 0.5:
|
||||
for i in range(4):
|
||||
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
last_blink = time.time()
|
||||
|
||||
|
||||
|
||||
350
EyeTrackApp/settings_widget.py
Normal file
350
EyeTrackApp/settings_widget.py
Normal file
@ -0,0 +1,350 @@
|
||||
import PySimpleGUI as sg
|
||||
from config import EyeTrackConfig
|
||||
from threading import Event, Thread
|
||||
from eye_processor import EyeProcessor, InformationOrigin
|
||||
from enum import Enum
|
||||
from queue import Queue, Empty
|
||||
from camera import Camera, CameraState
|
||||
import cv2
|
||||
from osc import EyeId
|
||||
|
||||
class SettingsWidget:
|
||||
def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue):
|
||||
|
||||
self.gui_show_color_image = f"-SETTING1{widget_id}-"
|
||||
self.gui_camera_addr = f"-CAMERAADDR{widget_id}-"
|
||||
self.gui_threshold_slider = f"-THREADHOLDSLIDER{widget_id}-"
|
||||
|
||||
self.gui_roi_button = f"-ROIMODE{widget_id}-"
|
||||
self.gui_roi_layout = f"-ROILAYOUT{widget_id}-"
|
||||
self.gui_roi_selection = f"-GRAPH{widget_id}-"
|
||||
self.gui_tracking_button = f"-TRACKINGMODE{widget_id}-"
|
||||
self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-"
|
||||
|
||||
self.gui_tracking_image = f"-IMAGE{widget_id}-"
|
||||
self.gui_output_graph = f"-OUTPUTGRAPH{widget_id}-"
|
||||
self.gui_restart_calibration = f"-RESTARTCALIBRATION{widget_id}-"
|
||||
self.gui_recenter_eye = f"-RECENTEREYE{widget_id}-"
|
||||
self.gui_mode_readout = f"-APPMODE{widget_id}-"
|
||||
self.gui_show_color_image = f"-SHOWCOLORIMAGE{widget_id}-"
|
||||
|
||||
self.gui_flip_x_axis_left = f"-FLIPXAXISLEFT{widget_id}-"
|
||||
self.gui_flip_x_axis_right = f"-FLIPXAXISRIGHT{widget_id}-"
|
||||
self.gui_flip_y_axis = f"-FLIPYAXIS{widget_id}-"
|
||||
|
||||
self.gui_roi_message = f"-ROIMESSAGE{widget_id}-"
|
||||
self.gui_save_button = f"-SAVE{widget_id}-"
|
||||
|
||||
self.gui_general_settings_layout = f"-GENERALSETTINGSLAYOUT{widget_id}-"
|
||||
|
||||
self.gui_osc_address = f"-OSCADDRESS{widget_id}-"
|
||||
self.gui_osc_port = f"-OSCPORT{widget_id}-"
|
||||
# self.gui_algo_settings_layout = f"-ALGOSETTINGSLAYOUT{widget_id}-"
|
||||
|
||||
self.gui_blob_fallback = f"-BLOBFALLBACK{widget_id}-"
|
||||
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
|
||||
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
|
||||
self.main_config = main_config
|
||||
|
||||
self.configr = main_config.right_eye
|
||||
|
||||
self.configl = main_config.left_eye
|
||||
|
||||
self.config = main_config.settings
|
||||
|
||||
self.osc_queue = osc_queue
|
||||
# Define the window's contents
|
||||
self.general_settings_layout = [
|
||||
|
||||
# [
|
||||
# sg.Button("Recenter Eye", key=self.gui_recenter_eye, button_color = '#6f4ca1'),
|
||||
|
||||
# ],
|
||||
[
|
||||
sg.Checkbox(
|
||||
"Flip Left Eye X Axis",
|
||||
default=self.config.gui_flip_x_axis_left,
|
||||
key=self.gui_flip_x_axis_left,
|
||||
background_color='#424042',
|
||||
),
|
||||
sg.Checkbox(
|
||||
"Flip Right Eye X Axis",
|
||||
default=self.config.gui_flip_x_axis_left,
|
||||
key=self.gui_flip_x_axis_right,
|
||||
background_color='#424042',
|
||||
),
|
||||
|
||||
],
|
||||
[sg.Checkbox(
|
||||
"Flip Y Axis",
|
||||
default=self.config.gui_flip_y_axis,
|
||||
key=self.gui_flip_y_axis,
|
||||
background_color='#424042',
|
||||
),
|
||||
],
|
||||
|
||||
[
|
||||
sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
|
||||
# sg.InputText(self.config.capture_source, key=self.gui_camera_addr),
|
||||
],
|
||||
|
||||
[sg.Checkbox(
|
||||
"Blob Fallback",
|
||||
default=self.config.gui_blob_fallback,
|
||||
key=self.gui_blob_fallback,
|
||||
background_color='#424042',
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Text("Filter Paramaters:", background_color='#242224'),
|
||||
|
||||
# sg.InputText(self.config.capture_source, key=self.gui_camera_addr),
|
||||
],
|
||||
[
|
||||
|
||||
sg.Text("Min Frequency Cutoff", background_color='#424042'),
|
||||
sg.InputText(self.config.gui_min_cutoff, key=self.gui_min_cutoff),
|
||||
|
||||
|
||||
#sg.Slider(
|
||||
# range=(0, 10),
|
||||
# default_value=self.config.gui_min_cutoff,
|
||||
# orientation="h",
|
||||
# key=self.gui_min_cutoff,
|
||||
# background_color='#424042'
|
||||
#),
|
||||
],
|
||||
[
|
||||
sg.Text("Speed Coefficient", background_color='#424042'),
|
||||
sg.InputText(self.config.gui_speed_coefficient, key=self.gui_speed_coefficient),
|
||||
|
||||
# sg.Text("Speed Coefficient", background_color='#424042'),
|
||||
#sg.Slider(
|
||||
# range=(0, 20),
|
||||
# default_value=9,
|
||||
# orientation="h",
|
||||
# key=self.gui_speed_coefficient_slider,
|
||||
# background_color='#424042'
|
||||
# ),
|
||||
],
|
||||
[
|
||||
sg.Text("OSC Settings:", background_color='#242224'),
|
||||
],
|
||||
[
|
||||
sg.Text("OSC Address:", background_color='#424042'),
|
||||
sg.InputText(self.config.gui_osc_address, key=self.gui_osc_address),
|
||||
],
|
||||
[
|
||||
sg.Text("OSC Port:", background_color='#424042'),
|
||||
sg.InputText(self.config.gui_osc_port, key=self.gui_osc_port),
|
||||
],
|
||||
|
||||
|
||||
#[sg.Image(filename="", key=self.gui_tracking_image)],
|
||||
# [
|
||||
# sg.Graph(
|
||||
# (200, 200),
|
||||
# (-100, 100),
|
||||
# (100, -100),
|
||||
# background_color="white",
|
||||
# key=self.gui_output_graph,
|
||||
# drag_submits=True,
|
||||
# enable_events=True,
|
||||
# ),
|
||||
# sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', visible=False),
|
||||
# ],
|
||||
]
|
||||
|
||||
|
||||
self.widget_layout = [
|
||||
[
|
||||
sg.Text("General Settings:", background_color='#242224'),
|
||||
# sg.InputText(self.config.capture_source, key=self.gui_camera_addr),
|
||||
],
|
||||
[
|
||||
sg.Column(self.general_settings_layout, key=self.gui_general_settings_layout, background_color='#424042' ),
|
||||
#sg.Column(self.algo_settings_layout, key=self.gui_algo_settings_layout, background_color='#424042' ),
|
||||
# sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False),
|
||||
],
|
||||
|
||||
# [
|
||||
# sg.Button(
|
||||
# "Save Settings", key=self.gui_save_button, button_color = '#6f4ca1'
|
||||
# ),
|
||||
#],
|
||||
]
|
||||
|
||||
self.cancellation_event = Event()
|
||||
# Set the event until start is called, otherwise we can block if shutdown is called.
|
||||
self.cancellation_event.set()
|
||||
# self.capture_event = Event()
|
||||
# self.capture_queue = Queue()
|
||||
# self.roi_queue = Queue()
|
||||
|
||||
self.image_queue = Queue()
|
||||
|
||||
# self.ransac = EyeProcessor(
|
||||
# self.config,
|
||||
# self.cancellation_event,
|
||||
# self.capture_event,
|
||||
# self.capture_queue,
|
||||
# self.image_queue,
|
||||
# )
|
||||
|
||||
# self.camera_status_queue = Queue()
|
||||
#self.camera = Camera(
|
||||
# self.config,
|
||||
# 0,
|
||||
# self.cancellation_event,
|
||||
#self.capture_event,
|
||||
# self.camera_status_queue,
|
||||
#self.capture_queue,
|
||||
#)
|
||||
|
||||
|
||||
def started(self):
|
||||
return not self.cancellation_event.is_set()
|
||||
|
||||
def start(self):
|
||||
# If we're already running, bail
|
||||
if not self.cancellation_event.is_set():
|
||||
return
|
||||
self.cancellation_event.clear()
|
||||
#self.ransac_thread = Thread(target=self.ransac.run)
|
||||
# self.ransac_thread.start()
|
||||
# self.camera_thread = Thread(target=self.camera.run)
|
||||
# self.camera_thread.start()
|
||||
|
||||
def stop(self):
|
||||
# If we're not running yet, bail
|
||||
if self.cancellation_event.is_set():
|
||||
return
|
||||
self.cancellation_event.set()
|
||||
# self.ransac_thread.join()
|
||||
#self.camera_thread.join()
|
||||
|
||||
def render(self, window, event, values):
|
||||
changed = False
|
||||
# If anything has changed in our configuration settings, change/update those.
|
||||
# if (
|
||||
# event == self.gui_save_tracking_button
|
||||
# and values[self.gui_camera_addr] != self.config.capture_source
|
||||
#):
|
||||
# print("New value: {}".format(values[self.gui_camera_addr]))
|
||||
# try:
|
||||
# Try storing ints as ints, for those using wired cameras.
|
||||
# # self.config.capture_source = int(values[self.gui_camera_addr])
|
||||
# except ValueError:
|
||||
# if values[self.gui_camera_addr] == "":
|
||||
# self.config.capture_source = None
|
||||
# else:
|
||||
# self.config.capture_source = values[self.gui_camera_addr]
|
||||
# changed = True
|
||||
|
||||
if self.config.gui_min_cutoff != values[self.gui_min_cutoff]:
|
||||
self.config.gui_min_cutoff = values[self.gui_min_cutoff]
|
||||
self.configl.gui_min_cutoff = values[self.gui_min_cutoff]
|
||||
self.configr.gui_min_cutoff = values[self.gui_min_cutoff]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_speed_coefficient != values[self.gui_speed_coefficient]:
|
||||
self.config.gui_speed_coefficient = values[self.gui_speed_coefficient]
|
||||
self.configl.gui_speed_coefficient = values[self.gui_speed_coefficient]
|
||||
self.configr.gui_speed_coefficient = values[self.gui_speed_coefficient]
|
||||
changed = True
|
||||
|
||||
|
||||
#if self.config.rotation_angle != values[self.gui_rotation_slider]:
|
||||
# self.config.rotation_angle = int(values[self.gui_rotation_slider])
|
||||
# changed = True
|
||||
# print(self.config.gui_flip_x_axis, values[self.gui_flip_x_axis])
|
||||
if self.config.gui_flip_x_axis_right != values[self.gui_flip_x_axis_right]:
|
||||
self.config.gui_flip_x_axis_right = values[self.gui_flip_x_axis_right]
|
||||
self.configl.gui_flip_x_axis_right = values[self.gui_flip_x_axis_right]
|
||||
self.configr.gui_flip_x_axis_right = values[self.gui_flip_x_axis_right]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_flip_x_axis_left != values[self.gui_flip_x_axis_left]:
|
||||
self.config.gui_flip_x_axis_left = values[self.gui_flip_x_axis_left]
|
||||
self.configl.gui_flip_x_axis_left = values[self.gui_flip_x_axis_left]
|
||||
self.configr.gui_flip_x_axis_left = values[self.gui_flip_x_axis_left]
|
||||
changed = True
|
||||
|
||||
|
||||
if self.config.gui_flip_y_axis != values[self.gui_flip_y_axis]:
|
||||
self.config.gui_flip_y_axis = values[self.gui_flip_y_axis]
|
||||
self.configl.gui_flip_y_axis = values[self.gui_flip_y_axis]
|
||||
self.configr.gui_flip_y_axis = values[self.gui_flip_y_axis]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_blob_fallback != values[self.gui_blob_fallback]:
|
||||
self.config.gui_blob_fallback = values[self.gui_blob_fallback]
|
||||
self.configl.gui_blob_fallback = values[self.gui_blob_fallback]
|
||||
self.configr.gui_blob_fallback = values[self.gui_blob_fallback]
|
||||
changed = True
|
||||
|
||||
|
||||
#print(self.config.gui_flip_x_axis, values[self.gui_flip_x_axis])
|
||||
# if values[self.gui_flip_x_axis] != self.config.gui_flip_x_axis:
|
||||
# values[self.gui_flip_x_axis] = self.config.gui_flip_x_axis
|
||||
|
||||
|
||||
|
||||
if changed:
|
||||
self.main_config.save()
|
||||
|
||||
|
||||
|
||||
# elif self.camera.camera_status == CameraState.CONNECTING:
|
||||
# window[self.gui_mode_readout].update("Camera Connecting")
|
||||
# elif self.camera.camera_status == CameraState.DISCONNECTED:
|
||||
# window[self.gui_mode_readout].update("CAMERA DISCONNECTED")
|
||||
#elif needs_roi_set:
|
||||
# window[self.gui_mode_readout].update("Awaiting Eye Cropping Setting")
|
||||
#elif self.ransac.calibration_frame_counter != None:
|
||||
# window[self.gui_mode_readout].update("Calibration")
|
||||
# else:
|
||||
# window[self.gui_mode_readout].update("Tracking")
|
||||
|
||||
# if self.in_roi_mode:
|
||||
# try:
|
||||
# if self.roi_queue.empty():
|
||||
# self.capture_event.set()
|
||||
# maybe_image = self.roi_queue.get(block=False)
|
||||
# imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes()
|
||||
# graph = window[self.gui_roi_selection]
|
||||
# if self.figure:
|
||||
# graph.delete_figure(self.figure)
|
||||
# INCREDIBLY IMPORTANT ERASE. Drawing images does NOT overwrite the buffer, the fucking
|
||||
# graph keeps every image fed in until you call this. Therefore we have to make sure we
|
||||
# erase before we redraw, otherwise we'll leak memory *very* quickly.
|
||||
# graph.erase()
|
||||
# graph.draw_image(data=imgbytes, location=(0, 0))
|
||||
# if None not in (self.x0, self.y0, self.x1, self.y1):
|
||||
# self.figure = graph.draw_rectangle(
|
||||
# (self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1"
|
||||
# )
|
||||
# except Empty:
|
||||
# pass
|
||||
# else:
|
||||
# if needs_roi_set:
|
||||
# window[self.gui_roi_message].update(visible=True)
|
||||
# window[self.gui_output_graph].update(visible=False)
|
||||
# return
|
||||
# try:
|
||||
# window[self.gui_roi_message].update(visible=False)
|
||||
# window[self.gui_output_graph].update(visible=True)
|
||||
#(maybe_image, eye_info) = self.image_queue.get(block=False)
|
||||
# imgbytes = cv2.imencode(".ppm", maybe_image)[1].tobytes()
|
||||
# window[self.gui_tracking_image].update(data=imgbytes)
|
||||
|
||||
# Update the GUI
|
||||
# graph = window[self.gui_output_graph]
|
||||
# graph.erase()
|
||||
|
||||
# Relay information to OSC
|
||||
# if eye_info.info_type != InformationOrigin.FAILURE:
|
||||
self.osc_queue.put((EyeId.SETTINGS))
|
||||
#self.osc_queue.put((EyeId.SETTINGS))
|
||||
# except Empty:
|
||||
# return
|
||||
Loading…
Reference in New Issue
Block a user