mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix bugs, add basic blink sync, adjust default values, bump ver, fix pyinstaller, osc
This commit is contained in:
parent
3bad2f13c9
commit
b1472949c7
@ -7,8 +7,8 @@ CONFIG_FILE_NAME: str = "eyetrack_settings.json"
|
|||||||
|
|
||||||
|
|
||||||
class EyeTrackCameraConfig(BaseModel):
|
class EyeTrackCameraConfig(BaseModel):
|
||||||
threshold: int = 0
|
threshold: int = 50
|
||||||
rotation_angle: int = 50
|
rotation_angle: int = 0
|
||||||
roi_window_x: int = 0
|
roi_window_x: int = 0
|
||||||
roi_window_y: int = 0
|
roi_window_y: int = 0
|
||||||
roi_window_w: int = 0
|
roi_window_w: int = 0
|
||||||
@ -34,7 +34,8 @@ class EyeTrackSettingsConfig(BaseModel):
|
|||||||
gui_blob_minsize: float = 10
|
gui_blob_minsize: float = 10
|
||||||
gui_recenter_eyes: bool = False
|
gui_recenter_eyes: bool = False
|
||||||
gui_eye_falloff: bool = False
|
gui_eye_falloff: bool = False
|
||||||
tracker_single_eye: float = 0
|
tracker_single_eye: int = 0
|
||||||
|
gui_blink_sync: bool = False
|
||||||
|
|
||||||
|
|
||||||
class EyeTrackConfig(BaseModel):
|
class EyeTrackConfig(BaseModel):
|
||||||
|
|||||||
@ -268,16 +268,12 @@ class EyeProcessor:
|
|||||||
# draw filled circle in white on black background as mask
|
# draw filled circle in white on black background as mask
|
||||||
mask = np.zeros((ht, wd), dtype=np.uint8)
|
mask = np.zeros((ht, wd), dtype=np.uint8)
|
||||||
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
||||||
|
|
||||||
# create white colored background
|
# create white colored background
|
||||||
color = np.full_like(self.current_image_gray, (255))
|
color = np.full_like(self.current_image_gray, (255))
|
||||||
|
|
||||||
# apply mask to image
|
# apply mask to image
|
||||||
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
||||||
|
|
||||||
# apply inverse mask to colored image
|
# apply inverse mask to colored image
|
||||||
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
||||||
|
|
||||||
# combine the two masked images
|
# combine the two masked images
|
||||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||||
except:
|
except:
|
||||||
@ -285,15 +281,7 @@ class EyeProcessor:
|
|||||||
else:
|
else:
|
||||||
self.cct = self.cct - 1
|
self.cct = self.cct - 1
|
||||||
|
|
||||||
# Increase our threshold value slightly, in order to have a better possibility of getting back
|
_, larger_threshold = cv2.threshold(self.current_image_gray, int(self.config.threshold + 12), 255, cv2.THRESH_BINARY)
|
||||||
# something to do blob tracking on.
|
|
||||||
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
|
|
||||||
histr = hist.ravel()
|
|
||||||
peaks, properties = sp.find_peaks(histr, distance=5)
|
|
||||||
minpeak = np.min(peaks)
|
|
||||||
thresholdoptics = np.array(minpeak + int(self.config.threshold + 12))
|
|
||||||
larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
|
|
||||||
larger_threshold = cv2.bitwise_not(larger_threshold)
|
|
||||||
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
|
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
|
||||||
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
|
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
|
||||||
# order to have a projected sphere.
|
# order to have a projected sphere.
|
||||||
@ -507,23 +495,18 @@ class EyeProcessor:
|
|||||||
if self.cct == 0:
|
if self.cct == 0:
|
||||||
try:
|
try:
|
||||||
ht, wd = self.current_image_gray.shape[:2]
|
ht, wd = self.current_image_gray.shape[:2]
|
||||||
|
|
||||||
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
||||||
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
|
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
|
||||||
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
|
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
|
||||||
# draw filled circle in white on black background as mask
|
# draw filled circle in white on black background as mask
|
||||||
mask = np.zeros((ht, wd), dtype=np.uint8)
|
mask = np.zeros((ht, wd), dtype=np.uint8)
|
||||||
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
||||||
|
|
||||||
# create white colored background
|
# create white colored background
|
||||||
color = np.full_like(self.current_image_gray, (255))
|
color = np.full_like(self.current_image_gray, (255))
|
||||||
|
|
||||||
# apply mask to image
|
# apply mask to image
|
||||||
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
||||||
|
|
||||||
# apply inverse mask to colored image
|
# apply inverse mask to colored image
|
||||||
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
||||||
|
|
||||||
# combine the two masked images
|
# combine the two masked images
|
||||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||||
except:
|
except:
|
||||||
@ -533,14 +516,12 @@ class EyeProcessor:
|
|||||||
else:
|
else:
|
||||||
self.cct = 300
|
self.cct = 300
|
||||||
|
|
||||||
# Using Histogram based thresholding. Improves robustness insanely
|
_, thresh = cv2.threshold(
|
||||||
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
|
self.current_image_gray,
|
||||||
histr = hist.ravel()
|
int(self.config.threshold),
|
||||||
peaks, properties = sp.find_peaks(histr, distance=5)
|
255,
|
||||||
minpeak = np.min(peaks)
|
cv2.THRESH_BINARY,
|
||||||
thresholdoptics = np.array(minpeak + int(self.config.threshold))
|
)
|
||||||
thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
|
|
||||||
thresh = cv2.bitwise_not(thresh)
|
|
||||||
|
|
||||||
# Set up morphological transforms, for smoothing and clearing the image we get out of the
|
# Set up morphological transforms, for smoothing and clearing the image we get out of the
|
||||||
# thresholding operation. After this, we'd really like to just have a black blob in the middle
|
# thresholding operation. After this, we'd really like to just have a black blob in the middle
|
||||||
|
|||||||
@ -119,7 +119,7 @@ def main():
|
|||||||
osc_receiver_thread.start()
|
osc_receiver_thread.start()
|
||||||
|
|
||||||
# Create the window
|
# Create the window
|
||||||
window = sg.Window("EyeTrackVR v0.1.7", layout, icon='Images/logo.ico', background_color='#292929')
|
window = sg.Window("EyeTrackVR v0.1.7.1", layout, icon='Images/logo.ico', background_color='#292929')
|
||||||
|
|
||||||
# GUI Render loop
|
# GUI Render loop
|
||||||
while True:
|
while True:
|
||||||
|
|||||||
@ -6,7 +6,7 @@ block_cipher = None
|
|||||||
a = Analysis(['eyetrackapp.py'],
|
a = Analysis(['eyetrackapp.py'],
|
||||||
pathex=[],
|
pathex=[],
|
||||||
binaries=[],
|
binaries=[],
|
||||||
datas=[("Audio/*", "Audio"), ("Images/*", "Images/")],
|
datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("pye3dcustom/refraction_models/*", "pye3dcustom/refraction_models")],
|
||||||
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
|
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
|
||||||
hookspath=[],
|
hookspath=[],
|
||||||
hooksconfig={},
|
hooksconfig={},
|
||||||
|
|||||||
@ -54,6 +54,12 @@ class VRChatOSC:
|
|||||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||||
|
if self.config.gui_blink_sync and not rb and not lb:
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||||
|
|
||||||
else:
|
else:
|
||||||
if eye_id in [EyeId.RIGHT]:
|
if eye_id in [EyeId.RIGHT]:
|
||||||
yr = eye_info.y
|
yr = eye_info.y
|
||||||
@ -61,8 +67,9 @@ class VRChatOSC:
|
|||||||
sy = eye_info.y
|
sy = eye_info.y
|
||||||
rb = False
|
rb = False
|
||||||
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
|
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
|
||||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not lb:
|
||||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
|
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
|
||||||
|
|
||||||
if eye_id in [EyeId.LEFT]:
|
if eye_id in [EyeId.LEFT]:
|
||||||
yl = eye_info.y
|
yl = eye_info.y
|
||||||
@ -70,27 +77,21 @@ class VRChatOSC:
|
|||||||
sy = eye_info.y
|
sy = eye_info.y
|
||||||
lb = False
|
lb = False
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
|
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not rb:
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||||
|
|
||||||
if (yr != 621 and yl != 621) and (lb == False and rb == False):
|
if (yr != 621 and yl != 621) and (lb == False and rb == False):
|
||||||
y = (yr + yl) / 2
|
y = (yr + yl) / 2
|
||||||
self.client.send_message("/avatar/parameters/EyesY", y)
|
self.client.send_message("/avatar/parameters/EyesY", y)
|
||||||
else:
|
else:
|
||||||
if self.config.gui_eye_falloff:
|
|
||||||
|
if self.config.gui_blink_sync:
|
||||||
if eye_id in [EyeId.LEFT]:
|
if eye_id in [EyeId.LEFT]:
|
||||||
lb = True
|
lb = True
|
||||||
if eye_id in [EyeId.RIGHT]:
|
if eye_id in [EyeId.RIGHT]:
|
||||||
rb = True
|
rb = True
|
||||||
if rb == True or lb == True: # If one eye closed and fall off is enabled, mirror data
|
if rb == True and lb == True : # If both eyes are closed, blink
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
|
|
||||||
self.client.send_message("/avatar/parameters/RightEyeX", sx)
|
|
||||||
self.client.send_message("/avatar/parameters/EyesY", sy)
|
|
||||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
|
||||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
|
||||||
if rb == True and lb == True: # If both eyes are closed, blink
|
|
||||||
if last_blink > 0.5:
|
if last_blink > 0.5:
|
||||||
for i in range(4):
|
for i in range(4):
|
||||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||||
@ -107,7 +108,8 @@ class VRChatOSC:
|
|||||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||||
last_blink = time.time()
|
last_blink = time.time()
|
||||||
else:
|
|
||||||
|
if not self.config.gui_eye_falloff:
|
||||||
if eye_id in [EyeId.LEFT]:
|
if eye_id in [EyeId.LEFT]:
|
||||||
lb = True
|
lb = True
|
||||||
if last_blink > 0.5:
|
if last_blink > 0.5:
|
||||||
@ -124,6 +126,28 @@ class VRChatOSC:
|
|||||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||||
last_blink = time.time()
|
last_blink = time.time()
|
||||||
|
|
||||||
|
else:
|
||||||
|
if eye_id in [EyeId.LEFT]:
|
||||||
|
lb = True
|
||||||
|
if eye_id in [EyeId.RIGHT]:
|
||||||
|
rb = True
|
||||||
|
if rb or lb: # If one eye closed and fall off is enabled, mirror data
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeX", sx)
|
||||||
|
self.client.send_message("/avatar/parameters/EyesY", sy)
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||||
|
if rb and lb: # If both eyes are closed, blink
|
||||||
|
if last_blink > 0.5:
|
||||||
|
for i in range(4):
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||||
|
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||||
|
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||||
|
last_blink = time.time()
|
||||||
|
|
||||||
|
|
||||||
class VRChatOSCReceiver:
|
class VRChatOSCReceiver:
|
||||||
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
|
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
|
||||||
@ -151,9 +175,14 @@ class VRChatOSCReceiver:
|
|||||||
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
|
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
|
|
||||||
# bind what function to run when specified OSC message is received
|
# bind what function to run when specified OSC message is received
|
||||||
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
|
try:
|
||||||
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
|
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
|
||||||
# start the server
|
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
|
||||||
self.server.serve_forever()
|
# start the server
|
||||||
|
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
|
||||||
|
self.server.serve_forever()
|
||||||
|
|
||||||
|
except:
|
||||||
|
print("[WARN] OSC Receive port taken! Please use another OSC port.")
|
||||||
|
|||||||
@ -26,6 +26,7 @@ class SettingsWidget:
|
|||||||
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
|
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
|
||||||
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
|
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
|
||||||
self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-"
|
self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-"
|
||||||
|
self.gui_blink_sync = f"-BLINKSYNC{widget_id}-"
|
||||||
self.main_config = main_config
|
self.main_config = main_config
|
||||||
self.config = main_config.settings
|
self.config = main_config.settings
|
||||||
self.osc_queue = osc_queue
|
self.osc_queue = osc_queue
|
||||||
@ -62,6 +63,13 @@ class SettingsWidget:
|
|||||||
background_color='#424042',
|
background_color='#424042',
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
|
[sg.Checkbox(
|
||||||
|
"Sync Blinks (disables winking)",
|
||||||
|
default=self.config.gui_blink_sync,
|
||||||
|
key=self.gui_blink_sync,
|
||||||
|
background_color='#424042',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
|
||||||
[
|
[
|
||||||
sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
|
sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
|
||||||
@ -221,6 +229,11 @@ class SettingsWidget:
|
|||||||
self.config.gui_eye_falloff = values[self.gui_eye_falloff]
|
self.config.gui_eye_falloff = values[self.gui_eye_falloff]
|
||||||
changed = True
|
changed = True
|
||||||
|
|
||||||
|
if self.config.gui_blink_sync != values[self.gui_blink_sync]:
|
||||||
|
self.config.gui_blink_sync = values[self.gui_blink_sync]
|
||||||
|
changed = True
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]:
|
if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]:
|
||||||
self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]
|
self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user