fix bugs, add basic blink sync, adjust default values, bump ver, fix pyinstaller, osc

This commit is contained in:
Prohurtz 2022-11-07 11:49:23 -08:00
parent 3bad2f13c9
commit b1472949c7
6 changed files with 76 additions and 52 deletions

View File

@ -7,8 +7,8 @@ CONFIG_FILE_NAME: str = "eyetrack_settings.json"
class EyeTrackCameraConfig(BaseModel): class EyeTrackCameraConfig(BaseModel):
threshold: int = 0 threshold: int = 50
rotation_angle: int = 50 rotation_angle: int = 0
roi_window_x: int = 0 roi_window_x: int = 0
roi_window_y: int = 0 roi_window_y: int = 0
roi_window_w: int = 0 roi_window_w: int = 0
@ -34,7 +34,8 @@ class EyeTrackSettingsConfig(BaseModel):
gui_blob_minsize: float = 10 gui_blob_minsize: float = 10
gui_recenter_eyes: bool = False gui_recenter_eyes: bool = False
gui_eye_falloff: bool = False gui_eye_falloff: bool = False
tracker_single_eye: float = 0 tracker_single_eye: int = 0
gui_blink_sync: bool = False
class EyeTrackConfig(BaseModel): class EyeTrackConfig(BaseModel):

View File

@ -268,16 +268,12 @@ class EyeProcessor:
# draw filled circle in white on black background as mask # draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8) mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background # create white colored background
color = np.full_like(self.current_image_gray, (255)) color = np.full_like(self.current_image_gray, (255))
# apply mask to image # apply mask to image
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image # apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask) masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images # combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color) self.current_image_gray = cv2.add(masked_img, masked_color)
except: except:
@ -285,15 +281,7 @@ class EyeProcessor:
else: else:
self.cct = self.cct - 1 self.cct = self.cct - 1
# Increase our threshold value slightly, in order to have a better possibility of getting back _, larger_threshold = cv2.threshold(self.current_image_gray, int(self.config.threshold + 12), 255, cv2.THRESH_BINARY)
# something to do blob tracking on.
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
histr = hist.ravel()
peaks, properties = sp.find_peaks(histr, distance=5)
minpeak = np.min(peaks)
thresholdoptics = np.array(minpeak + int(self.config.threshold + 12))
larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
larger_threshold = cv2.bitwise_not(larger_threshold)
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This # Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in # means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
# order to have a projected sphere. # order to have a projected sphere.
@ -507,23 +495,18 @@ class EyeProcessor:
if self.cct == 0: if self.cct == 0:
try: try:
ht, wd = self.current_image_gray.shape[:2] ht, wd = self.current_image_gray.shape[:2]
radius = int(float(self.lkg_projected_sphere["axes"][0])) radius = int(float(self.lkg_projected_sphere["axes"][0]))
self.xc = int(float(self.lkg_projected_sphere["center"][0])) self.xc = int(float(self.lkg_projected_sphere["center"][0]))
self.yc = int(float(self.lkg_projected_sphere["center"][1])) self.yc = int(float(self.lkg_projected_sphere["center"][1]))
# draw filled circle in white on black background as mask # draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8) mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background # create white colored background
color = np.full_like(self.current_image_gray, (255)) color = np.full_like(self.current_image_gray, (255))
# apply mask to image # apply mask to image
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image # apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask) masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images # combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color) self.current_image_gray = cv2.add(masked_img, masked_color)
except: except:
@ -533,14 +516,12 @@ class EyeProcessor:
else: else:
self.cct = 300 self.cct = 300
# Using Histogram based thresholding. Improves robustness insanely _, thresh = cv2.threshold(
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) self.current_image_gray,
histr = hist.ravel() int(self.config.threshold),
peaks, properties = sp.find_peaks(histr, distance=5) 255,
minpeak = np.min(peaks) cv2.THRESH_BINARY,
thresholdoptics = np.array(minpeak + int(self.config.threshold)) )
thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
thresh = cv2.bitwise_not(thresh)
# Set up morphological transforms, for smoothing and clearing the image we get out of the # Set up morphological transforms, for smoothing and clearing the image we get out of the
# thresholding operation. After this, we'd really like to just have a black blob in the middle # thresholding operation. After this, we'd really like to just have a black blob in the middle

View File

@ -119,7 +119,7 @@ def main():
osc_receiver_thread.start() osc_receiver_thread.start()
# Create the window # Create the window
window = sg.Window("EyeTrackVR v0.1.7", layout, icon='Images/logo.ico', background_color='#292929') window = sg.Window("EyeTrackVR v0.1.7.1", layout, icon='Images/logo.ico', background_color='#292929')
# GUI Render loop # GUI Render loop
while True: while True:

View File

@ -6,7 +6,7 @@ block_cipher = None
a = Analysis(['eyetrackapp.py'], a = Analysis(['eyetrackapp.py'],
pathex=[], pathex=[],
binaries=[], binaries=[],
datas=[("Audio/*", "Audio"), ("Images/*", "Images/")], datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("pye3dcustom/refraction_models/*", "pye3dcustom/refraction_models")],
hiddenimports=['cv2', 'numpy', 'PySimpleGui'], hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
hookspath=[], hookspath=[],
hooksconfig={}, hooksconfig={},

View File

@ -54,15 +54,22 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if self.config.gui_blink_sync and not rb and not lb:
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
else: else:
if eye_id in [EyeId.RIGHT]: if eye_id in [EyeId.RIGHT]:
yr = eye_info.y yr = eye_info.y
sx = eye_info.x sx = eye_info.x
sy = eye_info.y sy = eye_info.y
rb = False rb = False
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x) self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right if not self.config.gui_blink_sync or self.config.gui_blink_sync and not lb:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
if eye_id in [EyeId.LEFT]: if eye_id in [EyeId.LEFT]:
yl = eye_info.y yl = eye_info.y
@ -70,27 +77,21 @@ class VRChatOSC:
sy = eye_info.y sy = eye_info.y
lb = False lb = False
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x) self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left if not self.config.gui_blink_sync or self.config.gui_blink_sync and not rb:
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if (yr != 621 and yl != 621) and (lb == False and rb == False): if (yr != 621 and yl != 621) and (lb == False and rb == False):
y = (yr + yl) / 2 y = (yr + yl) / 2
self.client.send_message("/avatar/parameters/EyesY", y) self.client.send_message("/avatar/parameters/EyesY", y)
else: else:
if self.config.gui_eye_falloff:
if self.config.gui_blink_sync:
if eye_id in [EyeId.LEFT]: if eye_id in [EyeId.LEFT]:
lb = True lb = True
if eye_id in [EyeId.RIGHT]: if eye_id in [EyeId.RIGHT]:
rb = True rb = True
if rb == True or lb == True: # If one eye closed and fall off is enabled, mirror data if rb == True and lb == True : # If both eyes are closed, blink
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
self.client.send_message("/avatar/parameters/RightEyeX", sx)
self.client.send_message("/avatar/parameters/EyesY", sy)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if rb == True and lb == True: # If both eyes are closed, blink
if last_blink > 0.5: if last_blink > 0.5:
for i in range(4): for i in range(4):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
@ -107,7 +108,8 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0)) self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
last_blink = time.time() last_blink = time.time()
else:
if not self.config.gui_eye_falloff:
if eye_id in [EyeId.LEFT]: if eye_id in [EyeId.LEFT]:
lb = True lb = True
if last_blink > 0.5: if last_blink > 0.5:
@ -124,6 +126,28 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
last_blink = time.time() last_blink = time.time()
else:
if eye_id in [EyeId.LEFT]:
lb = True
if eye_id in [EyeId.RIGHT]:
rb = True
if rb or lb: # If one eye closed and fall off is enabled, mirror data
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
self.client.send_message("/avatar/parameters/RightEyeX", sx)
self.client.send_message("/avatar/parameters/EyesY", sy)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if rb and lb: # If both eyes are closed, blink
if last_blink > 0.5:
for i in range(4):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
last_blink = time.time()
class VRChatOSCReceiver: class VRChatOSCReceiver:
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []): def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
@ -151,9 +175,14 @@ class VRChatOSCReceiver:
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
def run(self): def run(self):
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
# bind what function to run when specified OSC message is received # bind what function to run when specified OSC message is received
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes) try:
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes) self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
# start the server self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
self.server.serve_forever() # start the server
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
self.server.serve_forever()
except:
print("[WARN] OSC Receive port taken! Please use another OSC port.")

View File

@ -26,6 +26,7 @@ class SettingsWidget:
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-" self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-" self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-" self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-"
self.gui_blink_sync = f"-BLINKSYNC{widget_id}-"
self.main_config = main_config self.main_config = main_config
self.config = main_config.settings self.config = main_config.settings
self.osc_queue = osc_queue self.osc_queue = osc_queue
@ -62,6 +63,13 @@ class SettingsWidget:
background_color='#424042', background_color='#424042',
), ),
], ],
[sg.Checkbox(
"Sync Blinks (disables winking)",
default=self.config.gui_blink_sync,
key=self.gui_blink_sync,
background_color='#424042',
),
],
[ [
sg.Text("Tracking Algorithim Settings:", background_color='#242224'), sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
@ -221,6 +229,11 @@ class SettingsWidget:
self.config.gui_eye_falloff = values[self.gui_eye_falloff] self.config.gui_eye_falloff = values[self.gui_eye_falloff]
changed = True changed = True
if self.config.gui_blink_sync != values[self.gui_blink_sync]:
self.config.gui_blink_sync = values[self.gui_blink_sync]
changed = True
if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]: if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]:
self.config.gui_blob_maxsize = values[self.gui_blob_maxsize] self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]