fix bugs, add basic blink sync, adjust default values, bump ver, fix pyinstaller, osc

This commit is contained in:
Prohurtz 2022-11-07 11:49:23 -08:00
parent 3bad2f13c9
commit b1472949c7
6 changed files with 76 additions and 52 deletions

View File

@ -7,8 +7,8 @@ CONFIG_FILE_NAME: str = "eyetrack_settings.json"
class EyeTrackCameraConfig(BaseModel):
threshold: int = 0
rotation_angle: int = 50
threshold: int = 50
rotation_angle: int = 0
roi_window_x: int = 0
roi_window_y: int = 0
roi_window_w: int = 0
@ -34,7 +34,8 @@ class EyeTrackSettingsConfig(BaseModel):
gui_blob_minsize: float = 10
gui_recenter_eyes: bool = False
gui_eye_falloff: bool = False
tracker_single_eye: float = 0
tracker_single_eye: int = 0
gui_blink_sync: bool = False
class EyeTrackConfig(BaseModel):

View File

@ -268,16 +268,12 @@ class EyeProcessor:
# draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background
color = np.full_like(self.current_image_gray, (255))
# apply mask to image
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color)
except:
@ -285,15 +281,7 @@ class EyeProcessor:
else:
self.cct = self.cct - 1
# Increase our threshold value slightly, in order to have a better possibility of getting back
# something to do blob tracking on.
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
histr = hist.ravel()
peaks, properties = sp.find_peaks(histr, distance=5)
minpeak = np.min(peaks)
thresholdoptics = np.array(minpeak + int(self.config.threshold + 12))
larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
larger_threshold = cv2.bitwise_not(larger_threshold)
_, larger_threshold = cv2.threshold(self.current_image_gray, int(self.config.threshold + 12), 255, cv2.THRESH_BINARY)
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
# order to have a projected sphere.
@ -507,23 +495,18 @@ class EyeProcessor:
if self.cct == 0:
try:
ht, wd = self.current_image_gray.shape[:2]
radius = int(float(self.lkg_projected_sphere["axes"][0]))
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
# draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background
color = np.full_like(self.current_image_gray, (255))
# apply mask to image
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color)
except:
@ -533,14 +516,12 @@ class EyeProcessor:
else:
self.cct = 300
# Using Histogram based thresholding. Improves robustness insanely
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
histr = hist.ravel()
peaks, properties = sp.find_peaks(histr, distance=5)
minpeak = np.min(peaks)
thresholdoptics = np.array(minpeak + int(self.config.threshold))
thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
thresh = cv2.bitwise_not(thresh)
_, thresh = cv2.threshold(
self.current_image_gray,
int(self.config.threshold),
255,
cv2.THRESH_BINARY,
)
# Set up morphological transforms, for smoothing and clearing the image we get out of the
# thresholding operation. After this, we'd really like to just have a black blob in the middle

View File

@ -119,7 +119,7 @@ def main():
osc_receiver_thread.start()
# Create the window
window = sg.Window("EyeTrackVR v0.1.7", layout, icon='Images/logo.ico', background_color='#292929')
window = sg.Window("EyeTrackVR v0.1.7.1", layout, icon='Images/logo.ico', background_color='#292929')
# GUI Render loop
while True:

View File

@ -6,7 +6,7 @@ block_cipher = None
a = Analysis(['eyetrackapp.py'],
pathex=[],
binaries=[],
datas=[("Audio/*", "Audio"), ("Images/*", "Images/")],
datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("pye3dcustom/refraction_models/*", "pye3dcustom/refraction_models")],
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
hookspath=[],
hooksconfig={},

View File

@ -54,15 +54,22 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if self.config.gui_blink_sync and not rb and not lb:
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
else:
if eye_id in [EyeId.RIGHT]:
yr = eye_info.y
sx = eye_info.x
sy = eye_info.y
rb = False
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not lb:
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
if eye_id in [EyeId.LEFT]:
yl = eye_info.y
@ -70,27 +77,21 @@ class VRChatOSC:
sy = eye_info.y
lb = False
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not rb:
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if (yr != 621 and yl != 621) and (lb == False and rb == False):
y = (yr + yl) / 2
self.client.send_message("/avatar/parameters/EyesY", y)
else:
if self.config.gui_eye_falloff:
if self.config.gui_blink_sync:
if eye_id in [EyeId.LEFT]:
lb = True
if eye_id in [EyeId.RIGHT]:
rb = True
if rb == True or lb == True: # If one eye closed and fall off is enabled, mirror data
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
self.client.send_message("/avatar/parameters/RightEyeX", sx)
self.client.send_message("/avatar/parameters/EyesY", sy)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if rb == True and lb == True: # If both eyes are closed, blink
if rb == True and lb == True : # If both eyes are closed, blink
if last_blink > 0.5:
for i in range(4):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
@ -107,7 +108,8 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
last_blink = time.time()
else:
if not self.config.gui_eye_falloff:
if eye_id in [EyeId.LEFT]:
lb = True
if last_blink > 0.5:
@ -124,6 +126,28 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
last_blink = time.time()
else:
if eye_id in [EyeId.LEFT]:
lb = True
if eye_id in [EyeId.RIGHT]:
rb = True
if rb or lb: # If one eye closed and fall off is enabled, mirror data
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
self.client.send_message("/avatar/parameters/RightEyeX", sx)
self.client.send_message("/avatar/parameters/EyesY", sy)
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
if rb and lb: # If both eyes are closed, blink
if last_blink > 0.5:
for i in range(4):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
last_blink = time.time()
class VRChatOSCReceiver:
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
@ -151,9 +175,14 @@ class VRChatOSCReceiver:
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
def run(self):
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
# bind what function to run when specified OSC message is received
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
# start the server
self.server.serve_forever()
try:
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
# start the server
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
self.server.serve_forever()
except:
print("[WARN] OSC Receive port taken! Please use another OSC port.")

View File

@ -26,6 +26,7 @@ class SettingsWidget:
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-"
self.gui_blink_sync = f"-BLINKSYNC{widget_id}-"
self.main_config = main_config
self.config = main_config.settings
self.osc_queue = osc_queue
@ -62,6 +63,13 @@ class SettingsWidget:
background_color='#424042',
),
],
[sg.Checkbox(
"Sync Blinks (disables winking)",
default=self.config.gui_blink_sync,
key=self.gui_blink_sync,
background_color='#424042',
),
],
[
sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
@ -221,6 +229,11 @@ class SettingsWidget:
self.config.gui_eye_falloff = values[self.gui_eye_falloff]
changed = True
if self.config.gui_blink_sync != values[self.gui_blink_sync]:
self.config.gui_blink_sync = values[self.gui_blink_sync]
changed = True
if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]:
self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]