mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix bugs, add basic blink sync, adjust default values, bump ver, fix pyinstaller, osc
This commit is contained in:
parent
3bad2f13c9
commit
b1472949c7
@ -7,8 +7,8 @@ CONFIG_FILE_NAME: str = "eyetrack_settings.json"
|
||||
|
||||
|
||||
class EyeTrackCameraConfig(BaseModel):
|
||||
threshold: int = 0
|
||||
rotation_angle: int = 50
|
||||
threshold: int = 50
|
||||
rotation_angle: int = 0
|
||||
roi_window_x: int = 0
|
||||
roi_window_y: int = 0
|
||||
roi_window_w: int = 0
|
||||
@ -34,7 +34,8 @@ class EyeTrackSettingsConfig(BaseModel):
|
||||
gui_blob_minsize: float = 10
|
||||
gui_recenter_eyes: bool = False
|
||||
gui_eye_falloff: bool = False
|
||||
tracker_single_eye: float = 0
|
||||
tracker_single_eye: int = 0
|
||||
gui_blink_sync: bool = False
|
||||
|
||||
|
||||
class EyeTrackConfig(BaseModel):
|
||||
|
||||
@ -268,16 +268,12 @@ class EyeProcessor:
|
||||
# draw filled circle in white on black background as mask
|
||||
mask = np.zeros((ht, wd), dtype=np.uint8)
|
||||
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
||||
|
||||
# create white colored background
|
||||
color = np.full_like(self.current_image_gray, (255))
|
||||
|
||||
# apply mask to image
|
||||
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
||||
|
||||
# apply inverse mask to colored image
|
||||
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
||||
|
||||
# combine the two masked images
|
||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||
except:
|
||||
@ -285,15 +281,7 @@ class EyeProcessor:
|
||||
else:
|
||||
self.cct = self.cct - 1
|
||||
|
||||
# Increase our threshold value slightly, in order to have a better possibility of getting back
|
||||
# something to do blob tracking on.
|
||||
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
|
||||
histr = hist.ravel()
|
||||
peaks, properties = sp.find_peaks(histr, distance=5)
|
||||
minpeak = np.min(peaks)
|
||||
thresholdoptics = np.array(minpeak + int(self.config.threshold + 12))
|
||||
larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
|
||||
larger_threshold = cv2.bitwise_not(larger_threshold)
|
||||
_, larger_threshold = cv2.threshold(self.current_image_gray, int(self.config.threshold + 12), 255, cv2.THRESH_BINARY)
|
||||
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
|
||||
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
|
||||
# order to have a projected sphere.
|
||||
@ -507,23 +495,18 @@ class EyeProcessor:
|
||||
if self.cct == 0:
|
||||
try:
|
||||
ht, wd = self.current_image_gray.shape[:2]
|
||||
|
||||
radius = int(float(self.lkg_projected_sphere["axes"][0]))
|
||||
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
|
||||
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
|
||||
# draw filled circle in white on black background as mask
|
||||
mask = np.zeros((ht, wd), dtype=np.uint8)
|
||||
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
|
||||
|
||||
# create white colored background
|
||||
color = np.full_like(self.current_image_gray, (255))
|
||||
|
||||
# apply mask to image
|
||||
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
|
||||
|
||||
# apply inverse mask to colored image
|
||||
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
|
||||
|
||||
# combine the two masked images
|
||||
self.current_image_gray = cv2.add(masked_img, masked_color)
|
||||
except:
|
||||
@ -533,14 +516,12 @@ class EyeProcessor:
|
||||
else:
|
||||
self.cct = 300
|
||||
|
||||
# Using Histogram based thresholding. Improves robustness insanely
|
||||
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
|
||||
histr = hist.ravel()
|
||||
peaks, properties = sp.find_peaks(histr, distance=5)
|
||||
minpeak = np.min(peaks)
|
||||
thresholdoptics = np.array(minpeak + int(self.config.threshold))
|
||||
thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
|
||||
thresh = cv2.bitwise_not(thresh)
|
||||
_, thresh = cv2.threshold(
|
||||
self.current_image_gray,
|
||||
int(self.config.threshold),
|
||||
255,
|
||||
cv2.THRESH_BINARY,
|
||||
)
|
||||
|
||||
# Set up morphological transforms, for smoothing and clearing the image we get out of the
|
||||
# thresholding operation. After this, we'd really like to just have a black blob in the middle
|
||||
|
||||
@ -119,7 +119,7 @@ def main():
|
||||
osc_receiver_thread.start()
|
||||
|
||||
# Create the window
|
||||
window = sg.Window("EyeTrackVR v0.1.7", layout, icon='Images/logo.ico', background_color='#292929')
|
||||
window = sg.Window("EyeTrackVR v0.1.7.1", layout, icon='Images/logo.ico', background_color='#292929')
|
||||
|
||||
# GUI Render loop
|
||||
while True:
|
||||
|
||||
@ -6,7 +6,7 @@ block_cipher = None
|
||||
a = Analysis(['eyetrackapp.py'],
|
||||
pathex=[],
|
||||
binaries=[],
|
||||
datas=[("Audio/*", "Audio"), ("Images/*", "Images/")],
|
||||
datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("pye3dcustom/refraction_models/*", "pye3dcustom/refraction_models")],
|
||||
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
|
||||
@ -54,15 +54,22 @@ class VRChatOSC:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
if self.config.gui_blink_sync and not rb and not lb:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
|
||||
else:
|
||||
if eye_id in [EyeId.RIGHT]:
|
||||
yr = eye_info.y
|
||||
sx = eye_info.x
|
||||
sy = eye_info.y
|
||||
rb = False
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
|
||||
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not lb:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open right eye
|
||||
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
yl = eye_info.y
|
||||
@ -70,27 +77,21 @@ class VRChatOSC:
|
||||
sy = eye_info.y
|
||||
lb = False
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
if not self.config.gui_blink_sync or self.config.gui_blink_sync and not rb:
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
|
||||
if (yr != 621 and yl != 621) and (lb == False and rb == False):
|
||||
y = (yr + yl) / 2
|
||||
self.client.send_message("/avatar/parameters/EyesY", y)
|
||||
else:
|
||||
if self.config.gui_eye_falloff:
|
||||
|
||||
if self.config.gui_blink_sync:
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
lb = True
|
||||
if eye_id in [EyeId.RIGHT]:
|
||||
rb = True
|
||||
if rb == True or lb == True: # If one eye closed and fall off is enabled, mirror data
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", sx)
|
||||
self.client.send_message("/avatar/parameters/EyesY", sy)
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
if rb == True and lb == True: # If both eyes are closed, blink
|
||||
if rb == True and lb == True : # If both eyes are closed, blink
|
||||
if last_blink > 0.5:
|
||||
for i in range(4):
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||
@ -107,7 +108,8 @@ class VRChatOSC:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
last_blink = time.time()
|
||||
else:
|
||||
|
||||
if not self.config.gui_eye_falloff:
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
lb = True
|
||||
if last_blink > 0.5:
|
||||
@ -124,6 +126,28 @@ class VRChatOSC:
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
last_blink = time.time()
|
||||
|
||||
else:
|
||||
if eye_id in [EyeId.LEFT]:
|
||||
lb = True
|
||||
if eye_id in [EyeId.RIGHT]:
|
||||
rb = True
|
||||
if rb or lb: # If one eye closed and fall off is enabled, mirror data
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", sx) #Send mirrored data to both eyes.
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", sx)
|
||||
self.client.send_message("/avatar/parameters/EyesY", sy)
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(0))# old param open right
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0.8)) # open r
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0.8)) # open left eye
|
||||
if rb and lb: # If both eyes are closed, blink
|
||||
if last_blink > 0.5:
|
||||
for i in range(4):
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(0)) # close eye
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(0))
|
||||
last_blink = time.time()
|
||||
|
||||
|
||||
class VRChatOSCReceiver:
|
||||
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
|
||||
@ -151,9 +175,14 @@ class VRChatOSCReceiver:
|
||||
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
|
||||
|
||||
def run(self):
|
||||
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
|
||||
|
||||
# bind what function to run when specified OSC message is received
|
||||
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
|
||||
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
|
||||
# start the server
|
||||
self.server.serve_forever()
|
||||
try:
|
||||
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
|
||||
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
|
||||
# start the server
|
||||
print("VRChatOSCReceiver serving on {}".format(self.server.server_address))
|
||||
self.server.serve_forever()
|
||||
|
||||
except:
|
||||
print("[WARN] OSC Receive port taken! Please use another OSC port.")
|
||||
|
||||
@ -26,6 +26,7 @@ class SettingsWidget:
|
||||
self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
|
||||
self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
|
||||
self.gui_eye_falloff = f"-EYEFALLOFF{widget_id}-"
|
||||
self.gui_blink_sync = f"-BLINKSYNC{widget_id}-"
|
||||
self.main_config = main_config
|
||||
self.config = main_config.settings
|
||||
self.osc_queue = osc_queue
|
||||
@ -62,6 +63,13 @@ class SettingsWidget:
|
||||
background_color='#424042',
|
||||
),
|
||||
],
|
||||
[sg.Checkbox(
|
||||
"Sync Blinks (disables winking)",
|
||||
default=self.config.gui_blink_sync,
|
||||
key=self.gui_blink_sync,
|
||||
background_color='#424042',
|
||||
),
|
||||
],
|
||||
|
||||
[
|
||||
sg.Text("Tracking Algorithim Settings:", background_color='#242224'),
|
||||
@ -221,6 +229,11 @@ class SettingsWidget:
|
||||
self.config.gui_eye_falloff = values[self.gui_eye_falloff]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_blink_sync != values[self.gui_blink_sync]:
|
||||
self.config.gui_blink_sync = values[self.gui_blink_sync]
|
||||
changed = True
|
||||
|
||||
|
||||
|
||||
if self.config.gui_blob_maxsize != values[self.gui_blob_maxsize]:
|
||||
self.config.gui_blob_maxsize = values[self.gui_blob_maxsize]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user