mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
Fix blob tracking positioning, resume sending packets over OSC
This commit is contained in:
parent
c1cd4583a3
commit
def7297e05
@ -1,7 +1,7 @@
|
||||
from osc import VRChatOSC
|
||||
from config import RansacConfig
|
||||
from speech_engine import SpeechEngine
|
||||
from ransac import Ransac
|
||||
from ransac import Ransac, InformationOrigin
|
||||
import queue
|
||||
import threading
|
||||
import cv2
|
||||
@ -19,6 +19,9 @@ TRACKING_BUTTON_NAME = "-TRACKINGMODE-"
|
||||
TRACKING_LAYOUT_NAME = "-TRACKINGLAYOUT-"
|
||||
TRACKING_IMAGE_NAME = "-IMAGE-"
|
||||
OUTPUT_GRAPH_NAME = "-OUTPUTGRAPH-"
|
||||
RESTART_CALIBRATION_NAME = "-RESTARTCALIBRATION-"
|
||||
RECENTER_EYE_NAME = "-RECENTEREYE-"
|
||||
MODE_READOUT_NAME = "-APPMODE-"
|
||||
|
||||
def main():
|
||||
in_roi_mode = False
|
||||
@ -35,8 +38,10 @@ def main():
|
||||
tracking_layout = [
|
||||
[sg.Text("Threshold"), sg.Slider(range=(0, 100), default_value=config.threshold, orientation = 'h', key=THRESHOLD_SLIDER_NAME)],
|
||||
[sg.Text("Rotation"), sg.Slider(range=(0, 360), default_value=config.rotation_angle, orientation = 'h', key=ROTATION_SLIDER_NAME)],
|
||||
[sg.Button("Restart Calibration", key=RESTART_CALIBRATION_NAME), sg.Button("Recenter Eye", key=RECENTER_EYE_NAME)],
|
||||
[sg.Text("Mode:"), sg.Text("Calibrating", key=MODE_READOUT_NAME)],
|
||||
[sg.Image(filename="", key=TRACKING_IMAGE_NAME)],
|
||||
[sg.Graph((200,200), (-100, 100), (100, -100), key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)]
|
||||
[sg.Graph((200,200), (-100, 100), (100, -100), background_color='white', key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)]
|
||||
]
|
||||
|
||||
layout = [[sg.Text("Camera Address"), sg.InputText(config.capture_source, key=CAMERA_ADDR_NAME)],
|
||||
@ -149,6 +154,15 @@ def main():
|
||||
is_mouse_up = False
|
||||
x0, y0 = values['-GRAPH-']
|
||||
x1, y1 = values['-GRAPH-']
|
||||
elif event == RESTART_CALIBRATION_NAME:
|
||||
ransac.calibration_frame_counter = 300
|
||||
elif event == RECENTER_EYE_NAME:
|
||||
ransac.recenter_eye = True
|
||||
|
||||
if ransac.calibration_frame_counter != None:
|
||||
window[MODE_READOUT_NAME].update("Calibration")
|
||||
else:
|
||||
window[MODE_READOUT_NAME].update("Tracking")
|
||||
|
||||
if in_roi_mode:
|
||||
try:
|
||||
@ -173,9 +187,14 @@ def main():
|
||||
window[TRACKING_IMAGE_NAME].update(data=imgbytes)
|
||||
graph = window[OUTPUT_GRAPH_NAME]
|
||||
graph.erase()
|
||||
if eye_info[0] and not eye_info[3]:
|
||||
graph.draw_circle((eye_info[1] * -100, eye_info[2] * -100), 25, fill_color='black',line_color='white')
|
||||
|
||||
if eye_info.info_type != InformationOrigin.FAILURE and not eye_info.blink:
|
||||
graph.background_color = 'white'
|
||||
graph.draw_circle((eye_info.x * -100, eye_info.y * -100), 25, fill_color='black',line_color='white')
|
||||
osc_queue.put(eye_info)
|
||||
elif eye_info.blink:
|
||||
graph.background_color = 'blue'
|
||||
elif eye_info.info_type == InformationOrigin.FAILURE:
|
||||
graph.background_color = 'red'
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
|
||||
@ -23,14 +23,14 @@ class VRChatOSC:
|
||||
print("Exiting OSC Queue")
|
||||
return
|
||||
try:
|
||||
item = self.msg_queue.get(block=True, timeout=0.1)
|
||||
eye_info = self.msg_queue.get(block=True, timeout=0.1)
|
||||
except queue.Empty:
|
||||
continue
|
||||
# If we're not blinking, set position
|
||||
if not item[0]:
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", item[1])
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", item[1])
|
||||
self.client.send_message("/avatar/parameters/EyesY", item[2])
|
||||
if not eye_info.blink:
|
||||
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
|
||||
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
|
||||
self.client.send_message("/avatar/parameters/EyesY", eye_info.y)
|
||||
if was_blinking:
|
||||
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
|
||||
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from dataclasses import dataclass
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
from config import RansacConfig
|
||||
@ -6,6 +7,19 @@ import queue
|
||||
import threading
|
||||
import numpy as np
|
||||
import cv2
|
||||
from enum import Enum
|
||||
|
||||
class InformationOrigin(Enum):
|
||||
RANSAC = 1
|
||||
BLOB = 2
|
||||
FAILURE = 3
|
||||
|
||||
@dataclass
|
||||
class EyeInformation:
|
||||
info_type: InformationOrigin
|
||||
x: float
|
||||
y: float
|
||||
blink: bool
|
||||
|
||||
def fit_rotated_ellipse_ransac(
|
||||
data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80
|
||||
@ -127,10 +141,11 @@ class Ransac:
|
||||
self.ymax = 69420
|
||||
self.ymin = -69420
|
||||
self.previous_rotation = self.config.rotation_angle
|
||||
self.recenter_eye = False
|
||||
|
||||
def output_images_and_update(self, threshold_image, output_tuple):
|
||||
def output_images_and_update(self, threshold_image, output_information: EyeInformation):
|
||||
image_stack = np.concatenate((self.current_image, cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), cv2.cvtColor(threshold_image, cv2.COLOR_GRAY2BGR)), axis=1)
|
||||
self.image_queue_outgoing.put((image_stack, output_tuple))
|
||||
self.image_queue_outgoing.put((image_stack, output_information))
|
||||
self.previous_image = self.current_image
|
||||
self.previous_rotation = self.config.rotation_angle
|
||||
|
||||
@ -165,7 +180,7 @@ class Ransac:
|
||||
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
|
||||
# order to have a projected sphere.
|
||||
if self.lkg_projected_sphere == None:
|
||||
self.output_images_and_update(larger_threshold, None)
|
||||
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False))
|
||||
return
|
||||
|
||||
try:
|
||||
@ -176,7 +191,7 @@ class Ransac:
|
||||
if len(contours) == 0:
|
||||
raise RuntimeError("No contours found for image")
|
||||
except:
|
||||
self.output_images_and_update(larger_threshold, None)
|
||||
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False))
|
||||
return
|
||||
|
||||
rows, cols = larger_threshold.shape
|
||||
@ -197,22 +212,28 @@ class Ransac:
|
||||
cv2.drawContours(self.current_image_gray, [cnt], -1, (255, 0, 0), 3)
|
||||
cv2.rectangle(self.current_image_gray, (x, y), (x + w, y + h), (255, 0, 0), 2)
|
||||
|
||||
eye_position_scalar = 3000
|
||||
# TODO These calculations were wrong in RANSAC3d, need to be fixed anyways.
|
||||
if xrlb >= 0:
|
||||
pass
|
||||
# client.send_message("/avatar/parameters/RightEyeX", -abs(xrl))
|
||||
# client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl))
|
||||
if eyeyb <= 0:
|
||||
pass
|
||||
# client.send_message("/avatar/parameters/RightEyeX", -abs(xrl))
|
||||
# client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl))
|
||||
self.output_images_and_update(larger_threshold, (True, -abs(xrlb) if xrlb >= 0 else abs(xrlb), -abs(xrlb) if eyeyb <= 0 else abs(xrlb), False))
|
||||
# If we've sent something, just return.
|
||||
return
|
||||
# If we haven't returned yet, consider this blinking
|
||||
# client.send_message("/avatar/parameters/LeftEyeLid", float(0))
|
||||
# client.send_message("/avatar/parameters/RightEyeLid", float(0))
|
||||
self.output_images_and_update(larger_threshold, (True, 0, 0, True))
|
||||
xl = float(((xt - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff))
|
||||
xr = float(((xt - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff))
|
||||
yu = float(((yt - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff))
|
||||
yd = float(((yt - self.yoff) * eye_position_scalar) / (self.ymax - self.yoff))
|
||||
# print(f"{exm} {eym} {xl} {xr} {yu} {yd}")
|
||||
|
||||
out_x = 0
|
||||
out_y = 0
|
||||
if xr > 0:
|
||||
out_x = max(0.0, min(1.0, xr))
|
||||
if xl > 0:
|
||||
out_x = -abs(max(0.0, min(1.0, xl)))
|
||||
if yd > 0:
|
||||
out_y = -abs(max(0.0, min(1.0, yd)))
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
|
||||
output_tuple = (True, out_x, out_y, False)
|
||||
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, False))
|
||||
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, 0, 0, True))
|
||||
print('[INFO] BLINK Detected.')
|
||||
|
||||
def run(self):
|
||||
@ -277,10 +298,7 @@ class Ransac:
|
||||
#
|
||||
# TODO Reimplement Prohurtz's blob tracking fallback
|
||||
if len(convex_hulls) == 0:
|
||||
# print("No contours found, eye not detected, continuing")
|
||||
# Draw our image and stack it for visual output
|
||||
self.blob_tracking_fallback()
|
||||
# self.output_images_and_update(thresh)
|
||||
continue
|
||||
|
||||
# Find our largest hull, which we expect will probably be the ellipse that represents the 2d
|
||||
@ -293,8 +311,6 @@ class Ransac:
|
||||
try:
|
||||
cx, cy, w, h, theta = fit_rotated_ellipse_ransac(largest_hull.reshape(-1, 2))
|
||||
except:
|
||||
# If we don't find anything, and we don't have a sphere to calculate off of. Give up.
|
||||
# self.output_images_and_update(thresh)
|
||||
self.blob_tracking_fallback()
|
||||
continue
|
||||
|
||||
@ -321,18 +337,17 @@ class Ransac:
|
||||
# And our eyeball that the pupil is on the surface of
|
||||
self.lkg_projected_sphere = result_3d["projected_sphere"]
|
||||
|
||||
|
||||
# Record our pupil center
|
||||
exm = ellipse_3d["center"][0]
|
||||
eym = ellipse_3d["center"][1]
|
||||
|
||||
|
||||
if self.calibration_frame_counter == 0:
|
||||
print("FINISHING")
|
||||
if self.calibration_frame_counter == 0 or self.recenter_eye:
|
||||
self.calibration_frame_counter = None
|
||||
self.xoff = ellipse_3d["center"][0]
|
||||
self.yoff = ellipse_3d["center"][1]
|
||||
self.recenter_eye = False
|
||||
self.xoff = exm
|
||||
self.yoff = eym
|
||||
elif self.calibration_frame_counter != None:
|
||||
print("CALIBRATING")
|
||||
if exm > self.xmax:
|
||||
self.xmax = exm
|
||||
if exm < self.xmin:
|
||||
@ -342,7 +357,7 @@ class Ransac:
|
||||
if eym < self.xmin:
|
||||
self.ymin = eym
|
||||
self.calibration_frame_counter -= 1
|
||||
eye_position_scalar = 1000
|
||||
eye_position_scalar = 3000
|
||||
xl = float(((cx - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff))
|
||||
xr = float(((cx - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff))
|
||||
yu = float(((cy - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff))
|
||||
@ -360,7 +375,7 @@ class Ransac:
|
||||
if yu > 0:
|
||||
out_y = max(0.0, min(1.0, yu))
|
||||
|
||||
output_tuple = (True, out_x, out_y, False)
|
||||
output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, False)
|
||||
|
||||
# # So now we get the offset of the center of the eyeball
|
||||
# xrl = (cx - self.lkg_projected_sphere["center"][0]) / self.lkg_projected_sphere["axes"][0]
|
||||
@ -403,5 +418,5 @@ class Ransac:
|
||||
)
|
||||
|
||||
# Shove a concatenated image out to the main GUI thread for rendering
|
||||
self.output_images_and_update(thresh, output_tuple)
|
||||
self.output_images_and_update(thresh, output_info)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user