Fix blob tracking positioning, resume sending packets over OSC

This commit is contained in:
Kyle Machulis 2022-06-04 19:14:23 -07:00
parent c1cd4583a3
commit def7297e05
3 changed files with 77 additions and 43 deletions

View File

@ -1,7 +1,7 @@
from osc import VRChatOSC
from config import RansacConfig
from speech_engine import SpeechEngine
from ransac import Ransac
from ransac import Ransac, InformationOrigin
import queue
import threading
import cv2
@ -19,6 +19,9 @@ TRACKING_BUTTON_NAME = "-TRACKINGMODE-"
TRACKING_LAYOUT_NAME = "-TRACKINGLAYOUT-"
TRACKING_IMAGE_NAME = "-IMAGE-"
OUTPUT_GRAPH_NAME = "-OUTPUTGRAPH-"
RESTART_CALIBRATION_NAME = "-RESTARTCALIBRATION-"
RECENTER_EYE_NAME = "-RECENTEREYE-"
MODE_READOUT_NAME = "-APPMODE-"
def main():
in_roi_mode = False
@ -35,8 +38,10 @@ def main():
tracking_layout = [
[sg.Text("Threshold"), sg.Slider(range=(0, 100), default_value=config.threshold, orientation = 'h', key=THRESHOLD_SLIDER_NAME)],
[sg.Text("Rotation"), sg.Slider(range=(0, 360), default_value=config.rotation_angle, orientation = 'h', key=ROTATION_SLIDER_NAME)],
[sg.Button("Restart Calibration", key=RESTART_CALIBRATION_NAME), sg.Button("Recenter Eye", key=RECENTER_EYE_NAME)],
[sg.Text("Mode:"), sg.Text("Calibrating", key=MODE_READOUT_NAME)],
[sg.Image(filename="", key=TRACKING_IMAGE_NAME)],
[sg.Graph((200,200), (-100, 100), (100, -100), key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)]
[sg.Graph((200,200), (-100, 100), (100, -100), background_color='white', key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)]
]
layout = [[sg.Text("Camera Address"), sg.InputText(config.capture_source, key=CAMERA_ADDR_NAME)],
@ -149,6 +154,15 @@ def main():
is_mouse_up = False
x0, y0 = values['-GRAPH-']
x1, y1 = values['-GRAPH-']
elif event == RESTART_CALIBRATION_NAME:
ransac.calibration_frame_counter = 300
elif event == RECENTER_EYE_NAME:
ransac.recenter_eye = True
if ransac.calibration_frame_counter != None:
window[MODE_READOUT_NAME].update("Calibration")
else:
window[MODE_READOUT_NAME].update("Tracking")
if in_roi_mode:
try:
@ -173,9 +187,14 @@ def main():
window[TRACKING_IMAGE_NAME].update(data=imgbytes)
graph = window[OUTPUT_GRAPH_NAME]
graph.erase()
if eye_info[0] and not eye_info[3]:
graph.draw_circle((eye_info[1] * -100, eye_info[2] * -100), 25, fill_color='black',line_color='white')
if eye_info.info_type != InformationOrigin.FAILURE and not eye_info.blink:
graph.background_color = 'white'
graph.draw_circle((eye_info.x * -100, eye_info.y * -100), 25, fill_color='black',line_color='white')
osc_queue.put(eye_info)
elif eye_info.blink:
graph.background_color = 'blue'
elif eye_info.info_type == InformationOrigin.FAILURE:
graph.background_color = 'red'
except queue.Empty:
pass

View File

@ -23,14 +23,14 @@ class VRChatOSC:
print("Exiting OSC Queue")
return
try:
item = self.msg_queue.get(block=True, timeout=0.1)
eye_info = self.msg_queue.get(block=True, timeout=0.1)
except queue.Empty:
continue
# If we're not blinking, set position
if not item[0]:
self.client.send_message("/avatar/parameters/RightEyeX", item[1])
self.client.send_message("/avatar/parameters/LeftEyeX", item[1])
self.client.send_message("/avatar/parameters/EyesY", item[2])
if not eye_info.blink:
self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x)
self.client.send_message("/avatar/parameters/EyesY", eye_info.y)
if was_blinking:
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))

View File

@ -1,3 +1,4 @@
from dataclasses import dataclass
import sys
sys.path.append(".")
from config import RansacConfig
@ -6,6 +7,19 @@ import queue
import threading
import numpy as np
import cv2
from enum import Enum
class InformationOrigin(Enum):
RANSAC = 1
BLOB = 2
FAILURE = 3
@dataclass
class EyeInformation:
info_type: InformationOrigin
x: float
y: float
blink: bool
def fit_rotated_ellipse_ransac(
data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80
@ -127,10 +141,11 @@ class Ransac:
self.ymax = 69420
self.ymin = -69420
self.previous_rotation = self.config.rotation_angle
self.recenter_eye = False
def output_images_and_update(self, threshold_image, output_tuple):
def output_images_and_update(self, threshold_image, output_information: EyeInformation):
image_stack = np.concatenate((self.current_image, cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), cv2.cvtColor(threshold_image, cv2.COLOR_GRAY2BGR)), axis=1)
self.image_queue_outgoing.put((image_stack, output_tuple))
self.image_queue_outgoing.put((image_stack, output_information))
self.previous_image = self.current_image
self.previous_rotation = self.config.rotation_angle
@ -165,7 +180,7 @@ class Ransac:
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
# order to have a projected sphere.
if self.lkg_projected_sphere == None:
self.output_images_and_update(larger_threshold, None)
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False))
return
try:
@ -176,7 +191,7 @@ class Ransac:
if len(contours) == 0:
raise RuntimeError("No contours found for image")
except:
self.output_images_and_update(larger_threshold, None)
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False))
return
rows, cols = larger_threshold.shape
@ -197,22 +212,28 @@ class Ransac:
cv2.drawContours(self.current_image_gray, [cnt], -1, (255, 0, 0), 3)
cv2.rectangle(self.current_image_gray, (x, y), (x + w, y + h), (255, 0, 0), 2)
eye_position_scalar = 3000
# TODO These calculations were wrong in RANSAC3d, need to be fixed anyways.
if xrlb >= 0:
pass
# client.send_message("/avatar/parameters/RightEyeX", -abs(xrl))
# client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl))
if eyeyb <= 0:
pass
# client.send_message("/avatar/parameters/RightEyeX", -abs(xrl))
# client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl))
self.output_images_and_update(larger_threshold, (True, -abs(xrlb) if xrlb >= 0 else abs(xrlb), -abs(xrlb) if eyeyb <= 0 else abs(xrlb), False))
# If we've sent something, just return.
return
# If we haven't returned yet, consider this blinking
# client.send_message("/avatar/parameters/LeftEyeLid", float(0))
# client.send_message("/avatar/parameters/RightEyeLid", float(0))
self.output_images_and_update(larger_threshold, (True, 0, 0, True))
xl = float(((xt - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff))
xr = float(((xt - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff))
yu = float(((yt - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff))
yd = float(((yt - self.yoff) * eye_position_scalar) / (self.ymax - self.yoff))
# print(f"{exm} {eym} {xl} {xr} {yu} {yd}")
out_x = 0
out_y = 0
if xr > 0:
out_x = max(0.0, min(1.0, xr))
if xl > 0:
out_x = -abs(max(0.0, min(1.0, xl)))
if yd > 0:
out_y = -abs(max(0.0, min(1.0, yd)))
if yu > 0:
out_y = max(0.0, min(1.0, yu))
output_tuple = (True, out_x, out_y, False)
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, False))
self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, 0, 0, True))
print('[INFO] BLINK Detected.')
def run(self):
@ -277,10 +298,7 @@ class Ransac:
#
# TODO Reimplement Prohurtz's blob tracking fallback
if len(convex_hulls) == 0:
# print("No contours found, eye not detected, continuing")
# Draw our image and stack it for visual output
self.blob_tracking_fallback()
# self.output_images_and_update(thresh)
continue
# Find our largest hull, which we expect will probably be the ellipse that represents the 2d
@ -293,8 +311,6 @@ class Ransac:
try:
cx, cy, w, h, theta = fit_rotated_ellipse_ransac(largest_hull.reshape(-1, 2))
except:
# If we don't find anything, and we don't have a sphere to calculate off of. Give up.
# self.output_images_and_update(thresh)
self.blob_tracking_fallback()
continue
@ -321,18 +337,17 @@ class Ransac:
# And our eyeball that the pupil is on the surface of
self.lkg_projected_sphere = result_3d["projected_sphere"]
# Record our pupil center
exm = ellipse_3d["center"][0]
eym = ellipse_3d["center"][1]
if self.calibration_frame_counter == 0:
print("FINISHING")
if self.calibration_frame_counter == 0 or self.recenter_eye:
self.calibration_frame_counter = None
self.xoff = ellipse_3d["center"][0]
self.yoff = ellipse_3d["center"][1]
self.recenter_eye = False
self.xoff = exm
self.yoff = eym
elif self.calibration_frame_counter != None:
print("CALIBRATING")
if exm > self.xmax:
self.xmax = exm
if exm < self.xmin:
@ -342,7 +357,7 @@ class Ransac:
if eym < self.xmin:
self.ymin = eym
self.calibration_frame_counter -= 1
eye_position_scalar = 1000
eye_position_scalar = 3000
xl = float(((cx - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff))
xr = float(((cx - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff))
yu = float(((cy - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff))
@ -360,7 +375,7 @@ class Ransac:
if yu > 0:
out_y = max(0.0, min(1.0, yu))
output_tuple = (True, out_x, out_y, False)
output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, False)
# # So now we get the offset of the center of the eyeball
# xrl = (cx - self.lkg_projected_sphere["center"][0]) / self.lkg_projected_sphere["axes"][0]
@ -403,5 +418,5 @@ class Ransac:
)
# Shove a concatenated image out to the main GUI thread for rendering
self.output_images_and_update(thresh, output_tuple)
self.output_images_and_update(thresh, output_info)