diff --git a/RANSACApp/main.py b/RANSACApp/main.py index da971c4..24214e3 100644 --- a/RANSACApp/main.py +++ b/RANSACApp/main.py @@ -1,7 +1,7 @@ from osc import VRChatOSC from config import RansacConfig from speech_engine import SpeechEngine -from ransac import Ransac +from ransac import Ransac, InformationOrigin import queue import threading import cv2 @@ -19,6 +19,9 @@ TRACKING_BUTTON_NAME = "-TRACKINGMODE-" TRACKING_LAYOUT_NAME = "-TRACKINGLAYOUT-" TRACKING_IMAGE_NAME = "-IMAGE-" OUTPUT_GRAPH_NAME = "-OUTPUTGRAPH-" +RESTART_CALIBRATION_NAME = "-RESTARTCALIBRATION-" +RECENTER_EYE_NAME = "-RECENTEREYE-" +MODE_READOUT_NAME = "-APPMODE-" def main(): in_roi_mode = False @@ -35,8 +38,10 @@ def main(): tracking_layout = [ [sg.Text("Threshold"), sg.Slider(range=(0, 100), default_value=config.threshold, orientation = 'h', key=THRESHOLD_SLIDER_NAME)], [sg.Text("Rotation"), sg.Slider(range=(0, 360), default_value=config.rotation_angle, orientation = 'h', key=ROTATION_SLIDER_NAME)], + [sg.Button("Restart Calibration", key=RESTART_CALIBRATION_NAME), sg.Button("Recenter Eye", key=RECENTER_EYE_NAME)], + [sg.Text("Mode:"), sg.Text("Calibrating", key=MODE_READOUT_NAME)], [sg.Image(filename="", key=TRACKING_IMAGE_NAME)], - [sg.Graph((200,200), (-100, 100), (100, -100), key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)] + [sg.Graph((200,200), (-100, 100), (100, -100), background_color='white', key=OUTPUT_GRAPH_NAME,drag_submits=True, enable_events=True)] ] layout = [[sg.Text("Camera Address"), sg.InputText(config.capture_source, key=CAMERA_ADDR_NAME)], @@ -149,6 +154,15 @@ def main(): is_mouse_up = False x0, y0 = values['-GRAPH-'] x1, y1 = values['-GRAPH-'] + elif event == RESTART_CALIBRATION_NAME: + ransac.calibration_frame_counter = 300 + elif event == RECENTER_EYE_NAME: + ransac.recenter_eye = True + + if ransac.calibration_frame_counter != None: + window[MODE_READOUT_NAME].update("Calibration") + else: + window[MODE_READOUT_NAME].update("Tracking") if in_roi_mode: try: @@ -173,9 +187,14 @@ def main(): window[TRACKING_IMAGE_NAME].update(data=imgbytes) graph = window[OUTPUT_GRAPH_NAME] graph.erase() - if eye_info[0] and not eye_info[3]: - graph.draw_circle((eye_info[1] * -100, eye_info[2] * -100), 25, fill_color='black',line_color='white') - + if eye_info.info_type != InformationOrigin.FAILURE and not eye_info.blink: + graph.background_color = 'white' + graph.draw_circle((eye_info.x * -100, eye_info.y * -100), 25, fill_color='black',line_color='white') + osc_queue.put(eye_info) + elif eye_info.blink: + graph.background_color = 'blue' + elif eye_info.info_type == InformationOrigin.FAILURE: + graph.background_color = 'red' except queue.Empty: pass diff --git a/RANSACApp/osc.py b/RANSACApp/osc.py index 086d882..e99f7f1 100644 --- a/RANSACApp/osc.py +++ b/RANSACApp/osc.py @@ -23,14 +23,14 @@ class VRChatOSC: print("Exiting OSC Queue") return try: - item = self.msg_queue.get(block=True, timeout=0.1) + eye_info = self.msg_queue.get(block=True, timeout=0.1) except queue.Empty: continue # If we're not blinking, set position - if not item[0]: - self.client.send_message("/avatar/parameters/RightEyeX", item[1]) - self.client.send_message("/avatar/parameters/LeftEyeX", item[1]) - self.client.send_message("/avatar/parameters/EyesY", item[2]) + if not eye_info.blink: + self.client.send_message("/avatar/parameters/RightEyeX", eye_info.x) + self.client.send_message("/avatar/parameters/LeftEyeX", eye_info.x) + self.client.send_message("/avatar/parameters/EyesY", eye_info.y) if was_blinking: self.client.send_message("/avatar/parameters/LeftEyeLid", float(1)) self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) diff --git a/RANSACApp/ransac.py b/RANSACApp/ransac.py index 9c84b5d..b8942ad 100644 --- a/RANSACApp/ransac.py +++ b/RANSACApp/ransac.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass import sys sys.path.append(".") from config import RansacConfig @@ -6,6 +7,19 @@ import queue import threading import numpy as np import cv2 +from enum import Enum + +class InformationOrigin(Enum): + RANSAC = 1 + BLOB = 2 + FAILURE = 3 + +@dataclass +class EyeInformation: + info_type: InformationOrigin + x: float + y: float + blink: bool def fit_rotated_ellipse_ransac( data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 @@ -127,10 +141,11 @@ class Ransac: self.ymax = 69420 self.ymin = -69420 self.previous_rotation = self.config.rotation_angle + self.recenter_eye = False - def output_images_and_update(self, threshold_image, output_tuple): + def output_images_and_update(self, threshold_image, output_information: EyeInformation): image_stack = np.concatenate((self.current_image, cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), cv2.cvtColor(threshold_image, cv2.COLOR_GRAY2BGR)), axis=1) - self.image_queue_outgoing.put((image_stack, output_tuple)) + self.image_queue_outgoing.put((image_stack, output_information)) self.previous_image = self.current_image self.previous_rotation = self.config.rotation_angle @@ -165,7 +180,7 @@ class Ransac: # means we need to have had at least one successful runthrough of the Pupil Labs algorithm in # order to have a projected sphere. if self.lkg_projected_sphere == None: - self.output_images_and_update(larger_threshold, None) + self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False)) return try: @@ -176,7 +191,7 @@ class Ransac: if len(contours) == 0: raise RuntimeError("No contours found for image") except: - self.output_images_and_update(larger_threshold, None) + self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.FAILURE, 0, 0, False)) return rows, cols = larger_threshold.shape @@ -197,22 +212,28 @@ class Ransac: cv2.drawContours(self.current_image_gray, [cnt], -1, (255, 0, 0), 3) cv2.rectangle(self.current_image_gray, (x, y), (x + w, y + h), (255, 0, 0), 2) + eye_position_scalar = 3000 # TODO These calculations were wrong in RANSAC3d, need to be fixed anyways. - if xrlb >= 0: - pass - # client.send_message("/avatar/parameters/RightEyeX", -abs(xrl)) - # client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl)) - if eyeyb <= 0: - pass - # client.send_message("/avatar/parameters/RightEyeX", -abs(xrl)) - # client.send_message("/avatar/parameters/LeftEyeX", -abs(xrl)) - self.output_images_and_update(larger_threshold, (True, -abs(xrlb) if xrlb >= 0 else abs(xrlb), -abs(xrlb) if eyeyb <= 0 else abs(xrlb), False)) - # If we've sent something, just return. - return - # If we haven't returned yet, consider this blinking - # client.send_message("/avatar/parameters/LeftEyeLid", float(0)) - # client.send_message("/avatar/parameters/RightEyeLid", float(0)) - self.output_images_and_update(larger_threshold, (True, 0, 0, True)) + xl = float(((xt - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff)) + xr = float(((xt - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff)) + yu = float(((yt - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff)) + yd = float(((yt - self.yoff) * eye_position_scalar) / (self.ymax - self.yoff)) + # print(f"{exm} {eym} {xl} {xr} {yu} {yd}") + + out_x = 0 + out_y = 0 + if xr > 0: + out_x = max(0.0, min(1.0, xr)) + if xl > 0: + out_x = -abs(max(0.0, min(1.0, xl))) + if yd > 0: + out_y = -abs(max(0.0, min(1.0, yd))) + if yu > 0: + out_y = max(0.0, min(1.0, yu)) + + output_tuple = (True, out_x, out_y, False) + self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, False)) + self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, 0, 0, True)) print('[INFO] BLINK Detected.') def run(self): @@ -277,10 +298,7 @@ class Ransac: # # TODO Reimplement Prohurtz's blob tracking fallback if len(convex_hulls) == 0: - # print("No contours found, eye not detected, continuing") - # Draw our image and stack it for visual output self.blob_tracking_fallback() - # self.output_images_and_update(thresh) continue # Find our largest hull, which we expect will probably be the ellipse that represents the 2d @@ -293,8 +311,6 @@ class Ransac: try: cx, cy, w, h, theta = fit_rotated_ellipse_ransac(largest_hull.reshape(-1, 2)) except: - # If we don't find anything, and we don't have a sphere to calculate off of. Give up. - # self.output_images_and_update(thresh) self.blob_tracking_fallback() continue @@ -321,18 +337,17 @@ class Ransac: # And our eyeball that the pupil is on the surface of self.lkg_projected_sphere = result_3d["projected_sphere"] + # Record our pupil center exm = ellipse_3d["center"][0] eym = ellipse_3d["center"][1] - - if self.calibration_frame_counter == 0: - print("FINISHING") + if self.calibration_frame_counter == 0 or self.recenter_eye: self.calibration_frame_counter = None - self.xoff = ellipse_3d["center"][0] - self.yoff = ellipse_3d["center"][1] + self.recenter_eye = False + self.xoff = exm + self.yoff = eym elif self.calibration_frame_counter != None: - print("CALIBRATING") if exm > self.xmax: self.xmax = exm if exm < self.xmin: @@ -342,7 +357,7 @@ class Ransac: if eym < self.xmin: self.ymin = eym self.calibration_frame_counter -= 1 - eye_position_scalar = 1000 + eye_position_scalar = 3000 xl = float(((cx - self.xoff) * eye_position_scalar) / (self.xmax - self.xoff)) xr = float(((cx - self.xoff) * eye_position_scalar) / (self.xmin - self.xoff)) yu = float(((cy - self.yoff) * eye_position_scalar) / (self.ymin - self.yoff)) @@ -360,7 +375,7 @@ class Ransac: if yu > 0: out_y = max(0.0, min(1.0, yu)) - output_tuple = (True, out_x, out_y, False) + output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, False) # # So now we get the offset of the center of the eyeball # xrl = (cx - self.lkg_projected_sphere["center"][0]) / self.lkg_projected_sphere["axes"][0] @@ -403,5 +418,5 @@ class Ransac: ) # Shove a concatenated image out to the main GUI thread for rendering - self.output_images_and_update(thresh, output_tuple) + self.output_images_and_update(thresh, output_info)