From c769144ac4e422089cc7e08f08bf4068361df777 Mon Sep 17 00:00:00 2001 From: Assassinsorrow <49711232+ShyAssassin@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:35:13 +0200 Subject: [PATCH 1/4] Update deprecated dependency --- EyeTrackApp/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EyeTrackApp/requirements.txt b/EyeTrackApp/requirements.txt index d93b831..17840cf 100644 --- a/EyeTrackApp/requirements.txt +++ b/EyeTrackApp/requirements.txt @@ -9,7 +9,7 @@ future==0.18.2 idna==3.3 importlib-metadata==4.8.3 joblib==1.1.0 -msgpack-python==0.5.6 +msgpack==1.0.4 numpy==1.19.5 opencv-python==4.5.3.56 pefile==2022.5.30 From f7e736c65560862c28dc1c79ed894de31b543427 Mon Sep 17 00:00:00 2001 From: Assassinsorrow <49711232+ShyAssassin@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:39:44 +0200 Subject: [PATCH 2/4] General cleanup --- EyeTrackApp/camera_widget.py | 67 +++++---- EyeTrackApp/eye_processor.py | 261 ++++++++++++----------------------- EyeTrackApp/eyetrackapp.py | 32 ++--- 3 files changed, 132 insertions(+), 228 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index bd010d0..ed00649 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -9,6 +9,8 @@ from camera import Camera, CameraState from osc import EyeId import cv2 from winsound import PlaySound, SND_FILENAME, SND_ASYNC +import traceback + class CameraWidget: def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue): @@ -27,7 +29,6 @@ class CameraWidget: self.gui_recenter_eyes = f"-RECENTEREYES{widget_id}-" self.gui_mode_readout = f"-APPMODE{widget_id}-" self.gui_circular_crop = f"-CIRCLECROP{widget_id}-" - # self.gui_show_color_image = f"-SHOWCOLORIMAGE{widget_id}-" self.gui_roi_message = f"-ROIMESSAGE{widget_id}-" self.osc_queue = osc_queue @@ -41,7 +42,6 @@ class CameraWidget: self.config = main_config.right_eye elif self.eye_id == EyeId.LEFT: self.config = main_config.left_eye - else: raise RuntimeError("Cannot have a camera widget represent both eyes!") @@ -54,7 +54,7 @@ class CameraWidget: key=self.gui_roi_selection, drag_submits=True, enable_events=True, - background_color='#424042', + background_color='#424042', ) ] ] @@ -82,11 +82,13 @@ class CameraWidget: ), ], [ - sg.Button("Restart Calibration", key=self.gui_restart_calibration, button_color = '#6f4ca1'), - sg.Button("Recenter Eyes", key=self.gui_recenter_eyes, button_color = '#6f4ca1'), - + sg.Button("Restart Calibration", key=self.gui_restart_calibration, button_color='#6f4ca1'), + sg.Button("Recenter Eyes", key=self.gui_recenter_eyes, button_color='#6f4ca1'), + ], - [sg.Text("Mode:", background_color='#424042'), sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#424042'), + [ + sg.Text("Mode:", background_color='#424042'), + sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#424042'), sg.Checkbox( "Circle crop:", default=self.config.gui_circular_crop, @@ -105,7 +107,8 @@ class CameraWidget: drag_submits=True, enable_events=True, ), - sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', visible=False), + sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', + visible=False), ], ] @@ -116,15 +119,15 @@ class CameraWidget: ], [ sg.Button( - "Save and Restart Tracking", key=self.gui_save_tracking_button, button_color = '#6f4ca1' + "Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#6f4ca1' ), ], [ - sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color = '#6f4ca1'), - sg.Button("Cropping Mode", key=self.gui_roi_button, button_color = '#6f4ca1'), + sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color='#6f4ca1'), + sg.Button("Cropping Mode", key=self.gui_roi_button, button_color='#6f4ca1'), ], [ - sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042' ), + sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042'), sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False), ], ] @@ -189,8 +192,8 @@ class CameraWidget: changed = False # If anything has changed in our configuration settings, change/update those. if ( - event == self.gui_save_tracking_button - and values[self.gui_camera_addr] != self.config.capture_source + event == self.gui_save_tracking_button + and values[self.gui_camera_addr] != self.config.capture_source ): print("New value: {}".format(values[self.gui_camera_addr])) try: @@ -211,9 +214,9 @@ class CameraWidget: self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True - # if self.config.show_color_image != values[self.gui_show_color_image]: - # self.config.show_color_image = values[self.gui_show_color_image] - # changed = True + if self.config.gui_circular_crop != values[self.gui_circular_crop]: + self.config.gui_circular_crop = values[self.gui_circular_crop] + changed = True if changed: self.main_config.save() @@ -224,13 +227,15 @@ class CameraWidget: self.camera.set_output_queue(self.capture_queue) window[self.gui_roi_layout].update(visible=False) window[self.gui_tracking_layout].update(visible=True) - elif event == self.gui_roi_button: + + if event == self.gui_roi_button: print("Move to roi mode") self.in_roi_mode = True self.camera.set_output_queue(self.roi_queue) window[self.gui_roi_layout].update(visible=True) window[self.gui_tracking_layout].update(visible=False) - elif event == "{}+UP".format(self.gui_roi_selection): + + if event == "{}+UP".format(self.gui_roi_selection): # Event for mouse button up in ROI mode self.is_mouse_up = True if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: @@ -239,24 +244,20 @@ class CameraWidget: self.config.roi_window_w = abs(self.x0 - self.x1) self.config.roi_window_h = abs(self.y0 - self.y1) self.main_config.save() - elif event == self.gui_roi_selection: + + if event == self.gui_roi_selection: # Event for mouse button down or mouse drag in ROI mode if self.is_mouse_up: self.is_mouse_up = False self.x0, self.y0 = values[self.gui_roi_selection] self.x1, self.y1 = values[self.gui_roi_selection] - elif event == self.gui_restart_calibration: + + if event == self.gui_restart_calibration: self.ransac.calibration_frame_counter = 300 - PlaySound('Audio/start.wav', SND_FILENAME|SND_ASYNC) - + PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) - elif event == self.gui_recenter_eyes: + if event == self.gui_recenter_eyes: self.settings.gui_recenter_eyes = True - if self.config.gui_circular_crop != values[self.gui_circular_crop]: - self.config.gui_circular_crop = values[self.gui_circular_crop] - changed = True - - #self.ransac.recenter_eye = True needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 @@ -312,13 +313,12 @@ class CameraWidget: graph.erase() if ( - eye_info.info_type != InformationOrigin.FAILURE - and not eye_info.blink + eye_info.info_type != InformationOrigin.FAILURE + and not eye_info.blink ): graph.update(background_color="white") try: - graph.draw_circle( (eye_info.x * -100, eye_info.y * -100), 25, @@ -333,7 +333,6 @@ class CameraWidget: graph.update(background_color="red") # Relay information to OSC if eye_info.info_type != InformationOrigin.FAILURE: - self.osc_queue.put((self.eye_id, eye_info)) except Empty: - return + pass diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 0fd66c5..c18293a 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -2,6 +2,7 @@ from operator import truth from dataclasses import dataclass import sys import asyncio + sys.path.append(".") from config import EyeTrackCameraConfig from config import EyeTrackSettingsConfig @@ -17,6 +18,8 @@ from one_euro_filter import OneEuroFilter from sympy import symbols, Eq, solve from winsound import PlaySound, SND_FILENAME, SND_ASYNC import scipy.signal as sp + + class InformationOrigin(Enum): RANSAC = 1 BLOB = 2 @@ -30,13 +33,17 @@ class EyeInformation: y: float pupil_dialation: int blink: bool + + lowb = np.array(0) + def run_once(f): def wrapper(*args, **kwargs): if not wrapper.has_run: wrapper.has_run = True return f(*args, **kwargs) + wrapper.has_run = False return wrapper @@ -44,10 +51,11 @@ def run_once(f): async def delayed_setting_change(setting, value): await asyncio.sleep(5) setting = value - PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) + def fit_rotated_ellipse_ransac( - data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 + data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 ): # before changing these values, please read up on the ransac algorithm # However if you want to change any value just know that higher iterations will make processing frames slower count_max = 0 @@ -65,9 +73,9 @@ def fit_rotated_ellipse_ransac( ys = data[sample][:, 1].reshape(-1, 1) J = np.mat( - np.hstack((xs * ys, ys**2, xs, ys, np.ones_like(xs, dtype=np.float))) + np.hstack((xs * ys, ys ** 2, xs, ys, np.ones_like(xs, dtype=np.float))) ) - Y = np.mat(-1 * xs**2) + Y = np.mat(-1 * xs ** 2) P = (J.T * J).I * J.T * Y # fitter a*x**2 + b*x*y + c*y**2 + d*x + e*y + f = 0 @@ -78,7 +86,7 @@ def fit_rotated_ellipse_ransac( e = P[3, 0] f = P[4, 0] ellipse_model = ( - lambda x, y: a * x**2 + b * x * y + c * y**2 + d * x + e * y + f + lambda x, y: a * x ** 2 + b * x * y + c * y ** 2 + d * x + e * y + f ) # thresh @@ -97,8 +105,8 @@ def fit_rotated_ellipse(data): xs = data[:, 0].reshape(-1, 1) ys = data[:, 1].reshape(-1, 1) - J = np.mat(np.hstack((xs * ys, ys**2, xs, ys, np.ones_like(xs, dtype=np.float)))) - Y = np.mat(-1 * xs**2) + J = np.mat(np.hstack((xs * ys, ys ** 2, xs, ys, np.ones_like(xs, dtype=np.float)))) + Y = np.mat(-1 * xs ** 2) P = (J.T * J).I * J.T * Y a = 1.0 @@ -109,28 +117,28 @@ def fit_rotated_ellipse(data): f = P[4, 0] theta = 0.5 * np.arctan(b / (a - c)) - cx = (2 * c * d - b * e) / (b**2 - 4 * a * c) - cy = (2 * a * e - b * d) / (b**2 - 4 * a * c) + cx = (2 * c * d - b * e) / (b ** 2 - 4 * a * c) + cy = (2 * a * e - b * d) / (b ** 2 - 4 * a * c) - cu = a * cx**2 + b * cx * cy + c * cy**2 - f + cu = a * cx ** 2 + b * cx * cy + c * cy ** 2 - f w = np.sqrt( cu / ( - a * np.cos(theta) ** 2 - + b * np.cos(theta) * np.sin(theta) - + c * np.sin(theta) ** 2 + a * np.cos(theta) ** 2 + + b * np.cos(theta) * np.sin(theta) + + c * np.sin(theta) ** 2 ) ) h = np.sqrt( cu / ( - a * np.sin(theta) ** 2 - - b * np.cos(theta) * np.sin(theta) - + c * np.cos(theta) ** 2 + a * np.sin(theta) ** 2 + - b * np.cos(theta) * np.sin(theta) + + c * np.cos(theta) ** 2 ) ) - ellipse_model = lambda x, y: a * x**2 + b * x * y + c * y**2 + d * x + e * y + f + ellipse_model = lambda x, y: a * x ** 2 + b * x * y + c * y ** 2 + d * x + e * y + f error_sum = np.sum([ellipse_model(x, y) for x, y in data]) @@ -139,19 +147,18 @@ def fit_rotated_ellipse(data): class EyeProcessor: def __init__( - self, - config: "EyeTrackCameraConfig", - settings: "EyeTrackSettingsConfig", - cancellation_event: "threading.Event", - capture_event: "threading.Event", - capture_queue_incoming: "queue.Queue", - image_queue_outgoing: "queue.Queue", - eye_id, + self, + config: "EyeTrackCameraConfig", + settings: "EyeTrackSettingsConfig", + cancellation_event: "threading.Event", + capture_event: "threading.Event", + capture_queue_incoming: "queue.Queue", + image_queue_outgoing: "queue.Queue", + eye_id, ): self.config = config self.settings = settings - - + # Cross-thread communication management self.capture_queue_incoming = capture_queue_incoming self.image_queue_outgoing = image_queue_outgoing @@ -160,7 +167,6 @@ class EyeProcessor: self.eye_id = eye_id # Cross algo state - self.lkg_projected_sphere = None self.xc = None self.yc = None @@ -191,45 +197,20 @@ class EyeProcessor: self.calibration_frame_counter try: - min_cutoff = float(self.settings.gui_min_cutoff) #0.0004 - beta = float(self.settings.gui_speed_coefficient) #0.9 + min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004 + beta = float(self.settings.gui_speed_coefficient) # 0.9 except: print('[WARN] OneEuroFilter values must be a legal number.') - min_cutoff = 0.0004 + min_cutoff = 0.0004 beta = 0.9 noisy_point = np.array([1, 1]) self.one_euro_filter = OneEuroFilter( noisy_point, min_cutoff=min_cutoff, beta=beta - ) - + ) - - - - - - - - - - - - - def output_images_and_update( - self, threshold_image, output_information: EyeInformation - ): - # if self.config.show_color_image: - # image_stack = np.concatenate( - # ( - # self.current_image, - # cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), - # cv2.cvtColor(threshold_image, cv2.COLOR_GRAY2BGR), - # ), - # axis=1, - # ) - # else: + def output_images_and_update(self, threshold_image, output_information: EyeInformation): image_stack = np.concatenate( ( cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), @@ -246,13 +227,13 @@ class EyeProcessor: try: # Get frame from capture source, crop to ROI self.current_image = self.current_image[ - int(self.config.roi_window_y) : int( - self.config.roi_window_y + self.config.roi_window_h - ), - int(self.config.roi_window_x) : int( - self.config.roi_window_x + self.config.roi_window_w - ), - ] + int(self.config.roi_window_y): int( + self.config.roi_window_y + self.config.roi_window_h + ), + int(self.config.roi_window_x): int( + self.config.roi_window_x + self.config.roi_window_w + ), + ] except: # Failure to process frame, reuse previous frame. self.current_image = self.previous_image @@ -275,20 +256,17 @@ class EyeProcessor: return True def blob_tracking_fallback(self): - - -# define circle - - if self.config.gui_circular_crop == True: + # define circle + if self.config.gui_circular_crop: if self.cct == 0: try: ht, wd = self.current_image_gray.shape[:2] radius = int(float(self.lkg_projected_sphere["axes"][0])) - + # draw filled circle in white on black background as mask - mask = np.zeros((ht,wd), dtype=np.uint8) - mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1) + mask = np.zeros((ht, wd), dtype=np.uint8) + mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) # create white colored background color = np.full_like(self.current_image_gray, (255)) @@ -297,7 +275,7 @@ class EyeProcessor: masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) # apply inverse mask to colored image - masked_color = cv2.bitwise_and(color, color, mask=255-mask) + masked_color = cv2.bitwise_and(color, color, mask=255 - mask) # combine the two masked images self.current_image_gray = cv2.add(masked_img, masked_color) @@ -306,15 +284,14 @@ class EyeProcessor: else: self.cct = self.cct - 1 - # Increase our threshold value slightly, in order to have a better possibility of getting back # something to do blob tracking on. hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) histr = hist.ravel() - peaks, properties = sp.find_peaks(histr, distance=5) + peaks, properties = sp.find_peaks(histr, distance=5) minpeak = np.min(peaks) thresholdoptics = np.array(minpeak + int(self.config.threshold + 12)) - larger_threshold = cv2.inRange(self.current_image_gray,lowb,thresholdoptics) #faster than cv2.threshold + larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold larger_threshold = cv2.bitwise_not(larger_threshold) # Blob tracking requires that we have a vague idea of where the eye may be at the moment. This # means we need to have had at least one successful runthrough of the Pupil Labs algorithm in @@ -325,11 +302,6 @@ class EyeProcessor: ) return - - - - - try: # Try rebuilding our contours contours, _ = cv2.findContours( @@ -352,21 +324,20 @@ class EyeProcessor: # if our blob width/height are within suitable (yet arbitrary) boundaries, call that good. # # TODO This should be scaled based on camera resolution. - - if not self.settings.gui_blob_minsize <= h <= self.settings.gui_blob_maxsize or not self.settings.gui_blob_minsize <= w <= self.settings.gui_blob_maxsize: + if not self.settings.gui_blob_minsize <= h <= self.settings.gui_blob_maxsize or not self.settings.gui_blob_minsize <= w <= self.settings.gui_blob_maxsize: continue cx = x + int(w / 2) - + cy = y + int(h / 2) - + xrlb = ( - cx - self.lkg_projected_sphere["center"][0] - ) / self.lkg_projected_sphere["axes"][0] + cx - self.lkg_projected_sphere["center"][0] + ) / self.lkg_projected_sphere["axes"][0] eyeyb = ( - cy - self.lkg_projected_sphere["center"][1] - ) / self.lkg_projected_sphere["axes"][1] + cy - self.lkg_projected_sphere["center"][1] + ) / self.lkg_projected_sphere["axes"][1] cv2.line( self.current_image_gray, (x + int(w / 2), 0), @@ -389,15 +360,15 @@ class EyeProcessor: if self.calibration_frame_counter == 0: self.calibration_frame_counter = None self.xoff = cx - self.yoff = cy - PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) + self.yoff = cy + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) elif self.calibration_frame_counter != None: self.settings.gui_recenter_eyes = False if cx > self.xmax: self.xmax = cx if cx < self.xmin: self.xmin = cx - if cy> self.ymax: + if cy > self.ymax: self.ymax = cy if cy < self.ymin: self.ymin = cy @@ -407,15 +378,11 @@ class EyeProcessor: self.yoff = cy if self.ts == 0: self.settings.gui_recenter_eyes = False - PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) else: self.ts = self.ts - 1 else: - self.ts = 10 - - - - + self.ts = 10 xl = float( ((cx - self.xoff)) / (self.xmax - self.xoff) @@ -430,10 +397,9 @@ class EyeProcessor: ((cy - self.yoff)) / (self.ymax - self.yoff) ) - # print(self.) out_x = 0 out_y = 0 - if self.settings.gui_flip_y_axis == True: #check config on flipped values settings and apply accordingly + if self.settings.gui_flip_y_axis == True: # check config on flipped values settings and apply accordingly if yd > 0: out_y = max(0.0, min(1.0, yd)) if yu > 0: @@ -456,16 +422,13 @@ class EyeProcessor: out_x = -abs(max(0.0, min(1.0, xl))) try: - noisy_point = np.array([out_x, out_y]) #fliter our values with a One Euro Filter + noisy_point = np.array([out_x, out_y]) # fliter our values with a One Euro Filter point_hat = self.one_euro_filter(noisy_point) out_x = point_hat[0] out_y = point_hat[1] except: pass - - - self.output_images_and_update( larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False), @@ -479,18 +442,14 @@ class EyeProcessor: def run(self): camera_model = None detector_3d = None - out_pupil_dialation = 1 - + if self.eye_id == "EyeId.RIGHT": flipx = self.settings.gui_flip_x_axis_right - #elif self.eye_id == "EyeId.LEFT": - # flipx = self.config.gui_flip_x_axis_left - else: + else: flipx = self.settings.gui_flip_x_axis_left + while True: - # oef = init_filter() - # Check to make sure we haven't been requested to close if self.cancellation_event.is_set(): print("Exiting RANSAC thread") @@ -504,15 +463,8 @@ class EyeProcessor: continue # If our ROI configuration has changed, reset our model and detector - if ( - camera_model is None - or detector_3d is None - or camera_model.resolution - != ( - self.config.roi_window_w, - self.config.roi_window_h, - ) - ): + if (camera_model is None or detector_3d is None or camera_model.resolution != ( + self.config.roi_window_w, self.config.roi_window_h,)): camera_model = CameraModel( focal_length=self.config.focal_length, resolution=(self.config.roi_window_w, self.config.roi_window_h), @@ -549,20 +501,17 @@ class EyeProcessor: self.current_image, cv2.COLOR_BGR2GRAY ) - #print(self.config.gui_circular_crop) - # print(self.cct) if self.config.gui_circular_crop == True: if self.cct == 0: try: ht, wd = self.current_image_gray.shape[:2] - radius = int(float(self.lkg_projected_sphere["axes"][0])) self.xc = int(float(self.lkg_projected_sphere["center"][0])) self.yc = int(float(self.lkg_projected_sphere["center"][1])) # draw filled circle in white on black background as mask - mask = np.zeros((ht,wd), dtype=np.uint8) - mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1) + mask = np.zeros((ht, wd), dtype=np.uint8) + mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1) # create white colored background color = np.full_like(self.current_image_gray, (255)) @@ -571,7 +520,7 @@ class EyeProcessor: masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) # apply inverse mask to colored image - masked_color = cv2.bitwise_and(color, color, mask=255-mask) + masked_color = cv2.bitwise_and(color, color, mask=255 - mask) # combine the two masked images self.current_image_gray = cv2.add(masked_img, masked_color) @@ -582,22 +531,15 @@ class EyeProcessor: else: self.cct = 300 - - #Using Histogram based thresholding. Improves robustness insanely + # Using Histogram based thresholding. Improves robustness insanely hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) histr = hist.ravel() - peaks, properties = sp.find_peaks(histr, distance=5) + peaks, properties = sp.find_peaks(histr, distance=5) minpeak = np.min(peaks) thresholdoptics = np.array(minpeak + int(self.config.threshold)) - thresh = cv2.inRange(self.current_image_gray,lowb,thresholdoptics) #faster than cv2.threshold + thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold thresh = cv2.bitwise_not(thresh) - - - - - - # Set up morphological transforms, for smoothing and clearing the image we get out of the # thresholding operation. After this, we'd really like to just have a black blob in the middle # of a bunch of white area. @@ -618,7 +560,6 @@ class EyeProcessor: # If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to # using blob tracking. - # if len(convex_hulls) == 0: if self.settings.gui_blob_fallback == True: self.blob_tracking_fallback() @@ -674,17 +615,14 @@ class EyeProcessor: # Record our pupil center exm = ellipse_3d["center"][0] eym = ellipse_3d["center"][1] - d = result_3d["diameter_3d"] - - if self.calibration_frame_counter == 0: self.calibration_frame_counter = None self.xoff = cx - self.yoff = cy - PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) + self.yoff = cy + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) elif self.calibration_frame_counter != None: # TODO reset calibration values on button press if exm > self.xmax: self.xmax = exm @@ -695,26 +633,16 @@ class EyeProcessor: if eym < self.ymin: self.ymin = eym self.calibration_frame_counter -= 1 - if self.settings.gui_recenter_eyes == True: + if self.settings.gui_recenter_eyes: self.xoff = cx self.yoff = cy if self.ts == 0: self.settings.gui_recenter_eyes = False - PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) else: self.ts = self.ts - 1 else: - self.ts = 20 - - - #print(self.yoff) - - - # noisy_point = np.array([cx, cy]) #fliter our values with a One Euro Filter - # point_hat = self.one_euro_filter(noisy_point) - # cx = point_hat[0] - # cy = point_hat[1] - + self.ts = 20 xl = float( ((cx - self.xoff)) / (self.xmax - self.xoff) @@ -728,8 +656,6 @@ class EyeProcessor: yd = float( ((cy - self.yoff)) / (self.ymax - self.yoff) ) - - out_x = 0 out_y = 0 @@ -756,30 +682,23 @@ class EyeProcessor: if xl > 0: out_x = -abs(max(0.0, min(1.0, xl))) - try: - noisy_point = np.array([out_x, out_y]) #fliter our values with a One Euro Filter + noisy_point = np.array([out_x, out_y]) # fliter our values with a One Euro Filter point_hat = self.one_euro_filter(noisy_point) out_x = point_hat[0] out_y = point_hat[1] except: pass - # print(cy, self.yoff, self.ymin, self.ymax, out_y) - # print(out_y, yu, yd) - output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, out_pupil_dialation, False) # Draw our image and stack it for visual output - try: + try: cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) cv2.circle(self.current_image_gray, (int(cx), int(cy)), 2, (0, 0, 255), -1) - # draw pupil except: pass - - try: cv2.ellipse( self.current_image_gray, @@ -795,9 +714,8 @@ class EyeProcessor: # validity beforehand, but for now just pass. It usually fixes itself on the next frame. pass - try: - # print(self.lkg_projected_sphere["angle"], self.lkg_projected_sphere["axes"], self.lkg_projected_sphere["center"]) + # print(self.lkg_projected_sphere["angle"], self.lkg_projected_sphere["axes"], self.lkg_projected_sphere["center"]) cv2.ellipse( self.current_image_gray, tuple(int(v) for v in self.lkg_projected_sphere["center"]), @@ -807,13 +725,9 @@ class EyeProcessor: 360, # start/end angle for drawing (0, 255, 0), # color (BGR): red ) - except: pass - - - # draw line from center of eyeball to center of pupil cv2.line( self.current_image_gray, @@ -824,4 +738,3 @@ class EyeProcessor: # Shove a concatenated image out to the main GUI thread for rendering self.output_images_and_update(thresh, output_info) - diff --git a/EyeTrackApp/eyetrackapp.py b/EyeTrackApp/eyetrackapp.py index 00d86b3..bb87c9c 100644 --- a/EyeTrackApp/eyetrackapp.py +++ b/EyeTrackApp/eyetrackapp.py @@ -21,6 +21,7 @@ RIGHT_EYE_RADIO_NAME = "-RIGHTEYERADIO-" BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-" SETTINGS_RADIO_NAME = '-SETTINGSRADIO-' + def main(): # Get Configuration config: EyeTrackConfig = EyeTrackConfig.load() @@ -40,48 +41,41 @@ def main(): # start worker threads osc_thread.start() - # t2s_queue: "queue.Queue[str | None]" = queue.Queue() - # t2s_engine = SpeechEngine(t2s_queue) - # t2s_thread = threading.Thread(target=t2s_engine.run) - # t2s_thread.start() - # t2s_queue.put("App Starting") - eyes = [ CameraWidget(EyeId.RIGHT, config, osc_queue), CameraWidget(EyeId.LEFT, config, osc_queue), - # CameraWidget(EyeId.SETTINGS, config, osc_queue), ] settings = [ SettingsWidget(EyeId.SETTINGS, config, osc_queue), ] - + layout = [ [ sg.Radio( "Right Eye", - "EYESELECTRADIO", + "EYESELECTRADIO", background_color='#292929', default=(config.eye_display_id == EyeId.RIGHT), key=RIGHT_EYE_RADIO_NAME, ), sg.Radio( "Left Eye", - "EYESELECTRADIO", + "EYESELECTRADIO", background_color='#292929', default=(config.eye_display_id == EyeId.LEFT), key=LEFT_EYE_RADIO_NAME, ), sg.Radio( "Both Eyes", - "EYESELECTRADIO", + "EYESELECTRADIO", background_color='#292929', default=(config.eye_display_id == EyeId.BOTH), key=BOTH_EYE_RADIO_NAME, ), sg.Radio( "Settings", - "EYESELECTRADIO", + "EYESELECTRADIO", background_color='#292929', default=(config.eye_display_id == EyeId.SETTINGS), key=SETTINGS_RADIO_NAME, @@ -136,21 +130,18 @@ def main(): # If we're in either mode and someone hits q, quit immediately if event == "Exit" or event == sg.WIN_CLOSED: - # eyes[2].stop() + # eyes[2].stop() for eye in eyes: eye.stop() cancellation_event.set() # shut down worker threads osc_thread.join() - # TODO: find a way to have this function run on join maybe?? - # threading.Event() wont work because pythonosc spawns its own thread. - # only way i can see to get around this is an ugly while loop that only checks if a threading event is trigggered - # and then call the pythonosc shutdown function + # TODO: find a way to have this function run on join maybe?? + # threading.Event() wont work because pythonosc spawns its own thread. + # only way i can see to get around this is an ugly while loop that only checks if a threading event is trigggered + # and then call the pythonosc shutdown function osc_receiver.shutdown() osc_receiver_thread.join() - # t2s_engine.force_stop() - # t2s_queue.put(None) - # t2s_thread.join() print("Exiting EyeTrackApp") return @@ -206,3 +197,4 @@ def main(): if __name__ == "__main__": main() + \ No newline at end of file From 5cfa972aa5e3ecb4a9729efaf1b922c8c5f2a011 Mon Sep 17 00:00:00 2001 From: Assassinsorrow <49711232+ShyAssassin@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:58:30 +0200 Subject: [PATCH 3/4] Update indentation --- EyeTrackApp/camera_widget.py | 5 +-- EyeTrackApp/eye_processor.py | 75 ++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 41 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index ed00649..4f4df3f 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -312,10 +312,7 @@ class CameraWidget: graph = window[self.gui_output_graph] graph.erase() - if ( - eye_info.info_type != InformationOrigin.FAILURE - and not eye_info.blink - ): + if eye_info.info_type != InformationOrigin.FAILURE and not eye_info.blink: graph.update(background_color="white") try: diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index c18293a..a5ee050 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -55,7 +55,7 @@ async def delayed_setting_change(setting, value): def fit_rotated_ellipse_ransac( - data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 + data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 ): # before changing these values, please read up on the ransac algorithm # However if you want to change any value just know that higher iterations will make processing frames slower count_max = 0 @@ -86,7 +86,7 @@ def fit_rotated_ellipse_ransac( e = P[3, 0] f = P[4, 0] ellipse_model = ( - lambda x, y: a * x ** 2 + b * x * y + c * y ** 2 + d * x + e * y + f + lambda x, y: a * x**2 + b * x * y + c * y**2 + d * x + e * y + f ) # thresh @@ -105,8 +105,8 @@ def fit_rotated_ellipse(data): xs = data[:, 0].reshape(-1, 1) ys = data[:, 1].reshape(-1, 1) - J = np.mat(np.hstack((xs * ys, ys ** 2, xs, ys, np.ones_like(xs, dtype=np.float)))) - Y = np.mat(-1 * xs ** 2) + J = np.mat(np.hstack((xs * ys, ys**2, xs, ys, np.ones_like(xs, dtype=np.float)))) + Y = np.mat(-1 * xs**2) P = (J.T * J).I * J.T * Y a = 1.0 @@ -120,25 +120,25 @@ def fit_rotated_ellipse(data): cx = (2 * c * d - b * e) / (b ** 2 - 4 * a * c) cy = (2 * a * e - b * d) / (b ** 2 - 4 * a * c) - cu = a * cx ** 2 + b * cx * cy + c * cy ** 2 - f + cu = a * cx**2 + b * cx * cy + c * cy**2 - f w = np.sqrt( cu / ( - a * np.cos(theta) ** 2 - + b * np.cos(theta) * np.sin(theta) - + c * np.sin(theta) ** 2 + a * np.cos(theta)**2 + + b * np.cos(theta) * np.sin(theta) + + c * np.sin(theta)**2 ) ) h = np.sqrt( cu / ( - a * np.sin(theta) ** 2 - - b * np.cos(theta) * np.sin(theta) - + c * np.cos(theta) ** 2 + a * np.sin(theta)**2 + - b * np.cos(theta) * np.sin(theta) + + c * np.cos(theta)**2 ) ) - ellipse_model = lambda x, y: a * x ** 2 + b * x * y + c * y ** 2 + d * x + e * y + f + ellipse_model = lambda x, y: a * x**2 + b * x * y + c * y**2 + d * x + e * y + f error_sum = np.sum([ellipse_model(x, y) for x, y in data]) @@ -147,14 +147,14 @@ def fit_rotated_ellipse(data): class EyeProcessor: def __init__( - self, - config: "EyeTrackCameraConfig", - settings: "EyeTrackSettingsConfig", - cancellation_event: "threading.Event", - capture_event: "threading.Event", - capture_queue_incoming: "queue.Queue", - image_queue_outgoing: "queue.Queue", - eye_id, + self, + config: "EyeTrackCameraConfig", + settings: "EyeTrackSettingsConfig", + cancellation_event: "threading.Event", + capture_event: "threading.Event", + capture_queue_incoming: "queue.Queue", + image_queue_outgoing: "queue.Queue", + eye_id, ): self.config = config self.settings = settings @@ -227,13 +227,13 @@ class EyeProcessor: try: # Get frame from capture source, crop to ROI self.current_image = self.current_image[ - int(self.config.roi_window_y): int( - self.config.roi_window_y + self.config.roi_window_h - ), - int(self.config.roi_window_x): int( - self.config.roi_window_x + self.config.roi_window_w - ), - ] + int(self.config.roi_window_y): int( + self.config.roi_window_y + self.config.roi_window_h + ), + int(self.config.roi_window_x): int( + self.config.roi_window_x + self.config.roi_window_w + ), + ] except: # Failure to process frame, reuse previous frame. self.current_image = self.previous_image @@ -332,12 +332,8 @@ class EyeProcessor: cy = y + int(h / 2) - xrlb = ( - cx - self.lkg_projected_sphere["center"][0] - ) / self.lkg_projected_sphere["axes"][0] - eyeyb = ( - cy - self.lkg_projected_sphere["center"][1] - ) / self.lkg_projected_sphere["axes"][1] + xrlb = (cx - self.lkg_projected_sphere["center"][0]) / self.lkg_projected_sphere["axes"][0] + eyeyb = (cy - self.lkg_projected_sphere["center"][1]) / self.lkg_projected_sphere["axes"][1] cv2.line( self.current_image_gray, (x + int(w / 2), 0), @@ -463,8 +459,13 @@ class EyeProcessor: continue # If our ROI configuration has changed, reset our model and detector - if (camera_model is None or detector_3d is None or camera_model.resolution != ( - self.config.roi_window_w, self.config.roi_window_h,)): + if (camera_model is None + or detector_3d is None + or camera_model.resolution != ( + self.config.roi_window_w, + self.config.roi_window_h, + ) + ): camera_model = CameraModel( focal_length=self.config.focal_length, resolution=(self.config.roi_window_w, self.config.roi_window_h), @@ -561,7 +562,7 @@ class EyeProcessor: # If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to # using blob tracking. if len(convex_hulls) == 0: - if self.settings.gui_blob_fallback == True: + if self.settings.gui_blob_fallback: self.blob_tracking_fallback() else: print("[INFO] Blob fallback disabled. Assuming blink.") @@ -580,7 +581,7 @@ class EyeProcessor: largest_hull.reshape(-1, 2) ) except: - if self.settings.gui_blob_fallback == True: + if self.settings.gui_blob_fallback: self.blob_tracking_fallback() else: print("[INFO] Blob fallback disabled. Assuming blink.") From 7a89adb687060d7d594df7e261867736758390c0 Mon Sep 17 00:00:00 2001 From: Assassinsorrow <49711232+ShyAssassin@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:17:15 +0200 Subject: [PATCH 4/4] Update indentation --- EyeTrackApp/camera_widget.py | 11 ++++------- EyeTrackApp/eye_processor.py | 32 ++++++++++++++++---------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 4f4df3f..1be72a4 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -107,8 +107,7 @@ class CameraWidget: drag_submits=True, enable_events=True, ), - sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', - visible=False), + sg.Text("Please set an Eye Cropping.", key=self.gui_roi_message, background_color='#424042', visible=False), ], ] @@ -118,9 +117,7 @@ class CameraWidget: sg.InputText(self.config.capture_source, key=self.gui_camera_addr), ], [ - sg.Button( - "Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#6f4ca1' - ), + sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#6f4ca1'), ], [ sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color='#6f4ca1'), @@ -192,8 +189,8 @@ class CameraWidget: changed = False # If anything has changed in our configuration settings, change/update those. if ( - event == self.gui_save_tracking_button - and values[self.gui_camera_addr] != self.config.capture_source + event == self.gui_save_tracking_button + and values[self.gui_camera_addr] != self.config.capture_source ): print("New value: {}".format(values[self.gui_camera_addr])) try: diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index a5ee050..51c9c89 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -73,9 +73,9 @@ def fit_rotated_ellipse_ransac( ys = data[sample][:, 1].reshape(-1, 1) J = np.mat( - np.hstack((xs * ys, ys ** 2, xs, ys, np.ones_like(xs, dtype=np.float))) + np.hstack((xs * ys, ys**2, xs, ys, np.ones_like(xs, dtype=np.float))) ) - Y = np.mat(-1 * xs ** 2) + Y = np.mat(-1 * xs**2) P = (J.T * J).I * J.T * Y # fitter a*x**2 + b*x*y + c*y**2 + d*x + e*y + f = 0 @@ -117,8 +117,8 @@ def fit_rotated_ellipse(data): f = P[4, 0] theta = 0.5 * np.arctan(b / (a - c)) - cx = (2 * c * d - b * e) / (b ** 2 - 4 * a * c) - cy = (2 * a * e - b * d) / (b ** 2 - 4 * a * c) + cx = (2 * c * d - b * e) / (b**2 - 4 * a * c) + cy = (2 * a * e - b * d) / (b**2 - 4 * a * c) cu = a * cx**2 + b * cx * cy + c * cy**2 - f w = np.sqrt( @@ -381,21 +381,21 @@ class EyeProcessor: self.ts = 10 xl = float( - ((cx - self.xoff)) / (self.xmax - self.xoff) + (cx - self.xoff) / (self.xmax - self.xoff) ) xr = float( - ((cx - self.xoff)) / (self.xmin - self.xoff) + (cx - self.xoff) / (self.xmin - self.xoff) ) yu = float( - ((cy - self.yoff)) / (self.ymin - self.yoff) + (cy - self.yoff) / (self.ymin - self.yoff) ) yd = float( - ((cy - self.yoff)) / (self.ymax - self.yoff) + (cy - self.yoff) / (self.ymax - self.yoff) ) out_x = 0 out_y = 0 - if self.settings.gui_flip_y_axis == True: # check config on flipped values settings and apply accordingly + if self.settings.gui_flip_y_axis: # check config on flipped values settings and apply accordingly if yd > 0: out_y = max(0.0, min(1.0, yd)) if yu > 0: @@ -406,7 +406,7 @@ class EyeProcessor: if yu > 0: out_y = max(0.0, min(1.0, yu)) - if self.settings.gui_flip_x_axis_right == True: + if self.settings.gui_flip_x_axis_right: if xr > 0: out_x = -abs(max(0.0, min(1.0, xr))) if xl > 0: @@ -646,22 +646,22 @@ class EyeProcessor: self.ts = 20 xl = float( - ((cx - self.xoff)) / (self.xmax - self.xoff) + (cx - self.xoff) / (self.xmax - self.xoff) ) xr = float( - ((cx - self.xoff)) / (self.xmin - self.xoff) + (cx - self.xoff) / (self.xmin - self.xoff) ) yu = float( - ((cy - self.yoff)) / (self.ymin - self.yoff) + (cy - self.yoff) / (self.ymin - self.yoff) ) yd = float( - ((cy - self.yoff)) / (self.ymax - self.yoff) + (cy - self.yoff) / (self.ymax - self.yoff) ) out_x = 0 out_y = 0 - if self.settings.gui_flip_y_axis == True: + if self.settings.gui_flip_y_axis: if yd > 0: out_y = max(0.0, min(1.0, yd)) if yu > 0: @@ -672,7 +672,7 @@ class EyeProcessor: if yu > 0: out_y = max(0.0, min(1.0, yu)) - if flipx == True: + if flipx: if xr > 0: out_x = -abs(max(0.0, min(1.0, xr))) if xl > 0: