From 1b961ae4775ac4ff4bcb7b72d24ae5beeecbf2de Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 00:31:25 +0000 Subject: [PATCH 01/14] Crop and rotate image in one operation --- EyeTrackApp/eye_processor.py | 98 ++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index d178191..72951c5 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -204,51 +204,83 @@ class EyeProcessor: def capture_crop_rotate_image(self): # Get our current frame - try: - # Get frame from capture source, crop to ROI - self.current_image = self.current_image[ - int(self.config.roi_window_y) : int( - self.config.roi_window_y + self.config.roi_window_h - ), - int(self.config.roi_window_x) : int( - self.config.roi_window_x + self.config.roi_window_w - ), - ] - self.ibo.change_roi(self.config.dict(include=self.roi_include_set)) + self.ibo.change_roi(self.config.dict(include=self.roi_include_set)) - except: - # Failure to process frame, reuse previous frame. - self.current_image = self.previous_image - print("\033[91m[ERROR] Frame capture issue detected.\033[0m") + roi_x = self.config.roi_window_x + roi_y = self.config.roi_window_y + roi_w = self.config.roi_window_w + roi_h = self.config.roi_window_h + + img_w, img_h, _ = self.current_image.shape try: # Apply rotation to cropped area. For any rotation area outside of the bounds of the image, - # fill with white. - try: - rows, cols, _ = self.current_image.shape - except: - rows, cols, _ = self.previous_image.shape - img_center = (cols / 2, rows / 2) + # fill with white (self.current_image_white) and average in-bounds color (self.current_image). + + crop_matrix = np.float32([[1, 0, -roi_x], + [0, 1, -roi_y], + [0, 0, 1]]) + img_center = (roi_w / 2, roi_h / 2) rotation_matrix = cv2.getRotationMatrix2D( img_center, self.config.rotation_angle, 1 ) - avg_color_per_row = np.average(self.current_image, axis=0) - avg_color = np.average(avg_color_per_row, axis=0) - ar, ag, ab = avg_color - self.current_image = cv2.warpAffine( - self.current_image, - rotation_matrix, - (cols, rows), - borderMode=cv2.BORDER_CONSTANT, - borderValue=(ar + 10, ag + 10, ab + 10), # (255, 255, 255), - ) + matrix = np.matmul(rotation_matrix, crop_matrix) + self.current_image_white = cv2.warpAffine( self.current_image, - rotation_matrix, - (cols, rows), + matrix, + (roi_w, roi_h), borderMode=cv2.BORDER_CONSTANT, borderValue=(255, 255, 255), ) + + # calculate position of all four corners of crop, and check if any are out of bounds + + # add w-preserve row to make matrix square, invert, and remove again + inv_matrix = np.linalg.inv(np.vstack((matrix, [0, 0, 1])))[:-1] + # calculate crop corner locations in original image space + corners = np.matmul([[0, 0, 1], + [roi_w, 0, 1], + [0, roi_h, 1], + [roi_w, roi_h, 1]], + np.transpose(inv_matrix)) + fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h + for (x, y) in corners) + + if fits_in_bounds: + # crop is entirely within original image bounds so average color and white are identical + self.current_image = self.current_image_white + return True + + # image does not fit in bounds, so warp, calculate average color of covered pixels, and apply that to the outside region. + + # warp image with alpha + alpha = np.full(self.current_image.shape[:2], 255, dtype=np.uint8) + self.current_image = np.dstack((self.current_image, alpha)) + + self.current_image = cv2.warpAffine( + self.current_image, + matrix, + (roi_w, roi_h), + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0, 0), + ) + + # calculate average color of crop, excluding alpha + avg_color_per_row = np.average(self.current_image, axis=0) + avg_color = np.average(avg_color_per_row, axis=0) + avg_color_norm = avg_color[0:3] / avg_color[3] + ar, ag, ab = np.clip(avg_color_norm, 0, 1) + + # add border color to image masked by alpha and discard alpha channel + rgb_ch = self.current_image[:, :, :3] + inv_alpha_ch = 255 - self.current_image[:, :, 3] + self.current_image = rgb_ch + np.stack( + np.uint8([inv_alpha_ch * ar, + inv_alpha_ch * ag, + inv_alpha_ch * ab]), + axis=-1) + return True except: pass From de764e57ca45a08698185ae9f121f86884c94ca3 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 06:15:40 +0000 Subject: [PATCH 02/14] Improve UX of rotated ROI selection --- EyeTrackApp/camera_widget.py | 116 ++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 21 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 0d412de..6fa1275 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -12,6 +12,7 @@ import cv2 import sys from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path import traceback +import math import numpy as np @@ -114,17 +115,6 @@ class CameraWidget: # Define the window's contents self.tracking_layout = [ - [ - sg.Text("Rotation", background_color="#424042"), - sg.Slider( - range=(0, 360), - default_value=self.config.rotation_angle, - orientation="h", - key=self.gui_rotation_slider, - background_color="#424042", - tooltip="Adjust the rotation of your cameras, make them level.", - ), - ], [ sg.Button( "Start Calibration", @@ -210,6 +200,17 @@ class CameraWidget: tooltip="Go here to crop out your eye.", ), ], + [ + sg.Text("Rotation", background_color="#424042"), + sg.Slider( + range=(0, 360), + default_value=self.config.rotation_angle, + orientation="h", + key=self.gui_rotation_slider, + background_color="#424042", + tooltip="Adjust the rotation of your cameras, make them level.", + ), + ], [ sg.Column( self.tracking_layout, @@ -225,8 +226,16 @@ class CameraWidget: ], ] + # cartesian co-ordinates in widget space are used during selection self.x0, self.y0 = None, None self.x1, self.y1 = None, None + self.cartesian_needs_update = False + # polar co-ordinates from the image center are the canonical representation + self.cr, self.ca = None, None + self.w, self.h = None, None + self.pad_x, self.pad_y = None, None + self.roi_image_center = (None, None) + self.figure = None self.is_mouse_up = True self.in_roi_mode = False @@ -243,6 +252,35 @@ class CameraWidget: self.movavg_bps_queue.append(next_bps) return f"{sum(self.movavg_bps_queue) / len(self.movavg_bps_queue) * 0.001 * 0.001 * 8:.3f} Mbps" + def _cartesian_to_polar(self): + if None not in (self.x0, self.y0, self.x1, self.y1): + image_center_x, image_center_y = self.roi_image_center + roi_center_x = image_center_x - (self.x0 + self.x1) / 2. + roi_center_y = image_center_y - (self.y0 + self.y1) / 2. + self.cr = (roi_center_x**2 + roi_center_y**2)**0.5 + self.ca = math.atan2(roi_center_x, roi_center_y) - \ + math.radians(self.config.rotation_angle) + self.w = abs(self.x1 - self.x0) + self.h = abs(self.y1 - self.y0) + + def _polar_to_cartesian_at_angle(self, rotation_angle_radians): + if None not in (self.cr, self.ca, self.w, self.h): + image_center_x, image_center_y = self.roi_image_center + ca = self.ca + rotation_angle_radians + cx = -math.sin(ca) * self.cr + image_center_x + cy = -math.cos(ca) * self.cr + image_center_y + return ((int(cx - self.w/2), int(cy - self.h/2)), + (int(cx + self.w/2), int(cy + self.h/2))) + else: + return 4 * (None,) + + def _polar_to_cartesian(self): + if None not in (self.cr, self.ca, self.w, self.h): + (self.x0, self.y0), (self.x1, self.y1) = \ + self._polar_to_cartesian_at_angle( + math.radians(self.config.rotation_angle)) + + def started(self): return not self.cancellation_event.is_set() @@ -296,9 +334,10 @@ class CameraWidget: self.config.capture_source = values[self.gui_camera_addr] changed = True - if self.config.rotation_angle != values[self.gui_rotation_slider]: + if self.config.rotation_angle != int(values[self.gui_rotation_slider]): self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True + self.cartesian_needs_update = True # if self.config.gui_circular_crop != values[self.gui_circular_crop]: # self.config.gui_circular_crop = values[self.gui_circular_crop] @@ -325,15 +364,14 @@ class CameraWidget: # Event for mouse button up in ROI mode self.is_mouse_up = True print("UP") - if self.x1 < 0: - self.x1 = 0 - if self.y1 < 0: - self.y1 = 0 + # TODO keep rect in bounds of rotated image if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: - self.config.roi_window_x = min([self.x0, self.x1]) - self.config.roi_window_y = min([self.y0, self.y1]) - self.config.roi_window_w = abs(self.x0 - self.x1) - self.config.roi_window_h = abs(self.y0 - self.y1) + (x0, y0), (x1, y1) = self._polar_to_cartesian_at_angle(0) + + self.config.roi_window_x = min([x0, x1]) - self.pad_x + self.config.roi_window_y = min([y0, y1]) - self.pad_y + self.config.roi_window_w = abs(x0 - x1) + self.config.roi_window_h = abs(y0 - y1) self.main_config.save() if event == self.gui_roi_selection: @@ -344,6 +382,8 @@ class CameraWidget: self.x1, self.y1 = values[self.gui_roi_selection] + self._cartesian_to_polar() + if event == self.gui_restart_calibration: self.ransac.calibration_frame_counter = self.settings.calibration_samples self.ransac.ibo.clear_filter() @@ -401,6 +441,41 @@ class CameraWidget: if self.roi_queue.empty(): self.capture_event.set() maybe_image = self.roi_queue.get(block=False) + + if maybe_image: + image = maybe_image[0] + + img_w, img_h, _ = image.shape + + # TODO: rotation matrix -> bounding box corners -> crop matrix + hyp = math.ceil((img_w**2 + img_h**2)**0.5) + self.pad_x = (hyp - img_w)/2 + self.pad_y = (hyp - img_h)/2 + self.roi_image_center = (hyp / 2, hyp / 2) + + # deferred to after roi_image_center is updated + if self.cartesian_needs_update: + self._polar_to_cartesian() + self.cartesian_needs_update = False + + crop_matrix = np.float32([[1, 0, self.pad_x], + [0, 1, self.pad_y], + [0, 0, 1]]) + rotation_matrix = cv2.getRotationMatrix2D( + self.roi_image_center, self.config.rotation_angle, 1 + ) + matrix = np.matmul(rotation_matrix, crop_matrix) + + image = cv2.warpAffine( + image, + matrix, + (hyp, hyp), + borderMode=cv2.BORDER_CONSTANT, + borderValue=(128, 128, 128), + ) + + maybe_image = (image, *maybe_image[1:]) + imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes() graph = window[self.gui_roi_selection] if self.figure: @@ -411,7 +486,6 @@ class CameraWidget: graph.erase() graph.draw_image(data=imgbytes, location=(0, 0)) if None not in (self.x0, self.y0, self.x1, self.y1): - self.figure = graph.draw_rectangle( (self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1" ) From 666022d0e50dcb6bc14cb3aa339f6887e7984e59 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 07:02:24 +0000 Subject: [PATCH 03/14] Dynamically adjust image padding --- EyeTrackApp/camera_widget.py | 45 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 6fa1275..cbcfd88 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -233,7 +233,8 @@ class CameraWidget: # polar co-ordinates from the image center are the canonical representation self.cr, self.ca = None, None self.w, self.h = None, None - self.pad_x, self.pad_y = None, None + self.pad_w, self.pad_h = None, None + self.pad_left, self.pad_top = None, None self.roi_image_center = (None, None) self.figure = None @@ -368,8 +369,8 @@ class CameraWidget: if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: (x0, y0), (x1, y1) = self._polar_to_cartesian_at_angle(0) - self.config.roi_window_x = min([x0, x1]) - self.pad_x - self.config.roi_window_y = min([y0, y1]) - self.pad_y + self.config.roi_window_x = min([x0, x1]) - self.pad_left + self.config.roi_window_y = min([y0, y1]) - self.pad_top self.config.roi_window_w = abs(x0 - x1) self.config.roi_window_h = abs(y0 - y1) self.main_config.save() @@ -447,29 +448,45 @@ class CameraWidget: img_w, img_h, _ = image.shape - # TODO: rotation matrix -> bounding box corners -> crop matrix - hyp = math.ceil((img_w**2 + img_h**2)**0.5) - self.pad_x = (hyp - img_w)/2 - self.pad_y = (hyp - img_h)/2 - self.roi_image_center = (hyp / 2, hyp / 2) + rotation_matrix = cv2.getRotationMatrix2D( + ((img_w/2), (img_h/2)), self.config.rotation_angle, 1 + ) + + # calculate position of all four corners of image + + # calculate crop corner locations in original image space + x_coords, y_coords = np.matmul( + rotation_matrix, + np.transpose([ + [0, 0, 1], + [img_w, 0, 1], + [0, img_h, 1], + [img_w, img_h, 1]]), + ) + self.pad_w = math.ceil(max(x_coords) - min(x_coords)) + self.pad_h = math.ceil(max(y_coords) - min(y_coords)) + + self.pad_left = round((self.pad_w - img_w)/2) + self.pad_top = round((self.pad_h - img_h)/2) + self.roi_image_center = (self.pad_w / 2, self.pad_h / 2) # deferred to after roi_image_center is updated if self.cartesian_needs_update: self._polar_to_cartesian() self.cartesian_needs_update = False - crop_matrix = np.float32([[1, 0, self.pad_x], - [0, 1, self.pad_y], - [0, 0, 1]]) - rotation_matrix = cv2.getRotationMatrix2D( + pad_matrix = np.float32([[1, 0, self.pad_left], + [0, 1, self.pad_top], + [0, 0, 1]]) + rotation_matrix_padded = cv2.getRotationMatrix2D( self.roi_image_center, self.config.rotation_angle, 1 ) - matrix = np.matmul(rotation_matrix, crop_matrix) + matrix = np.matmul(rotation_matrix_padded, pad_matrix) image = cv2.warpAffine( image, matrix, - (hyp, hyp), + (self.pad_w, self.pad_h), borderMode=cv2.BORDER_CONSTANT, borderValue=(128, 128, 128), ) From 826afb2f30960442700e298c1af4dee011fd2db9 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 07:03:45 +0000 Subject: [PATCH 04/14] Limit ROI crop to padded region --- EyeTrackApp/camera_widget.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index cbcfd88..47e11f7 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -365,7 +365,11 @@ class CameraWidget: # Event for mouse button up in ROI mode self.is_mouse_up = True print("UP") - # TODO keep rect in bounds of rotated image + self.x0 = np.clip(self.x0, 0, self.pad_w) + self.y0 = np.clip(self.y0, 0, self.pad_h) + self.x1 = np.clip(self.x1, 0, self.pad_w) + self.y1 = np.clip(self.y1, 0, self.pad_h) + self._cartesian_to_polar() if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: (x0, y0), (x1, y1) = self._polar_to_cartesian_at_angle(0) From 41e1d33a19cf5b02dcc70a3f5439ce1d5902b076 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 13:45:56 +0000 Subject: [PATCH 05/14] Add crosshair to ROI editor --- EyeTrackApp/camera_widget.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 47e11f7..e43d3b5 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -108,6 +108,7 @@ class CameraWidget: key=self.gui_roi_selection, drag_submits=True, enable_events=True, + motion_events=True, background_color="#424042", ), ], @@ -226,6 +227,8 @@ class CameraWidget: ], ] + self.hover_x, self.hover_y = None, None + # cartesian co-ordinates in widget space are used during selection self.x0, self.y0 = None, None self.x1, self.y1 = None, None @@ -237,7 +240,6 @@ class CameraWidget: self.pad_left, self.pad_top = None, None self.roi_image_center = (None, None) - self.figure = None self.is_mouse_up = True self.in_roi_mode = False self.movavg_fps_queue = deque(maxlen=120) @@ -381,6 +383,8 @@ class CameraWidget: if event == self.gui_roi_selection: # Event for mouse button down or mouse drag in ROI mode + (self.hover_x, self.hover_y) = (None, None) + if self.is_mouse_up: self.is_mouse_up = False self.x0, self.y0 = values[self.gui_roi_selection] @@ -389,6 +393,13 @@ class CameraWidget: self._cartesian_to_polar() + if event == "{}+MOVE".format(self.gui_roi_selection): + if self.is_mouse_up: + (self.hover_x, self.hover_y) = values[self.gui_roi_selection] + + if self.hover_x > self.pad_w or self.hover_y > self.pad_h: + (self.hover_x, self.hover_y) = (None, None) + if event == self.gui_restart_calibration: self.ransac.calibration_frame_counter = self.settings.calibration_samples self.ransac.ibo.clear_filter() @@ -499,17 +510,22 @@ class CameraWidget: imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes() graph = window[self.gui_roi_selection] - if self.figure: - graph.delete_figure(self.figure) # INCREDIBLY IMPORTANT ERASE. Drawing images does NOT overwrite the buffer, the fucking # graph keeps every image fed in until you call this. Therefore we have to make sure we # erase before we redraw, otherwise we'll leak memory *very* quickly. graph.erase() graph.draw_image(data=imgbytes, location=(0, 0)) if None not in (self.x0, self.y0, self.x1, self.y1): - self.figure = graph.draw_rectangle( + graph.draw_rectangle( (self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1" ) + if self.is_mouse_up and None not in (self.hover_x, self.hover_y): + graph.draw_line( + (self.hover_x, 0), (self.hover_x, self.pad_h), color="#6f4ca1" + ) + graph.draw_line( + (0, self.hover_y), (self.pad_w, self.hover_y), color="#6f4ca1" + ) except Empty: pass From 29084baa29163e06e96f215ffd505e45145e044d Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 14:13:31 +0000 Subject: [PATCH 06/14] Style the ROI cursor and crop lines with dashes --- EyeTrackApp/camera_widget.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index e43d3b5..91f139c 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -515,17 +515,27 @@ class CameraWidget: # erase before we redraw, otherwise we'll leak memory *very* quickly. graph.erase() graph.draw_image(data=imgbytes, location=(0, 0)) + + def make_dashed(spawn_item, dark="#000000", light="#ffffff", duty=1): + pixel_duty = math.floor(4 * duty) + for (color, dashoffset) in [(dark, 0), (light, 4)]: + item = spawn_item(color) + graph._TKCanvas2.itemconfig(item, dash=(pixel_duty, 8 - pixel_duty), dashoffset=dashoffset) + if None not in (self.x0, self.y0, self.x1, self.y1): - graph.draw_rectangle( - (self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1" - ) + style = {} + if self.is_mouse_up: + style = {"dark": "#7f78ff", "light": "#d002ff", "duty": 0.5} + make_dashed(lambda color: graph.draw_rectangle( + (self.x0, self.y0), (self.x1, self.y1), line_color=color, + ), **style) if self.is_mouse_up and None not in (self.hover_x, self.hover_y): - graph.draw_line( - (self.hover_x, 0), (self.hover_x, self.pad_h), color="#6f4ca1" - ) - graph.draw_line( - (0, self.hover_y), (self.pad_w, self.hover_y), color="#6f4ca1" - ) + make_dashed(lambda color: graph.draw_line( + (self.hover_x, 0), (self.hover_x, self.pad_h), color=color + )) + make_dashed(lambda color: graph.draw_line( + (0, self.hover_y), (self.pad_w, self.hover_y), color=color + )) except Empty: pass From 8eae8d89d318c800790e5cf3b5de0c0b06762e70 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Mon, 6 Nov 2023 15:36:13 +0000 Subject: [PATCH 07/14] Add option to pad ROI widget for rotation --- EyeTrackApp/camera_widget.py | 41 ++++++++++++++++++++++++++++++------ EyeTrackApp/config.py | 1 + 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 91f139c..9e3d712 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -20,6 +20,7 @@ class CameraWidget: def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue): self.gui_camera_addr = f"-CAMERAADDR{widget_id}-" self.gui_rotation_slider = f"-ROTATIONSLIDER{widget_id}-" + self.gui_rotation_ui_padding = f"-ROTATIONUIPADDING{widget_id}-" self.gui_roi_button = f"-ROIMODE{widget_id}-" self.gui_roi_layout = f"-ROILAYOUT{widget_id}-" self.gui_roi_selection = f"-GRAPH{widget_id}-" @@ -99,6 +100,13 @@ class CameraWidget: button_color="#6f4ca1", tooltip="Lighten shadowed areas.", ), + sg.Checkbox( + "Camera Widget Padding", + default=self.config.gui_rotation_ui_padding, + tooltip="Pad the camera view widget enough to allow a full rotation.", + key=self.gui_rotation_ui_padding, + background_color="#424042", + ), ], [ sg.Graph( @@ -236,6 +244,8 @@ class CameraWidget: # polar co-ordinates from the image center are the canonical representation self.cr, self.ca = None, None self.w, self.h = None, None + self.clip_w, self.clip_h = None, None + self.clip_left, self.clip_top = None, None self.pad_w, self.pad_h = None, None self.pad_left, self.pad_top = None, None self.roi_image_center = (None, None) @@ -342,6 +352,12 @@ class CameraWidget: changed = True self.cartesian_needs_update = True + if self.config.gui_rotation_ui_padding != bool(values[self.gui_rotation_ui_padding]): + self.config.gui_rotation_ui_padding = bool(values[self.gui_rotation_ui_padding]) + changed = True + self.cartesian_needs_update = True + + # if self.config.gui_circular_crop != values[self.gui_circular_crop]: # self.config.gui_circular_crop = values[self.gui_circular_crop] # changed = True @@ -367,10 +383,10 @@ class CameraWidget: # Event for mouse button up in ROI mode self.is_mouse_up = True print("UP") - self.x0 = np.clip(self.x0, 0, self.pad_w) - self.y0 = np.clip(self.y0, 0, self.pad_h) - self.x1 = np.clip(self.x1, 0, self.pad_w) - self.y1 = np.clip(self.y1, 0, self.pad_h) + self.x0 = np.clip(self.x0, self.clip_left, self.clip_left + self.clip_w) + self.y0 = np.clip(self.y0, self.clip_top, self.clip_top + self.clip_h) + self.x1 = np.clip(self.x1, self.clip_left, self.clip_left + self.clip_w) + self.y1 = np.clip(self.y1, self.clip_top, self.clip_top + self.clip_h) self._cartesian_to_polar() if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: (x0, y0), (x1, y1) = self._polar_to_cartesian_at_angle(0) @@ -463,6 +479,7 @@ class CameraWidget: img_w, img_h, _ = image.shape + hyp = math.ceil((img_w**2 + img_h**2)**0.5) rotation_matrix = cv2.getRotationMatrix2D( ((img_w/2), (img_h/2)), self.config.rotation_angle, 1 ) @@ -478,11 +495,23 @@ class CameraWidget: [0, img_h, 1], [img_w, img_h, 1]]), ) - self.pad_w = math.ceil(max(x_coords) - min(x_coords)) - self.pad_h = math.ceil(max(y_coords) - min(y_coords)) + + self.clip_w = math.ceil(max(x_coords) - min(x_coords)) + self.clip_h = math.ceil(max(y_coords) - min(y_coords)) + if self.config.gui_rotation_ui_padding: + self.pad_w = hyp + self.pad_h = hyp + else: + self.pad_w = self.clip_w + self.pad_h = self.clip_h + self.pad_left = round((self.pad_w - img_w)/2) self.pad_top = round((self.pad_h - img_h)/2) + + self.clip_left = round((self.pad_w - self.clip_w)/2) + self.clip_top = round((self.pad_h - self.clip_h)/2) + self.roi_image_center = (self.pad_w / 2, self.pad_h / 2) # deferred to after roi_image_center is updated diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index 889be2e..7907764 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -11,6 +11,7 @@ BACKUP_CONFIG_FILE_NAME: str = "eyetrack_settings.backup" class EyeTrackCameraConfig(BaseModel): rotation_angle: int = 0 + gui_rotation_ui_padding: bool = True roi_window_x: int = 0 roi_window_y: int = 0 roi_window_w: int = 240 From 3641751a223ab79dc46eacd109d360cf9343ac53 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Tue, 7 Nov 2023 19:54:48 +0000 Subject: [PATCH 08/14] Load ROI from config.roi_window --- EyeTrackApp/camera_widget.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 9e3d712..1a3d02b 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -551,13 +551,23 @@ class CameraWidget: item = spawn_item(color) graph._TKCanvas2.itemconfig(item, dash=(pixel_duty, 8 - pixel_duty), dashoffset=dashoffset) - if None not in (self.x0, self.y0, self.x1, self.y1): - style = {} - if self.is_mouse_up: - style = {"dark": "#7f78ff", "light": "#d002ff", "duty": 0.5} - make_dashed(lambda color: graph.draw_rectangle( - (self.x0, self.y0), (self.x1, self.y1), line_color=color, - ), **style) + if None in (self.x0, self.y0, self.x1, self.y1): + # roi_window rotates around roi center, we rotate around image center + # TODO: it would be nice if they were more consistent + self.x0 = self.config.roi_window_x + self.pad_left + self.y0 = self.config.roi_window_y + self.pad_top + self.x1 = self.x0 + self.config.roi_window_w + self.y1 = self.y0 + self.config.roi_window_h + self._cartesian_to_polar() + self.ca += math.radians(self.config.rotation_angle) + self._polar_to_cartesian() + + style = {} + if self.is_mouse_up: + style = {"dark": "#7f78ff", "light": "#d002ff", "duty": 0.5} + make_dashed(lambda color: graph.draw_rectangle( + (self.x0, self.y0), (self.x1, self.y1), line_color=color, + ), **style) if self.is_mouse_up and None not in (self.hover_x, self.hover_y): make_dashed(lambda color: graph.draw_line( (self.hover_x, 0), (self.hover_x, self.pad_h), color=color From 98a2448f5ca23b02f243eb9dd1a4c4829122d140 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Tue, 7 Nov 2023 22:41:03 +0000 Subject: [PATCH 09/14] Use vec2s where possible --- EyeTrackApp/camera_widget.py | 123 ++++++++++++++++------------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 1a3d02b..620c303 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -15,6 +15,9 @@ import traceback import math import numpy as np +# for clarity when indexing +X = 0 +Y = 1 class CameraWidget: def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue): @@ -235,22 +238,23 @@ class CameraWidget: ], ] - self.hover_x, self.hover_y = None, None + self.hover = None # cartesian co-ordinates in widget space are used during selection - self.x0, self.y0 = None, None - self.x1, self.y1 = None, None + self.xy0 = None + self.xy1 = None self.cartesian_needs_update = False # polar co-ordinates from the image center are the canonical representation self.cr, self.ca = None, None - self.w, self.h = None, None - self.clip_w, self.clip_h = None, None - self.clip_left, self.clip_top = None, None - self.pad_w, self.pad_h = None, None - self.pad_left, self.pad_top = None, None - self.roi_image_center = (None, None) + self.roi_size = None + self.clip_size = None + self.clip_pos = None + self.padded_size = None + self.img_pos = None + self.roi_image_center = None self.is_mouse_up = True + self.hover_pos = None self.in_roi_mode = False self.movavg_fps_queue = deque(maxlen=120) self.movavg_bps_queue = deque(maxlen=120) @@ -266,30 +270,26 @@ class CameraWidget: return f"{sum(self.movavg_bps_queue) / len(self.movavg_bps_queue) * 0.001 * 0.001 * 8:.3f} Mbps" def _cartesian_to_polar(self): - if None not in (self.x0, self.y0, self.x1, self.y1): - image_center_x, image_center_y = self.roi_image_center - roi_center_x = image_center_x - (self.x0 + self.x1) / 2. - roi_center_y = image_center_y - (self.y0 + self.y1) / 2. - self.cr = (roi_center_x**2 + roi_center_y**2)**0.5 - self.ca = math.atan2(roi_center_x, roi_center_y) - \ + if not (self.xy0 is None or self.xy1 is None): + roi_center = self.roi_image_center - (self.xy0 + self.xy1) / 2. + self.cr = np.linalg.norm(roi_center) + self.ca = math.atan2(roi_center[X], roi_center[Y]) - \ math.radians(self.config.rotation_angle) - self.w = abs(self.x1 - self.x0) - self.h = abs(self.y1 - self.y0) + self.roi_size = np.abs(self.xy1 - self.xy0) def _polar_to_cartesian_at_angle(self, rotation_angle_radians): - if None not in (self.cr, self.ca, self.w, self.h): - image_center_x, image_center_y = self.roi_image_center + if not (self.cr is None or self.ca is None or self.roi_size is None): ca = self.ca + rotation_angle_radians - cx = -math.sin(ca) * self.cr + image_center_x - cy = -math.cos(ca) * self.cr + image_center_y - return ((int(cx - self.w/2), int(cy - self.h/2)), - (int(cx + self.w/2), int(cy + self.h/2))) + cx = -math.sin(ca) * self.cr + self.roi_image_center[X] + cy = -math.cos(ca) * self.cr + self.roi_image_center[Y] + roi_pos = np.array((int(cx), int(cy))) - self.roi_size//2 + return (roi_pos, roi_pos + self.roi_size) else: - return 4 * (None,) + return (None, None) def _polar_to_cartesian(self): - if None not in (self.cr, self.ca, self.w, self.h): - (self.x0, self.y0), (self.x1, self.y1) = \ + if not (self.cr is None or self.ca is None or self.roi_size is None): + (self.xy0), (self.xy1) = \ self._polar_to_cartesian_at_angle( math.radians(self.config.rotation_angle)) @@ -383,38 +383,34 @@ class CameraWidget: # Event for mouse button up in ROI mode self.is_mouse_up = True print("UP") - self.x0 = np.clip(self.x0, self.clip_left, self.clip_left + self.clip_w) - self.y0 = np.clip(self.y0, self.clip_top, self.clip_top + self.clip_h) - self.x1 = np.clip(self.x1, self.clip_left, self.clip_left + self.clip_w) - self.y1 = np.clip(self.y1, self.clip_top, self.clip_top + self.clip_h) + self.xy0 = np.clip(self.xy0, self.clip_pos, self.clip_pos + self.clip_size) + self.xy1 = np.clip(self.xy1, self.clip_pos, self.clip_pos + self.clip_size) self._cartesian_to_polar() - if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: - (x0, y0), (x1, y1) = self._polar_to_cartesian_at_angle(0) + if all(abs(self.xy0 - self.xy1) != 0): + xy0, xy1 = self._polar_to_cartesian_at_angle(0) - self.config.roi_window_x = min([x0, x1]) - self.pad_left - self.config.roi_window_y = min([y0, y1]) - self.pad_top - self.config.roi_window_w = abs(x0 - x1) - self.config.roi_window_h = abs(y0 - y1) + self.config.roi_window_x, self.config.roi_window_y = (np.minimum(xy0, xy1) - self.img_pos).tolist() + self.config.roi_window_w, self.config.roi_window_h = (np.abs(xy0 - xy1)).tolist() self.main_config.save() if event == self.gui_roi_selection: # Event for mouse button down or mouse drag in ROI mode - (self.hover_x, self.hover_y) = (None, None) + self.hover_pos = None if self.is_mouse_up: self.is_mouse_up = False - self.x0, self.y0 = values[self.gui_roi_selection] + self.xy0 = np.array(values[self.gui_roi_selection]) - self.x1, self.y1 = values[self.gui_roi_selection] + self.xy1 = np.array(values[self.gui_roi_selection]) self._cartesian_to_polar() if event == "{}+MOVE".format(self.gui_roi_selection): if self.is_mouse_up: - (self.hover_x, self.hover_y) = values[self.gui_roi_selection] + self.hover_pos = np.array(values[self.gui_roi_selection]) - if self.hover_x > self.pad_w or self.hover_y > self.pad_h: - (self.hover_x, self.hover_y) = (None, None) + if any(self.hover_pos > self.padded_size): + self.hover_pos = None if event == self.gui_restart_calibration: self.ransac.calibration_frame_counter = self.settings.calibration_samples @@ -496,31 +492,26 @@ class CameraWidget: [img_w, img_h, 1]]), ) - self.clip_w = math.ceil(max(x_coords) - min(x_coords)) - self.clip_h = math.ceil(max(y_coords) - min(y_coords)) + self.clip_size = np.array([math.ceil(max(x_coords) - min(x_coords)), + math.ceil(max(y_coords) - min(y_coords))]) if self.config.gui_rotation_ui_padding: - self.pad_w = hyp - self.pad_h = hyp + self.padded_size = np.array([hyp, hyp]) else: - self.pad_w = self.clip_w - self.pad_h = self.clip_h + self.padded_size = self.clip_size + self.img_pos = ((self.padded_size - (img_w, img_h))/2).astype(np.int32) - self.pad_left = round((self.pad_w - img_w)/2) - self.pad_top = round((self.pad_h - img_h)/2) + self.clip_pos = ((self.padded_size - self.clip_size)/2).astype(np.int32) - self.clip_left = round((self.pad_w - self.clip_w)/2) - self.clip_top = round((self.pad_h - self.clip_h)/2) - - self.roi_image_center = (self.pad_w / 2, self.pad_h / 2) + self.roi_image_center = self.padded_size / 2 # deferred to after roi_image_center is updated if self.cartesian_needs_update: self._polar_to_cartesian() self.cartesian_needs_update = False - pad_matrix = np.float32([[1, 0, self.pad_left], - [0, 1, self.pad_top], + pad_matrix = np.float32([[1, 0, self.img_pos[X]], + [0, 1, self.img_pos[Y]], [0, 0, 1]]) rotation_matrix_padded = cv2.getRotationMatrix2D( self.roi_image_center, self.config.rotation_angle, 1 @@ -530,7 +521,7 @@ class CameraWidget: image = cv2.warpAffine( image, matrix, - (self.pad_w, self.pad_h), + self.padded_size, borderMode=cv2.BORDER_CONSTANT, borderValue=(128, 128, 128), ) @@ -551,13 +542,13 @@ class CameraWidget: item = spawn_item(color) graph._TKCanvas2.itemconfig(item, dash=(pixel_duty, 8 - pixel_duty), dashoffset=dashoffset) - if None in (self.x0, self.y0, self.x1, self.y1): + if (self.xy0 is None or self.xy1 is None): # roi_window rotates around roi center, we rotate around image center # TODO: it would be nice if they were more consistent - self.x0 = self.config.roi_window_x + self.pad_left - self.y0 = self.config.roi_window_y + self.pad_top - self.x1 = self.x0 + self.config.roi_window_w - self.y1 = self.y0 + self.config.roi_window_h + roi_window_pos = (self.config.roi_window_x, self.config.roi_window_y) + roi_window_size = (self.config.roi_window_w, self.config.roi_window_h) + self.xy0 = roi_window_pos + self.img_pos + self.xy1 = self.xy0 + roi_window_size self._cartesian_to_polar() self.ca += math.radians(self.config.rotation_angle) self._polar_to_cartesian() @@ -566,14 +557,14 @@ class CameraWidget: if self.is_mouse_up: style = {"dark": "#7f78ff", "light": "#d002ff", "duty": 0.5} make_dashed(lambda color: graph.draw_rectangle( - (self.x0, self.y0), (self.x1, self.y1), line_color=color, + self.xy0, self.xy1, line_color=color, ), **style) - if self.is_mouse_up and None not in (self.hover_x, self.hover_y): + if self.is_mouse_up and self.hover_pos is not None: make_dashed(lambda color: graph.draw_line( - (self.hover_x, 0), (self.hover_x, self.pad_h), color=color + (self.hover_pos[X], 0), (self.hover_pos[X], self.padded_size[Y]), color=color )) make_dashed(lambda color: graph.draw_line( - (0, self.hover_y), (self.pad_w, self.hover_y), color=color + (0, self.hover_pos[Y]), (self.padded_size[X], self.hover_pos[Y]), color=color )) except Empty: From c231b004b9017730d48c74317b26a5fa7984f74d Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Tue, 7 Nov 2023 23:08:13 +0000 Subject: [PATCH 10/14] Fix some mutually destructive sign issues --- EyeTrackApp/camera_widget.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 620c303..60384ae 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -271,17 +271,17 @@ class CameraWidget: def _cartesian_to_polar(self): if not (self.xy0 is None or self.xy1 is None): - roi_center = self.roi_image_center - (self.xy0 + self.xy1) / 2. + roi_center = (self.xy0 + self.xy1) / 2 - self.roi_image_center self.cr = np.linalg.norm(roi_center) - self.ca = math.atan2(roi_center[X], roi_center[Y]) - \ + self.ca = math.atan2(roi_center[Y], roi_center[X]) + \ math.radians(self.config.rotation_angle) self.roi_size = np.abs(self.xy1 - self.xy0) def _polar_to_cartesian_at_angle(self, rotation_angle_radians): if not (self.cr is None or self.ca is None or self.roi_size is None): - ca = self.ca + rotation_angle_radians - cx = -math.sin(ca) * self.cr + self.roi_image_center[X] - cy = -math.cos(ca) * self.cr + self.roi_image_center[Y] + ca = self.ca - rotation_angle_radians + cx = math.cos(ca) * self.cr + self.roi_image_center[X] + cy = math.sin(ca) * self.cr + self.roi_image_center[Y] roi_pos = np.array((int(cx), int(cy))) - self.roi_size//2 return (roi_pos, roi_pos + self.roi_size) else: @@ -550,7 +550,7 @@ class CameraWidget: self.xy0 = roi_window_pos + self.img_pos self.xy1 = self.xy0 + roi_window_size self._cartesian_to_polar() - self.ca += math.radians(self.config.rotation_angle) + self.ca -= math.radians(self.config.rotation_angle) self._polar_to_cartesian() style = {} From 8f0acab6ce68d4f8eb1f9c269c589f488a25e615 Mon Sep 17 00:00:00 2001 From: Charlton Rodda Date: Sat, 11 Nov 2023 19:49:56 +0000 Subject: [PATCH 11/14] Fix image shape axis order --- EyeTrackApp/camera_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 60384ae..e8a462b 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -473,7 +473,7 @@ class CameraWidget: if maybe_image: image = maybe_image[0] - img_w, img_h, _ = image.shape + img_h, img_w, _ = image.shape hyp = math.ceil((img_w**2 + img_h**2)**0.5) rotation_matrix = cv2.getRotationMatrix2D( From 9c98f62ce38966b9b050ae4cbff3b0cff39f4496 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:44:54 -0700 Subject: [PATCH 12/14] Update camera_widget.py to new (fixes merge conflicts due to very old imp) --- EyeTrackApp/camera_widget.py | 66 ++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index e8a462b..bfe36c3 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -1,20 +1,46 @@ +""" +------------------------------------------------------------------------------------------------------ + + ,@@@@@@ + @@@@@@@@@@@ @@@ + @@@@@@@@@@@@ @@@@@@@@@@@ + @@@@@@@@@@@@@ @@@@@@@@@@@@@@ + @@@@@@@/ ,@@@@@@@@@@@@@ + /@@@@@@@@@@@@@@@ @@@@@@@@ + @@@@@@@@@@@@@@@@@@@@@@@@ @@@@@ + @@@@@@@@ @@@@@ + ,@@@ @@@@& + @@@@@@. @@@@ + @@@ @@@@@@@@@/ @@@@@ + ,@@@. @@@@@@((@ @@@@( + //@@@ ,, @@@@ @@@@@ + @@@( @@@@@@@ + @@@ @ @@@@@@@@# + @@@@@@@@@@@@@@@@@ + @@@@@@@@@@@@@( + +Copyright (c) 2023 EyeTrackVR <3 +LICENSE: GNU GPLv3 +------------------------------------------------------------------------------------------------------ +""" + import PySimpleGUI as sg from config import EyeTrackConfig -from config import EyeTrackSettingsConfig from collections import deque from threading import Event, Thread +import math +from eye import EyeId from eye_processor import EyeProcessor, EyeInfoOrigin -from enum import Enum from queue import Queue, Empty from camera import Camera, CameraState -from osc import EyeId import cv2 -import sys + +from osc.OSCMessage import OSCMessageType, OSCMessage from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path -import traceback -import math import numpy as np + + # for clarity when indexing X = 0 Y = 1 @@ -315,6 +341,28 @@ class CameraWidget: self.ransac_thread.join() self.camera_thread.join() + def on_config_update(self, data): + keys = set(data.keys()) + model_keys = set(self.config.model_fields.keys()) + # we only want to restart our stuff, if our stuff got updated + # at the model level + if model_keys.intersection(keys): + self.stop() + self.start() + + def recenter_eyes(self, osc_message: OSCMessage): + if osc_message.data is not bool: + return # just incase we get anything other than bool + + def recalibrate_eyes(self, osc_message: OSCMessage): + if osc_message.data is not bool: + return # just incase we get anything other than bool + + if osc_message.data: + self.ransac.ibo.clear_filter() + self.ransac.calibration_frame_counter = self.config.calibration_samples + PlaySound("Audio/start.wav", SND_FILENAME | SND_ASYNC) + def render(self, window, event, values): changed = False @@ -626,6 +674,10 @@ class CameraWidget: graph.update(background_color="red") # Relay information to OSC if eye_info.info_type != EyeInfoOrigin.FAILURE: - self.osc_queue.put((self.eye_id, eye_info)) + osc_message = OSCMessage( + type=OSCMessageType.EYE_INFO, + data=(self.eye_id, eye_info), + ) + self.osc_queue.put(osc_message) except Empty: pass From 1d69fc110fcce42eff70abd602c27a7161d4f479 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:45:25 -0700 Subject: [PATCH 13/14] Update config.py --- EyeTrackApp/config.py | 212 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 192 insertions(+), 20 deletions(-) diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index 7907764..8dbc9e2 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -1,29 +1,124 @@ +""" +------------------------------------------------------------------------------------------------------ + + ,@@@@@@ + @@@@@@@@@@@ @@@ + @@@@@@@@@@@@ @@@@@@@@@@@ + @@@@@@@@@@@@@ @@@@@@@@@@@@@@ + @@@@@@@/ ,@@@@@@@@@@@@@ + /@@@@@@@@@@@@@@@ @@@@@@@@ + @@@@@@@@@@@@@@@@@@@@@@@@ @@@@@ + @@@@@@@@ @@@@@ + ,@@@ @@@@& + @@@@@@. @@@@ + @@@ @@@@@@@@@/ @@@@@ + ,@@@. @@@@@@((@ @@@@( + //@@@ ,, @@@@ @@@@@ + @@@( @@@@@@@ + @@@ @ @@@@@@@@# + @@@@@@@@@@@@@@@@@ + @@@@@@@@@@@@@( + +Copyright (c) 2023 EyeTrackVR <3 +LICENSE: GNU GPLv3 +------------------------------------------------------------------------------------------------------ +""" + import json import os.path import shutil -from eye import EyeId + +from colorama import Fore from pydantic import BaseModel -from typing import Union +from typing import Any, Union, List +import os + +from eye import EyeId CONFIG_FILE_NAME: str = "eyetrack_settings.json" BACKUP_CONFIG_FILE_NAME: str = "eyetrack_settings.backup" class EyeTrackCameraConfig(BaseModel): - rotation_angle: int = 0 gui_rotation_ui_padding: bool = True + rotation_angle: int = 0 roi_window_x: int = 0 roi_window_y: int = 0 roi_window_w: int = 240 roi_window_h: int = 240 focal_length: int = 30 capture_source: Union[int, str, None] = None - calib_XMAX: Union[int, None] = None - calib_XMIN: Union[int, None] = None - calib_YMAX: Union[int, None] = None - calib_YMIN: Union[int, None] = None - calib_XOFF: Union[int, None] = None - calib_YOFF: Union[int, None] = None + calib_XMAX: Union[float, None] = None + calib_XMIN: Union[float, None] = None + calib_YMAX: Union[float, None] = None + calib_YMIN: Union[float, None] = None + calib_XOFF: Union[float, None] = None + calib_YOFF: Union[float, None] = None + calibration_points: List[List[Union[float, None]]] = [] + calibration_points_3d: List[List[Union[float, None]]] = [] + + + def update_capture_source(self, new_camera_address: str): + if not new_camera_address: + self.capture_source = None + return + + if new_camera_address.isnumeric(): + self.capture_source = int(new_camera_address) + return + + # we were passed an IP, probably, lets add HTTP:// to it + if len(new_camera_address) > 5 and not ( + not new_camera_address.startswith(("http", "/dev")) or not new_camera_address.endswith(".mp4") + ): + self.capture_source = f"http://{new_camera_address}" + return + + self.capture_source = new_camera_address + + def update(self, data: dict[str, Any]) -> bool: + """ + Updates the model one field at a time based on the provided data dict. + The dict has to be defined like + ``` + data = { + "model_field": value + } + ``` + + If stale data is provided, + ex. User clicked on save and restart but didn't provide a new field + + we skip it, assuming that it was just a call to restart the tracking, or a miss-click. + + Some fields may require more validation, we take care of that with special methods. + defining a method like + + ``` + def update_custom_field(value: type): + pass + ``` + + will cause it to be picked up by this method and called with the current value. + Return values are ignored. + + """ + for key, value in data.items(): + old_value = getattr(self, key, None) + # no reason to update if it's the same value + if old_value == value: + return False + + if hasattr(self, f"update_{key}"): + update_attr = getattr(self, f"update_{key}") + if callable(update_attr): + update_attr(value) + else: + setattr(self, "key", value) + return True + else: + print(f"\033[93m[WARN] Field {key} does not exist on {self}.\033[0m") + return False class EyeTrackSettingsConfig(BaseModel): @@ -34,7 +129,9 @@ class EyeTrackSettingsConfig(BaseModel): gui_HSF: bool = False gui_BLOB: bool = False gui_BLINK: bool = False - gui_HSRAC: bool = True + gui_HSRAC: bool = False + gui_AHSFRAC: bool = True + gui_AHSF: bool = False gui_DADDY: bool = False gui_LEAP: bool = False gui_HSF_radius: int = 15 @@ -50,14 +147,18 @@ class EyeTrackSettingsConfig(BaseModel): gui_blob_maxsize: float = 25 gui_blob_minsize: float = 10 gui_recenter_eyes: bool = False + gui_3d_calibration: bool = False + grab_3d_point: bool = False tracker_single_eye: int = 0 gui_threshold: int = 65 - gui_HSRACP: int = 1 - gui_HSFP: int = 2 - gui_DADDYP: int = 3 - gui_RANSAC3DP: int = 4 - gui_BLOBP: int = 5 - gui_LEAPP: int = 6 + gui_AHSFRACP: int = 1 + gui_AHSFP: int = 2 + gui_HSRACP: int = 3 + gui_HSFP: int = 4 + gui_DADDYP: int = 5 + gui_RANSAC3DP: int = 6 + gui_BLOBP: int = 7 + gui_LEAPP: int = 8 gui_IBO: bool = True gui_skip_autoradius: bool = False gui_thresh_add: int = 11 @@ -74,12 +175,13 @@ class EyeTrackSettingsConfig(BaseModel): osc_left_eye_x_address: str = "/avatar/parameters/LeftEyeX" osc_right_eye_x_address: str = "/avatar/parameters/RightEyeX" osc_eyes_y_address: str = "/avatar/parameters/EyesY" + osc_eyes_pupil_dilation_address: str = "/avatar/parameters/EyesDilation" osc_invert_eye_close: bool = False gui_RANSACBLINK: bool = False gui_right_eye_dominant: bool = False gui_left_eye_dominant: bool = False - gui_outer_side_falloff: bool = True + gui_outer_side_falloff: bool = False gui_eye_dominant_diff_thresh: float = 0.3 gui_legacy_ransac: bool = False @@ -91,6 +193,24 @@ class EyeTrackSettingsConfig(BaseModel): gui_vrc_native: bool = True gui_pupil_dilation: bool = True + gui_VRCFTModulePort: int = 8889 + gui_VRCFTModuleIPAddress: str = "127.0.0.1" + gui_ShouldEmulateEyeWiden: bool = False + gui_ShouldEmulateEyeSquint: bool = False + gui_ShouldEmulateEyebrows: bool = False + gui_WidenThresholdV1_min: float = 0.60 + gui_WidenThresholdV1_max: float = 1 + gui_WidenThresholdV2_min: float = 0.60 + gui_WidenThresholdV2_max: float = 1.05 + gui_SqueezeThresholdV1_min: float = 0.07 + gui_SqueezeThresholdV1_max: float = 0.5 + gui_SqueezeThresholdV2_min: float = 0.07 + gui_SqueezeThresholdV2_max: float = -1 + gui_EyebrowThresholdRising: float = 0.8 + gui_EyebrowThresholdLowering: float = 0.15 + gui_OutputMultiplier: float = 1 + gui_use_module: bool = False + class EyeTrackConfig(BaseModel): version: int = 1 @@ -123,10 +243,62 @@ class EyeTrackConfig(BaseModel): load_config = EyeTrackConfig() return load_config + def validate_camera_address_conflict(self, eye_id, capture_source): + match eye_id: + case EyeId.RIGHT: + if self.left_eye.capture_source == capture_source: + print( + f"{Fore.YELLOW}[WARN] Capture source {capture_source} already in use by the left camera.{Fore.RESET}" + ) + return False + case EyeId.LEFT: + if self.right_eye.capture_source == capture_source: + print( + f"{Fore.YELLOW}[WARN] Capture source {capture_source} already in use by the right camera.{Fore.RESET}" + ) + return False + case _: + return False + return True + + def update_eye_model_config(self, eye_id: EyeId, data: dict, should_save=True, should_notify=True) -> bool: + """ + A more granular method for updating a particular model so that everything that relies on it + will get notified about any changes. Note, it acts a bit like pub-sub, + we don't care what changes got passed, we will notify the listeners with them. + + It's the listeners job to check if they want that update. + """ + + # The app really doesn't like address clashes, so we have to validate it as soon as possible + # otherwise we crash + if "capture_source" in data and not self.validate_camera_address_conflict(eye_id, data["capture_source"]): + return False + + match eye_id: + case EyeId.RIGHT: + changed = self.right_eye.update(data) + case EyeId.LEFT: + changed = self.left_eye.update(data) + case _: + return False + + if should_save: + self.save() + + if should_notify: + self.__notify_listeners(data) + + return changed + def update(self, data, save=False): + """ + More of an internal method for modules to be able to update the config + and have other parts of the system react to changes + """ for field, value in data.items(): setattr(self.settings, field, value) - self.__notify_listeners() + self.__notify_listeners(data) if save: self.save() @@ -150,6 +322,6 @@ class EyeTrackConfig(BaseModel): print(f"[DEBUG] Registering listener {callback}") self.__listeners.append(callback) - def __notify_listeners(self): + def __notify_listeners(self, data: dict): for listener in self.__listeners: - listener() + listener(data) From f8961d86162a6d5833ea6e2e1f1503e078f2d5fc Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:45:39 -0700 Subject: [PATCH 14/14] Update eye_processor.py --- EyeTrackApp/eye_processor.py | 266 ++++++++++++++++++++++------------- 1 file changed, 168 insertions(+), 98 deletions(-) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 72951c5..9ebcd97 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -18,37 +18,28 @@ @@@ @ @@@@@@@@# @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( - -HSR By: PallasNeko (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) -BLOB By: Prohurtz (Main App Developer) -Algorithm App Implementations By: Prohurtz, qdot (Initial App Creator), PallasNeko + +Algorithm App Implementations By: Prohurtz, qdot (GUI, Initial Implementations), PallasNeko (Optimizations), Summer (Algorithim Engineer) Additional Contributors: [Assassin], Summer404NotFound, lorow, ZanzyTHEbar Copyright (c) 2023 EyeTrackVR <3 +LICENSE: GNU GPLv3 ------------------------------------------------------------------------------------------------------ """ -from operator import truth -from dataclasses import dataclass import sys import asyncio +import os +os.environ["OMP_NUM_THREADS"] = "1" sys.path.append(".") from config import EyeTrackCameraConfig from config import EyeTrackSettingsConfig from pye3d.camera import CameraModel from pye3d.detector_3d import Detector3D, DetectorMode import queue -import threading -import numpy as np -import cv2 -from enum import Enum -from one_euro_filter import OneEuroFilter -from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path -import importlib -from osc import EyeId +from eye import EyeId from osc_calibrate_filter import * from daddy import External_Run_DADDY from leap import External_Run_LEAP @@ -60,6 +51,7 @@ from utils.img_utils import circle_crop from eye import EyeInfo, EyeInfoOrigin from intensity_based_openness import * from ellipse_based_pupil_dilation import * +from AHSF import * def run_once(f): @@ -86,8 +78,8 @@ class EyeProcessor: baseconfig: "EyetrackConfig", cancellation_event: "threading.Event", capture_event: "threading.Event", - capture_queue_incoming: "queue.Queue", - image_queue_outgoing: "queue.Queue", + capture_queue_incoming: "queue.Queue(maxsize=2)", + image_queue_outgoing: "queue.Queue(maxsize=2)", eye_id, ): self.main_config = EyeTrackSettingsConfig @@ -102,6 +94,8 @@ class EyeProcessor: self.eye_id = eye_id self.baseconfig = baseconfig self.filterlist = [] + self.left_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data + self.right_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data # Cross algo state self.lkg_projected_sphere = None @@ -122,9 +116,10 @@ class EyeProcessor: self.yoff = 1 # Keep large in order to recenter correctly self.calibration_frame_counter = None + self.calibration_3d_frame_counter = None self.eyeoffx = 1 self.printcal = True - + self.grab_3d_point = False self.xmax = -69420 self.xmin = 69420 self.ymax = -69420 @@ -166,9 +161,10 @@ class EyeProcessor: self.ran_blink_check_for_file = True self.bd_blink = False self.current_algo = EyeInfoOrigin.HSRAC - self.puipil_width = 0.0 + self.pupil_width = 0.0 self.pupil_height = 0.0 self.avg_velocity = 0.0 + self.angle = 621 try: min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004 @@ -178,9 +174,7 @@ class EyeProcessor: min_cutoff = 0.0004 beta = 0.9 noisy_point = np.array([1, 1]) - self.one_euro_filter = OneEuroFilter( - noisy_point, min_cutoff=min_cutoff, beta=beta - ) + self.one_euro_filter = OneEuroFilter(noisy_point, min_cutoff=min_cutoff, beta=beta) def output_images_and_update(self, threshold_image, output_information: EyeInfo): try: @@ -195,17 +189,12 @@ class EyeProcessor: self.previous_image = self.current_image self.previous_rotation = self.config.rotation_angle except: # If this fails it likely means that the images are not the same size for some reason. - print( - "\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m" - ) - - pass + print("\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m") def capture_crop_rotate_image(self): # Get our current frame self.ibo.change_roi(self.config.dict(include=self.roi_include_set)) - roi_x = self.config.roi_window_x roi_y = self.config.roi_window_y roi_w = self.config.roi_window_w @@ -215,16 +204,38 @@ class EyeProcessor: try: # Apply rotation to cropped area. For any rotation area outside of the bounds of the image, + # fill with avg color + 10. # fill with white (self.current_image_white) and average in-bounds color (self.current_image). crop_matrix = np.float32([[1, 0, -roi_x], [0, 1, -roi_y], - [0, 0, 1]]) + [0, 0, 1]]) img_center = (roi_w / 2, roi_h / 2) + rotation_matrix = cv2.getRotationMatrix2D( img_center, self.config.rotation_angle, 1 ) + + # rows, cols = self.current_image.shape[:2] + # rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), self.config.rotation_angle, 1) + #cos_theta = np.abs(rotation_matrix[0, 0]) + # sin_theta = np.abs(rotation_matrix[0, 1]) + # new_cols = int((cols * cos_theta) + (rows * sin_theta)) + # new_rows = int((cols * sin_theta) + (rows * cos_theta)) + # rotation_matrix[0, 2] += (new_cols - cols) / 2 + # rotation_matrix[1, 2] += (new_rows - rows) / 2 + + + + matrix = np.matmul(rotation_matrix, crop_matrix) + self.current_image_white = cv2.warpAffine( + self.current_image, + matrix, + (roi_w, roi_h), + borderMode=cv2.BORDER_CONSTANT, + borderValue=(255, 255, 255), + ) self.current_image_white = cv2.warpAffine( self.current_image, @@ -234,14 +245,11 @@ class EyeProcessor: borderValue=(255, 255, 255), ) - # calculate position of all four corners of crop, and check if any are out of bounds - - # add w-preserve row to make matrix square, invert, and remove again inv_matrix = np.linalg.inv(np.vstack((matrix, [0, 0, 1])))[:-1] # calculate crop corner locations in original image space - corners = np.matmul([[0, 0, 1], - [roi_w, 0, 1], - [0, roi_h, 1], + corners = np.matmul([[0, 0, 1], + [roi_w, 0, 1], + [0, roi_h, 1], [roi_w, roi_h, 1]], np.transpose(inv_matrix)) fits_in_bounds = all(0 <= x <= img_w and 0 <= y <= img_h @@ -266,7 +274,6 @@ class EyeProcessor: borderValue=(0, 0, 0, 0), ) - # calculate average color of crop, excluding alpha avg_color_per_row = np.average(self.current_image, axis=0) avg_color = np.average(avg_color_per_row, axis=0) avg_color_norm = avg_color[0:3] / avg_color[3] @@ -281,6 +288,7 @@ class EyeProcessor: inv_alpha_ch * ab]), axis=-1) + return True except: pass @@ -301,9 +309,8 @@ class EyeProcessor: self.settings.ibo_filter_samples, self.settings.ibo_average_output_samples, ) - if self.eyeopen < float( - self.settings.ibo_fully_close_eye_threshold - ): # threshold so the eye fully closes + # threshold so the eye fully closes + if self.eyeopen < float(self.settings.ibo_fully_close_eye_threshold): self.eyeopen = 0.0 if self.bd_blink == True: @@ -325,12 +332,10 @@ class EyeProcessor: self.rawx, self.rawy, self.eyeopen, - ) = self.er_leap.run(self.current_image_gray) + ) = self.er_leap.run(self.current_image_gray, self.current_image_gray_clean) # print(self.eyeopen) - if ( - len(self.prev_y_list) >= 100 - ): # "lock" eye when close/blink IN TESTING, kinda broke + if len(self.prev_y_list) >= 100: # "lock" eye when close/blink IN TESTING, kinda broke self.prev_y_list.pop(0) self.prev_y_list.append(self.out_y) else: @@ -383,18 +388,12 @@ class EyeProcessor: def LEAPM(self): self.thresh = self.current_image_gray.copy() - ( - self.current_image_gray, - self.rawx, - self.rawy, - self.eyeopen, - ) = self.er_leap.run( - self.current_image_gray + (self.current_image_gray, self.rawx, self.rawy, self.eyeopen,) = self.er_leap.run( + self.current_image_gray, self.current_image_gray_clean ) # TODO: make own self var and LEAP toggle self.thresh = self.current_image_gray.copy() - self.out_x, self.out_y, self.avg_velocity = cal.cal_osc( - self, self.rawx, self.rawy - ) + # todo: lorow, fix this as well + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.LEAP # print(self.eyeopen) @@ -405,11 +404,54 @@ class EyeProcessor: self.thresh = self.current_image_gray.copy() self.rawx, self.rawy, self.radius = self.er_daddy.run(self.current_image_gray) # Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it. - self.out_x, self.out_y, self.avg_velocity = cal.cal_osc( - self, self.rawx, self.rawy - ) + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.DADDY + def AHSFRACM(self): + if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left: + self.current_image_gray, self.cct = circle_crop( + self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct + ) + else: + pass + if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right: + self.current_image_gray, self.cct = circle_crop( + self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct + ) + else: + pass + + self.hasrac_en = True + ( + self.current_image_gray, + resize_img, + self.rawx, + self.rawy, + self.radius, + ) = External_Run_AHSF(self.current_image_gray) + self.current_image_gray_clean = resize_img.copy() + + self.thresh = resize_img + ( + self.rawx, + self.rawy, + self.angle, + self.thresh, + ranblink, + self.pupil_width, + self.pupil_height, + ) = RANSAC3D(self, True) + if self.settings.gui_RANSACBLINK: # might be redundant + self.eyeopen = ranblink + # print("RANBLINK", ranblink) + + # print(self.radius) + # if self.prev_x is None: + # self.prev_x = self.rawx + # self.prev_y = self.rawy + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) + self.current_algorithm = EyeInfoOrigin.HSRAC + def HSRACM(self): if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left: self.current_image_gray, self.cct = circle_crop( @@ -425,13 +467,11 @@ class EyeProcessor: pass self.hasrac_en = True - # todo: add process to initialise er_hsrac when resolution changes - self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run( - self.current_image_gray - ) + self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray) ( self.rawx, self.rawy, + self.angle, self.thresh, ranblink, self.pupil_width, @@ -445,9 +485,7 @@ class EyeProcessor: # if self.prev_x is None: # self.prev_x = self.rawx # self.prev_y = self.rawy - self.out_x, self.out_y, self.avg_velocity = cal.cal_osc( - self, self.rawx, self.rawy - ) + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.HSRAC def HSFM(self): @@ -464,12 +502,8 @@ class EyeProcessor: else: pass # todo: add process to initialise er_hsf when resolution changes - self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run( - self.current_image_gray - ) - self.out_x, self.out_y, self.avg_velocity = cal.cal_osc( - self, self.rawx, self.rawy - ) + self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray) + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.HSF def RANSAC3DM(self): @@ -486,12 +520,11 @@ class EyeProcessor: else: pass self.hasrac_en = False - current_image_gray_copy = ( - self.current_image_gray.copy() - ) # Duplicate before overwriting in RANSAC3D. + current_image_gray_copy = self.current_image_gray.copy() # Duplicate before overwriting in RANSAC3D. ( self.rawx, self.rawy, + self.angle, self.thresh, ranblink, self.pupil_width, @@ -499,11 +532,33 @@ class EyeProcessor: ) = RANSAC3D(self, True) if self.settings.gui_RANSACBLINK: self.eyeopen = ranblink - self.out_x, self.out_y, self.avg_velocity = cal.cal_osc( - self, self.rawx, self.rawy - ) + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.RANSAC + def AHSFM(self): + if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left: + self.current_image_gray, self.cct = circle_crop( + self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct + ) + else: + pass + if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right: + self.current_image_gray, self.cct = circle_crop( + self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct + ) + else: + pass + ( + self.current_image_gray, + resize_img, + self.rawx, + self.rawy, + self.radius, + ) = External_Run_AHSF(self.current_image_gray) + self.thresh = self.current_image_gray + self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) + self.current_algorithm = EyeInfoOrigin.HSF + def BLOBM(self): if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left: self.current_image_gray, self.cct = circle_crop( @@ -519,41 +574,41 @@ class EyeProcessor: pass self.rawx, self.rawy, self.thresh = BLOB(self) - self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) + self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.BLOB def ALGOSELECT(self): - # self.DADDYM() + # send the tracking algos previous fail number, in algo if we pass set to 0, if fail, + 1 if self.failed == 0 and self.firstalgo != None: self.firstalgo() else: self.failed = self.failed + 1 - - if ( - self.failed == 1 and self.secondalgo != None - ): # send the tracking algos previous fail number, in algo if we pass set to 0, if fail, + 1 + if self.failed == 1 and self.secondalgo != None: self.secondalgo() else: self.failed = self.failed + 1 - if self.failed == 2 and self.thirdalgo != None: self.thirdalgo() else: self.failed = self.failed + 1 - if self.failed == 3 and self.fourthalgo != None: self.fourthalgo() else: self.failed = self.failed + 1 - if self.failed == 4 and self.fithalgo != None: self.fithalgo() else: self.failed = self.failed + 1 - if self.failed == 5 and self.sixthalgo != None: self.sixthalgo() - + else: + self.failed = self.failed + 1 + if self.failed == 6 and self.seventhalgo != None: + self.seventhalgo() + else: + self.failed = self.failed + 1 + if self.failed == 7 and self.eigthalgo != None: + self.eigthalgo() else: self.failed = 0 # we have reached last possible algo and it is disabled, move to first algo @@ -567,11 +622,22 @@ class EyeProcessor: self.fourthalgo = None self.fithalgo = None self.sixthalgo = None - algolist = [None, None, None, None, None, None, None] + self.seventhalgo = None + self.eigthalgo = None + algolist = [None, None, None, None, None, None, None, None, None] + # clear HSF values when page is opened to correctly reflect setting changes self.er_hsf = None + # algolist[self.settings.gui_HSFP] = self.HSFM + # set algo priorities + if self.settings.gui_AHSFRAC: + algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM + + if self.settings.gui_AHSF: + algolist[self.settings.gui_AHSFP] = self.AHSFM + if self.settings.gui_HSF: if self.er_hsf is None: if self.eye_id in [EyeId.LEFT]: @@ -588,16 +654,16 @@ class EyeProcessor: ) else: pass + algolist[self.settings.gui_HSFP] = self.HSFM + else: if self.er_hsf is not None: self.er_hsf = None if self.settings.gui_HSRAC: if self.er_hsf is None: - if self.eye_id in [EyeId.LEFT]: - self.er_hsf = External_Run_HSF( self.settings.gui_skip_autoradius, self.settings.gui_HSF_radius_left, @@ -611,6 +677,7 @@ class EyeProcessor: ) else: pass + algolist[self.settings.gui_HSRACP] = self.HSRACM else: if not self.settings.gui_HSF and self.er_hsf is not None: @@ -646,11 +713,14 @@ class EyeProcessor: self.fourthalgo, self.fithalgo, self.sixthalgo, + self.seventhalgo, + self.eigthalgo, ) = algolist f = True while True: # f = True + # print(self.capture_queue_incoming.qsize()) # Check to make sure we haven't been requested to close if self.cancellation_event.is_set(): print("\033[94m[INFO] Exiting Tracking thread\033[0m") @@ -676,9 +746,7 @@ class EyeProcessor: focal_length=self.config.focal_length, resolution=(self.config.roi_window_w, self.config.roi_window_h), ) - self.detector_3d = Detector3D( - camera=self.camera_model, long_term_mode=DetectorMode.blocking - ) + self.detector_3d = Detector3D(camera=self.camera_model, long_term_mode=DetectorMode.blocking) try: if self.capture_queue_incoming.empty(): @@ -688,7 +756,7 @@ class EyeProcessor: self.current_image, self.current_frame_number, self.current_fps, - ) = self.capture_queue_incoming.get(block=True, timeout=0.2) + ) = self.capture_queue_incoming.get(block=True, timeout=0.1) except queue.Empty: # print("No image available") continue @@ -696,12 +764,14 @@ class EyeProcessor: if not self.capture_crop_rotate_image(): continue - self.current_image_gray = cv2.cvtColor( - self.current_image, cv2.COLOR_BGR2GRAY - ) + self.current_image_gray = cv2.cvtColor(self.current_image, cv2.COLOR_BGR2GRAY) self.current_image_gray_clean = ( self.current_image_gray.copy() ) # copy this frame to have a clean image for blink algo - self.ALGOSELECT() # run our algos in priority order set in settings - self.UPDATE() + if self.cancellation_event.is_set(): + print("\033[94m[INFO] Exiting Tracking thread\033[0m") + return + else: + self.ALGOSELECT() # run our algos in priority order set in settings + self.UPDATE()