From dd23dcf83827053c7a8d0bcdad219962f3bd5c82 Mon Sep 17 00:00:00 2001 From: PallasNeko <124042774+PallasNeko@users.noreply.github.com> Date: Wed, 1 Feb 2023 23:50:08 +0900 Subject: [PATCH 1/5] fix and move --- EyeTrackApp/haar_surround_feature.py | 34 ++- EyeTrackApp/hsrac.py | 250 ++++++++++++------ EyeTrackApp/utils/__init__.py | 0 EyeTrackApp/{ => utils}/img_utils.py | 6 +- EyeTrackApp/{utils.py => utils/misc_utils.py} | 0 EyeTrackApp/{ => utils}/time_utils.py | 0 6 files changed, 197 insertions(+), 93 deletions(-) create mode 100644 EyeTrackApp/utils/__init__.py rename EyeTrackApp/{ => utils}/img_utils.py (72%) rename EyeTrackApp/{utils.py => utils/misc_utils.py} (100%) rename EyeTrackApp/{ => utils}/time_utils.py (100%) diff --git a/EyeTrackApp/haar_surround_feature.py b/EyeTrackApp/haar_surround_feature.py index 16bec9b..be0cc7e 100644 --- a/EyeTrackApp/haar_surround_feature.py +++ b/EyeTrackApp/haar_surround_feature.py @@ -7,12 +7,13 @@ from functools import lru_cache import cv2 import numpy as np -from img_utils import safe_crop +from utils.misc_utils import clamp +from utils.img_utils import safe_crop # from line_profiler_pycharm import profile video_path = "ezgif.com-gif-maker.avi" -imshow_enable = True +imshow_enable = False calc_print_enable = True save_video = False skip_autoradius = False @@ -619,8 +620,10 @@ class HSF_cls(object): def single_run(self): # Temporary implementation to run - ## default_radius = 14 - + # default_radius = 14 + + # cropbox=[] # debug code + frame = self.current_image_gray if self.now_modeo == self.cv_modeo[1]: # adjustment of radius @@ -673,6 +676,9 @@ class HSF_cls(object): # Crop the image using the calculated bounds cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y) + # cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in + # zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code + if self.now_modeo == self.cv_modeo[0] or self.now_modeo == self.cv_modeo[1]: # If mode is first_frame or radius_adjust, record current radius and response self.auto_radius_calc.add_response(radius, response) @@ -686,7 +692,7 @@ class HSF_cls(object): upper_y = center_y + self.center_correct.center_q1_radius lower_y = center_y - self.center_correct.center_q1_radius self.center_q1.add_response( - cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y))[ + cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y,keepsize=False))[ 0 ] ) @@ -714,6 +720,12 @@ class HSF_cls(object): self.center_correct.init_array( gray_frame.shape, self.center_q1.quartile_1, radius ) + elif self.center_correct.frame_shape!=gray_frame.shape: + """The resolution should have changed and the statistics should have changed, so essentially the statistics + need to be reworked, but implementation will be postponed as viability is the highest priority. """ + self.center_correct.init_array( + gray_frame.shape, self.center_q1.quartile_1, radius + ) center_x, center_y = self.center_correct.correction( gray_frame, center_x, center_y @@ -728,6 +740,9 @@ class HSF_cls(object): cropped_image = safe_crop( gray_frame, lower_x, lower_y, upper_x, upper_y ) + # cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in + # zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code + # if imshow_enable or save_video: # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) # cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1) @@ -765,20 +780,27 @@ class HSF_cls(object): self.now_modeo = self.cv_modeo[2] else: self.now_modeo = self.cv_modeo[1] - + + # debug code + # return center_x,center_y,cropbox,frame return center_x, center_y, frame + class External_Run_HSF(object): def __init__(self): self.algo = HSF_cls() def run(self, current_image_gray): self.algo.current_image_gray = current_image_gray + # debug code + # center_x, center_y,cropbox, frame = self.algo.single_run() + # return center_x, center_y,cropbox, frame center_x, center_y, frame = self.algo.single_run() return center_x, center_y, frame + if __name__ == "__main__": hsf = HSF_cls() hsf.open_video(video_path) diff --git a/EyeTrackApp/hsrac.py b/EyeTrackApp/hsrac.py index 521d896..6d8ab57 100644 --- a/EyeTrackApp/hsrac.py +++ b/EyeTrackApp/hsrac.py @@ -1,7 +1,4 @@ -import math -import sys import timeit -from functools import lru_cache import cv2 import numpy as np @@ -9,13 +6,12 @@ import numpy as np from haar_surround_feature import ( AutoRadiusCalc, BlinkDetector, - CenterCorrection, - CvParameters, - conv_int, + CvParameters, conv_int, frameint_get_xy_step, ) -from img_utils import safe_crop -from utils import clamp +from utils.img_utils import safe_crop +from utils.misc_utils import clamp + # from line_profiler_pycharm import profile @@ -190,7 +186,6 @@ class HSRAC_cls(object): self.auto_radius_calc = AutoRadiusCalc() self.blink_detector = BlinkDetector() self.center_q1 = BlinkDetector() - self.center_correct = CenterCorrection() self.cap = None @@ -204,7 +199,10 @@ class HSRAC_cls(object): # ransac self.rng = np.random.default_rng() - self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) + + # self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) + # or + self.kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3)) def open_video(self, video_path): # Temporary implementation to run @@ -221,17 +219,20 @@ class HSRAC_cls(object): ret, frame = self.cap.read() if ret: # I have set it to grayscale (1ch) just in case, but if the frame is 1ch, this line can be commented out. + # self.current_image=frame # debug code self.current_image_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) return True return False def single_run(self): # Temporary implementation to run - - ## default_radius = 14 - + # default_radius = 14 + + # ori_frame = self.current_image.copy()# debug code + # cropbox=[] # debug code + + blink_bd = False frame = self.current_image_gray - if self.now_modeo == self.cv_modeo[1]: # adjustment of radius @@ -282,7 +283,9 @@ class HSRAC_cls(object): # Crop the image using the calculated bounds cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y) - + + # cropbox=[clamp(val, 0, gray_frame.shape[i]) for i,val in zip([1,0,1,0],[lower_x,lower_y,upper_x,upper_y])] # debug code + if self.now_modeo == self.cv_modeo[0] or self.now_modeo == self.cv_modeo[1]: # If mode is first_frame or radius_adjust, record current radius and response self.auto_radius_calc.add_response(radius, response) @@ -291,12 +294,12 @@ class HSRAC_cls(object): if self.blink_detector.response_len() < blink_init_frames: self.blink_detector.add_response(cv2.mean(cropped_image)[0]) - upper_x = center_x + self.center_correct.center_q1_radius - lower_x = center_x - self.center_correct.center_q1_radius - upper_y = center_y + self.center_correct.center_q1_radius - lower_y = center_y - self.center_correct.center_q1_radius + upper_x = center_x + 20 + lower_x = center_x - 20 + upper_y = center_y + 20 + lower_y = center_y - 20 self.center_q1.add_response( - cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y))[ + cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y,keepsize=False))[ 0 ] ) @@ -307,7 +310,7 @@ class HSRAC_cls(object): self.center_q1.calc_thresh() self.now_modeo = self.cv_modeo[3] else: - if 0 in cropped_image.shape: + if 0 in cropped_image.shape: # This line may not be needed. The image will be cropped using safecrop. # If shape contains 0, it is not detected well. print("Something's wrong.") else: @@ -317,30 +320,11 @@ class HSRAC_cls(object): # (i.e., if the cropimage is whitish if self.blink_detector.detect(cv2.mean(cropped_image)[0]): # blink - pass - else: - # pass - if not self.center_correct.setup_comp: - self.center_correct.init_array( - gray_frame.shape, self.center_q1.quartile_1, radius - ) - - center_x, center_y = self.center_correct.correction( - gray_frame, center_x, center_y - ) - # Define the center point and radius - center_xy = (center_x, center_y) - upper_x = center_x + radius - lower_x = center_x - radius - upper_y = center_y + radius - lower_y = center_y - radius - # Crop the image using the calculated bounds - cropped_image = safe_crop( - gray_frame, lower_x, lower_y, upper_x, upper_y - ) + print("BLINK BD") + blink_bd=True # if imshow_enable or save_video: # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) - # cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1) + # cv2.circle(ori_frame, (center_x, center_y), 7, (255, 0, 0), -1) # If you want to update response_max. it may be more cost-effective to rewrite response_list in the following way # https://stackoverflow.com/questions/42771110/fastest-way-to-left-cycle-a-numpy-array-like-pop-push-for-a-queue @@ -362,7 +346,6 @@ class HSRAC_cls(object): # If shape contains 0, it is not detected well. pass else: - cv2.imshow("crop", cropped_image) cv2.imshow("frame", frame) if cv2.waitKey(1) & 0xFF == ord("q"): @@ -377,8 +360,6 @@ class HSRAC_cls(object): else: self.now_modeo = self.cv_modeo[1] - newFrame2 = frame.copy() - # frame = cropped_image # For measuring processing time of image processing cv_start_time = timeit.default_timer() # Crop first to reduce the amount of data to process. @@ -386,37 +367,56 @@ class HSRAC_cls(object): # To reduce the processing data, first convert to 1-channel and then blur. # The processing results were the same when I swapped the order of blurring and 1-channelization. frame_gray = cv2.GaussianBlur(frame, (5, 5), 0) - - upper_x = center_x + 20 - lower_x = center_x - 20 - upper_y = center_y + 20 - lower_y = center_y - 20 + hsf_center_x, hsf_center_y = center_x.copy(), center_y.copy() + ransac_xy_offset = (hsf_center_x-20, hsf_center_y-20) + upper_x = hsf_center_x + 20 + lower_x = hsf_center_x - 20 + upper_y = hsf_center_y + 20 + lower_y = hsf_center_y - 20 # Crop the image using the calculated bounds - frame_gray = safe_crop(frame_gray, lower_x, lower_y, upper_x, upper_y) - frame = frame_gray + frame_gray_crop = safe_crop(frame_gray, lower_x, lower_y, upper_x, upper_y) + frame = frame_gray_crop # this will need to be adjusted everytime hardware is changed (brightness of IR, Camera postion, etc)m - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(frame_gray) - - maxloc0_hf, maxloc1_hf = int(0.5 * max_loc[0]), int(0.5 * max_loc[1]) - - # crop 15% sqare around min_loc - # frame_gray = frame_gray[max_loc[1] - maxloc1_hf:max_loc[1] + maxloc1_hf, - # max_loc[0] - maxloc0_hf:max_loc[0] + maxloc0_hf] + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(frame_gray_crop) threshold_value = min_val + thresh_add - _, thresh = cv2.threshold(frame_gray, threshold_value, 255, cv2.THRESH_BINARY) + _, thresh = cv2.threshold(frame_gray_crop, threshold_value, 255, cv2.THRESH_BINARY) # print(thresh.shape, frame_gray.shape) try: + opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, self.kernel) closing = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, self.kernel) th_frame = 255 - closing except: # I want to eliminate try here because try tends to be slow in execution. - th_frame = 255 - frame_gray + th_frame = 255 - frame_gray_crop contours, _ = cv2.findContours(th_frame, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) + # or + # contours, _=cv2.findContours(th_frame, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + + if not blink_bd and self.blink_detector.enable_detect_flg: + threshold_value = self.center_q1.quartile_1 + if threshold_value Date: Tue, 31 Jan 2023 12:02:25 -0800 Subject: [PATCH 2/5] Revert "Cleaning+SafeCrop" --- EyeTrackApp/eye_processor.py | 304 ++++++++++++++++------------------- EyeTrackApp/ransac.py | 2 +- 2 files changed, 143 insertions(+), 163 deletions(-) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index c7d8903..ac6cef5 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -1,4 +1,4 @@ -""" +''' ------------------------------------------------------------------------------------------------------ ,@@@@@@ @@ -19,8 +19,8 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -HSR By: PallasNeko (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) +HSR By: Sean.Denka (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) BLOB By: Prohurtz#0001 (Main App Developer) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) @@ -28,7 +28,7 @@ Additional Contributors: [Assassin], Summer404NotFound, lorow, ZanzyTHEbar Copyright (c) 2022 EyeTrackVR <3 ------------------------------------------------------------------------------------------------------ -""" +''' from operator import truth from dataclasses import dataclass @@ -46,20 +46,19 @@ import numpy as np import cv2 from enum import Enum from one_euro_filter import OneEuroFilter - if sys.platform.startswith("win"): from winsound import PlaySound, SND_FILENAME, SND_ASYNC import importlib from osc_calibrate_filter import * -from haar_surround_feature import External_Run_HSF +from haar_surround_feature import * from blob import * from ransac import * -from hsrac import External_Run_HSRACS +from hsrac import * from blink import * -from intensity_eye_open import * +from intensity_eye_open import * class InformationOrigin(Enum): RANSAC = 1 @@ -68,10 +67,7 @@ class InformationOrigin(Enum): HSF = 4 HSRAC = 5 - bbb = 0 - - @dataclass class EyeInformation: info_type: InformationOrigin @@ -89,7 +85,7 @@ def run_once(f): if not wrapper.has_run: wrapper.has_run = True return f(*args, **kwargs) - + wrapper.has_run = False return wrapper @@ -98,35 +94,36 @@ async def delayed_setting_change(setting, value): await asyncio.sleep(5) setting = value if sys.platform.startswith("win"): - PlaySound("Audio/compleated.wav", SND_FILENAME | SND_ASYNC) + PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC) + class EyeProcessor: def __init__( - self, - config: "EyeTrackCameraConfig", - settings: "EyeTrackSettingsConfig", - cancellation_event: "threading.Event", - capture_event: "threading.Event", - capture_queue_incoming: "queue.Queue", - image_queue_outgoing: "queue.Queue", - eye_id, + self, + config: "EyeTrackCameraConfig", + settings: "EyeTrackSettingsConfig", + cancellation_event: "threading.Event", + capture_event: "threading.Event", + capture_queue_incoming: "queue.Queue", + image_queue_outgoing: "queue.Queue", + eye_id, ): self.config = config self.settings = settings - + # Cross-thread communication management self.capture_queue_incoming = capture_queue_incoming self.image_queue_outgoing = image_queue_outgoing self.cancellation_event = cancellation_event self.capture_event = capture_event self.eye_id = eye_id - + # Cross algo state self.lkg_projected_sphere = None self.xc = None self.yc = None - + # Image state self.previous_image = None self.current_image = None @@ -134,14 +131,14 @@ class EyeProcessor: self.current_frame_number = None self.current_fps = None self.threshold_image = None - + # Calibration Values self.xoff = 1 self.yoff = 1 # Keep large in order to recenter correctly self.calibration_frame_counter = None self.eyeoffx = 1 - + self.xmax = -69420 self.xmin = 69420 self.ymax = -69420 @@ -150,41 +147,44 @@ class EyeProcessor: self.cccs = False self.ts = 10 self.previous_rotation = self.config.rotation_angle + self.calibration_frame_counter self.camera_model = None self.detector_3d = None - + self.camera_model = None self.detector_3d = None - + self.failed = 0 - + self.skip_blink_detect = False - - # blink + + #blink self.max_ints = [] self.max_int = 0 self.min_int = 4000000000000 - self.frames = 0 + self.frames = 0 self.blinkvalue = False self.prev_x = None self.prev_y = None + + try: min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004 beta = float(self.settings.gui_speed_coefficient) # 0.9 except: - print("\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m") + print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m') min_cutoff = 0.0004 beta = 0.9 noisy_point = np.array([1, 1]) self.one_euro_filter = OneEuroFilter( - noisy_point, min_cutoff=min_cutoff, beta=beta + noisy_point, + min_cutoff=min_cutoff, + beta=beta ) - - def output_images_and_update( - self, threshold_image, output_information: EyeInformation - ): + + def output_images_and_update(self, threshold_image, output_information: EyeInformation): try: image_stack = np.concatenate( ( @@ -196,32 +196,29 @@ class EyeProcessor: self.image_queue_outgoing.put((image_stack, output_information)) self.previous_image = self.current_image self.previous_rotation = self.config.rotation_angle - except: # If this fails it likely means that the images are not the same size for some reason. - print( - "\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m" - ) - + except: # If this fails it likely means that the images are not the same size for some reason. + print('\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m') + pass - def capture_crop_rotate_image(self): # Get our current frame try: # Get frame from capture source, crop to ROI self.current_image = self.current_image[ - int(self.config.roi_window_y): int( - self.config.roi_window_y + self.config.roi_window_h - ), - int(self.config.roi_window_x): int( - self.config.roi_window_x + self.config.roi_window_w - ), - ] - + int(self.config.roi_window_y): int( + self.config.roi_window_y + self.config.roi_window_h + ), + int(self.config.roi_window_x): int( + self.config.roi_window_x + self.config.roi_window_w + ), + ] + except: # Failure to process frame, reuse previous frame. self.current_image = self.previous_image print("\033[91m[ERROR] Frame capture issue detected.\033[0m") - + try: # Apply rotation to cropped area. For any rotation area outside of the bounds of the image, # fill with white. @@ -243,119 +240,96 @@ class EyeProcessor: return True except: pass - + def BLINKM(self): self.blinkvalue = BLINK(self) - + + def HSRACM(self): - # temporary implementation - cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS().run( - self.current_image_gray - ) + cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS.HSRACS(self) self.current_image_gray = gray_frame - if self.prev_x is None: + if self.prev_x == None: self.prev_x = cx self.prev_y = cy - # print(self.prev_x, self.prev_y, cx, cy) + #print(self.prev_x, self.prev_y, cx, cy) # #filter values with too much movement - # if (cx - self.prev_x) <= 45 and (cy - self.prev_y) <= 45 : - # self.prev_x = cx - # self.prev_y = cy + # if (cx - self.prev_x) <= 45 and (cy - self.prev_y) <= 45 : + # self.prev_x = cx + # self.prev_y = cy eyeopen = intense(cx, cy, uncropframe) out_x, out_y = cal_osc(self, cx, cy) - + if cx == 0: - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen), - ) # update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app else: - - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen), - ) - - # else: - # print("EYE MOVED TOO FAST") - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False)) + + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) + # else: + # print("EYE MOVED TOO FAST") + # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False)) def HSFM(self): - # temporary implementation - cx, cy, frame = External_Run_HSF().run(self.current_image_gray) + cx, cy, frame = External_Run_HSF.HSFS(self) eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update( - frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen) - ) # update app + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) #update app else: - self.output_images_and_update( - frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen) - ) - + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) + def RANSAC3DM(self): cx, cy, thresh = RANSAC3D(self) eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen), - ) # update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) #update app else: - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen), - ) - + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) + def BLOBM(self): cx, cy, thresh = BLOB(self) eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen), - ) # update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app else: - self.output_images_and_update( - thresh, - EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen), - ) - - def ALGOSELECT(self): + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) - if self.failed == 0 and self.firstalgo is not None: + + + def ALGOSELECT(self): + + if self.failed == 0 and self.firstalgo != None: self.firstalgo() else: self.failed = self.failed + 1 - - if ( - self.failed == 1 and self.secondalgo is not None - ): # send the tracking algos previous fail number, in algo if we pass set to 0, if fail, + 1 + + if self.failed == 1 and self.secondalgo != None: #send the tracking algos previous fail number, in algo if we pass set to 0, if fail, + 1 self.secondalgo() else: self.failed = self.failed + 1 - - if self.failed == 2 and self.thirdalgo is not None: + + if self.failed == 2 and self.thirdalgo != None: self.thirdalgo() else: self.failed = self.failed + 1 - - if self.failed == 3 and self.fourthalgo is not None: + + if self.failed == 3 and self.fourthalgo != None: self.fourthalgo() else: - self.failed = 0 # we have reached last possible algo and it is disabled, move to first algo - + self.failed = 0 # we have reached last possible algo and it is disabled, move to first algo + + + + def run(self): - + self.firstalgo = None self.secondalgo = None self.thirdalgo = None self.fourthalgo = None - # set algo priorities - - if (self.settings.gui_HSF and self.settings.gui_HSFP == 1): # I feel like this is super innefficient though it only runs at startup and no solution is coming to me atm + #set algo priorities + + if self.settings.gui_HSF and self.settings.gui_HSFP == 1: #I feel like this is super innefficient though it only runs at startup and no solution is coming to me atm self.firstalgo = self.HSFM elif self.settings.gui_HSF and self.settings.gui_HSFP == 2: self.secondalgo = self.HSFM @@ -363,7 +337,7 @@ class EyeProcessor: self.thirdalgo = self.HSFM elif self.settings.gui_HSF and self.settings.gui_HSFP == 4: self.fourthalgo = self.HSFM - + if self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 1: self.firstalgo = self.RANSAC3DM elif self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 2: @@ -372,8 +346,8 @@ class EyeProcessor: self.thirdalgo = self.RANSAC3DM elif self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 4: self.fourthalgo = self.RANSAC3DM - - if self.settings.gui_HSRAC and self.settings.gui_HSRACP == 1: + + if self.settings.gui_HSRAC == True and self.settings.gui_HSRACP == 1: self.firstalgo = self.HSRACM elif self.settings.gui_HSRAC and self.settings.gui_HSRACP == 2: self.secondalgo = self.HSRACM @@ -381,7 +355,7 @@ class EyeProcessor: self.thirdalgo = self.HSRACM elif self.settings.gui_HSRAC and self.settings.gui_HSRACP == 4: self.fourthalgo = self.HSRACM - + if self.settings.gui_BLOB and self.settings.gui_BLOBP == 1: self.firstalgo = self.BLOBM elif self.settings.gui_BLOB and self.settings.gui_BLOBP == 2: @@ -390,31 +364,31 @@ class EyeProcessor: self.thirdalgo = self.BLOBM elif self.settings.gui_BLOB and self.settings.gui_BLOBP == 4: self.fourthalgo = self.BLOBM - + f = True while True: - # f = True - # Check to make sure we haven't been requested to close + # f = True + # Check to make sure we haven't been requested to close if self.cancellation_event.is_set(): print("\033[94m[INFO] Exiting Tracking thread\033[0m") return - + if self.config.roi_window_w <= 0 or self.config.roi_window_h <= 0: # At this point, we're waiting for the user to set up the ROI window in the GUI. # Sleep a bit while we wait. if self.cancellation_event.wait(0.1): return continue + + # If our ROI configuration has changed, reset our model and detector - if ( - self.camera_model is None - or self.detector_3d is None - or self.camera_model.resolution - != ( + if (self.camera_model is None + or self.detector_3d is None + or self.camera_model.resolution != ( self.config.roi_window_w, self.config.roi_window_h, - ) + ) ): self.camera_model = CameraModel( focal_length=self.config.focal_length, @@ -423,7 +397,7 @@ class EyeProcessor: self.detector_3d = Detector3D( camera=self.camera_model, long_term_mode=DetectorMode.blocking ) - + try: if self.capture_queue_incoming.empty(): self.capture_event.set() @@ -439,35 +413,41 @@ class EyeProcessor: if not self.capture_crop_rotate_image(): continue + self.current_image_gray = cv2.cvtColor( - self.current_image, cv2.COLOR_BGR2GRAY + self.current_image, cv2.COLOR_BGR2GRAY ) - self.current_image_gray_clean = ( - self.current_image_gray.copy() - ) # copy this frame to have a clean image for blink algo - # print(self.settings.gui_RANSAC3D) + self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo + # print(self.settings.gui_RANSAC3D) + + # BLINK(self) + + # cx, cy, thresh = HSRAC(self) + # out_x, out_y = cal_osc(self, cx, cy) + # if cx == 0: + # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, True)) #update app + # else: + # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.blinkvalue)) - # BLINK(self) + + # cx, cy, thresh = RANSAC3D(self) + # out_x, out_y = cal_osc(self, cx, cy) + # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, False)) #update app + + + # cx, cy, larger_threshold = BLOB(self) + # out_x, out_y = cal_osc(self, cx, cy) + # self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False)) #update app + + #center_x, center_y, frame = HSF(self) #run algo + #out_x, out_y = cal_osc(self, center_x, center_y) #filter and calibrate + #self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, False)) #update app - # cx, cy, thresh = HSRAC(self) - # out_x, out_y = cal_osc(self, cx, cy) - # if cx == 0: - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, True)) #update app - # else: - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.blinkvalue)) - - # cx, cy, thresh = RANSAC3D(self) - # out_x, out_y = cal_osc(self, cx, cy) - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, False)) #update app - - # cx, cy, larger_threshold = BLOB(self) - # out_x, out_y = cal_osc(self, cx, cy) - # self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False)) #update app - - # center_x, center_y, frame = HSF(self) #run algo - # out_x, out_y = cal_osc(self, center_x, center_y) #filter and calibrate - # self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, False)) #update app - - self.ALGOSELECT() # run our algos in priority order set in settings + self.ALGOSELECT() #run our algos in priority order set in settings self.BLINKM() + + + + + diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 06d7c40..355600a 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -19,7 +19,7 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) Copyright (c) 2022 EyeTrackVR <3 From 2624080d58f071c510437fd020ed7f68c29d6981 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Tue, 31 Jan 2023 16:05:47 -0600 Subject: [PATCH 3/5] intensity dual eye & bug hunting --- EyeTrackApp/eye_processor.py | 26 +++++++++++++------------- EyeTrackApp/intensity_eye_open.py | 26 +++++++++++++++++++++----- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index ac6cef5..1434e67 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -242,7 +242,7 @@ class EyeProcessor: pass def BLINKM(self): - self.blinkvalue = BLINK(self) + self.eyeoffx = BLINK(self) def HSRACM(self): @@ -256,43 +256,43 @@ class EyeProcessor: # if (cx - self.prev_x) <= 45 and (cy - self.prev_y) <= 45 : # self.prev_x = cx # self.prev_y = cy - eyeopen = intense(cx, cy, uncropframe) + self.eyeopen = intense(cx, cy, uncropframe) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) # else: # print("EYE MOVED TOO FAST") # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False)) def HSFM(self): cx, cy, frame = External_Run_HSF.HSFS(self) - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen)) def RANSAC3DM(self): cx, cy, thresh = RANSAC3D(self) - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen)) def BLOBM(self): cx, cy, thresh = BLOB(self) - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) diff --git a/EyeTrackApp/intensity_eye_open.py b/EyeTrackApp/intensity_eye_open.py index 0dcb48f..fdf0c68 100644 --- a/EyeTrackApp/intensity_eye_open.py +++ b/EyeTrackApp/intensity_eye_open.py @@ -1,11 +1,17 @@ import pandas as pd import numpy as np import time + +from enum import IntEnum #higher intensity means more closed/ more white/less pupil #Hm I need an acronym for this, any ideas? #IBO Intensity Based Openess - +class EyeId(IntEnum): + RIGHT = 0 + LEFT = 1 + BOTH = 2 + SETTINGS = 3 # HOW THIS WORKS: # we get the intensity of pupil area from HSF crop, When the eyelid starts to close, the pupil starts being obstructed by skin which is generally lighter than the pupil. @@ -13,8 +19,14 @@ import time # ex. when you look up there is less pupil visible, which results in an uncalculated change in intensity even though the eyelid has not moved in a meaningful way. # We compare the darkest intensity of that area, to the lightest (global) intensity to find the appropriate openness state via a float. -fname = "IBO.csv" #TODO Expose as setting + +if EyeId.RIGHT: + fname = "IBO_RIGHT.csv" +if EyeId.LEFT: + fname = "IBO_LEFT.csv" + lct = time.time() + try: data = pd.read_csv(fname, sep=",") except: @@ -27,15 +39,18 @@ except: def intense(x, y, frame): global lct + upper_x = int(x) + 25 #TODO make this a setting lower_x = int(x) - 25 upper_y = int(y) + 25 lower_y = int(y) - 25 frame = frame[lower_y:upper_y, lower_x:upper_x] + print(x, y, int(x), int(y), upper_x, upper_y, lower_x, lower_y) try: xy = int(str(int(x)) + str(int(y)) + str(int(x)+int(y))) - intensity = np.sum(frame) + intensity = np.sum(frame) #why is this outputting 0s? + # print(intensity, upper_x, upper_y, lower_x, lower_y) except: return 0.0 #TODO find how on earth a hyphen gets thrown into this @@ -78,9 +93,10 @@ def intense(x, y, frame): #eyeopen = (intensity - minp) / (maxp - minp) eyeopen = (intensity - maxp) / (minp - maxp) eyeopen = 1 - eyeopen + eyeopen = eyeopen - 0.2 # print(intensity, maxp, minp, x, y) - print(f"EYEOPEN: {eyeopen}") - + # print(f"EYEOPEN: {eyeopen}") + # print(int(x), int(y), eyeopen, maxp, minp) except: print('[INFO] Something went wrong, assuming blink.') From e3b47f0b68e11d502e978e52cf908c5dd0abd3a3 Mon Sep 17 00:00:00 2001 From: PallasNeko <124042774+PallasNeko@users.noreply.github.com> Date: Thu, 2 Feb 2023 01:21:52 +0900 Subject: [PATCH 4/5] Fixing the call --- EyeTrackApp/eye_processor.py | 17 ++++++++--------- EyeTrackApp/haar_surround_feature.py | 5 +---- EyeTrackApp/ransac.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 1434e67..2fff820 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -19,8 +19,8 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -HSR By: Sean.Denka (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) +HSR By: PallasNeko (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) BLOB By: Prohurtz#0001 (Main App Developer) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) @@ -51,10 +51,10 @@ if sys.platform.startswith("win"): import importlib from osc_calibrate_filter import * -from haar_surround_feature import * +from haar_surround_feature import External_Run_HSF from blob import * from ransac import * -from hsrac import * +from hsrac import External_Run_HSRACS from blink import * @@ -147,7 +147,6 @@ class EyeProcessor: self.cccs = False self.ts = 10 self.previous_rotation = self.config.rotation_angle - self.calibration_frame_counter self.camera_model = None self.detector_3d = None @@ -246,9 +245,9 @@ class EyeProcessor: def HSRACM(self): - cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS.HSRACS(self) + cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS().run(self.current_image_gray) self.current_image_gray = gray_frame - if self.prev_x == None: + if self.prev_x is None: self.prev_x = cx self.prev_y = cy #print(self.prev_x, self.prev_y, cx, cy) @@ -268,7 +267,7 @@ class EyeProcessor: # print("EYE MOVED TOO FAST") # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False)) def HSFM(self): - cx, cy, frame = External_Run_HSF.HSFS(self) + cx, cy, frame = External_Run_HSF().run(self.current_image_gray) self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: @@ -347,7 +346,7 @@ class EyeProcessor: elif self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 4: self.fourthalgo = self.RANSAC3DM - if self.settings.gui_HSRAC == True and self.settings.gui_HSRACP == 1: + if self.settings.gui_HSRAC and self.settings.gui_HSRACP == 1: self.firstalgo = self.HSRACM elif self.settings.gui_HSRAC and self.settings.gui_HSRACP == 2: self.secondalgo = self.HSRACM diff --git a/EyeTrackApp/haar_surround_feature.py b/EyeTrackApp/haar_surround_feature.py index be0cc7e..4c03139 100644 --- a/EyeTrackApp/haar_surround_feature.py +++ b/EyeTrackApp/haar_surround_feature.py @@ -1,6 +1,3 @@ -import functools -import math -import sys import timeit from functools import lru_cache @@ -720,7 +717,7 @@ class HSF_cls(object): self.center_correct.init_array( gray_frame.shape, self.center_q1.quartile_1, radius ) - elif self.center_correct.frame_shape!=gray_frame.shape: + elif self.center_correct.frame_shape != gray_frame.shape: """The resolution should have changed and the statistics should have changed, so essentially the statistics need to be reworked, but implementation will be postponed as viability is the highest priority. """ self.center_correct.init_array( diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 355600a..06d7c40 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -19,7 +19,7 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) Copyright (c) 2022 EyeTrackVR <3 From fc31c0c0544cb0aa97e2fb1c0ffaabd70a8d9a3e Mon Sep 17 00:00:00 2001 From: PallasNeko <124042774+PallasNeko@users.noreply.github.com> Date: Thu, 2 Feb 2023 01:32:59 +0900 Subject: [PATCH 5/5] Fixing the call --- EyeTrackApp/haar_surround_feature.py | 187 +----- EyeTrackApp/hsrac.py | 847 +-------------------------- EyeTrackApp/ransac.py | 2 +- 3 files changed, 32 insertions(+), 1004 deletions(-) diff --git a/EyeTrackApp/haar_surround_feature.py b/EyeTrackApp/haar_surround_feature.py index 5b6b653..2e66daf 100644 --- a/EyeTrackApp/haar_surround_feature.py +++ b/EyeTrackApp/haar_surround_feature.py @@ -29,181 +29,6 @@ blink_init_frames = 60 * 3 # 60fps*3sec,Number of blink statistical frames # step==(x,y) default_step = (5, 5) # bigger the steps,lower the processing time! ofc acc also takes an impact -""" -Attention. -If using cv2.filter2D in this code, be careful with the kernel -https://stackoverflow.com/questions/39457468/convolution-without-any-padding-opencv-python -""" - - -def TimeitWrapper(*args, **kwargs): - """ - This decorator @TimeitWrapper() prints the function name and execution time in seconds. - :param args: - :param kwargs: - :return: - """ - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - start = timeit.default_timer() - results = function(*args, **kwargs) - end = timeit.default_timer() - print('{} execution time: {:.10f} s'.format(function.__name__, end - start)) - return results - - return wrapper - - return decorator - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - - class CvParameters: # It may be a little slower because a dict named "self" is read for each function call. def __init__(self, radius, step): @@ -421,7 +246,7 @@ def conv_int(frame_int, kernel, xy_step, padding, xy_steps_list): return frame_conv, min_response, center -class Auto_Radius_Calc(object): +class AutoRadiusCalc(object): def __init__(self): self.response_list = [] self.radius_cand_list = [] @@ -524,7 +349,7 @@ class Auto_Radius_Calc(object): return None -class Blink_Detector(object): +class BlinkDetector(object): def __init__(self): self.response_list = [] self.response_max = None @@ -667,7 +492,7 @@ class CenterCorrection(object): return out_x, out_y -class HSRAC_cls(object): +class HSF_cls(object): def __init__(self): # I'd like to take into account things like print, end_time - start_time processing time, etc., but it's too much trouble. @@ -681,9 +506,9 @@ class HSRAC_cls(object): self.cv_modeo = ["first_frame", "radius_adjust", "blink_adjust", "normal"] self.now_modeo = self.cv_modeo[0] - self.auto_radius_calc = Auto_Radius_Calc() - self.blink_detector = Blink_Detector() - self.center_q1 = Blink_Detector() + self.auto_radius_calc = AutoRadiusCalc() + self.blink_detector = BlinkDetector() + self.center_q1 = BlinkDetector() self.center_correct = CenterCorrection() self.cap = None diff --git a/EyeTrackApp/hsrac.py b/EyeTrackApp/hsrac.py index 760d9ad..cc4265b 100644 --- a/EyeTrackApp/hsrac.py +++ b/EyeTrackApp/hsrac.py @@ -15,6 +15,10 @@ from utils.misc_utils import clamp # from line_profiler_pycharm import profile +#RANSAC + +thresh_add = 10 + video_path = "ezgif.com-gif-maker.avi" imshow_enable = True calc_print_enable = True @@ -33,799 +37,6 @@ blink_init_frames = 60 * 3 # 60fps*3sec,Number of blink statistical frames # step==(x,y) default_step = (5, 5) # bigger the steps,lower the processing time! ofc acc also takes an impact -""" -Attention. -If using cv2.filter2D in this code, be careful with the kernel -https://stackoverflow.com/questions/39457468/convolution-without-any-padding-opencv-python -""" - - -def TimeitWrapper(*args, **kwargs): - """ - This decorator @TimeitWrapper() prints the function name and execution time in seconds. - :param args: - :param kwargs: - :return: - """ - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - start = timeit.default_timer() - results = function(*args, **kwargs) - end = timeit.default_timer() - print('{} execution time: {:.10f} s'.format(function.__name__, end - start)) - return results - - return wrapper - - return decorator - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - - -class CvParameters: - # It may be a little slower because a dict named "self" is read for each function call. - def __init__(self, radius, step): - # self.prev_radius=radius - self._radius = radius - self.pad = 2 * radius - # self.prev_step=step - self._step = step - self._hsf = HaarSurroundFeature(radius) - - def get_rpsh(self): - return self._radius, self.pad, self._step, self._hsf - # Essentially, the following would be preferable, but it would take twice as long to call. - # return self.radius, self.pad, self.step, self.hsf - - @property - def radius(self): - return self._radius - - @radius.setter - def radius(self, now_radius): - # self.prev_radius=self._radius - self._radius = now_radius - self.pad = 2 * now_radius - self.hsf = now_radius - - @property - def step(self): - return self._step - - @step.setter - def step(self, now_step): - # self.prev_step=self.step - self._step = now_step - - @property - def hsf(self): - return self._hsf - - @hsf.setter - def hsf(self, now_radius): - self._hsf = HaarSurroundFeature(now_radius) - - -class HaarSurroundFeature: - - def __init__(self, r_inner, r_outer=None, val=None): - if r_outer is None: - r_outer = r_inner * 3 - # print(r_outer) - r_inner2 = r_inner * r_inner - count_inner = r_inner2 - count_outer = r_outer * r_outer - r_inner2 - - if val is None: - val_inner = 1.0 / r_inner2 - val_outer = -val_inner * count_inner / count_outer - - else: - val_inner = val[0] - val_outer = val[1] - - self.val_in = np.array(val_inner, dtype=np.float64) - self.val_out = np.array(val_outer, dtype=np.float64) - self.r_in = r_inner - self.r_out = r_outer - - def get_kernel(self): - # Defined here, but not yet used? - # Create a kernel filled with the value of self.val_out - kernel = np.ones(shape=(2 * self.r_out - 1, 2 * self.r_out - 1), dtype=np.float64) * self.val_out - - # Set the values of the inner area of the kernel using array slicing - start = (self.r_out - self.r_in) - end = (self.r_out + self.r_in - 1) - kernel[start:end, start:end] = self.val_in - - return kernel - - -def to_gray(frame): - # Faster by quitting checking if the input image is already grayscale - # Perhaps it would be faster with less overhead to call cv2.cvtColor directly instead of using this function - return cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - - -@lru_cache(maxsize=lru_maxsize_vs) -def frameint_get_xy_step(imageshape, xysteps, pad, start_offset=None, end_offset=None): - """ - :param imageshape: (height(row),width(col)). row==y,cal==x - :param xysteps: (x,y) - :param pad: int - :param start_offset: (x,y) or None - :param end_offset: (x,y) or None - :return: xy_np:tuple(x,y) - """ - row, col = imageshape - row -= 1 - col -= 1 - x_step, y_step = xysteps - - # This is not beautiful. - start_pad_x = start_pad_y = end_pad_x = end_pad_y = pad - - if start_offset is not None: - start_pad_x += start_offset[0] - start_pad_y += start_offset[1] - if end_offset is not None: - end_pad_x += end_offset[0] - end_pad_y += end_offset[1] - y_np = np.arange(start_pad_y, row - end_pad_y, y_step) - x_np = np.arange(start_pad_x, col - end_pad_x, x_step) - - xy_np = (x_np, y_np) - - return xy_np - - -@lru_cache(maxsize=lru_maxsize_vvs) -def get_hsf_empty_array(len_syx, frameint_x, frame_int_dtype, fcshape): - # Function to reduce array allocation by providing an empty array first and recycling it with lru - inner_sum = np.empty(len_syx, dtype=frame_int_dtype) - outer_sum = np.empty(len_syx, dtype=frame_int_dtype) - p_temp = np.empty((len_syx[0], frameint_x), dtype=frame_int_dtype) - p00 = np.empty(len_syx, dtype=frame_int_dtype) - p11 = np.empty(len_syx, dtype=frame_int_dtype) - p01 = np.empty(len_syx, dtype=frame_int_dtype) - p10 = np.empty(len_syx, dtype=frame_int_dtype) - response_list = np.empty(len_syx, dtype=np.float64) - frame_conv = np.zeros(shape=fcshape[0], dtype=np.uint8) - frame_conv_stride = frame_conv[::fcshape[1], ::fcshape[2]] - return (inner_sum, outer_sum), p_temp, (p00, p11, p01, p10), response_list, (frame_conv, frame_conv_stride) - - -# @profile -def conv_int(frame_int, kernel, xy_step, padding, xy_steps_list): - """ - :param frame_int: - :param kernel: hsf - :param step: (x,y) - :param padding: int - :return: - """ - row, col = frame_int.shape - row -= 1 - col -= 1 - x_step, y_step = xy_step - # padding2 = 2 * padding - f_shape = row - 2 * padding, col - 2 * padding - r_in = kernel.r_in - - len_sx, len_sy = len(xy_steps_list[0]), len(xy_steps_list[1]) - inout_sum, p_temp, p_list, response_list, frameconvlist = get_hsf_empty_array((len_sy, len_sx), col + 1, - frame_int.dtype, (f_shape, y_step, x_step)) - inner_sum, outer_sum = inout_sum - p00, p11, p01, p10 = p_list - frame_conv, frame_conv_stride = frameconvlist - - y_rin_m = xy_steps_list[1] - r_in - x_rin_m = xy_steps_list[0] - r_in - y_rin_p = xy_steps_list[1] + r_in - x_rin_p = xy_steps_list[0] + r_in - # xx==(y,x),m==MINUS,p==PLUS, ex: mm==(y-,x-) - inarr_mm = frame_int[y_rin_m[0]:y_rin_m[-1] + 1:y_step, x_rin_m[0]:x_rin_m[-1] + 1:x_step] - inarr_mp = frame_int[y_rin_m[0]:y_rin_m[-1] + 1:y_step, x_rin_p[0]:x_rin_p[-1] + 1:x_step] - inarr_pm = frame_int[y_rin_p[0]:y_rin_p[-1] + 1:y_step, x_rin_m[0]:x_rin_m[-1] + 1:x_step] - inarr_pp = frame_int[y_rin_p[0]:y_rin_p[-1] + 1:y_step, x_rin_p[0]:x_rin_p[-1] + 1:x_step] - - # == inarr_mm + inarr_pp - inarr_mp - inarr_pm - inner_sum[:, :] = inarr_mm - inner_sum += inarr_pp - inner_sum -= inarr_mp - inner_sum -= inarr_pm - - # Bottleneck here, I want to make it smarter. Someone do it. - # (y,x) - # p00=max(y_ro_m,0),max(x_ro_m,0) - # p11=min(y_ro_p,ylim),min(x_ro_p,xlim) - # p01=max(y_ro_m,0),min(x_ro_p,xlim) - # p10=min(y_ro_p,ylim),max(x_ro_m,0) - y_ro_m = xy_steps_list[1] - kernel.r_out - x_ro_m = xy_steps_list[0] - kernel.r_out - y_ro_p = xy_steps_list[1] + kernel.r_out - x_ro_p = xy_steps_list[0] + kernel.r_out - # p00 calc - np.take(frame_int, y_ro_m, axis=0, mode="clip", out=p_temp) - np.take(p_temp, x_ro_m, axis=1, mode="clip", out=p00) - # p01 calc - np.take(p_temp, x_ro_p, axis=1, mode="clip", out=p01) - # p11 calc - np.take(frame_int, y_ro_p, axis=0, mode="clip", out=p_temp) - np.take(p_temp, x_ro_p, axis=1, mode="clip", out=p11) - # p10 calc - np.take(p_temp, x_ro_m, axis=1, mode="clip", out=p10) - # the point is this - # p00=np.take(np.take(frame_int, y_ro_m, axis=0, mode="clip"), x_ro_m, axis=1, mode="clip") - # p11=np.take(np.take(frame_int, y_ro_p, axis=0, mode="clip"), x_ro_p, axis=1, mode="clip") - # p01=np.take(np.take(frame_int, y_ro_m, axis=0, mode="clip"), x_ro_p, axis=1, mode="clip") - # p10=np.take(np.take(frame_int, y_ro_p, axis=0, mode="clip"), x_ro_m, axis=1, mode="clip") - - outer_sum[:, :] = p00 + p11 - p01 - p10 - inner_sum - - np.multiply(kernel.val_in, inner_sum, dtype=np.float64, out=response_list) - response_list += kernel.val_out * outer_sum - - # min_response, max_val, min_loc, max_loc = cv2.minMaxLoc(response_list) - min_response, _, min_loc, _ = cv2.minMaxLoc(response_list) - - center = ((xy_steps_list[0][min_loc[0]] - padding), (xy_steps_list[1][min_loc[1]] - padding)) - - frame_conv_stride[:, :] = response_list - # or - # frame_conv_stride[:, :] = response_list.astype(np.uint8) - - return frame_conv, min_response, center - - -class Auto_Radius_Calc(object): - def __init__(self): - self.response_list = [] - self.radius_cand_list = [] - self.adj_comp_flag = False - - self.radius_middle_index = None - - self.left_item = None - self.right_item = None - self.left_index = None - self.right_index = None - - def get_radius(self): - prev_res_len = len(self.response_list) - # adjustment of radius - if prev_res_len == 1: - # len==1==response_list==[default_radius] - self.adj_comp_flag = False - return auto_radius_range[0] - elif prev_res_len == 2: - # len==2==response_list==[default_radius, auto_radius_range[0]] - self.adj_comp_flag = False - return auto_radius_range[1] - elif prev_res_len == 3: - # len==3==response_list==[default_radius,auto_radius_range[0],auto_radius_range[1]] - if self.response_list[1][1] < self.response_list[2][1]: - self.left_item = self.response_list[1] - self.right_item = self.response_list[0] - else: - self.left_item = self.response_list[0] - self.right_item = self.response_list[2] - self.radius_cand_list = [i for i in range(self.left_item[0], self.right_item[0] + auto_radius_step, auto_radius_step)] - self.left_index = 0 - self.right_index = len(self.radius_cand_list) - 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - else: - if self.left_index <= self.right_index and self.left_index != self.radius_middle_index: - if (self.left_item[1] + self.response_list[-1][1]) < (self.right_item[1] + self.response_list[-1][1]): - self.right_item = self.response_list[-1] - self.right_index = self.radius_middle_index - 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - if (self.left_item[1] + self.response_list[-1][1]) > (self.right_item[1] + self.response_list[-1][1]): - self.left_item = self.response_list[-1] - self.left_index = self.radius_middle_index + 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - self.adj_comp_flag = True - return self.radius_cand_list[self.radius_middle_index] - - def get_radius_base(self): - """ - Use it when the new version doesn't work well. - :return: - """ - - prev_res_len = len(self.response_list) - # adjustment of radius - if prev_res_len == 1: - # len==1==response_list==[default_radius] - self.adj_comp_flag = False - return auto_radius_range[0] - elif prev_res_len == 2: - # len==2==response_list==[default_radius, auto_radius_range[0]] - self.adj_comp_flag = False - return auto_radius_range[1] - elif prev_res_len == 3: - # len==3==response_list==[default_radius,auto_radius_range[0],auto_radius_range[1]] - sort_res = sorted(self.response_list, key=lambda x: x[1])[0] - # Extract the radius with the lowest response value - if sort_res[0] == default_radius: - # If the default value is best, change now_mode to init after setting radius to the default value. - self.adj_comp_flag = True - return default_radius - elif sort_res[0] == auto_radius_range[0]: - self.radius_cand_list = [i for i in range(auto_radius_range[0], default_radius, auto_radius_step)][1:] - self.adj_comp_flag = False - return self.radius_cand_list.pop() - else: - self.radius_cand_list = [i for i in range(default_radius, auto_radius_range[1], auto_radius_step)][1:] - self.adj_comp_flag = False - return self.radius_cand_list.pop() - else: - # Try the contents of the radius_cand_list in order until the radius_cand_list runs out - # Better make it a binary search. - if len(self.radius_cand_list) == 0: - sort_res = sorted(self.response_list, key=lambda x: x[1])[0] - self.adj_comp_flag = True - return sort_res[0] - else: - self.adj_comp_flag = False - return self.radius_cand_list.pop() - - def add_response(self, radius, response): - self.response_list.append((radius, response)) - return None - - -class Blink_Detector(object): - def __init__(self): - self.response_list = [] - self.response_max = None - self.enable_detect_flg = False - self.quartile_1 = None - - def calc_thresh(self): - # Calculate response_max by computing interquartile range, IQR - # self.response_listo = np.array(self.response_listo) - # 25%,75% - # This value may need to be adjusted depending on the environment. - # quartile_1, quartile_3 = np.percentile(self.response_listo, [25, 75]) - # iqr = quartile_3 - quartile_1 - # self.response_maxo = quartile_3 + (iqr * 1.5) - - # quartile_1, quartile_3 = np.percentile(self.response_list, [25, 75]) - # or - quartile_1, quartile_3 = np.percentile(np.array(self.response_list), [25, 75]) - self.quartile_1 = quartile_1 - iqr = quartile_3 - quartile_1 - # response_min = quartile_1 - (iqr * 1.5) - - self.response_max = float(quartile_3 + (iqr * 1.5)) - # or - # self.response_max = quartile_3 + (iqr * 1.5) - - self.enable_detect_flg = True - return None - - def detect(self, now_response): - return now_response > self.response_max - - def add_response(self, response): - self.response_list.append(response) - return None - - def response_len(self): - return len(self.response_list) - - -class CenterCorrection(object): - def __init__(self): - # Tunable parameters - kernel_size = 7 # 3 or 5 or 7 - self.hist_thr = float(4) # 4% - self.center_q1_radius = 20 - - self.setup_comp = False - self.quartile_1 = None - self.radius = None - self.frame_shape = None - self.frame_mask = None - self.frame_bin = None - self.frame_final = None - self.morph_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_size, kernel_size)) - self.morph_kernel2 = np.ones((3, 3)) - self.hist_index = np.arange(256) - self.hist = np.empty((256, 1)) - self.hist_norm = np.empty((256, 1)) - - def init_array(self, gray_shape, quartile_1, radius): - self.frame_shape = gray_shape - self.frame_mask = np.empty(gray_shape, dtype=np.uint8) - self.frame_bin = np.empty(gray_shape, dtype=np.uint8) - self.frame_final = np.empty(gray_shape, dtype=np.uint8) - self.quartile_1 = quartile_1 - self.radius = radius - self.setup_comp = True - - # def reset_array(self): - # self.frame_mask.fill(0) - - def correction(self, gray_frame, orig_x, orig_y): - center_x, center_y = orig_x, orig_y - self.frame_mask.fill(0) - - # cv2.circle(self.frame_mask, center=(center_x, center_y), radius=int(self.radius * 2), color=255, thickness=-1) - - # bottleneck - cv2.calcHist([gray_frame], [0], None, [256], [0, 256], hist=self.hist) - - cv2.normalize(self.hist, self.hist_norm, alpha=100.0, norm_type=cv2.NORM_L1) - hist_per = self.hist_norm.cumsum() - hist_index_list = self.hist_index[hist_per >= self.hist_thr] - frame_thr = hist_index_list[0] if len(hist_index_list) else np.percentile(cv2.bitwise_or(255 - self.frame_mask, gray_frame), 4) - - # bottleneck - self.frame_bin = cv2.threshold(gray_frame, frame_thr, 1, cv2.THRESH_BINARY_INV)[1] - cropped_x, cropped_y, cropped_w, cropped_h = cv2.boundingRect(self.frame_bin) - - self.frame_final = cv2.bitwise_and(self.frame_bin, self.frame_mask) - - # bottleneck - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_CLOSE, self.morph_kernel) - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_OPEN, self.morph_kernel) - - if (cropped_h, cropped_w) == self.frame_shape: - # Not detected. - base_x, base_y = center_x, center_y - else: - base_x = cropped_x + cropped_w // 2 - base_y = cropped_y + cropped_h // 2 - if self.frame_final[base_y, base_x] != 1: - if self.frame_final[center_y, center_x] != 1: - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_DILATE, self.morph_kernel2, iterations=3) - else: - base_x, base_y = center_x, center_y - - contours, _ = cv2.findContours(self.frame_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - contours_box = [cv2.boundingRect(cnt) for cnt in contours] - contours_dist = np.array( - [abs(base_x - (cnt_x + cnt_w / 2)) + abs(base_y - (cnt_y + cnt_h / 2)) for cnt_x, cnt_y, cnt_w, cnt_h in contours_box]) - - if len(contours_box): - cropped_x2, cropped_y2, cropped_w2, cropped_h2 = contours_box[contours_dist.argmin()] - x = cropped_x2 + cropped_w2 // 2 - y = cropped_y2 + cropped_h2 // 2 - else: - x = center_x - y = center_y - - # if imshow_enable: - # cv2.circle(frame, (orig_x, orig_y), 10, (255, 0, 0), -1) - # cv2.circle(frame, (x, y), 7, (0, 0, 255), -1) - - # - # out_x = center_x if abs(x - center_x) > radius else x - # out_y = center_y if abs(y - center_y) > radius else y - out_x, out_y = orig_x, orig_y - if gray_frame[int(max(y - 5, 0)):int(min(y + 5, self.frame_shape[0])), - int(max(x - 5, 0)):int(min(x + 5, self.frame_shape[1]))].min() < self.quartile_1: - out_x = x - out_y = y - - # if imshow_enable: - # cv2.circle(frame, (out_x, out_y), 5, (0, 255, 0), -1) - # - # cv2.imshow("frame_bin", self.frame_bin * 255) - # cv2.imshow("frame_final", self.frame_final * 255) - return out_x, out_y - - -#RANSAC - - -video_path = "demo2.mp4" -imshow_enable = True -save_video = False - -thresh_add = 10 - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - def ellipse_model(data, y, f): """ @@ -931,11 +142,6 @@ def fit_rotated_ellipse(data, P): return (cx, cy, w, h, theta) - - - - - class HSRAC_cls(object): def __init__(self): # I'd like to take into account things like print, end_time - start_time processing time, etc., but it's too much trouble. @@ -1090,21 +296,21 @@ class HSRAC_cls(object): self.timedict["total_cv"].append(cv_end_time - cv_start_time) # if calc_print_enable: - # the lower the response the better the likelyhood of there being a pupil. you can adujst the radius and steps accordingly - # print('Kernel response:', response) - # print('Pixel position:', center_xy) + # the lower the response the better the likelyhood of there being a pupil. you can adujst the radius and steps accordingly + # print('Kernel response:', response) + # print('Pixel position:', center_xy) - - # if imshow_enable: - # if self.now_modeo != self.cv_modeo[0] and self.now_modeo != self.cv_modeo[1]: - # if 0 in cropped_image.shape: - # If shape contains 0, it is not detected well. - pass - else: - cv2.imshow("crop", cropped_image) - cv2.imshow("frame", frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - pass + # + # if imshow_enable: + # if self.now_modeo != self.cv_modeo[0] and self.now_modeo != self.cv_modeo[1]: + # if 0 in cropped_image.shape: + # If shape contains 0, it is not detected well. + # pass + # else: + # cv2.imshow("crop", cropped_image) + # cv2.imshow("frame", frame) + # if cv2.waitKey(1) & 0xFF == ord("q"): + # pass if self.now_modeo == self.cv_modeo[0]: # Moving from first_frame to the next mode @@ -1119,7 +325,7 @@ class HSRAC_cls(object): # For measuring processing time of image processing cv_start_time = timeit.default_timer() # Crop first to reduce the amount of data to process. - # frame = cropped_image[0:len(cropped_image) - 10, :] + # frame = cropped_image[0:len(cropped_image) - 10, :] # To reduce the processing data, first convert to 1-channel and then blur. # The processing results were the same when I swapped the order of blurring and 1-channelization. frame_gray = cv2.GaussianBlur(frame, (5, 5), 0) @@ -1196,16 +402,13 @@ class HSRAC_cls(object): crop_start_time = timeit.default_timer() cx, cy, w, h, theta = ransac_data - # print(cx, cy) + # print(cx, cy) if w >= 2.1 * h: #new blink detection algo lmao this works pretty good actually print("RAN BLINK") #return center_x, center_y, frame, frame, True - csx = frame.shape[0] - csy = frame.shape[1] - - #cx = center_x - (csx - cx) # we find the difference between the crop size and ransac point, and subtract from the center point from HSF - # cy = center_y - (csy - cy) + # cx = center_x - (csx - cx) # we find the difference between the crop size and ransac point, and subtract from the center point from HSF + # cy = center_y - (csy - cy) # csy = frame.shape[0] # csx = frame.shape[1] @@ -1264,9 +467,9 @@ class HSRAC_cls(object): -class External_Run_HSRACS: - - hsrac = HSRAC_cls() +class External_Run_HSRACS(object): + def __init__(self): + self.algo = HSRAC_cls() def run(self, current_image_gray): self.algo.current_image_gray = current_image_gray diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 355600a..06d7c40 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -19,7 +19,7 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) Copyright (c) 2022 EyeTrackVR <3