diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 4a91b7c..4d4a5f6 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -19,8 +19,8 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -HSR By: Sean.Denka (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) +HSR By: PallasNeko (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) BLOB By: Prohurtz#0001 (Main App Developer) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) @@ -51,10 +51,10 @@ if sys.platform.startswith("win"): import importlib from osc_calibrate_filter import * -from haar_surround_feature import * +from haar_surround_feature import External_Run_HSF from blob import * from ransac import * -from hsrac import * +from hsrac import External_Run_HSRACS from blink import * @@ -147,7 +147,6 @@ class EyeProcessor: self.cccs = False self.ts = 10 self.previous_rotation = self.config.rotation_angle - self.calibration_frame_counter self.camera_model = None self.detector_3d = None @@ -242,13 +241,13 @@ class EyeProcessor: pass def BLINKM(self): - self.blinkvalue = BLINK(self) + self.eyeoffx = BLINK(self) def HSRACM(self): - cx, cy, thresh, gray_frame = External_Run_HSRACS.HSRACS(self) + cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS().run(self.current_image_gray) self.current_image_gray = gray_frame - if self.prev_x == None: + if self.prev_x is None: self.prev_x = cx self.prev_y = cy #print(self.prev_x, self.prev_y, cx, cy) @@ -256,43 +255,43 @@ class EyeProcessor: # if (cx - self.prev_x) <= 45 and (cy - self.prev_y) <= 45 : # self.prev_x = cx # self.prev_y = cy - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, uncropframe) out_x, out_y = cal_osc(self, cx, cy) + #print(self.eyeoffx, self.eyeopen) - if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app + if self.eyeoffx: + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, 0.0)) #update app else: - - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) # else: # print("EYE MOVED TOO FAST") # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False)) def HSFM(self): - cx, cy, frame = External_Run_HSF.HSFS(self) - eyeopen = intense(cx, cy, self.current_image_gray) + cx, cy, frame = External_Run_HSF().run(self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen)) def RANSAC3DM(self): cx, cy, thresh = RANSAC3D(self) - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen)) def BLOBM(self): cx, cy, thresh = BLOB(self) - eyeopen = intense(cx, cy, self.current_image_gray) + self.eyeopen = intense(cx, cy, self.current_image_gray) out_x, out_y = cal_osc(self, cx, cy) if cx == 0: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) #update app else: - self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) + self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) @@ -347,7 +346,7 @@ class EyeProcessor: elif self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 4: self.fourthalgo = self.RANSAC3DM - if self.settings.gui_HSRAC == True and self.settings.gui_HSRACP == 1: + if self.settings.gui_HSRAC and self.settings.gui_HSRACP == 1: self.firstalgo = self.HSRACM elif self.settings.gui_HSRAC and self.settings.gui_HSRACP == 2: self.secondalgo = self.HSRACM diff --git a/EyeTrackApp/eyetrackapp.py b/EyeTrackApp/eyetrackapp.py index 8476de6..c2c2ed5 100644 --- a/EyeTrackApp/eyetrackapp.py +++ b/EyeTrackApp/eyetrackapp.py @@ -7,8 +7,7 @@ import queue import threading import PySimpleGUI as sg import sys -from urllib.request import urlopen -from bs4 import BeautifulSoup +import urllib.request import webbrowser @@ -53,35 +52,33 @@ def main(): if config.settings.gui_update_check: print("\033[95m[INFO] Checking for updates...\033[0m") url = "https://raw.githubusercontent.com/RedHawk989/EyeTrackVR-Installer/master/Version-Data/Version_Num.txt" - html = urlopen(url).read() - soup = BeautifulSoup(html, features="html.parser") - for script in soup(["script", "style"]): - script.extract() - text = soup.get_text() - - # break into lines and remove leading and trailing space on each - lines = (line.strip() for line in text.splitlines()) - # break multi-headlines into a line each - chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) - # drop blank lines - latestversion = '\n'.join(chunk for chunk in chunks if chunk) - - if appversion == latestversion: # If what we scraped and hardcoded versions are same, assume we are up to date. - print(f"\033[92m[INFO] App is up to date! [{latestversion}]\033[0m") - else: - print(f"\033[93m[INFO] You have app version [{appversion}] installed. Please update to [{latestversion}] for the newest features.\033[0m") - if sys.platform.startswith("win"): - toaster = ToastNotifier() - toaster.show_toast( #show windows toast - "EyeTrackVR has an update.", - "Click to go to the latest version.", - icon_path= "Images/logo.ico", - duration=5, - threaded=True, - callback_on_click=open_url + req = urllib.request.Request(url) + try: + with urllib.request.urlopen(req, timeout=10) as res: + latestversion = res.read().decode("utf-8").strip() + except urllib.error.HTTPError as err: + print("Failed to check latest version.") + print("{} : {}".format(err.code,err.reason)) + except urllib.error.URLError as err: + print("Failed to check latest version.") + print(err.reason) + else: + if appversion == latestversion: # If what we scraped and hardcoded versions are same, assume we are up to date. + print(f"\033[92m[INFO] App is up to date! [{latestversion}]\033[0m") + else: + print( + f"\033[93m[INFO] You have app version [{appversion}] installed. Please update to [{latestversion}] for the newest features.\033[0m") + if sys.platform.startswith("win"): + toaster = ToastNotifier() + toaster.show_toast( # show windows toast + "EyeTrackVR has an update.", + "Click to go to the latest version.", + icon_path="Images/logo.ico", + duration=5, + threaded=True, + callback_on_click=open_url ) - - + # Check to see if we have an ROI. If not, bring up ROI finder GUI. # Spawn worker threads diff --git a/EyeTrackApp/haar_surround_feature.py b/EyeTrackApp/haar_surround_feature.py index bbbe36b..2e66daf 100644 --- a/EyeTrackApp/haar_surround_feature.py +++ b/EyeTrackApp/haar_surround_feature.py @@ -1,16 +1,18 @@ -import functools -import math -import sys import timeit from functools import lru_cache import cv2 import numpy as np + +from utils.misc_utils import clamp +from utils.img_utils import safe_crop + + # from line_profiler_pycharm import profile video_path = "ezgif.com-gif-maker.avi" -imshow_enable = True +imshow_enable = False calc_print_enable = True save_video = False skip_autoradius = False @@ -27,181 +29,6 @@ blink_init_frames = 60 * 3 # 60fps*3sec,Number of blink statistical frames # step==(x,y) default_step = (5, 5) # bigger the steps,lower the processing time! ofc acc also takes an impact -""" -Attention. -If using cv2.filter2D in this code, be careful with the kernel -https://stackoverflow.com/questions/39457468/convolution-without-any-padding-opencv-python -""" - - -def TimeitWrapper(*args, **kwargs): - """ - This decorator @TimeitWrapper() prints the function name and execution time in seconds. - :param args: - :param kwargs: - :return: - """ - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - start = timeit.default_timer() - results = function(*args, **kwargs) - end = timeit.default_timer() - print('{} execution time: {:.10f} s'.format(function.__name__, end - start)) - return results - - return wrapper - - return decorator - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - - class CvParameters: # It may be a little slower because a dict named "self" is read for each function call. def __init__(self, radius, step): @@ -419,7 +246,7 @@ def conv_int(frame_int, kernel, xy_step, padding, xy_steps_list): return frame_conv, min_response, center -class Auto_Radius_Calc(object): +class AutoRadiusCalc(object): def __init__(self): self.response_list = [] self.radius_cand_list = [] @@ -522,7 +349,7 @@ class Auto_Radius_Calc(object): return None -class Blink_Detector(object): +class BlinkDetector(object): def __init__(self): self.response_list = [] self.response_max = None @@ -665,7 +492,7 @@ class CenterCorrection(object): return out_x, out_y -class HSRAC_cls(object): +class HSF_cls(object): def __init__(self): # I'd like to take into account things like print, end_time - start_time processing time, etc., but it's too much trouble. @@ -679,9 +506,9 @@ class HSRAC_cls(object): self.cv_modeo = ["first_frame", "radius_adjust", "blink_adjust", "normal"] self.now_modeo = self.cv_modeo[0] - self.auto_radius_calc = Auto_Radius_Calc() - self.blink_detector = Blink_Detector() - self.center_q1 = Blink_Detector() + self.auto_radius_calc = AutoRadiusCalc() + self.blink_detector = BlinkDetector() + self.center_q1 = BlinkDetector() self.center_correct = CenterCorrection() self.cap = None @@ -709,9 +536,13 @@ class HSRAC_cls(object): def single_run(self): # Temporary implementation to run - + + ## default_radius = 14 + # cropbox=[] # debug code + + frame = self.current_image_gray if self.now_modeo == self.cv_modeo[1]: # adjustment of radius @@ -756,7 +587,12 @@ class HSRAC_cls(object): lower_y = center_y - radius # Crop the image using the calculated bounds - cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x] + + cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y) + + # cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in + # zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code + if self.now_modeo == self.cv_modeo[0] or self.now_modeo == self.cv_modeo[1]: # If mode is first_frame or radius_adjust, record current radius and response @@ -770,8 +606,14 @@ class HSRAC_cls(object): lower_x = center_x - self.center_correct.center_q1_radius upper_y = center_y + self.center_correct.center_q1_radius lower_y = center_y - self.center_correct.center_q1_radius - self.center_q1.add_response(cv2.mean(gray_frame[lower_y:upper_y, lower_x:upper_x])[0]) - + + self.center_q1.add_response( + cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y,keepsize=False))[ + 0 + ] + ) + + else: self.blink_detector.calc_thresh() @@ -792,9 +634,19 @@ class HSRAC_cls(object): else: # pass if not self.center_correct.setup_comp: - self.center_correct.init_array(gray_frame.shape, self.center_q1.quartile_1, radius) - - center_x, center_y = self.center_correct.correction(gray_frame, center_x, center_y) + self.center_correct.init_array( + gray_frame.shape, self.center_q1.quartile_1, radius + ) + elif self.center_correct.frame_shape != gray_frame.shape: + """The resolution should have changed and the statistics should have changed, so essentially the statistics + need to be reworked, but implementation will be postponed as viability is the highest priority. """ + self.center_correct.init_array( + gray_frame.shape, self.center_q1.quartile_1, radius + ) + + center_x, center_y = self.center_correct.correction( + gray_frame, center_x, center_y + ) # Define the center point and radius center_xy = (center_x, center_y) upper_x = center_x + radius @@ -802,10 +654,16 @@ class HSRAC_cls(object): upper_y = center_y + radius lower_y = center_y - radius # Crop the image using the calculated bounds - cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x] - # if imshow_enable or save_video: - # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) - # cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1) + cropped_image = safe_crop( + gray_frame, lower_x, lower_y, upper_x, upper_y + ) + # cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in + # zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code + + # if imshow_enable or save_video: + # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) + # cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1) + # If you want to update response_max. it may be more cost-effective to rewrite response_list in the following way # https://stackoverflow.com/questions/42771110/fastest-way-to-left-cycle-a-numpy-array-like-pop-push-for-a-queue @@ -837,20 +695,28 @@ class HSRAC_cls(object): self.now_modeo = self.cv_modeo[2] else: self.now_modeo = self.cv_modeo[1] - + + + # debug code + # return center_x,center_y,cropbox,frame return center_x, center_y, frame -class External_Run_HSF: +class External_Run_HSF(object): + def __init__(self): + self.algo = HSF_cls() - hsrac = HSRAC_cls() - - def HSFS(self): - External_Run_HSF.hsrac.current_image_gray = self.current_image_gray - center_x, center_y, frame = External_Run_HSF.hsrac.single_run() + def run(self, current_image_gray): + self.algo.current_image_gray = current_image_gray + # debug code + # center_x, center_y,cropbox, frame = self.algo.single_run() + # return center_x, center_y,cropbox, frame + center_x, center_y, frame = self.algo.single_run() return center_x, center_y, frame -if __name__ == '__main__': - hsrac = HSRAC_cls() - hsrac.open_video(video_path) - while hsrac.read_frame(): - _ = hsrac.single_run() \ No newline at end of file + + +if __name__ == "__main__": + hsf = HSF_cls() + hsf.open_video(video_path) + while hsf.read_frame(): + _ = hsf.single_run() diff --git a/EyeTrackApp/hsrac.py b/EyeTrackApp/hsrac.py index d07c17c..cc4265b 100644 --- a/EyeTrackApp/hsrac.py +++ b/EyeTrackApp/hsrac.py @@ -1,14 +1,24 @@ -import functools -import math -import sys import timeit -from functools import lru_cache import cv2 import numpy as np +from haar_surround_feature import ( + AutoRadiusCalc, + BlinkDetector, + CvParameters, conv_int, + frameint_get_xy_step, +) +from utils.img_utils import safe_crop +from utils.misc_utils import clamp + + # from line_profiler_pycharm import profile +#RANSAC + +thresh_add = 10 + video_path = "ezgif.com-gif-maker.avi" imshow_enable = True calc_print_enable = True @@ -27,799 +37,6 @@ blink_init_frames = 60 * 3 # 60fps*3sec,Number of blink statistical frames # step==(x,y) default_step = (5, 5) # bigger the steps,lower the processing time! ofc acc also takes an impact -""" -Attention. -If using cv2.filter2D in this code, be careful with the kernel -https://stackoverflow.com/questions/39457468/convolution-without-any-padding-opencv-python -""" - - -def TimeitWrapper(*args, **kwargs): - """ - This decorator @TimeitWrapper() prints the function name and execution time in seconds. - :param args: - :param kwargs: - :return: - """ - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - start = timeit.default_timer() - results = function(*args, **kwargs) - end = timeit.default_timer() - print('{} execution time: {:.10f} s'.format(function.__name__, end - start)) - return results - - return wrapper - - return decorator - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - - -class CvParameters: - # It may be a little slower because a dict named "self" is read for each function call. - def __init__(self, radius, step): - # self.prev_radius=radius - self._radius = radius - self.pad = 2 * radius - # self.prev_step=step - self._step = step - self._hsf = HaarSurroundFeature(radius) - - def get_rpsh(self): - return self._radius, self.pad, self._step, self._hsf - # Essentially, the following would be preferable, but it would take twice as long to call. - # return self.radius, self.pad, self.step, self.hsf - - @property - def radius(self): - return self._radius - - @radius.setter - def radius(self, now_radius): - # self.prev_radius=self._radius - self._radius = now_radius - self.pad = 2 * now_radius - self.hsf = now_radius - - @property - def step(self): - return self._step - - @step.setter - def step(self, now_step): - # self.prev_step=self.step - self._step = now_step - - @property - def hsf(self): - return self._hsf - - @hsf.setter - def hsf(self, now_radius): - self._hsf = HaarSurroundFeature(now_radius) - - -class HaarSurroundFeature: - - def __init__(self, r_inner, r_outer=None, val=None): - if r_outer is None: - r_outer = r_inner * 3 - # print(r_outer) - r_inner2 = r_inner * r_inner - count_inner = r_inner2 - count_outer = r_outer * r_outer - r_inner2 - - if val is None: - val_inner = 1.0 / r_inner2 - val_outer = -val_inner * count_inner / count_outer - - else: - val_inner = val[0] - val_outer = val[1] - - self.val_in = np.array(val_inner, dtype=np.float64) - self.val_out = np.array(val_outer, dtype=np.float64) - self.r_in = r_inner - self.r_out = r_outer - - def get_kernel(self): - # Defined here, but not yet used? - # Create a kernel filled with the value of self.val_out - kernel = np.ones(shape=(2 * self.r_out - 1, 2 * self.r_out - 1), dtype=np.float64) * self.val_out - - # Set the values of the inner area of the kernel using array slicing - start = (self.r_out - self.r_in) - end = (self.r_out + self.r_in - 1) - kernel[start:end, start:end] = self.val_in - - return kernel - - -def to_gray(frame): - # Faster by quitting checking if the input image is already grayscale - # Perhaps it would be faster with less overhead to call cv2.cvtColor directly instead of using this function - return cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - - -@lru_cache(maxsize=lru_maxsize_vs) -def frameint_get_xy_step(imageshape, xysteps, pad, start_offset=None, end_offset=None): - """ - :param imageshape: (height(row),width(col)). row==y,cal==x - :param xysteps: (x,y) - :param pad: int - :param start_offset: (x,y) or None - :param end_offset: (x,y) or None - :return: xy_np:tuple(x,y) - """ - row, col = imageshape - row -= 1 - col -= 1 - x_step, y_step = xysteps - - # This is not beautiful. - start_pad_x = start_pad_y = end_pad_x = end_pad_y = pad - - if start_offset is not None: - start_pad_x += start_offset[0] - start_pad_y += start_offset[1] - if end_offset is not None: - end_pad_x += end_offset[0] - end_pad_y += end_offset[1] - y_np = np.arange(start_pad_y, row - end_pad_y, y_step) - x_np = np.arange(start_pad_x, col - end_pad_x, x_step) - - xy_np = (x_np, y_np) - - return xy_np - - -@lru_cache(maxsize=lru_maxsize_vvs) -def get_hsf_empty_array(len_syx, frameint_x, frame_int_dtype, fcshape): - # Function to reduce array allocation by providing an empty array first and recycling it with lru - inner_sum = np.empty(len_syx, dtype=frame_int_dtype) - outer_sum = np.empty(len_syx, dtype=frame_int_dtype) - p_temp = np.empty((len_syx[0], frameint_x), dtype=frame_int_dtype) - p00 = np.empty(len_syx, dtype=frame_int_dtype) - p11 = np.empty(len_syx, dtype=frame_int_dtype) - p01 = np.empty(len_syx, dtype=frame_int_dtype) - p10 = np.empty(len_syx, dtype=frame_int_dtype) - response_list = np.empty(len_syx, dtype=np.float64) - frame_conv = np.zeros(shape=fcshape[0], dtype=np.uint8) - frame_conv_stride = frame_conv[::fcshape[1], ::fcshape[2]] - return (inner_sum, outer_sum), p_temp, (p00, p11, p01, p10), response_list, (frame_conv, frame_conv_stride) - - -# @profile -def conv_int(frame_int, kernel, xy_step, padding, xy_steps_list): - """ - :param frame_int: - :param kernel: hsf - :param step: (x,y) - :param padding: int - :return: - """ - row, col = frame_int.shape - row -= 1 - col -= 1 - x_step, y_step = xy_step - # padding2 = 2 * padding - f_shape = row - 2 * padding, col - 2 * padding - r_in = kernel.r_in - - len_sx, len_sy = len(xy_steps_list[0]), len(xy_steps_list[1]) - inout_sum, p_temp, p_list, response_list, frameconvlist = get_hsf_empty_array((len_sy, len_sx), col + 1, - frame_int.dtype, (f_shape, y_step, x_step)) - inner_sum, outer_sum = inout_sum - p00, p11, p01, p10 = p_list - frame_conv, frame_conv_stride = frameconvlist - - y_rin_m = xy_steps_list[1] - r_in - x_rin_m = xy_steps_list[0] - r_in - y_rin_p = xy_steps_list[1] + r_in - x_rin_p = xy_steps_list[0] + r_in - # xx==(y,x),m==MINUS,p==PLUS, ex: mm==(y-,x-) - inarr_mm = frame_int[y_rin_m[0]:y_rin_m[-1] + 1:y_step, x_rin_m[0]:x_rin_m[-1] + 1:x_step] - inarr_mp = frame_int[y_rin_m[0]:y_rin_m[-1] + 1:y_step, x_rin_p[0]:x_rin_p[-1] + 1:x_step] - inarr_pm = frame_int[y_rin_p[0]:y_rin_p[-1] + 1:y_step, x_rin_m[0]:x_rin_m[-1] + 1:x_step] - inarr_pp = frame_int[y_rin_p[0]:y_rin_p[-1] + 1:y_step, x_rin_p[0]:x_rin_p[-1] + 1:x_step] - - # == inarr_mm + inarr_pp - inarr_mp - inarr_pm - inner_sum[:, :] = inarr_mm - inner_sum += inarr_pp - inner_sum -= inarr_mp - inner_sum -= inarr_pm - - # Bottleneck here, I want to make it smarter. Someone do it. - # (y,x) - # p00=max(y_ro_m,0),max(x_ro_m,0) - # p11=min(y_ro_p,ylim),min(x_ro_p,xlim) - # p01=max(y_ro_m,0),min(x_ro_p,xlim) - # p10=min(y_ro_p,ylim),max(x_ro_m,0) - y_ro_m = xy_steps_list[1] - kernel.r_out - x_ro_m = xy_steps_list[0] - kernel.r_out - y_ro_p = xy_steps_list[1] + kernel.r_out - x_ro_p = xy_steps_list[0] + kernel.r_out - # p00 calc - np.take(frame_int, y_ro_m, axis=0, mode="clip", out=p_temp) - np.take(p_temp, x_ro_m, axis=1, mode="clip", out=p00) - # p01 calc - np.take(p_temp, x_ro_p, axis=1, mode="clip", out=p01) - # p11 calc - np.take(frame_int, y_ro_p, axis=0, mode="clip", out=p_temp) - np.take(p_temp, x_ro_p, axis=1, mode="clip", out=p11) - # p10 calc - np.take(p_temp, x_ro_m, axis=1, mode="clip", out=p10) - # the point is this - # p00=np.take(np.take(frame_int, y_ro_m, axis=0, mode="clip"), x_ro_m, axis=1, mode="clip") - # p11=np.take(np.take(frame_int, y_ro_p, axis=0, mode="clip"), x_ro_p, axis=1, mode="clip") - # p01=np.take(np.take(frame_int, y_ro_m, axis=0, mode="clip"), x_ro_p, axis=1, mode="clip") - # p10=np.take(np.take(frame_int, y_ro_p, axis=0, mode="clip"), x_ro_m, axis=1, mode="clip") - - outer_sum[:, :] = p00 + p11 - p01 - p10 - inner_sum - - np.multiply(kernel.val_in, inner_sum, dtype=np.float64, out=response_list) - response_list += kernel.val_out * outer_sum - - # min_response, max_val, min_loc, max_loc = cv2.minMaxLoc(response_list) - min_response, _, min_loc, _ = cv2.minMaxLoc(response_list) - - center = ((xy_steps_list[0][min_loc[0]] - padding), (xy_steps_list[1][min_loc[1]] - padding)) - - frame_conv_stride[:, :] = response_list - # or - # frame_conv_stride[:, :] = response_list.astype(np.uint8) - - return frame_conv, min_response, center - - -class Auto_Radius_Calc(object): - def __init__(self): - self.response_list = [] - self.radius_cand_list = [] - self.adj_comp_flag = False - - self.radius_middle_index = None - - self.left_item = None - self.right_item = None - self.left_index = None - self.right_index = None - - def get_radius(self): - prev_res_len = len(self.response_list) - # adjustment of radius - if prev_res_len == 1: - # len==1==response_list==[default_radius] - self.adj_comp_flag = False - return auto_radius_range[0] - elif prev_res_len == 2: - # len==2==response_list==[default_radius, auto_radius_range[0]] - self.adj_comp_flag = False - return auto_radius_range[1] - elif prev_res_len == 3: - # len==3==response_list==[default_radius,auto_radius_range[0],auto_radius_range[1]] - if self.response_list[1][1] < self.response_list[2][1]: - self.left_item = self.response_list[1] - self.right_item = self.response_list[0] - else: - self.left_item = self.response_list[0] - self.right_item = self.response_list[2] - self.radius_cand_list = [i for i in range(self.left_item[0], self.right_item[0] + auto_radius_step, auto_radius_step)] - self.left_index = 0 - self.right_index = len(self.radius_cand_list) - 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - else: - if self.left_index <= self.right_index and self.left_index != self.radius_middle_index: - if (self.left_item[1] + self.response_list[-1][1]) < (self.right_item[1] + self.response_list[-1][1]): - self.right_item = self.response_list[-1] - self.right_index = self.radius_middle_index - 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - if (self.left_item[1] + self.response_list[-1][1]) > (self.right_item[1] + self.response_list[-1][1]): - self.left_item = self.response_list[-1] - self.left_index = self.radius_middle_index + 1 - self.radius_middle_index = (self.left_index + self.right_index) // 2 - self.adj_comp_flag = False - return self.radius_cand_list[self.radius_middle_index] - self.adj_comp_flag = True - return self.radius_cand_list[self.radius_middle_index] - - def get_radius_base(self): - """ - Use it when the new version doesn't work well. - :return: - """ - - prev_res_len = len(self.response_list) - # adjustment of radius - if prev_res_len == 1: - # len==1==response_list==[default_radius] - self.adj_comp_flag = False - return auto_radius_range[0] - elif prev_res_len == 2: - # len==2==response_list==[default_radius, auto_radius_range[0]] - self.adj_comp_flag = False - return auto_radius_range[1] - elif prev_res_len == 3: - # len==3==response_list==[default_radius,auto_radius_range[0],auto_radius_range[1]] - sort_res = sorted(self.response_list, key=lambda x: x[1])[0] - # Extract the radius with the lowest response value - if sort_res[0] == default_radius: - # If the default value is best, change now_mode to init after setting radius to the default value. - self.adj_comp_flag = True - return default_radius - elif sort_res[0] == auto_radius_range[0]: - self.radius_cand_list = [i for i in range(auto_radius_range[0], default_radius, auto_radius_step)][1:] - self.adj_comp_flag = False - return self.radius_cand_list.pop() - else: - self.radius_cand_list = [i for i in range(default_radius, auto_radius_range[1], auto_radius_step)][1:] - self.adj_comp_flag = False - return self.radius_cand_list.pop() - else: - # Try the contents of the radius_cand_list in order until the radius_cand_list runs out - # Better make it a binary search. - if len(self.radius_cand_list) == 0: - sort_res = sorted(self.response_list, key=lambda x: x[1])[0] - self.adj_comp_flag = True - return sort_res[0] - else: - self.adj_comp_flag = False - return self.radius_cand_list.pop() - - def add_response(self, radius, response): - self.response_list.append((radius, response)) - return None - - -class Blink_Detector(object): - def __init__(self): - self.response_list = [] - self.response_max = None - self.enable_detect_flg = False - self.quartile_1 = None - - def calc_thresh(self): - # Calculate response_max by computing interquartile range, IQR - # self.response_listo = np.array(self.response_listo) - # 25%,75% - # This value may need to be adjusted depending on the environment. - # quartile_1, quartile_3 = np.percentile(self.response_listo, [25, 75]) - # iqr = quartile_3 - quartile_1 - # self.response_maxo = quartile_3 + (iqr * 1.5) - - # quartile_1, quartile_3 = np.percentile(self.response_list, [25, 75]) - # or - quartile_1, quartile_3 = np.percentile(np.array(self.response_list), [25, 75]) - self.quartile_1 = quartile_1 - iqr = quartile_3 - quartile_1 - # response_min = quartile_1 - (iqr * 1.5) - - self.response_max = float(quartile_3 + (iqr * 1.5)) - # or - # self.response_max = quartile_3 + (iqr * 1.5) - - self.enable_detect_flg = True - return None - - def detect(self, now_response): - return now_response > self.response_max - - def add_response(self, response): - self.response_list.append(response) - return None - - def response_len(self): - return len(self.response_list) - - -class CenterCorrection(object): - def __init__(self): - # Tunable parameters - kernel_size = 7 # 3 or 5 or 7 - self.hist_thr = float(4) # 4% - self.center_q1_radius = 20 - - self.setup_comp = False - self.quartile_1 = None - self.radius = None - self.frame_shape = None - self.frame_mask = None - self.frame_bin = None - self.frame_final = None - self.morph_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_size, kernel_size)) - self.morph_kernel2 = np.ones((3, 3)) - self.hist_index = np.arange(256) - self.hist = np.empty((256, 1)) - self.hist_norm = np.empty((256, 1)) - - def init_array(self, gray_shape, quartile_1, radius): - self.frame_shape = gray_shape - self.frame_mask = np.empty(gray_shape, dtype=np.uint8) - self.frame_bin = np.empty(gray_shape, dtype=np.uint8) - self.frame_final = np.empty(gray_shape, dtype=np.uint8) - self.quartile_1 = quartile_1 - self.radius = radius - self.setup_comp = True - - # def reset_array(self): - # self.frame_mask.fill(0) - - def correction(self, gray_frame, orig_x, orig_y): - center_x, center_y = orig_x, orig_y - self.frame_mask.fill(0) - - # cv2.circle(self.frame_mask, center=(center_x, center_y), radius=int(self.radius * 2), color=255, thickness=-1) - - # bottleneck - cv2.calcHist([gray_frame], [0], None, [256], [0, 256], hist=self.hist) - - cv2.normalize(self.hist, self.hist_norm, alpha=100.0, norm_type=cv2.NORM_L1) - hist_per = self.hist_norm.cumsum() - hist_index_list = self.hist_index[hist_per >= self.hist_thr] - frame_thr = hist_index_list[0] if len(hist_index_list) else np.percentile(cv2.bitwise_or(255 - self.frame_mask, gray_frame), 4) - - # bottleneck - self.frame_bin = cv2.threshold(gray_frame, frame_thr, 1, cv2.THRESH_BINARY_INV)[1] - cropped_x, cropped_y, cropped_w, cropped_h = cv2.boundingRect(self.frame_bin) - - self.frame_final = cv2.bitwise_and(self.frame_bin, self.frame_mask) - - # bottleneck - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_CLOSE, self.morph_kernel) - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_OPEN, self.morph_kernel) - - if (cropped_h, cropped_w) == self.frame_shape: - # Not detected. - base_x, base_y = center_x, center_y - else: - base_x = cropped_x + cropped_w // 2 - base_y = cropped_y + cropped_h // 2 - if self.frame_final[base_y, base_x] != 1: - if self.frame_final[center_y, center_x] != 1: - self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_DILATE, self.morph_kernel2, iterations=3) - else: - base_x, base_y = center_x, center_y - - contours, _ = cv2.findContours(self.frame_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - contours_box = [cv2.boundingRect(cnt) for cnt in contours] - contours_dist = np.array( - [abs(base_x - (cnt_x + cnt_w / 2)) + abs(base_y - (cnt_y + cnt_h / 2)) for cnt_x, cnt_y, cnt_w, cnt_h in contours_box]) - - if len(contours_box): - cropped_x2, cropped_y2, cropped_w2, cropped_h2 = contours_box[contours_dist.argmin()] - x = cropped_x2 + cropped_w2 // 2 - y = cropped_y2 + cropped_h2 // 2 - else: - x = center_x - y = center_y - - # if imshow_enable: - # cv2.circle(frame, (orig_x, orig_y), 10, (255, 0, 0), -1) - # cv2.circle(frame, (x, y), 7, (0, 0, 255), -1) - - # - # out_x = center_x if abs(x - center_x) > radius else x - # out_y = center_y if abs(y - center_y) > radius else y - out_x, out_y = orig_x, orig_y - if gray_frame[int(max(y - 5, 0)):int(min(y + 5, self.frame_shape[0])), - int(max(x - 5, 0)):int(min(x + 5, self.frame_shape[1]))].min() < self.quartile_1: - out_x = x - out_y = y - - # if imshow_enable: - # cv2.circle(frame, (out_x, out_y), 5, (0, 255, 0), -1) - # - # cv2.imshow("frame_bin", self.frame_bin * 255) - # cv2.imshow("frame_final", self.frame_final * 255) - return out_x, out_y - - -#RANSAC - - -video_path = "demo2.mp4" -imshow_enable = False -save_video = False - -thresh_add = 10 - - -class TimeitResult(object): - """ - from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - - Object returned by the timeit magic with info about the run. - Contains the following attributes : - loops: (int) number of loops done per measurement - repeat: (int) number of times the measurement has been repeated - best: (float) best execution time / number - all_runs: (list of float) execution time of each run (in s) - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = best - self.worst = worst - self.all_runs = all_runs - self._precision = precision - self.timings = [dt / self.loops for dt in all_runs] - - @property - def average(self): - return math.fsum(self.timings) / len(self.timings) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean=format_time(self.average, self._precision), - std=format_time(self.stdev, self._precision), - best=format_time(self.best, self._precision), - worst=format_time(self.worst, self._precision), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = '+-' - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb1'.encode(sys.stdout.encoding) - pm = u'\xb1' - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text(u'') - - -def format_time(timespan, precision=3): - """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form - """ - - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append(u'%s%s' % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) - - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = [u"s", u"ms", u'us', "ns"] # the save value - if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: - try: - u'\xb5'.encode(sys.stdout.encoding) - units = [u"s", u"ms", u'\xb5s', "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] - - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) - def ellipse_model(data, y, f): """ @@ -925,11 +142,6 @@ def fit_rotated_ellipse(data, P): return (cx, cy, w, h, theta) - - - - - class HSRAC_cls(object): def __init__(self): # I'd like to take into account things like print, end_time - start_time processing time, etc., but it's too much trouble. @@ -943,20 +155,23 @@ class HSRAC_cls(object): self.cv_modeo = ["first_frame", "radius_adjust", "blink_adjust", "normal"] self.now_modeo = self.cv_modeo[0] - - self.auto_radius_calc = Auto_Radius_Calc() - self.blink_detector = Blink_Detector() - self.center_q1 = Blink_Detector() - self.center_correct = CenterCorrection() - + + self.auto_radius_calc = AutoRadiusCalc() + self.blink_detector = BlinkDetector() + self.center_q1 = BlinkDetector() + + self.cap = None self.timedict = {"to_gray": [], "int_img": [], "conv_int": [], "crop": [], "total_cv": []} # ransac self.rng = np.random.default_rng() - self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) - + + # self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) + # or + self.kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3)) + def open_video(self, video_path): # Temporary implementation to run cap = cv2.VideoCapture(video_path) @@ -972,17 +187,20 @@ class HSRAC_cls(object): ret, frame = self.cap.read() if ret: # I have set it to grayscale (1ch) just in case, but if the frame is 1ch, this line can be commented out. + # self.current_image=frame # debug code self.current_image_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) return True return False def single_run(self): # Temporary implementation to run - ## default_radius = 14 + # ori_frame = self.current_image.copy()# debug code + # cropbox=[] # debug code + + blink_bd = False frame = self.current_image_gray - frame = cv2.copyMakeBorder(frame, 21, 21, 21, 21, cv2.BORDER_CONSTANT, value=[255, 255, 255]) # add a border to prevent overcropping the image. if self.now_modeo == self.cv_modeo[1]: # adjustment of radius @@ -1027,7 +245,9 @@ class HSRAC_cls(object): lower_y = center_y - radius # Crop the image using the calculated bounds - cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x] + cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y) + + # cropbox=[clamp(val, 0, gray_frame.shape[i]) for i,val in zip([1,0,1,0],[lower_x,lower_y,upper_x,upper_y])] # debug code if self.now_modeo == self.cv_modeo[0] or self.now_modeo == self.cv_modeo[1]: # If mode is first_frame or radius_adjust, record current radius and response @@ -1036,20 +256,24 @@ class HSRAC_cls(object): # Statistics for blink detection if self.blink_detector.response_len() < blink_init_frames: self.blink_detector.add_response(cv2.mean(cropped_image)[0]) - - upper_x = center_x + self.center_correct.center_q1_radius - lower_x = center_x - self.center_correct.center_q1_radius - upper_y = center_y + self.center_correct.center_q1_radius - lower_y = center_y - self.center_correct.center_q1_radius - self.center_q1.add_response(cv2.mean(gray_frame[lower_y:upper_y, lower_x:upper_x])[0]) - + + upper_x = center_x + 20 + lower_x = center_x - 20 + upper_y = center_y + 20 + lower_y = center_y - 20 + self.center_q1.add_response( + cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y,keepsize=False))[ + 0 + ] + ) + else: self.blink_detector.calc_thresh() self.center_q1.calc_thresh() self.now_modeo = self.cv_modeo[3] else: - if 0 in cropped_image.shape: + if 0 in cropped_image.shape: # This line may not be needed. The image will be cropped using safecrop. # If shape contains 0, it is not detected well. print("Something's wrong.") else: @@ -1059,24 +283,11 @@ class HSRAC_cls(object): # (i.e., if the cropimage is whitish if self.blink_detector.detect(cv2.mean(cropped_image)[0]): # blink - pass - else: - # pass - if not self.center_correct.setup_comp: - self.center_correct.init_array(gray_frame.shape, self.center_q1.quartile_1, radius) - - center_x, center_y = self.center_correct.correction(gray_frame, center_x, center_y) - # Define the center point and radius - center_xy = (center_x, center_y) - upper_x = center_x + radius - lower_x = center_x - radius - upper_y = center_y + radius - lower_y = center_y - radius - # Crop the image using the calculated bounds - cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x] - # if imshow_enable or save_video: - # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) - # cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1) + print("BLINK BD") + blink_bd=True + # if imshow_enable or save_video: + # cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1) + # cv2.circle(ori_frame, (center_x, center_y), 7, (255, 0, 0), -1) # If you want to update response_max. it may be more cost-effective to rewrite response_list in the following way # https://stackoverflow.com/questions/42771110/fastest-way-to-left-cycle-a-numpy-array-like-pop-push-for-a-queue @@ -1085,21 +296,22 @@ class HSRAC_cls(object): self.timedict["total_cv"].append(cv_end_time - cv_start_time) # if calc_print_enable: - # the lower the response the better the likelyhood of there being a pupil. you can adujst the radius and steps accordingly - # print('Kernel response:', response) - # print('Pixel position:', center_xy) - - if imshow_enable: - if self.now_modeo != self.cv_modeo[0] and self.now_modeo != self.cv_modeo[1]: - if 0 in cropped_image.shape: - # If shape contains 0, it is not detected well. - pass - else: - cv2.imshow("crop", cropped_image) - cv2.imshow("frame", frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - pass + # the lower the response the better the likelyhood of there being a pupil. you can adujst the radius and steps accordingly + # print('Kernel response:', response) + # print('Pixel position:', center_xy) + # + # if imshow_enable: + # if self.now_modeo != self.cv_modeo[0] and self.now_modeo != self.cv_modeo[1]: + # if 0 in cropped_image.shape: + # If shape contains 0, it is not detected well. + # pass + # else: + # cv2.imshow("crop", cropped_image) + # cv2.imshow("frame", frame) + # if cv2.waitKey(1) & 0xFF == ord("q"): + # pass + if self.now_modeo == self.cv_modeo[0]: # Moving from first_frame to the next mode if skip_autoradius and skip_blink_detect: @@ -1110,46 +322,63 @@ class HSRAC_cls(object): self.now_modeo = self.cv_modeo[1] - newFrame2 = frame.copy() - #frame = cropped_image # For measuring processing time of image processing cv_start_time = timeit.default_timer() # Crop first to reduce the amount of data to process. - # frame = cropped_image[0:len(cropped_image) - 10, :] + # frame = cropped_image[0:len(cropped_image) - 10, :] # To reduce the processing data, first convert to 1-channel and then blur. # The processing results were the same when I swapped the order of blurring and 1-channelization. frame_gray = cv2.GaussianBlur(frame, (5, 5), 0) - - upper_x = center_x + 20 - lower_x = center_x - 20 - upper_y = center_y + 20 - lower_y = center_y - 20 + hsf_center_x, hsf_center_y = center_x.copy(), center_y.copy() + ransac_xy_offset = (hsf_center_x-20, hsf_center_y-20) + upper_x = hsf_center_x + 20 + lower_x = hsf_center_x - 20 + upper_y = hsf_center_y + 20 + lower_y = hsf_center_y - 20 # Crop the image using the calculated bounds - - frame_gray = frame_gray[lower_y:upper_y, lower_x:upper_x] - frame = frame_gray + + frame_gray_crop = safe_crop(frame_gray, lower_x, lower_y, upper_x, upper_y) + frame = frame_gray_crop # this will need to be adjusted everytime hardware is changed (brightness of IR, Camera postion, etc)m - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(frame_gray) - - maxloc0_hf, maxloc1_hf = int(0.5 * max_loc[0]), int(0.5 * max_loc[1]) - - # crop 15% sqare around min_loc - # frame_gray = frame_gray[max_loc[1] - maxloc1_hf:max_loc[1] + maxloc1_hf, - # max_loc[0] - maxloc0_hf:max_loc[0] + maxloc0_hf] - + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(frame_gray_crop) + threshold_value = min_val + thresh_add - _, thresh = cv2.threshold(frame_gray, threshold_value, 255, cv2.THRESH_BINARY) - # print(thresh.shape, frame_gray.shape) + _, thresh = cv2.threshold(frame_gray_crop, threshold_value, 255, cv2.THRESH_BINARY) + # print(thresh.shape, frame_gray.shape) try: + opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, self.kernel) closing = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, self.kernel) th_frame = 255 - closing except: # I want to eliminate try here because try tends to be slow in execution. - th_frame = 255 - frame_gray + th_frame = 255 - frame_gray_crop contours, _ = cv2.findContours(th_frame, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) + # or + # contours, _=cv2.findContours(th_frame, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + + if not blink_bd and self.blink_detector.enable_detect_flg: + threshold_value = self.center_q1.quartile_1 + if threshold_value= 2.1 * h: #new blink detection algo lmao this works pretty good actually print("RAN BLINK") #return center_x, center_y, frame, frame, True - csx = frame.shape[0] - csy = frame.shape[1] + # cx = center_x - (csx - cx) # we find the difference between the crop size and ransac point, and subtract from the center point from HSF + # cy = center_y - (csy - cy) - cx = center_x - (csx - cx) # we find the difference between the crop size and ransac point, and subtract from the center point from HSF - cy = center_y - (csy - cy) + # csy = frame.shape[0] + # csx = frame.shape[1] + csy = gray_frame.shape[0] + csx = gray_frame.shape[1] + # cx = clamp((cx - 20) + center_x, 0, csx) + # cy = clamp((cy - 20) + center_y, 0, csy) + cx = int(clamp(cx + ransac_xy_offset[0], 0, csx)) + cy = int(clamp(cy + ransac_xy_offset[1], 0, csy)) + cv_end_time = timeit.default_timer() - if imshow_enable or save_video: - cv2.drawContours(frame_gray, contours, -1, (255, 0, 0), 1) - cv2.circle(frame_gray, (cx, cy), 2, (0, 0, 255), -1) - # cx1, cy1, w1, h1, theta1 = fit_rotated_ellipse(maxcnt.reshape(-1, 2)) - cv2.ellipse(frame_gray, (cx, cy), (w, h), theta * 180.0 / np.pi, 0.0, 360.0, (50, 250, 200), 1, ) - - except: + # if imshow_enable or save_video: + # + # cv2.circle(ori_frame, (orig_x, orig_y), 3, (0, 255, 0), -1) + # cv2.drawContours(ori_frame, contours, -1, (255, 0, 0), 1) + # cv2.circle(ori_frame, (cx, cy), 2, (0, 0, 255), -1) + # # cx1, cy1, w1, h1, theta1 = fit_rotated_ellipse(maxcnt.reshape(-1, 2)) + # cv2.ellipse( + # ori_frame, + # (cx, cy), + # (int(w), int(h)), + # theta * 180.0 / np.pi, + # 0.0, + # 360.0, + # (50, 250, 200), + # 1, + # ) + # cv2.imshow("crop", cropped_image) + # # cv2.imshow("frame", frame) + # cv2.imshow("ori_frame",ori_frame) + # if cv2.waitKey(1) & 0xFF == ord("q"): + # pass + + except Exception as e: + # print(e) pass - - - - # print(frame_gray.shape, thresh.shape) + # debug code + # try: + # if any([isinstance(val, float) for val in [cx, cy]]): + # print() + # return int(cx), int(cy),cropbox, ori_frame,thresh, frame, gray_frame + # except: + # if any([isinstance(val, float) for val in [center_x, center_y]]): + # print() + # return center_x, center_y,cropbox, ori_frame,thresh, frame, gray_frame + # print(frame_gray.shape, thresh.shape) + try: - return cx, cy, thresh, frame + return int(cx), int(cy), thresh, frame, gray_frame except: - return center_x, center_y, thresh, frame + return int(center_x), int(center_y), thresh, frame, gray_frame -class External_Run_HSRACS: - hsrac = HSRAC_cls() - def HSRACS(self): - External_Run_HSRACS.hsrac.current_image_gray = self.current_image_gray - center_x, center_y, thresh, frame = External_Run_HSRACS.hsrac.single_run() - return center_x, center_y, thresh, frame -if __name__ == '__main__': +class External_Run_HSRACS(object): + def __init__(self): + self.algo = HSRAC_cls() + + def run(self, current_image_gray): + self.algo.current_image_gray = current_image_gray + #debug code + # center_x, center_y,cropbox,ori_frame, thresh, frame, gray_frame = self.algo.single_run() + # return center_x, center_y,cropbox,ori_frame, thresh, frame, gray_frame + center_x, center_y, thresh, frame, gray_frame = self.algo.single_run() + return center_x, center_y, thresh, frame, gray_frame + + + +if __name__ == "__main__": hsrac = HSRAC_cls() hsrac.open_video(video_path) while hsrac.read_frame(): - _ = hsrac.single_run() \ No newline at end of file + _ = hsrac.single_run() + + # hsrac = HSRAC_cls() + # hsrac.open_video(video_path) + # hsf = HSF_cls() + # while hsrac.read_frame(): + # hsf.current_image_gray = hsrac.current_image_gray.copy() + # _ = hsrac.single_run() + # + # _ = hsf.single_run() + + # w_video=True + # + # er_hsracs=External_Run_HSRACS() + # er_hsracs.algo.open_video(video_path) + # er_hsf=External_Run_HSF() + # + # if w_video: + # filepath = 'test.mp4' + # codec = cv2.VideoWriter_fourcc(*"x264") + # video = cv2.VideoWriter(filepath, codec, 60.0, (200,150))#(60, 60)) # (150, 200)) + # while er_hsracs.algo.read_frame(): + # base_gray = er_hsracs.algo.current_image_gray.copy() + # base_img=er_hsracs.algo.current_image.copy() + # cv2.imshow("frame",base_gray) + # hsf_x, hsf_y, hsf_cropbox,*_ = er_hsf.run(base_gray) + # + # # hsrac_x, hsrac_y, hsrac_cropbox, *_ = er_hsracs.run(base_gray) + # if 0:#random.random()<0.1: + # hsrac_x, hsrac_y, hsrac_cropbox, *_ = er_hsracs.run(cv2.resize(base_gray,None,fx=0.75,fy=0.75).copy()) + # hsrac_x=int(hsrac_x*1.25) + # hsrac_y=int(hsrac_y*1.25) + # hsrac_cropbox=[int(val*1.25) for val in hsrac_cropbox] + # else: + # hsrac_x, hsrac_y, hsrac_cropbox,ori_frame, *_ = er_hsracs.run(base_gray) + # + # + # + # cv2.rectangle(base_img,hsf_cropbox[:2],hsf_cropbox[2:],(0, 0, 255),3) + # cv2.rectangle(base_img, hsrac_cropbox[:2], hsrac_cropbox[2:], (255, 0, 0), 1) + # cv2.circle(base_img, (hsf_x, hsf_y), 6, (0, 0, 255), -1) + # try: + # cv2.circle(base_img, (hsrac_x, hsrac_y), 3, (255, 0, 0), -1) + # except: + # print() + # cv2.imshow("hsf_hsrac",base_img) + # if cv2.waitKey(1) & 0xFF == ord("q"): + # pass + # if w_video: + # video.write(ori_frame) + # if w_video: + # video.release() + # # cv2.imwrite("b.png",er_hsracs.algo.result2) + # er_hsracs.algo.cap.release() + # cv2.destroyAllWindows() + \ No newline at end of file diff --git a/EyeTrackApp/intensity_eye_open.py b/EyeTrackApp/intensity_eye_open.py index 31a4371..726845f 100644 --- a/EyeTrackApp/intensity_eye_open.py +++ b/EyeTrackApp/intensity_eye_open.py @@ -1,13 +1,18 @@ -import pandas as pd import numpy as np +import time +import os import cv2 -from pythonosc import udp_client - + +from enum import IntEnum #higher intensity means more closed/ more white/less pupil #Hm I need an acronym for this, any ideas? -#IBO Intensity Based Openess - +#IBO Intensity Based Openess +class EyeId(IntEnum): + RIGHT = 0 + LEFT = 1 + BOTH = 2 + SETTINGS = 3 # HOW THIS WORKS: # we get the intensity of pupil area from HSF crop, When the eyelid starts to close, the pupil starts being obstructed by skin which is generally lighter than the pupil. @@ -15,79 +20,183 @@ from pythonosc import udp_client # ex. when you look up there is less pupil visible, which results in an uncalculated change in intensity even though the eyelid has not moved in a meaningful way. # We compare the darkest intensity of that area, to the lightest (global) intensity to find the appropriate openness state via a float. -fname = "IBO.csv" #TODO Expose as setting -try: - data = pd.read_csv(fname, sep=",") -except: - cf = open(fname, "w") - cf.write("xy,intensity") - cf.close() - data = pd.read_csv(fname, sep=",") +# Note. +# OpenCV on Windows will generate an error if the file path contains non-ASCII characters when using cv2.imread(), cv2.imwrite(), etc. +# https://stackoverflow.com/questions/43185605/how-do-i-read-an-image-from-a-path-with-unicode-characters +# https://github.com/opencv/opencv/issues/18305 + + +if EyeId.RIGHT: + fname = "IBO_RIGHT.png" +if EyeId.LEFT: + fname = "IBO_LEFT.png" + +lct = time.time() +data = None + + +def csv2data(frameshape, filepath): + # For data checking + frameshape = (frameshape[0], frameshape[1]+1) + out = np.zeros(frameshape, dtype=np.uint32) + xy_list = [] + val_list = [] + with open(filepath, mode="r", encoding="utf-8") as in_f: + # Skip header. + _ = in_f.readline() + for s in in_f: + xyval = [int(val) for val in s.strip().split(',')] + xy_list.append((xyval[0], xyval[1])) + val_list.append(xyval[2]) + xy_list = np.array(xy_list) + val_list = np.array(val_list) + out[xy_list[:, 1], xy_list[:, 0]] = val_list[:] + return out + +def data2csv(data_u32, filepath): + # For data checking + nonzero_index = np.nonzero(data_u32) #(row,col) + data_list = data_u32[nonzero_index].tolist() + datalines = ["{},{},{}\n".format(x, y, val) for y, x, val in zip(*nonzero_index, data_list)] + with open(filepath, 'w', encoding="utf-8") as out_f: + out_f.write("x,y,intensity\n") + out_f.writelines(datalines) + return + + +def u32_u16_1ch3ch(img): + img_copy = img.copy() + # In the case of bit operations. (img>>32)&0xffff,(img>>16)&0xffff,img&0xffff + out = np.zeros((*img.shape[:2], 3), dtype=np.uint16) + for i in range(3): + out[:, :, i] = img_copy % 0xffff + img_copy //= 0xffff + return out + + +def u16_u32_3ch_1ch(img): + # In the case of bit operations. ((img >> 32) & 0xffff) << 32 |((img >> 16) & 0xffff) << 16 | (img & 0xffff) + out = np.zeros(img.shape[:2], dtype=np.uint32) + for i in range(3): + out += img[:, :, i] if i == 0 else img[:, :, i] * (i * 0xffff) + return out + + +def newdata(frameshape): + print("Initialise data for blinking.") + return np.zeros(frameshape, dtype=np.uint32) + + +def check_and_load(frameshape, now_data): + # In the future, both eyes may be processed at the same time. Therefore, data should be passed as arguments. + req_newdata = False + # Not very clever, but increase the width by 1px to save the maximum value. + frameshape = (frameshape[0], frameshape[1]+1) + if now_data is None: + print("Load data for blinking: {}".format(fname)) + if os.path.isfile(fname): + img = cv2.imread(fname, flags=cv2.IMREAD_UNCHANGED) + if img.shape[:2] != frameshape: + print("size does not match the input frame.") + req_newdata = True + else: + now_data = u16_u32_3ch_1ch(img) + else: + print("File does not exist.") + req_newdata = True + else: + if now_data.shape != frameshape: + # Using the previous and current frame sizes and centre positions from the original, etc., the data can be ported to some extent, but there may be many areas where code changes are required. + print("Frame size changed.") + req_newdata = True + if req_newdata: + now_data = newdata(frameshape) + # data2csv(now_data, "a.csv") + # csv2data(frameshape,"a.csv") + return now_data -#TODO we need more pixel points for smooth operation, lets get this setup in hsrac def intense(x, y, frame): - # upper_x = x + 25 - # lower_x = x - 25 - # upper_y = y + 25 - # lower_y = y - 25 - # frame = frame[lower_y:upper_y, lower_x:upper_x] - - try: - xy = int(str(int(x)) + str(int(y)) + str(int(x)+int(y))) - intensity = np.sum(frame) - except: - return 0.0 #TODO find how on earth a hyphen gets thrown into this + global lct, data + + # 0 in data is used as the initial value. + # When assigning a value, +1 is added to the value to be assigned. + data = check_and_load(frame.shape[:2], data) + int_x, int_y = int(x), int(y) + # upper_x = min(int_x + 25, frame.shape[1]) #TODO make this a setting + #lower_x = max(int_x - 25, 0) + #upper_y = min(int_y + 25, frame.shape[0]) + #lower_y = max(int_y - 25, 0) + + #frame_crop = frame[lower_y:upper_y, lower_x:upper_x] + frame_crop = frame + + # The same can be done with cv2.integral, but since there is only one area of the rectangle for which we want to know the total value, there is no advantage in terms of computational complexity. + intensity = frame_crop.sum()+1 + # numpy:np.sum(),ndarray.sum() + # opencv:cv2.sumElems() + # I don't know which is faster. + print(frame.shape[1], frame.shape[0], int_x, int_y) changed = False - try: #max pupil per cord - dfb = data[data['xy']==xy].index.values.astype(int)[0] # find pandas index of line with matching xy value + newval_flg = False + if int_y >= frame.shape[1]: + data_val = 0 + print('CAUGHT Y OUT OF BOUNDS') - if intensity < data.at[dfb, 'intensity']: #if current intensity value is less (more pupil), save that - data.at[dfb, 'intensity'] = intensity # set value - changed = True - print("var adjusted") - - else: - intensitya = data.at[dfb, 'intensity'] - 3 #if current intensity value is less (more pupil), save that - data.at[dfb, 'intensity'] = intensitya # set value - changed = True + else: - except: # that value is not yet saved - data.loc[len(data.index)] = [xy, intensity] #create new data on last line of csv with current intesity + data_val = data[int_y, int_x] + + + # max pupil per cord + if data_val == 0: + # The value of the specified coordinates has not yet been recorded. + data[int_y, int_x] = intensity + changed = True + newval_flg = True + elif intensity < data_val: # if current intensity value is less (more pupil), save that + data[int_y, int_x] = intensity # set value + changed = True + print("var adjusted") + else: + intensitya = max(data_val - 3, 1) # if current intensity value is less (more pupil), save that + data[int_y, int_x] = intensitya # set value changed = True - try: # min pupil global - if intensity > data.at[0, 'intensity']: #if current intensity value is more (less pupil), save that NOTE: we have the - data.at[0, 'intensity'] = intensity # set value at 0 index - changed = True - print("new max", intensity) - - else: - intensityd = data.at[0, 'intensity'] - 10 #continuously adjust closed intensity, will be set when user blink, used to allow eyes to close when lighting changes - data.at[0, 'intensity'] = intensityd # set value at 0 index - changed = True - - except: # there is no max intensity yet, create - data.at[0, 'intensity'] = intensity # set value at 0 index + # min pupil global + if data[0, -1] == 0: # that value is not yet saved + data[0, -1] = intensity # set value at 0 index changed = True print("create max", intensity) - - try: - maxp = data.at[dfb, 'intensity'] - minp = data.at[0, 'intensity'] - #eyeopen = (intensity - minp) / (maxp - minp) - eyeopen = (intensity - maxp) / (minp - maxp) - eyeopen = 1 - eyeopen - print(intensity, maxp, minp, x, y) - # print(f"EYEOPEN: {eyeopen}") - - except: + elif intensity > data[0, -1]: # if current intensity value is more (less pupil), save that NOTE: we have the + data[0, -1] = intensity # set value at 0 index + changed = True + print("new max", intensity) + else: + intensityd = max(data[0, -1] - 10, 1) #continuously adjust closed intensity, will be set when user blink, used to allow eyes to close when lighting changes + data[0, -1] = intensityd # set value at 0 index + changed = True + + if newval_flg: + # Do the same thing as in the original version. print('[INFO] Something went wrong, assuming blink.') - eyeopen = 0.0 + eyeopen = 0.7 + else: + maxp = data[int_y, int_x] + minp = data[0, -1] + diffp = minp - maxp if (minp - maxp) != 0 else 1 + eyeopen = (intensity - maxp) / diffp + eyeopen = 1 - eyeopen + #eyeopen = eyeopen - 0.2 + # print(intensity, maxp, minp, x, y) + # print(f"EYEOPEN: {eyeopen}") + # print(int(x), int(y), eyeopen, maxp, minp) - if changed == True: - data.to_csv(fname, encoding='utf-8', index=False) #save file since we made a change + if changed and ((time.time() - lct) > 4): # save every 4 seconds if something changed to save disk usage + cv2.imwrite(fname, u32_u16_1ch3ch(data)) + lct = time.time() + print("SAVED") return eyeopen diff --git a/EyeTrackApp/osc.py b/EyeTrackApp/osc.py index f73f2a1..5bd639e 100644 --- a/EyeTrackApp/osc.py +++ b/EyeTrackApp/osc.py @@ -126,11 +126,9 @@ class VRChatOSC: if eye_id in [EyeId.RIGHT]: rb = True - if last_blink > 0.7: - for i in range(5): - self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) - self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye - last_blink = time.time() - last_blink + self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye + else: if eye_id in [EyeId.LEFT]: @@ -146,13 +144,10 @@ class VRChatOSC: self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink)) # open left eye if rb and lb: # If both eyes are closed, blink - if last_blink > 0.5: - for i in range(4): - self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye - self.client.send_message("/avatar/parameters/LeftEyeLid", float(1)) - self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye - self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink)) - last_blink = time.time() - last_blink + self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye + self.client.send_message("/avatar/parameters/LeftEyeLid", float(1)) + self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye + self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink)) class VRChatOSCReceiver: diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 355600a..06d7c40 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -19,7 +19,7 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization) +RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization) Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator) Copyright (c) 2022 EyeTrackVR <3 diff --git a/EyeTrackApp/test_list.txt b/EyeTrackApp/test_list.txt deleted file mode 100644 index 578de9c..0000000 --- a/EyeTrackApp/test_list.txt +++ /dev/null @@ -1,308 +0,0 @@ -xy,intensity -473582.0,357639.0 -473583.0,250971.0 -463582.0,259772.0 -463581.0,250001.0 -463481.0,262307.0 -473482.0,259063.0 -503485.0,253050.0 -513586.0,242916.0 -523486.0,245595.0 -523587.0,239752.0 -523588.0,235965.0 -533589.0,234207.0 -553792.0,227890.0 -573997.0,219112.0 -594099.0,211998.0 -6040100.0,97336.0 -6040101.0,204647.0 -6041101.0,206041.0 -553692.0,222899.0 -533689.0,234080.0 -523689.0,235235.0 -533588.0,234591.0 -513587.0,240166.0 -503586.0,242147.0 -483584.0,248543.0 -453681.0,252077.0 -453581.0,252706.0 -453580.0,238681.0 -443580.0,251563.0 -0.0,-282.0 -483583.0,246862.0 -513687.0,239926.0 -513486.0,244214.0 -543691.0,229066.0 -553691.0,224848.0 -563793.0,222710.0 -563693.0,221737.0 -563692.0,221532.0 -543589.0,233991.0 -523688.0,237634.0 -513688.0,240610.0 -503686.0,241587.0 -493686.0,243116.0 -483685.0,241468.0 -483785.0,242252.0 -473784.0,243209.0 -463784.0,244225.0 -453884.0,245899.0 -453883.0,245353.0 -443883.0,244192.0 -443882.0,245493.0 -433982.0,246043.0 -433882.0,246397.0 -423982.0,247620.0 -423981.0,248308.0 -433881.0,250237.0 -443782.0,255290.0 -453782.0,251893.0 -463783.0,247845.0 -443781.0,256185.0 -453783.0,252276.0 -463682.0,257657.0 -473683.0,248249.0 -473684.0,254666.0 -483684.0,253290.0 -493685.0,253745.0 -553590.0,160147.0 -563591.0,228955.0 -573592.0,227065.0 -583594.0,229045.0 -573593.0,229591.0 -563592.0,230609.0 -553490.0,235088.0 -543489.0,235599.0 -503585.0,235551.0 -493585.0,252256.0 -463683.0,249050.0 -453682.0,256510.0 -433780.0,257233.0 -423780.0,257321.0 -433781.0,254818.0 -423881.0,255737.0 -463884.0,245749.0 -473785.0,242568.0 -483786.0,249030.0 -493786.0,247564.0 -503787.0,244608.0 -543690.0,234709.0 -543590.0,233992.0 -553591.0,232013.0 -583593.0,228357.0 -583493.0,226658.0 -593594.0,228134.0 -583492.0,227921.0 -573391.0,231334.0 -573390.0,231444.0 -563390.0,234623.0 -563389.0,234593.0 -553388.0,237340.0 -543387.0,235194.0 -533386.0,239102.0 -523386.0,241225.0 -523385.0,241552.0 -513384.0,250384.0 -503383.0,253299.0 -493383.0,254572.0 -483382.0,258460.0 -473481.0,259955.0 -463480.0,258556.0 -453480.0,259863.0 -443680.0,261992.0 -433579.0,261555.0 -433679.0,259909.0 -433680.0,256797.0 -423880.0,258517.0 -424082.0,253054.0 -434083.0,250342.0 -434084.0,248101.0 -444084.0,246720.0 -444085.0,250595.0 -444488.0,265335.0 -464995.0,289119.0 -4954104.0,295429.0 -4953103.0,294742.0 -454893.0,283001.0 -464591.0,271508.0 -474491.0,264746.0 -484290.0,258544.0 -484190.0,255352.0 -484089.0,251344.0 -494089.0,245657.0 -493989.0,232298.0 -503990.0,230283.0 -513890.0,238264.0 -513990.0,226967.0 -523890.0,235722.0 -523891.0,234865.0 -533891.0,233355.0 -533791.0,232810.0 -543791.0,232428.0 -523790.0,237078.0 -513789.0,242689.0 -513788.0,242871.0 -533488.0,237010.0 -533487.0,238427.0 -533387.0,237412.0 -513385.0,244320.0 -503384.0,248585.0 -493382.0,255212.0 -483381.0,254841.0 -473381.0,258108.0 -473380.0,257087.0 -453479.0,260130.0 -443579.0,261332.0 -473885.0,242285.0 -483886.0,240662.0 -483887.0,238495.0 -493887.0,236504.0 -493988.0,234020.0 -503989.0,232407.0 -513991.0,225947.0 -523991.0,224778.0 -504090.0,229604.0 -494090.0,233456.0 -483988.0,236861.0 -484088.0,237088.0 -483987.0,239272.0 -473987.0,238618.0 -473986.0,240348.0 -473886.0,241339.0 -493484.0,249358.0 -503484.0,247653.0 -543388.0,233512.0 -523284.0,256409.0 -543084.0,269285.0 -542983.0,280529.0 -542883.0,291685.0 -9738135.0,213939.0 -9638134.0,219657.0 -9738136.0,213451.0 -9839137.0,204007.0 -9839138.0,204217.0 -9440134.0,228059.0 -9241134.0,240587.0 -9141133.0,246331.0 -9539134.0,224047.0 -9739136.0,211537.0 -533084.0,279261.0 -533184.0,262679.0 -523285.0,255018.0 -532983.0,274461.0 -532781.0,302302.0 -9637133.0,222471.0 -9537132.0,229626.0 -9737135.0,215710.0 -9637134.0,221660.0 -9537133.0,229847.0 -9737134.0,215498.0 -523184.0,257570.0 -494291.0,298100.0 -5250102.0,307350.0 -514192.0,302543.0 -523487.0,269115.0 -493483.0,259705.0 -483482.0,264756.0 -483483.0,263669.0 -542782.0,288667.0 -543185.0,253212.0 -533185.0,257473.0 -533285.0,252283.0 -513284.0,256989.0 -8552138.0,258478.0 -5151103.0,312095.0 -514799.0,313296.0 -544296.0,302653.0 -533892.0,289151.0 -543488.0,256997.0 -514395.0,299664.0 -5351104.0,304022.0 -485098.0,307963.0 -514294.0,304829.0 -503687.0,251585.0 -503788.0,248328.0 -493888.0,246838.0 -464086.0,252127.0 -454085.0,239326.0 -443984.0,255446.0 -433983.0,256770.0 -443681.0,264741.0 -543286.0,253677.0 -543186.0,251910.0 -513283.0,258668.0 -503283.0,259877.0 -493282.0,264590.0 -483281.0,266513.0 -463380.0,269924.0 -483280.0,270615.0 -493079.0,271830.0 -492979.0,271781.0 -492978.0,272371.0 -493080.0,271169.0 -513082.0,261750.0 -513183.0,260039.0 -523083.0,271107.0 -532881.0,278287.0 -9635131.0,228473.0 -9535130.0,233599.0 -9535131.0,233232.0 -9436130.0,236829.0 -9736134.0,218341.0 -8850138.0,238086.0 -5249101.0,310885.0 -494695.0,315451.0 -533286.0,252608.0 -504394.0,302886.0 -504293.0,306282.0 -522982.0,272159.0 -9635132.0,229019.0 -9736133.0,218560.0 -523082.0,298688.0 -493181.0,280463.0 -504696.0,307991.0 -9149141.0,221327.0 -504091.0,304619.0 -493787.0,253151.0 -513182.0,261244.0 -513081.0,263879.0 -503181.0,264893.0 -493584.0,256296.0 -503182.0,264151.0 -6050110.0,143710.0 -554095.0,79411.0 -10050150.0,83752.0 -6045105.0,146315.0 -6540105.0,48067.0 -6535100.0,137496.0 -653095.0,86445.0 -7535110.0,153158.0 -4040.0,-6.0 -7030100.0,163605.0 -652085.0,0.0 -65065.0,-15.0 -6565130.0,235622.0 -6555120.0,292006.0 -6560125.0,234630.0 -5560115.0,372229.0 -5055105.0,272624.0 -5550105.0,241914.0 -5050100.0,318630.0 -5555110.0,252782.0 -6055115.0,217787.0 -5545100.0,225673.0 -504595.0,236617.0 -405090.0,262150.0 -455095.0,272500.0 -454590.0,267193.0 -405595.0,267416.0 -404080.0,215254.0 -4060100.0,273388.0 -603595.0,269172.0 -404585.0,257809.0 -403575.0,253865.0 -403070.0,264744.0 -453075.0,265099.0 -503080.0,261448.0 -6550115.0,158537.0 -4065105.0,67964.0 diff --git a/EyeTrackApp/utils/__init__.py b/EyeTrackApp/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/EyeTrackApp/utils/img_utils.py b/EyeTrackApp/utils/img_utils.py new file mode 100644 index 0000000..b93d4b1 --- /dev/null +++ b/EyeTrackApp/utils/img_utils.py @@ -0,0 +1,12 @@ +import cv2 + + +def safe_crop(img, x, y, x2, y2, keepsize=True): + # The order of the arguments can be reconsidered. + img_h, img_w = img.shape[:2] + outimg = img[max(0, y) : min(img_h, y2), max(0, x) : min(img_w, x2)].copy() + reqsize_x, reqsize_y = abs(x2 - x), abs(y2 - y) + if keepsize and outimg.shape[:2] != (reqsize_y, reqsize_x): + # If the size is different from the expected size (smaller by the amount that is out of range) + outimg = cv2.resize(outimg, (reqsize_x, reqsize_y)) + return outimg diff --git a/EyeTrackApp/utils/misc_utils.py b/EyeTrackApp/utils/misc_utils.py new file mode 100644 index 0000000..13931a7 --- /dev/null +++ b/EyeTrackApp/utils/misc_utils.py @@ -0,0 +1,2 @@ +def clamp(x, low, high): + return max(low, min(x, high)) \ No newline at end of file diff --git a/EyeTrackApp/utils/time_utils.py b/EyeTrackApp/utils/time_utils.py new file mode 100644 index 0000000..ec46710 --- /dev/null +++ b/EyeTrackApp/utils/time_utils.py @@ -0,0 +1,171 @@ +import functools +import math +import sys +import timeit + +def TimeitWrapper(*args, **kwargs): + """ + This decorator @TimeitWrapper() prints the function name and execution time in seconds. + :param args: + :param kwargs: + :return: + """ + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + start = timeit.default_timer() + results = function(*args, **kwargs) + end = timeit.default_timer() + print('{} execution time: {:.10f} s'.format(function.__name__, end - start)) + return results + + return wrapper + + return decorator + + +class TimeitResult(object): + """ + from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 + + Object returned by the timeit magic with info about the run. + Contains the following attributes : + loops: (int) number of loops done per measurement + repeat: (int) number of times the measurement has been repeated + best: (float) best execution time / number + all_runs: (list of float) execution time of each run (in s) + """ + + def __init__(self, loops, repeat, best, worst, all_runs, precision): + self.loops = loops + self.repeat = repeat + self.best = best + self.worst = worst + self.all_runs = all_runs + self._precision = precision + self.timings = [dt / self.loops for dt in all_runs] + + @property + def average(self): + return math.fsum(self.timings) / len(self.timings) + + @property + def stdev(self): + mean = self.average + return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5 + + def __str__(self): + pm = '+-' + if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: + try: + u'\xb1'.encode(sys.stdout.encoding) + pm = u'\xb1' + except: + pass + return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( + pm=pm, + runs=self.repeat, + loops=self.loops, + loop_plural="" if self.loops == 1 else "s", + run_plural="" if self.repeat == 1 else "s", + mean=format_time(self.average, self._precision), + std=format_time(self.stdev, self._precision), + best=format_time(self.best, self._precision), + worst=format_time(self.worst, self._precision), + ) + + def _repr_pretty_(self, p, cycle): + unic = self.__str__() + p.text(u'') + + +class FPSResult(object): + """ + base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 + """ + + def __init__(self, loops, repeat, best, worst, all_runs, precision): + self.loops = loops + self.repeat = repeat + self.best = 1 / best + self.worst = 1 / worst + self.all_runs = all_runs + self._precision = precision + self.fps = [1 / dt for dt in all_runs] + self.unit = "fps" + + @property + def average(self): + return math.fsum(self.fps) / len(self.fps) + + @property + def stdev(self): + mean = self.average + return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 + + def __str__(self): + pm = '+-' + if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: + try: + u'\xb1'.encode(sys.stdout.encoding) + pm = u'\xb1' + except: + pass + return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( + pm=pm, + runs=self.repeat, + loops=self.loops, + loop_plural="" if self.loops == 1 else "s", + run_plural="" if self.repeat == 1 else "s", + mean="%.*g%s" % (self._precision, self.average, self.unit), + std="%.*g%s" % (self._precision, self.stdev, self.unit), + best="%.*g%s" % (self._precision, self.best, self.unit), + worst="%.*g%s" % (self._precision, self.worst, self.unit), + ) + + def _repr_pretty_(self, p, cycle): + unic = self.__str__() + p.text(u'') + + +def format_time(timespan, precision=3): + """ + https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 + Formats the timespan in a human readable form + """ + + if timespan >= 60.0: + # we have more than a minute, format that in a human readable form + # Idea from http://snipplr.com/view/5713/ + parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] + time = [] + leftover = timespan + for suffix, length in parts: + value = int(leftover / length) + if value > 0: + leftover = leftover % length + time.append(u'%s%s' % (str(value), suffix)) + if leftover < 1: + break + return " ".join(time) + + # Unfortunately the unicode 'micro' symbol can cause problems in + # certain terminals. + # See bug: https://bugs.launchpad.net/ipython/+bug/348466 + # Try to prevent crashes by being more secure than it needs to + # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. + units = [u"s", u"ms", u'us', "ns"] # the save value + if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding: + try: + u'\xb5'.encode(sys.stdout.encoding) + units = [u"s", u"ms", u'\xb5s', "ns"] + except: + pass + scaling = [1, 1e3, 1e6, 1e9] + + if timespan > 0.0: + order = min(-int(math.floor(math.log10(timespan)) // 3), 3) + else: + order = 3 + return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 222dc77..c313102 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,18 +12,6 @@ files = [ {file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"}, ] -[[package]] -name = "beautifulsoup4" -version = "4.11.1" -description = "Screen-scraping library" -category = "main" -optional = false -python-versions = ">=3.6.0" -files = [ - {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, - {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, -] - [package.dependencies] soupsieve = ">1.2" diff --git a/pyproject.toml b/pyproject.toml index 573b3cc..80641eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ pydantic = "^1.10.2" win10toast_click = [ { version = "^0.1.2", platform = 'win32' } ] -beautifulsoup4 = "^4.11.1" [tool.poetry.group.dev.dependencies] black = "^22.10.0"