From ab441f2ce020fc830917e1542a69c93a7d3f820e Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Tue, 6 May 2025 12:52:26 -0500 Subject: [PATCH] New AHSF code (fixed) ported from Summer --- EyeTrackApp/AHSF.py | 912 +++++++++++------------------------ EyeTrackApp/eye_processor.py | 34 +- 2 files changed, 294 insertions(+), 652 deletions(-) diff --git a/EyeTrackApp/AHSF.py b/EyeTrackApp/AHSF.py index 4970e5a..0bdee22 100644 --- a/EyeTrackApp/AHSF.py +++ b/EyeTrackApp/AHSF.py @@ -19,7 +19,7 @@ @@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@( -Adaptive Haar Surround Feature: Summer, PallasNeko (Optimization) +Adaptive Haar Surround Feature: Summer Algorithm App Implementations and Tweaks By: Prohurtz Copyright (c) 2025 EyeTrackVR <3 @@ -27,678 +27,328 @@ Copyright (c) 2025 EyeTrackVR <3 LICENSE: Summer Software Distribution License 1.0 ------------------------------------------------------------------------------------------------------ """ - -import functools -import math -import os -import sys -import time -import timeit -from logging import FileHandler, Formatter, INFO, StreamHandler, getLogger - -from functools import lru_cache +from __future__ import annotations +from typing import Tuple, Optional import cv2 import numpy as np +# ------------------------- utility helpers ------------------------- # +def _rect_scale(rect: Tuple[int, int, int, int], + ratio: float, + keep_center: bool = True, + square_outer: bool = False) -> Tuple[int, int, int, int]: + """Scale rectangle by *ratio* (optionally keep centre fixed).""" + x, y, w, h = rect + if square_outer: + w = h = int(max(w, h) * ratio) + else: + w = int(w * ratio) + h = int(h * ratio) + if keep_center: + cx, cy = x + rect[2] // 2, y + rect[3] // 2 + x = int(cx - w / 2) + y = int(cy - h / 2) + return (x, y, w, h) -# from line_profiler_pycharm import profile +def _clip_rect(rect: Tuple[int, int, int, int], + boundary: Tuple[int, int, int, int]) -> Tuple[int, int, int, int]: + """Clip *rect* to *boundary* = (x, y, w, h).""" + bx, by, bw, bh = boundary + x, y, w, h = rect + x = max(bx, x) + y = max(by, y) + w = min(x + w, bx + bw) - x + h = min(y + h, by + bh) - y + return (x, y, max(0, w), max(0, h)) +def _get_block_integral(ii: np.ndarray, + rect: Tuple[int, int, int, int]) -> int: + """Integral‑image sum over *rect*.""" + x, y, w, h = rect + return (ii[y+h, x+w] - ii[y, x+w] - ii[y+h, x] + ii[y, x]) +def _canny_pure(img: np.ndarray, + low: int = 64, + high_ratio: float = 2.0) -> np.ndarray: + """Lightweight Canny wrapper (imitates canny_pure()).""" + img_blur = cv2.GaussianBlur(img, (3, 3), 0) + return cv2.Canny(img_blur, low, int(low*high_ratio)) -class AHSF: - def __init__(self, video_src, save_logfile=False, imshow_enable=False, save_video=False): - self.this_file_basename = os.path.basename(__file__) - self.this_file_name = self.this_file_basename.replace(".py", "") - self.alg_ver = "PallasNekoV3" +# --------------------------- main class ---------------------------- # +class PupilDetectorHaar: + """ + Haar‑based coarse‑to‑fine pupil detector. - self.save_logfile = save_logfile - self.imshow_enable = imshow_enable - self.save_video = save_video + Parameters + ---------- + ratio_outer : float + Scaling factor for Haar outer rectangle. + kf : float + Weighting term in response function f = µ_outer − kf*µ_inner. + use_square_haar : bool + If True, outer Haar window is square; else horizontal rectangle. + use_init_rect : bool + If True, provide an approximate pupil box in *init_rect*. + init_rect : Tuple[int,int,int,int] | None + Initial pupil location on the full‑resolution frame. + target_resolution : Tuple[int,int] + Image is down‑sampled so the longer side ~320 px by default. + width_min / width_max / wh_step / xy_step + Search‑grid parameters for Haar scanning. + """ - self.VideoCapture_SRC = video_src - self.input_is_webcam = False - self.benchmark_flag = True if not self.input_is_webcam and not self.imshow_enable and not self.save_video else False - self.loop_num = 1 if self.imshow_enable or self.save_video else 10 - self.output_video_path = f"./{self.this_file_name}.mp4" - self.logfilename = f"./{self.this_file_name}.log" - self.print_enable = False + # -------- initialisation -------- # + def __init__(self, + ratio_outer: float = 1.4, + kf: float = 1.5, + use_square_haar: bool = False, + use_init_rect: bool = False, + init_rect: Optional[Tuple[int, int, int, int]] = None, + target_resolution: Tuple[int, int] = (320, 240), + width_min: int = 31, + width_max: int = 120, + wh_step: int = 2, + xy_step: int = 2): + self.ratio_outer = ratio_outer + self.kf = kf + self.use_square_haar = use_square_haar + self.use_init_rect = use_init_rect + self.init_rect = (0, 0, 0, 0) if init_rect is None else init_rect + self.target_resolution = target_resolution - self.lru_maxsize_vvs = 16 - self.lru_maxsize_vs = 64 - self.lru_maxsize_s = 128 + # search‑grid params (may be auto‑tuned after first frame) + self.width_min = width_min + self.width_max = width_max + self.wh_step = wh_step + self.xy_step = xy_step - self.logger = getLogger(__name__) - self.logger.setLevel(INFO) - formatter = Formatter("%(message)s") - handler = StreamHandler() - handler.setLevel(INFO) - handler.setFormatter(formatter) - self.logger.addHandler(handler) - if self.save_logfile: - handler = FileHandler(self.logfilename, encoding="utf8", mode="w") - handler.setLevel(INFO) - handler.setFormatter(formatter) - self.logger.addHandler(handler) - else: - self.save_logfile = False - self.video_wr = cv2.VideoWriter if self.save_video else None + # dynamic state + self.frame_num = 0 + self.mu_inner = 50 + self.mu_outer = 200 + self.mu_inner0 = 50 # first frame stats + self.mu_outer0 = 200 + # outputs (public) + self.pupil_rect_coarse = (0, 0, 0, 0) + self.outer_rect_coarse = (0, 0, 0, 0) + self.max_response_coarse = -255 + self.center_coarse = (0.0, 0.0) - def format_time(self, timespan, precision=3): + self.pupil_rect_fine = (0, 0, 0, 0) + self.center_fine = (0.0, 0.0) + + # private temp + self._ratio_down = 1.0 + self._img_boundary = (0, 0, 0, 0) + self._init_rect_down = (0, 0, 0, 0) + + # ---------------------------------------------------------------- # + # PUBLIC API # + # ---------------------------------------------------------------- # + def detect(self, img_gray: np.ndarray) -> Tuple[Tuple[int,int,int,int], Tuple[float,float]]: """ - https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 - Formats the timespan in a human readable form + Run detector on a single *uint8* gray image. + + Returns + ------- + pupil_rect_fine : (x,y,w,h) + center_fine : (cx,cy) -- both on full‑resolution image. """ + if img_gray.dtype != np.uint8: + raise TypeError("img_gray must be uint8 [0,255]") - if timespan >= 60.0: - # we have more than a minute, format that in a human readable form - # Idea from http://snipplr.com/view/5713/ - parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] - time = [] - leftover = timespan - for suffix, length in parts: - value = int(leftover / length) - if value > 0: - leftover = leftover % length - time.append("%s%s" % (str(value), suffix)) - if leftover < 1: - break - return " ".join(time) + self.frame_num += 1 + img_down = self._preprocess(img_gray) + self._coarse_detection(img_down) + self._fine_detection(img_down) + self._postprocess() - # Unfortunately the unicode 'micro' symbol can cause problems in - # certain terminals. - # See bug: https://bugs.launchpad.net/ipython/+bug/348466 - # Try to prevent crashes by being more secure than it needs to - # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. - units = ["s", "ms", "us", "ns"] # the save value - if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: - try: - "\xb5".encode(sys.stdout.encoding) - units = ["s", "ms", "\xb5s", "ns"] - except: - pass - scaling = [1, 1e3, 1e6, 1e9] + return self.pupil_rect_fine, self.center_fine - if timespan > 0.0: - order = min(-int(math.floor(math.log10(timespan)) // 3), 3) - else: - order = 3 - return "%.*g %s" % (precision, timespan * scaling[order], units[order]) + # --------------- optional helper for visual debugging ------------ # + def draw_debug(self, bgr: np.ndarray) -> None: + """Draw rectangular outputs on *bgr* in‑place.""" + cv2.rectangle(bgr, self.pupil_rect_fine, (0, 255, 0), 1) + cv2.rectangle(bgr, self.outer_rect_coarse, (255, 0, 0), 1) + cx, cy = map(int, self.center_fine) + cv2.drawMarker(bgr, (cx, cy), (0, 0, 255), + markerType=cv2.MARKER_CROSS, markerSize=10, thickness=1) + # ---------------------------------------------------------------- # + # INTERNAL STAGES # + # ---------------------------------------------------------------- # + def _preprocess(self, img_gray: np.ndarray) -> np.ndarray: + # down‑sample to target size (longer side ≈ target_resolution[0]) + h, w = img_gray.shape + self._ratio_down = max(w / self.target_resolution[0], + h / self.target_resolution[1], 1.0) + new_w = int(round(w / self._ratio_down)) + new_h = int(round(h / self._ratio_down)) + img_down = cv2.resize(img_gray, (new_w, new_h), + interpolation=cv2.INTER_AREA) - def filter_light(self, img_gray, img_blur, tau): - for i in range(img_gray.shape[1]): - for j in range(img_gray.shape[0]): - if img_gray[j, i] > tau: - img_blur[j, i] = tau - else: - img_blur[j, i] = img_gray[j, i] - return img_blur + self._img_boundary = (0, 0, new_w, new_h) + # optional high‑intensity suppression on first frame + if self.use_init_rect and self.frame_num == 1: + # Estimate µ_inner0 / µ_outer0 inside init box + x, y, rw, rh = self.init_rect + region = img_gray[y:y+rh, x:x+rw] + self.mu_inner0 = np.percentile(region, 25) + self.mu_outer0 = np.percentile(region, 75) - def pupil_detector_haar(self, img_gray, params): - frame_num = 0 - img_down = cv2.resize( - img_gray, - ( - img_gray.shape[1] // params["ratio_downsample"], - img_gray.shape[0] // params["ratio_downsample"], - ), - ) - img_boundary = (0, 0, img_down.shape[1], img_down.shape[0]) - - if params["use_init_rect"]: - tau = max(params["mu_outer"], params["mu_inner"] + 30) - self.filter_light(img_down, img_down, tau) - - # Coarse Detection - ( - pupil_rect_coarse, - outer_rect_coarse, - max_response_coarse, - mu_inner, - mu_outer, - ) = self.coarse_detection(img_down, params) - print( - "Coarse Detection: ", - pupil_rect_coarse, - outer_rect_coarse, - max_response_coarse, - mu_inner, - mu_outer, - ) - - if params["use_init_rect"] and frame_num == 0: - mu_inner0 = mu_inner - mu_outer0 = mu_outer - kf = 2 - 0.01 * mu_inner0 - - img_coarse = cv2.cvtColor(img_down, cv2.COLOR_GRAY2BGR) - # show image - - # Fine Detection - if mu_outer - mu_inner >= 5: - pupil_rect_fine = self.fine_detection(img_down, pupil_rect_coarse) - else: - pupil_rect_fine = pupil_rect_coarse - - # Postprocessing - pupil_rect_coarse = self.rect_scale(pupil_rect_coarse, params["ratio_downsample"], False) - outer_rect_coarse = self.rect_scale(outer_rect_coarse, params["ratio_downsample"], False) - pupil_rect_fine = self.rect_scale(pupil_rect_fine, params["ratio_downsample"], False) - - center_coarse = ( - pupil_rect_coarse[0] + pupil_rect_coarse[2] // 2, - pupil_rect_coarse[1] + pupil_rect_coarse[3] // 2, - ) - center_fine = ( - pupil_rect_fine[0] + pupil_rect_fine[2] // 2, - pupil_rect_fine[1] + pupil_rect_fine[3] // 2, - ) - - return ( - pupil_rect_coarse, - outer_rect_coarse, - pupil_rect_fine, - center_coarse, - center_fine, - ) - - - # @lru_cache(maxsize=self.lru_maxsize_vvs) - def get_empty_array(self, frame_shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer): - frame_int_dtype = np.intc - np_index_dtype = ( - np.intc - ) # memo: Better to use np.intp, but a little slower ref: https://numpy.org/doc/1.25/user/basics.indexing.html#detailed-notes - - row, col = frame_shape - - frame_int = np.empty((row + 1, col + 1), dtype=frame_int_dtype) - - w_arr = np.arange(width_min, width_max + 1, wh_step, dtype=np_index_dtype) - h_arr = (w_arr / ratio_outer).astype(np.int16) - - # memo: It is not smart code and needs to be changed. - y_out_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) for h in h_arr]) - x_out_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) for w in w_arr]) - y_out_h = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + h for h in h_arr]) - x_out_w = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + w for w in w_arr]) - out_h = y_out_h - y_out_n - out_w = x_out_w - x_out_n - - y_in_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) for h in h_arr]) - x_in_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) for w in w_arr]) - y_in_h = np.hstack( - [np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) + int(h / 2) for h in h_arr] - ) - x_in_w = np.hstack( - [np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) + int(w / 2) for w in w_arr] - ) - in_h = y_in_h - y_in_n - in_w = x_in_w - x_in_n - - # # memo: Unelegant code - # # memo: Non-transposed version - # wh_in_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,int(h/2),dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,int(w/2),dtype=np_index_dtype) for w in w_arr])[np.newaxis, :] - # wh_out_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,h,dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,w,dtype=np_index_dtype) for w in w_arr])[np.newaxis, :] - - # memo: Unelegant code - # memo: transposed version - - wh_in_arr = ( - np.hstack( - [ - np.full( - ((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1, - int(w / 2), - dtype=np_index_dtype, - ) - for w in w_arr - ] - )[:, np.newaxis] - * np.hstack( - [ - np.full( - ((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1, - int(h / 2), - dtype=np_index_dtype, - ) - for h in h_arr - ] - )[np.newaxis, :] - ) - wh_out_arr = ( - np.hstack( - [ - np.full( - ((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1, - w, - dtype=np_index_dtype, - ) - for w in w_arr - ] - )[:, np.newaxis] - * np.hstack( - [ - np.full( - ((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1, - h, - dtype=np_index_dtype, - ) - for h in h_arr - ] - )[np.newaxis, :] - ) - - mu_outer_rect = cv2.subtract( - wh_out_arr, wh_in_arr - ) # ,dst=) # == (outer_rect[2] * outer_rect[3] - inner_rect[2] * inner_rect[3]) - - wh_in_arr = 1 / wh_in_arr # .astype(np.float32) - # wh_out_arr=wh_out_arr.astype(np.float64) - mu_outer_rect = 1 / mu_outer_rect # .astype(np.float32) - mu_outer_rect2 = -1.0 * mu_outer_rect # cv2.merge([mu_outer_rect,-1.0*mu_outer_rect]) - - # 1/wh_in_arr == wh_in_arr_mul - return ( - frame_int, - y_out_n, - x_out_n, - y_out_h, - x_out_w, - out_h, - out_w, - y_in_n, - x_in_n, - y_in_h, - x_in_w, - in_h, - in_w, - wh_in_arr, - wh_out_arr, - mu_outer_rect, - mu_outer_rect2, - ) - - - # @profile - def coarse_detection(self, img_gray, params): - ratio_outer = params["ratio_outer"] - kf = params["kf"] - width_min = params["width_min"] - width_max = params["width_max"] - wh_step = params["wh_step"] - xy_step = params["xy_step"] - roi = params["roi"] - init_rect_flag = params["init_rect_flag"] - init_rect = params["init_rect"] - mu_inner = params["mu_inner"] - mu_outer = params["mu_outer"] - max_response_coarse = -255 - - imgboundary = (0, 0, img_gray.shape[1], img_gray.shape[0]) - img_blur = np.copy(img_gray) - rectlist = [] - response = [] - - # Assign values to avoid unassigned errors - pupil_rect_coarse = (10, 10, 10, 10) - outer_rect_coarse = (5, 5, 5, 5) - - if init_rect_flag: - init_rect_down = self.rect_scale(init_rect, params["ratio_downsample"], False) - init_rect_down = self.intersect_rect(init_rect_down, imgboundary) - img_blur = img_gray[ - init_rect_down[1]: init_rect_down[1] + init_rect_down[3], - init_rect_down[0]: init_rect_down[0] + init_rect_down[2], - ] - - ( - frame_int, - y_out_n, - x_out_n, - y_out_h, - x_out_w, - out_h, - out_w, - y_in_n, - x_in_n, - y_in_h, - x_in_w, - in_h, - in_w, - wh_in_arr, - wh_out_arr, - mu_outer_rect, - mu_outer_rect2, - ) = self.get_empty_array(img_blur.shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer) - cv2.integral( - img_blur, sum=frame_int, sdepth=cv2.CV_32S - ) - - out_p_temp = frame_int.take(y_out_n, axis=0, mode="clip") - out_p_temp = cv2.transpose(out_p_temp) - out_p00 = out_p_temp.take(x_out_n, axis=0, mode="clip") - out_p01 = out_p_temp.take(x_out_w, axis=0, mode="clip") - out_p_temp = frame_int.take(y_out_h, axis=0, mode="clip") - out_p_temp = cv2.transpose(out_p_temp) - out_p11 = out_p_temp.take(x_out_w, axis=0, mode="clip") - out_p10 = out_p_temp.take(x_out_n, axis=0, mode="clip") - - outer_sum = cv2.add(out_p00, out_p11) - cv2.subtract(outer_sum, out_p01, dst=outer_sum) - cv2.subtract(outer_sum, out_p10, dst=outer_sum) - - in_p_temp = frame_int.take(y_in_n, axis=0, mode="clip") - in_p_temp = cv2.transpose(in_p_temp) - in_p00 = in_p_temp.take(x_in_n, axis=0, mode="clip") - in_p01 = in_p_temp.take(x_in_w, axis=0, mode="clip") - in_p_temp = frame_int.take(y_in_h, axis=0, mode="clip") - in_p_temp = cv2.transpose(in_p_temp) - in_p11 = in_p_temp.take(x_in_w, axis=0, mode="clip") - in_p10 = in_p_temp.take(x_in_n, axis=0, mode="clip") - - inner_sum = cv2.add(in_p00, in_p11) - cv2.subtract(inner_sum, in_p01, dst=inner_sum) - cv2.subtract(inner_sum, in_p10, dst=inner_sum) - - inner_sum_f = inner_sum.astype(np.float64) - outer_sum_f = outer_sum.astype(np.float64) - - response_value = np.empty(outer_sum.shape, dtype=np.float64) - inout_rect_sum = mu_outer_rect2.copy() - inout_rect_mul = mu_outer_rect.copy() - - cv2.multiply(inner_sum_f, inout_rect_mul, inout_rect_mul) - cv2.multiply(outer_sum_f, inout_rect_sum, inout_rect_sum) - cv2.add(inout_rect_mul, inout_rect_sum, dst=inout_rect_sum) - - cv2.multiply(inner_sum_f, wh_in_arr, inner_sum_f, kf) - cv2.add(inout_rect_sum, inner_sum_f, dst=response_value) - - min_response, max_response, min_loc, max_loc = cv2.minMaxLoc(response_value) - - rec_o = ( - x_out_n[min_loc[1]], - y_out_n[min_loc[0]], - out_w[min_loc[1]], - out_h[min_loc[0]], - ) - rec_in = ( - x_in_n[min_loc[1]], - y_in_n[min_loc[0]], - in_w[min_loc[1]], - in_h[min_loc[0]], - ) - max_response_coarse = -min_response - pupil_rect_coarse = rec_in - outer_rect_coarse = rec_o - - return pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer - - - def fine_detection(self, img_gray, pupil_rect_coarse): - boundary = (0, 0, img_gray.shape[1], img_gray.shape[0]) - valid_ratio = 1.2 - valid_rect = self.intersect_rect(self.rect_scale(pupil_rect_coarse, valid_ratio), boundary) - img_pupil = img_gray[ - valid_rect[1] : valid_rect[1] + valid_rect[3], - valid_rect[0] : valid_rect[0] + valid_rect[2], - ] - img_pupil_blur = cv2.GaussianBlur(img_pupil, (5, 5), 0, 0) - edges_filter = self.detect_edges(img_pupil_blur) - # fit ellipse to edges - contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - # sort contours by area - contours = sorted(contours, key=lambda x: cv2.contourArea(x), reverse=True) - # fit ellipse to largest contour - try: - if len(contours) > 0 and len(contours[0]) >= 5: - pupil_contour = contours[0] - pupil_ellipse = cv2.fitEllipse(pupil_contour) - center_fitting = ( - int(pupil_ellipse[0][0] + valid_rect[0]), - int(pupil_ellipse[0][1] + valid_rect[1]), - ) - pupil_rect_fine = ( - int(pupil_ellipse[0][0] - pupil_ellipse[1][0] / 2), - int(pupil_ellipse[0][1] - pupil_ellipse[1][1] / 2), - int(pupil_ellipse[1][0]), - int(pupil_ellipse[1][1]), - ) - pupil_rect_fine = ( - pupil_rect_fine[0] + valid_rect[0], - pupil_rect_fine[1] + valid_rect[1], - pupil_rect_fine[2], - pupil_rect_fine[3], - ) - pupil_rect_fine = self.intersect_rect(pupil_rect_fine, boundary) - pupil_rect_fine = self.rect_scale(pupil_rect_fine, 1 / valid_ratio) + # adjust kf like original code + if self.mu_outer0 - self.mu_inner0 > 30: + tau = self.mu_outer0 else: - pupil_rect_fine = pupil_rect_coarse - center_fitting = ( - int(pupil_rect_fine[0] + pupil_rect_fine[2] / 2), - int(pupil_rect_fine[1] + pupil_rect_fine[3] / 2), - ) - except: - pass - try: - return pupil_rect_fine, center_fitting - except: - pass + tau = self.mu_inner0 + 30 + img_down = np.minimum(img_down, tau).astype(np.uint8) + return img_down - def detect_edges(self, img_pupil_blur): - tau1 = 1 - 20.0 / img_pupil_blur.shape[1] - edges = cv2.Canny(img_pupil_blur, 64, 128) + # ------------------------------------------------------------ # + # COARSE DETECTION # + # ------------------------------------------------------------ # + def _initial_search_range(self, img_down: np.ndarray) -> Tuple[int,int,int,int]: + """Compute ROI and width range for current frame (down‑sampled).""" + h, w = img_down.shape + margin = h // 10 // 2 + full = (margin, margin, w - 2*margin, h - 2*margin) - # img_bw = np.zeros_like(img_pupil_blur) - # img_bw[img_pupil_blur > 100] = 255 - img_bw = cv2.compare(img_pupil_blur, 100, cv2.CMP_GT) - kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - img_bw = cv2.dilate(img_bw, kernel) + if not self.use_init_rect: + self.roi = full + return - # edges_filter = edges & (~img_bw) - # or - edges_filter = cv2.bitwise_and(edges, cv2.bitwise_not(img_bw)) - return edges_filter + # scale init_rect to down resolution + self._init_rect_down = tuple(int(x / self._ratio_down) for x in self.init_rect) + ix, iy, iw, ih = self._init_rect_down + # grow ROI adaptively near borders (imitates C++ code) + rx, ry, rw, rh = full + enlarge = 35 + if ix < enlarge: rx, rw = 0, w + if iy < enlarge: ry, rh = 0, h + if ix+iw > w - enlarge: rx, rw = 0, w + if iy+ih > h - enlarge: ry, rh = 0, h + self.roi = (rx, ry, rw, rh) - def fit_pupil_ellipse_swirski(self, img_pupil, edges_filter): - contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - max_contour_area = 0 - max_contour = None - #print("contours: ", contours) - for contour in contours: - area = cv2.contourArea(contour) - if area > max_contour_area: - max_contour_area = area - max_contour = contour + # width search band tuned by first frame + self.width_min = max(int(iw*1.0), 24) + self.width_max = min(int(iw*1.5), 120) - if max_contour is None: - return (0, 0, 0, 0), None + def _coarse_detection(self, img_down: np.ndarray) -> None: + self._initial_search_range(img_down) + roi_x, roi_y, roi_w, roi_h = self.roi - ellipse = cv2.fitEllipse(max_contour) - return ellipse + # build integral image (cv2 adds +1 row/col) + ii = cv2.integral(img_down, sdepth=cv2.CV_32S) + best_f = -255 + best_pupil = (0, 0, 0, 0) + best_outer = (0, 0, 0, 0) + best_mu_in, best_mu_out = 0, 0 - def rect_scale(self, rect, scale, round_up=True): - x, y, width, height = rect - new_width = int(width * scale) - new_height = int(height * scale) - if round_up: - new_width = int(np.ceil(width * scale)) - new_height = int(np.ceil(height * scale)) - new_x = x + int((width - new_width) / 2) - new_y = y + int((height - new_height) / 2) - return new_x, new_y, new_width, new_height + for width in range(self.width_min, self.width_max+1, self.wh_step): + # height tied to width; rectangular pupils handled fine + for height in range(width, width+1, self.wh_step): + xmax = roi_x + roi_w - width + ymax = roi_y + roi_h - height + for x in range(roi_x, xmax+1, self.xy_step): + for y in range(roi_y, ymax+1, self.xy_step): + pupil = (x, y, width, height) + outer = _rect_scale(pupil, self.ratio_outer, + keep_center=True, + square_outer=self.use_square_haar) + outer = _clip_rect(outer, self._img_boundary) + mu_in, mu_out = 0.0, 0.0 + mu_out = (_get_block_integral(ii, outer) - + _get_block_integral(ii, pupil)) / \ + (outer[2]*outer[3] - width*height) + mu_in = _get_block_integral(ii, pupil) / (width*height) + f_val = mu_out - self.kf * mu_in + if f_val > best_f: + best_f = f_val + best_pupil = pupil + best_outer = outer + best_mu_in, best_mu_out = mu_in, mu_out - def intersect_rect(self, rect1, rect2): - x1, y1, w1, h1 = rect1 - x2, y2, w2, h2 = rect2 - x = max(x1, x2) - y = max(y1, y2) - w = min(x1 + w1, x2 + w2) - x - h = min(y1 + h1, y2 + h2) - y - return x, y, w, h + self.pupil_rect_coarse = best_pupil + self.outer_rect_coarse = best_outer + self.max_response_coarse = best_f + self.mu_inner, self.mu_outer = best_mu_in, best_mu_out + px, py, pw, ph = best_pupil + self.center_coarse = (px + pw / 2, py + ph / 2) - def rect_suppression(self, rectlist, response, rectlist_out, response_out): - for i in range(len(rectlist)): - flag_intersect = False - for j in range(len(rectlist_out)): - tmp = self.intersect_rect(rectlist[i], rectlist_out[j]) - if tmp[2] > 0 and tmp[3] > 0: - flag_intersect = True - if response[i] > response_out[j]: - rectlist_out[j] = rectlist[i] - response_out[j] = response[i] - else: - continue - if not flag_intersect: - rectlist_out.append(rectlist[i]) - response_out.append(response[i]) - return rectlist_out, response_out + # ------------------------------------------------------------ # + # FINE DETECTION # + # ------------------------------------------------------------ # + def _fine_detection(self, img_down: np.ndarray) -> None: + px, py, pw, ph = self.pupil_rect_coarse + expand = 1.42 + exp_rect = _clip_rect(_rect_scale(self.pupil_rect_coarse, expand, True), + self._img_boundary) + ex, ey, ew, eh = exp_rect + patch = img_down[ey:ey+eh, ex:ex+ew] + # threshold at µ_inner (same heuristic) + _, bw = cv2.threshold(patch, int(self.mu_inner), 255, + cv2.THRESH_BINARY_INV) - def put_number(self, img_bgr, number, position, color): - cv2.putText( - img_bgr, - str(number), - (int(position[0]) + 10, int(position[1]) - 10), - cv2.FONT_HERSHEY_SIMPLEX, - 0.5, - color, - 1, - cv2.LINE_AA, - ) + # dilate to merge gaps + bw = cv2.dilate(bw, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5,5))) + # connected components + n, labels, stats, centroids = cv2.connectedComponentsWithStats(bw) + if n <= 1: + # fall‑back: keep coarse rect + self.pupil_rect_fine = tuple(int(v) for v in + np.array(self.pupil_rect_coarse) * + self._ratio_down) + self.center_fine = tuple(v * self._ratio_down + for v in self.center_coarse) + return - def External_Run_AHSF(self, frame_gray): - average_color = np.mean(frame_gray) - height, width = frame_gray.shape - max_dimension = max(height, width) - square_background = np.full((max_dimension, max_dimension), average_color, dtype=np.uint8) - x_offset = (max_dimension - width) // 2 - y_offset = (max_dimension - height) // 2 - square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray - frame_gray = cv2.resize(square_background, (100, 100)) - frame_clear_resize = frame_gray.copy() + # discard tiny blobs (<4% of patch) + areas = stats[1:, cv2.CC_STAT_AREA] + mask = areas > 0.04 * bw.size + if not np.any(mask): + mask = areas.argmax()[None] # keep largest if all tiny - params = { - "ratio_downsample": 0.5, - "use_init_rect": False, - "mu_outer": 200, - "mu_inner": 50, - "ratio_outer": 1, - "kf": 1, - "width_min": 25, - "width_max": 50, - "wh_step": 1, - "xy_step": 5, - "roi": (0, 0, frame_gray.shape[1], frame_gray.shape[0]), - "init_rect_flag": False, - "init_rect": (0, 0, frame_gray.shape[1], frame_gray.shape[0]), - } - try: - pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer = self.coarse_detection(frame_gray, params) - # ellipse_rect, center_fitting = self.fine_detection(frame_gray, pupil_rect_coarse) + # choose component through image centre, else darkest centroid + cx_local = patch.shape[1] // 2 + cy_local = patch.shape[0] // 2 + comp_idx = labels[cy_local, cx_local] + if comp_idx == 0 or not mask[comp_idx-1]: + # pick darkest of two largest blobs (C++ heuristic) + dark = 255 + for idx in np.flatnonzero(mask) + 1: + cx_i, cy_i = centroids[idx] + val = patch[int(cy_i), int(cx_i)] + if val < dark: + dark = val + comp_idx = idx - except TypeError: - return frame_gray, frame_gray, 0, 0, 0 - - x_center = outer_rect_coarse[0] + outer_rect_coarse[2] / 2 - y_center = outer_rect_coarse[1] + outer_rect_coarse[3] / 2 - x, y, width, height = outer_rect_coarse - - - cv2.circle(frame_gray, (int(x_center), int(y_center)), 2, (255, 255, 255), -1) - thickness = 1 - - cv2.rectangle(frame_gray, (pupil_rect_coarse[0], pupil_rect_coarse[1]), - (pupil_rect_coarse[0] + pupil_rect_coarse[2], pupil_rect_coarse[1] + pupil_rect_coarse[3]), - (0, 255, 0), 2) - cv2.rectangle(frame_gray, (outer_rect_coarse[0], outer_rect_coarse[1]), - (outer_rect_coarse[0] + outer_rect_coarse[2], outer_rect_coarse[1] + outer_rect_coarse[3]), - (255, 0, 0), 2) - - - - - - - - - - - - - major_diameter = math.sqrt(width**2 + height**2) - minor_diameter = min(width, height) - average_diameter = (major_diameter + minor_diameter) / 2 - - return frame_gray, frame_clear_resize, x_center, y_center, abs(width - height) - - - - - - - - - - - - - -class FPSResult(object): - """ - base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 - """ - - def __init__(self, loops, repeat, best, worst, all_runs, precision): - self.loops = loops - self.repeat = repeat - self.best = 1 / best - self.worst = 1 / worst - self.all_runs = all_runs - self._precision = precision - self.fps = [1 / dt for dt in all_runs] - self.unit = "fps" - - @property - def average(self): - return math.fsum(self.fps) / len(self.fps) - - @property - def stdev(self): - mean = self.average - return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 - - def __str__(self): - pm = "+-" - if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: - try: - "\xb1".encode(sys.stdout.encoding) - pm = "\xb1" - except: - pass - return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( - pm=pm, - runs=self.repeat, - loops=self.loops, - loop_plural="" if self.loops == 1 else "s", - run_plural="" if self.repeat == 1 else "s", - mean="%.*g%s" % (self._precision, self.average, self.unit), - std="%.*g%s" % (self._precision, self.stdev, self.unit), - best="%.*g%s" % (self._precision, self.best, self.unit), - worst="%.*g%s" % (self._precision, self.worst, self.unit), - ) - - def _repr_pretty_(self, p, cycle): - unic = self.__str__() - p.text("") + # final bounding box in down‑scaled coords + x, y, w, h = stats[comp_idx, cv2.CC_STAT_LEFT : cv2.CC_STAT_HEIGHT+1] + x += ex + y += ey + self.pupil_rect_fine = (x, y, w, h) + self.center_fine = (x + w / 2, y + h / 2) + # ------------------------------------------------------------ # + # UPSAMPLE BACK # + # ------------------------------------------------------------ # + def _postprocess(self) -> None: + # scale coarse and fine rects + centres back to full resolution + scale = self._ratio_down + def _up(rect): + return tuple(int(round(v*scale)) for v in rect) + self.pupil_rect_coarse = _up(self.pupil_rect_coarse) + self.outer_rect_coarse = _up(self.outer_rect_coarse) + self.pupil_rect_fine = _up(self.pupil_rect_fine) + self.center_coarse = tuple(v*scale for v in self.center_coarse) + self.center_fine = tuple(v*scale for v in self.center_fine) diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 5de8350..ddc2eb1 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -165,7 +165,7 @@ class EyeProcessor: self.pupil_height = 0.0 self.avg_velocity = 0.0 self.angle = 621 - self.er_ahsf = None + self.det = PupilDetectorHaar(ratio_outer=1.4, kf=1.4) try: @@ -406,16 +406,14 @@ class EyeProcessor: pass self.hasrac_en = True - ( - self.current_image_gray, - resize_img, - self.rawx, - self.rawy, - self.radius, - ) = self.er_ahsf.External_Run_AHSF(self.current_image_gray) - self.current_image_gray_clean = resize_img.copy() - self.thresh = resize_img + self.current_image_gray_clean = self.current_image_gray.copy() + self.det.detect(self.current_image_gray) + cx, cy = map(int, self.det.center_fine) + cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1) + cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1) + + self.thresh = self.current_image_gray_clean ( self.rawx, self.rawy, @@ -522,13 +520,11 @@ class EyeProcessor: ) else: pass - ( - self.current_image_gray, - resize_img, - self.rawx, - self.rawy, - self.radius, - ) = self.er_ahsf.External_Run_AHSF(self.current_image_gray) + + self.det.detect(self.current_image_gray) # <- single call per frame + cx, cy = map(int, self.det.center_fine) # fine centre (upsampled) + cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1) + cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1) self.thresh = self.current_image_gray self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.current_algorithm = EyeInfoOrigin.HSF @@ -603,13 +599,9 @@ class EyeProcessor: # set algo priorities if self.settings.gui_AHSFRAC: - if self.er_ahsf is None: - self.er_ahsf = AHSF(self.current_image_gray) algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM if self.settings.gui_AHSF: - if self.er_ahsf is None: - self.er_ahsf = AHSF(self.current_image_gray) algolist[self.settings.gui_AHSFP] = self.AHSFM if self.settings.gui_HSF: