New AHSF code (fixed) ported from Summer
Some checks failed
App Builder / build (ubuntu-latest) (push) Has been cancelled
App Builder / build (windows-latest) (push) Has been cancelled
App Builder / Deploy (push) Has been cancelled
App Builder / Cleanup actions (push) Has been cancelled

This commit is contained in:
Prohurtz 2025-05-06 12:52:26 -05:00
parent 3fc82a43f2
commit ab441f2ce0
2 changed files with 294 additions and 652 deletions

View File

@ -19,7 +19,7 @@
@@@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@( @@@@@@@@@@@@@(
Adaptive Haar Surround Feature: Summer, PallasNeko (Optimization) Adaptive Haar Surround Feature: Summer
Algorithm App Implementations and Tweaks By: Prohurtz Algorithm App Implementations and Tweaks By: Prohurtz
Copyright (c) 2025 EyeTrackVR <3 Copyright (c) 2025 EyeTrackVR <3
@ -27,678 +27,328 @@ Copyright (c) 2025 EyeTrackVR <3
LICENSE: Summer Software Distribution License 1.0 LICENSE: Summer Software Distribution License 1.0
------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------
""" """
from __future__ import annotations
import functools from typing import Tuple, Optional
import math
import os
import sys
import time
import timeit
from logging import FileHandler, Formatter, INFO, StreamHandler, getLogger
from functools import lru_cache
import cv2 import cv2
import numpy as np import numpy as np
# ------------------------- utility helpers ------------------------- #
# from line_profiler_pycharm import profile def _rect_scale(rect: Tuple[int, int, int, int],
ratio: float,
keep_center: bool = True,
square_outer: bool = False) -> Tuple[int, int, int, int]:
class AHSF: """Scale rectangle by *ratio* (optionally keep centre fixed)."""
def __init__(self, video_src, save_logfile=False, imshow_enable=False, save_video=False): x, y, w, h = rect
self.this_file_basename = os.path.basename(__file__) if square_outer:
self.this_file_name = self.this_file_basename.replace(".py", "") w = h = int(max(w, h) * ratio)
self.alg_ver = "PallasNekoV3"
self.save_logfile = save_logfile
self.imshow_enable = imshow_enable
self.save_video = save_video
self.VideoCapture_SRC = video_src
self.input_is_webcam = False
self.benchmark_flag = True if not self.input_is_webcam and not self.imshow_enable and not self.save_video else False
self.loop_num = 1 if self.imshow_enable or self.save_video else 10
self.output_video_path = f"./{self.this_file_name}.mp4"
self.logfilename = f"./{self.this_file_name}.log"
self.print_enable = False
self.lru_maxsize_vvs = 16
self.lru_maxsize_vs = 64
self.lru_maxsize_s = 128
self.logger = getLogger(__name__)
self.logger.setLevel(INFO)
formatter = Formatter("%(message)s")
handler = StreamHandler()
handler.setLevel(INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
if self.save_logfile:
handler = FileHandler(self.logfilename, encoding="utf8", mode="w")
handler.setLevel(INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
else: else:
self.save_logfile = False w = int(w * ratio)
self.video_wr = cv2.VideoWriter if self.save_video else None h = int(h * ratio)
if keep_center:
cx, cy = x + rect[2] // 2, y + rect[3] // 2
x = int(cx - w / 2)
y = int(cy - h / 2)
return (x, y, w, h)
def _clip_rect(rect: Tuple[int, int, int, int],
boundary: Tuple[int, int, int, int]) -> Tuple[int, int, int, int]:
"""Clip *rect* to *boundary* = (x, y, w, h)."""
bx, by, bw, bh = boundary
x, y, w, h = rect
x = max(bx, x)
y = max(by, y)
w = min(x + w, bx + bw) - x
h = min(y + h, by + bh) - y
return (x, y, max(0, w), max(0, h))
def format_time(self, timespan, precision=3): def _get_block_integral(ii: np.ndarray,
rect: Tuple[int, int, int, int]) -> int:
"""Integralimage sum over *rect*."""
x, y, w, h = rect
return (ii[y+h, x+w] - ii[y, x+w] - ii[y+h, x] + ii[y, x])
def _canny_pure(img: np.ndarray,
low: int = 64,
high_ratio: float = 2.0) -> np.ndarray:
"""Lightweight Canny wrapper (imitates canny_pure())."""
img_blur = cv2.GaussianBlur(img, (3, 3), 0)
return cv2.Canny(img_blur, low, int(low*high_ratio))
# --------------------------- main class ---------------------------- #
class PupilDetectorHaar:
""" """
https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473 Haarbased coarsetofine pupil detector.
Formats the timespan in a human readable form
Parameters
----------
ratio_outer : float
Scaling factor for Haar outer rectangle.
kf : float
Weighting term in response function f = µ_outer kf*µ_inner.
use_square_haar : bool
If True, outer Haar window is square; else horizontal rectangle.
use_init_rect : bool
If True, provide an approximate pupil box in *init_rect*.
init_rect : Tuple[int,int,int,int] | None
Initial pupil location on the fullresolution frame.
target_resolution : Tuple[int,int]
Image is downsampled so the longer side ~320 px by default.
width_min / width_max / wh_step / xy_step
Searchgrid parameters for Haar scanning.
""" """
if timespan >= 60.0: # -------- initialisation -------- #
# we have more than a minute, format that in a human readable form def __init__(self,
# Idea from http://snipplr.com/view/5713/ ratio_outer: float = 1.4,
parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] kf: float = 1.5,
time = [] use_square_haar: bool = False,
leftover = timespan use_init_rect: bool = False,
for suffix, length in parts: init_rect: Optional[Tuple[int, int, int, int]] = None,
value = int(leftover / length) target_resolution: Tuple[int, int] = (320, 240),
if value > 0: width_min: int = 31,
leftover = leftover % length width_max: int = 120,
time.append("%s%s" % (str(value), suffix)) wh_step: int = 2,
if leftover < 1: xy_step: int = 2):
break self.ratio_outer = ratio_outer
return " ".join(time) self.kf = kf
self.use_square_haar = use_square_haar
# Unfortunately the unicode 'micro' symbol can cause problems in self.use_init_rect = use_init_rect
# certain terminals. self.init_rect = (0, 0, 0, 0) if init_rect is None else init_rect
# See bug: https://bugs.launchpad.net/ipython/+bug/348466 self.target_resolution = target_resolution
# Try to prevent crashes by being more secure than it needs to
# E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. # searchgrid params (may be autotuned after first frame)
units = ["s", "ms", "us", "ns"] # the save value self.width_min = width_min
if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: self.width_max = width_max
try: self.wh_step = wh_step
"\xb5".encode(sys.stdout.encoding) self.xy_step = xy_step
units = ["s", "ms", "\xb5s", "ns"]
except: # dynamic state
pass self.frame_num = 0
scaling = [1, 1e3, 1e6, 1e9] self.mu_inner = 50
self.mu_outer = 200
if timespan > 0.0: self.mu_inner0 = 50 # first frame stats
order = min(-int(math.floor(math.log10(timespan)) // 3), 3) self.mu_outer0 = 200
else:
order = 3 # outputs (public)
return "%.*g %s" % (precision, timespan * scaling[order], units[order]) self.pupil_rect_coarse = (0, 0, 0, 0)
self.outer_rect_coarse = (0, 0, 0, 0)
self.max_response_coarse = -255
def filter_light(self, img_gray, img_blur, tau): self.center_coarse = (0.0, 0.0)
for i in range(img_gray.shape[1]):
for j in range(img_gray.shape[0]): self.pupil_rect_fine = (0, 0, 0, 0)
if img_gray[j, i] > tau: self.center_fine = (0.0, 0.0)
img_blur[j, i] = tau
else: # private temp
img_blur[j, i] = img_gray[j, i] self._ratio_down = 1.0
return img_blur self._img_boundary = (0, 0, 0, 0)
self._init_rect_down = (0, 0, 0, 0)
def pupil_detector_haar(self, img_gray, params): # ---------------------------------------------------------------- #
frame_num = 0 # PUBLIC API #
img_down = cv2.resize( # ---------------------------------------------------------------- #
img_gray, def detect(self, img_gray: np.ndarray) -> Tuple[Tuple[int,int,int,int], Tuple[float,float]]:
(
img_gray.shape[1] // params["ratio_downsample"],
img_gray.shape[0] // params["ratio_downsample"],
),
)
img_boundary = (0, 0, img_down.shape[1], img_down.shape[0])
if params["use_init_rect"]:
tau = max(params["mu_outer"], params["mu_inner"] + 30)
self.filter_light(img_down, img_down, tau)
# Coarse Detection
(
pupil_rect_coarse,
outer_rect_coarse,
max_response_coarse,
mu_inner,
mu_outer,
) = self.coarse_detection(img_down, params)
print(
"Coarse Detection: ",
pupil_rect_coarse,
outer_rect_coarse,
max_response_coarse,
mu_inner,
mu_outer,
)
if params["use_init_rect"] and frame_num == 0:
mu_inner0 = mu_inner
mu_outer0 = mu_outer
kf = 2 - 0.01 * mu_inner0
img_coarse = cv2.cvtColor(img_down, cv2.COLOR_GRAY2BGR)
# show image
# Fine Detection
if mu_outer - mu_inner >= 5:
pupil_rect_fine = self.fine_detection(img_down, pupil_rect_coarse)
else:
pupil_rect_fine = pupil_rect_coarse
# Postprocessing
pupil_rect_coarse = self.rect_scale(pupil_rect_coarse, params["ratio_downsample"], False)
outer_rect_coarse = self.rect_scale(outer_rect_coarse, params["ratio_downsample"], False)
pupil_rect_fine = self.rect_scale(pupil_rect_fine, params["ratio_downsample"], False)
center_coarse = (
pupil_rect_coarse[0] + pupil_rect_coarse[2] // 2,
pupil_rect_coarse[1] + pupil_rect_coarse[3] // 2,
)
center_fine = (
pupil_rect_fine[0] + pupil_rect_fine[2] // 2,
pupil_rect_fine[1] + pupil_rect_fine[3] // 2,
)
return (
pupil_rect_coarse,
outer_rect_coarse,
pupil_rect_fine,
center_coarse,
center_fine,
)
# @lru_cache(maxsize=self.lru_maxsize_vvs)
def get_empty_array(self, frame_shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer):
frame_int_dtype = np.intc
np_index_dtype = (
np.intc
) # memo: Better to use np.intp, but a little slower ref: https://numpy.org/doc/1.25/user/basics.indexing.html#detailed-notes
row, col = frame_shape
frame_int = np.empty((row + 1, col + 1), dtype=frame_int_dtype)
w_arr = np.arange(width_min, width_max + 1, wh_step, dtype=np_index_dtype)
h_arr = (w_arr / ratio_outer).astype(np.int16)
# memo: It is not smart code and needs to be changed.
y_out_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) for h in h_arr])
x_out_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) for w in w_arr])
y_out_h = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + h for h in h_arr])
x_out_w = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + w for w in w_arr])
out_h = y_out_h - y_out_n
out_w = x_out_w - x_out_n
y_in_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) for h in h_arr])
x_in_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) for w in w_arr])
y_in_h = np.hstack(
[np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) + int(h / 2) for h in h_arr]
)
x_in_w = np.hstack(
[np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) + int(w / 2) for w in w_arr]
)
in_h = y_in_h - y_in_n
in_w = x_in_w - x_in_n
# # memo: Unelegant code
# # memo: Non-transposed version
# wh_in_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,int(h/2),dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,int(w/2),dtype=np_index_dtype) for w in w_arr])[np.newaxis, :]
# wh_out_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,h,dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,w,dtype=np_index_dtype) for w in w_arr])[np.newaxis, :]
# memo: Unelegant code
# memo: transposed version
wh_in_arr = (
np.hstack(
[
np.full(
((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,
int(w / 2),
dtype=np_index_dtype,
)
for w in w_arr
]
)[:, np.newaxis]
* np.hstack(
[
np.full(
((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,
int(h / 2),
dtype=np_index_dtype,
)
for h in h_arr
]
)[np.newaxis, :]
)
wh_out_arr = (
np.hstack(
[
np.full(
((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,
w,
dtype=np_index_dtype,
)
for w in w_arr
]
)[:, np.newaxis]
* np.hstack(
[
np.full(
((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,
h,
dtype=np_index_dtype,
)
for h in h_arr
]
)[np.newaxis, :]
)
mu_outer_rect = cv2.subtract(
wh_out_arr, wh_in_arr
) # ,dst=) # == (outer_rect[2] * outer_rect[3] - inner_rect[2] * inner_rect[3])
wh_in_arr = 1 / wh_in_arr # .astype(np.float32)
# wh_out_arr=wh_out_arr.astype(np.float64)
mu_outer_rect = 1 / mu_outer_rect # .astype(np.float32)
mu_outer_rect2 = -1.0 * mu_outer_rect # cv2.merge([mu_outer_rect,-1.0*mu_outer_rect])
# 1/wh_in_arr == wh_in_arr_mul
return (
frame_int,
y_out_n,
x_out_n,
y_out_h,
x_out_w,
out_h,
out_w,
y_in_n,
x_in_n,
y_in_h,
x_in_w,
in_h,
in_w,
wh_in_arr,
wh_out_arr,
mu_outer_rect,
mu_outer_rect2,
)
# @profile
def coarse_detection(self, img_gray, params):
ratio_outer = params["ratio_outer"]
kf = params["kf"]
width_min = params["width_min"]
width_max = params["width_max"]
wh_step = params["wh_step"]
xy_step = params["xy_step"]
roi = params["roi"]
init_rect_flag = params["init_rect_flag"]
init_rect = params["init_rect"]
mu_inner = params["mu_inner"]
mu_outer = params["mu_outer"]
max_response_coarse = -255
imgboundary = (0, 0, img_gray.shape[1], img_gray.shape[0])
img_blur = np.copy(img_gray)
rectlist = []
response = []
# Assign values to avoid unassigned errors
pupil_rect_coarse = (10, 10, 10, 10)
outer_rect_coarse = (5, 5, 5, 5)
if init_rect_flag:
init_rect_down = self.rect_scale(init_rect, params["ratio_downsample"], False)
init_rect_down = self.intersect_rect(init_rect_down, imgboundary)
img_blur = img_gray[
init_rect_down[1]: init_rect_down[1] + init_rect_down[3],
init_rect_down[0]: init_rect_down[0] + init_rect_down[2],
]
(
frame_int,
y_out_n,
x_out_n,
y_out_h,
x_out_w,
out_h,
out_w,
y_in_n,
x_in_n,
y_in_h,
x_in_w,
in_h,
in_w,
wh_in_arr,
wh_out_arr,
mu_outer_rect,
mu_outer_rect2,
) = self.get_empty_array(img_blur.shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer)
cv2.integral(
img_blur, sum=frame_int, sdepth=cv2.CV_32S
)
out_p_temp = frame_int.take(y_out_n, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p00 = out_p_temp.take(x_out_n, axis=0, mode="clip")
out_p01 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p_temp = frame_int.take(y_out_h, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p11 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p10 = out_p_temp.take(x_out_n, axis=0, mode="clip")
outer_sum = cv2.add(out_p00, out_p11)
cv2.subtract(outer_sum, out_p01, dst=outer_sum)
cv2.subtract(outer_sum, out_p10, dst=outer_sum)
in_p_temp = frame_int.take(y_in_n, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p00 = in_p_temp.take(x_in_n, axis=0, mode="clip")
in_p01 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p_temp = frame_int.take(y_in_h, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p11 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p10 = in_p_temp.take(x_in_n, axis=0, mode="clip")
inner_sum = cv2.add(in_p00, in_p11)
cv2.subtract(inner_sum, in_p01, dst=inner_sum)
cv2.subtract(inner_sum, in_p10, dst=inner_sum)
inner_sum_f = inner_sum.astype(np.float64)
outer_sum_f = outer_sum.astype(np.float64)
response_value = np.empty(outer_sum.shape, dtype=np.float64)
inout_rect_sum = mu_outer_rect2.copy()
inout_rect_mul = mu_outer_rect.copy()
cv2.multiply(inner_sum_f, inout_rect_mul, inout_rect_mul)
cv2.multiply(outer_sum_f, inout_rect_sum, inout_rect_sum)
cv2.add(inout_rect_mul, inout_rect_sum, dst=inout_rect_sum)
cv2.multiply(inner_sum_f, wh_in_arr, inner_sum_f, kf)
cv2.add(inout_rect_sum, inner_sum_f, dst=response_value)
min_response, max_response, min_loc, max_loc = cv2.minMaxLoc(response_value)
rec_o = (
x_out_n[min_loc[1]],
y_out_n[min_loc[0]],
out_w[min_loc[1]],
out_h[min_loc[0]],
)
rec_in = (
x_in_n[min_loc[1]],
y_in_n[min_loc[0]],
in_w[min_loc[1]],
in_h[min_loc[0]],
)
max_response_coarse = -min_response
pupil_rect_coarse = rec_in
outer_rect_coarse = rec_o
return pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer
def fine_detection(self, img_gray, pupil_rect_coarse):
boundary = (0, 0, img_gray.shape[1], img_gray.shape[0])
valid_ratio = 1.2
valid_rect = self.intersect_rect(self.rect_scale(pupil_rect_coarse, valid_ratio), boundary)
img_pupil = img_gray[
valid_rect[1] : valid_rect[1] + valid_rect[3],
valid_rect[0] : valid_rect[0] + valid_rect[2],
]
img_pupil_blur = cv2.GaussianBlur(img_pupil, (5, 5), 0, 0)
edges_filter = self.detect_edges(img_pupil_blur)
# fit ellipse to edges
contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
# sort contours by area
contours = sorted(contours, key=lambda x: cv2.contourArea(x), reverse=True)
# fit ellipse to largest contour
try:
if len(contours) > 0 and len(contours[0]) >= 5:
pupil_contour = contours[0]
pupil_ellipse = cv2.fitEllipse(pupil_contour)
center_fitting = (
int(pupil_ellipse[0][0] + valid_rect[0]),
int(pupil_ellipse[0][1] + valid_rect[1]),
)
pupil_rect_fine = (
int(pupil_ellipse[0][0] - pupil_ellipse[1][0] / 2),
int(pupil_ellipse[0][1] - pupil_ellipse[1][1] / 2),
int(pupil_ellipse[1][0]),
int(pupil_ellipse[1][1]),
)
pupil_rect_fine = (
pupil_rect_fine[0] + valid_rect[0],
pupil_rect_fine[1] + valid_rect[1],
pupil_rect_fine[2],
pupil_rect_fine[3],
)
pupil_rect_fine = self.intersect_rect(pupil_rect_fine, boundary)
pupil_rect_fine = self.rect_scale(pupil_rect_fine, 1 / valid_ratio)
else:
pupil_rect_fine = pupil_rect_coarse
center_fitting = (
int(pupil_rect_fine[0] + pupil_rect_fine[2] / 2),
int(pupil_rect_fine[1] + pupil_rect_fine[3] / 2),
)
except:
pass
try:
return pupil_rect_fine, center_fitting
except:
pass
def detect_edges(self, img_pupil_blur):
tau1 = 1 - 20.0 / img_pupil_blur.shape[1]
edges = cv2.Canny(img_pupil_blur, 64, 128)
# img_bw = np.zeros_like(img_pupil_blur)
# img_bw[img_pupil_blur > 100] = 255
img_bw = cv2.compare(img_pupil_blur, 100, cv2.CMP_GT)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
img_bw = cv2.dilate(img_bw, kernel)
# edges_filter = edges & (~img_bw)
# or
edges_filter = cv2.bitwise_and(edges, cv2.bitwise_not(img_bw))
return edges_filter
def fit_pupil_ellipse_swirski(self, img_pupil, edges_filter):
contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
max_contour_area = 0
max_contour = None
#print("contours: ", contours)
for contour in contours:
area = cv2.contourArea(contour)
if area > max_contour_area:
max_contour_area = area
max_contour = contour
if max_contour is None:
return (0, 0, 0, 0), None
ellipse = cv2.fitEllipse(max_contour)
return ellipse
def rect_scale(self, rect, scale, round_up=True):
x, y, width, height = rect
new_width = int(width * scale)
new_height = int(height * scale)
if round_up:
new_width = int(np.ceil(width * scale))
new_height = int(np.ceil(height * scale))
new_x = x + int((width - new_width) / 2)
new_y = y + int((height - new_height) / 2)
return new_x, new_y, new_width, new_height
def intersect_rect(self, rect1, rect2):
x1, y1, w1, h1 = rect1
x2, y2, w2, h2 = rect2
x = max(x1, x2)
y = max(y1, y2)
w = min(x1 + w1, x2 + w2) - x
h = min(y1 + h1, y2 + h2) - y
return x, y, w, h
def rect_suppression(self, rectlist, response, rectlist_out, response_out):
for i in range(len(rectlist)):
flag_intersect = False
for j in range(len(rectlist_out)):
tmp = self.intersect_rect(rectlist[i], rectlist_out[j])
if tmp[2] > 0 and tmp[3] > 0:
flag_intersect = True
if response[i] > response_out[j]:
rectlist_out[j] = rectlist[i]
response_out[j] = response[i]
else:
continue
if not flag_intersect:
rectlist_out.append(rectlist[i])
response_out.append(response[i])
return rectlist_out, response_out
def put_number(self, img_bgr, number, position, color):
cv2.putText(
img_bgr,
str(number),
(int(position[0]) + 10, int(position[1]) - 10),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
color,
1,
cv2.LINE_AA,
)
def External_Run_AHSF(self, frame_gray):
average_color = np.mean(frame_gray)
height, width = frame_gray.shape
max_dimension = max(height, width)
square_background = np.full((max_dimension, max_dimension), average_color, dtype=np.uint8)
x_offset = (max_dimension - width) // 2
y_offset = (max_dimension - height) // 2
square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray
frame_gray = cv2.resize(square_background, (100, 100))
frame_clear_resize = frame_gray.copy()
params = {
"ratio_downsample": 0.5,
"use_init_rect": False,
"mu_outer": 200,
"mu_inner": 50,
"ratio_outer": 1,
"kf": 1,
"width_min": 25,
"width_max": 50,
"wh_step": 1,
"xy_step": 5,
"roi": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
"init_rect_flag": False,
"init_rect": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
}
try:
pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer = self.coarse_detection(frame_gray, params)
# ellipse_rect, center_fitting = self.fine_detection(frame_gray, pupil_rect_coarse)
except TypeError:
return frame_gray, frame_gray, 0, 0, 0
x_center = outer_rect_coarse[0] + outer_rect_coarse[2] / 2
y_center = outer_rect_coarse[1] + outer_rect_coarse[3] / 2
x, y, width, height = outer_rect_coarse
cv2.circle(frame_gray, (int(x_center), int(y_center)), 2, (255, 255, 255), -1)
thickness = 1
cv2.rectangle(frame_gray, (pupil_rect_coarse[0], pupil_rect_coarse[1]),
(pupil_rect_coarse[0] + pupil_rect_coarse[2], pupil_rect_coarse[1] + pupil_rect_coarse[3]),
(0, 255, 0), 2)
cv2.rectangle(frame_gray, (outer_rect_coarse[0], outer_rect_coarse[1]),
(outer_rect_coarse[0] + outer_rect_coarse[2], outer_rect_coarse[1] + outer_rect_coarse[3]),
(255, 0, 0), 2)
major_diameter = math.sqrt(width**2 + height**2)
minor_diameter = min(width, height)
average_diameter = (major_diameter + minor_diameter) / 2
return frame_gray, frame_clear_resize, x_center, y_center, abs(width - height)
class FPSResult(object):
""" """
base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55 Run detector on a single *uint8* gray image.
Returns
-------
pupil_rect_fine : (x,y,w,h)
center_fine : (cx,cy) -- both on fullresolution image.
""" """
if img_gray.dtype != np.uint8:
raise TypeError("img_gray must be uint8 [0,255]")
def __init__(self, loops, repeat, best, worst, all_runs, precision): self.frame_num += 1
self.loops = loops img_down = self._preprocess(img_gray)
self.repeat = repeat self._coarse_detection(img_down)
self.best = 1 / best self._fine_detection(img_down)
self.worst = 1 / worst self._postprocess()
self.all_runs = all_runs
self._precision = precision
self.fps = [1 / dt for dt in all_runs]
self.unit = "fps"
@property return self.pupil_rect_fine, self.center_fine
def average(self):
return math.fsum(self.fps) / len(self.fps)
@property # --------------- optional helper for visual debugging ------------ #
def stdev(self): def draw_debug(self, bgr: np.ndarray) -> None:
mean = self.average """Draw rectangular outputs on *bgr* inplace."""
return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5 cv2.rectangle(bgr, self.pupil_rect_fine, (0, 255, 0), 1)
cv2.rectangle(bgr, self.outer_rect_coarse, (255, 0, 0), 1)
cx, cy = map(int, self.center_fine)
cv2.drawMarker(bgr, (cx, cy), (0, 0, 255),
markerType=cv2.MARKER_CROSS, markerSize=10, thickness=1)
def __str__(self): # ---------------------------------------------------------------- #
pm = "+-" # INTERNAL STAGES #
if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: # ---------------------------------------------------------------- #
try: def _preprocess(self, img_gray: np.ndarray) -> np.ndarray:
"\xb1".encode(sys.stdout.encoding) # downsample to target size (longer side ≈ target_resolution[0])
pm = "\xb1" h, w = img_gray.shape
except: self._ratio_down = max(w / self.target_resolution[0],
pass h / self.target_resolution[1], 1.0)
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format( new_w = int(round(w / self._ratio_down))
pm=pm, new_h = int(round(h / self._ratio_down))
runs=self.repeat, img_down = cv2.resize(img_gray, (new_w, new_h),
loops=self.loops, interpolation=cv2.INTER_AREA)
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean="%.*g%s" % (self._precision, self.average, self.unit),
std="%.*g%s" % (self._precision, self.stdev, self.unit),
best="%.*g%s" % (self._precision, self.best, self.unit),
worst="%.*g%s" % (self._precision, self.worst, self.unit),
)
def _repr_pretty_(self, p, cycle): self._img_boundary = (0, 0, new_w, new_h)
unic = self.__str__()
p.text("<FPSResult : " + unic + ">")
# optional highintensity suppression on first frame
if self.use_init_rect and self.frame_num == 1:
# Estimate µ_inner0 / µ_outer0 inside init box
x, y, rw, rh = self.init_rect
region = img_gray[y:y+rh, x:x+rw]
self.mu_inner0 = np.percentile(region, 25)
self.mu_outer0 = np.percentile(region, 75)
# adjust kf like original code
if self.mu_outer0 - self.mu_inner0 > 30:
tau = self.mu_outer0
else:
tau = self.mu_inner0 + 30
img_down = np.minimum(img_down, tau).astype(np.uint8)
return img_down
# ------------------------------------------------------------ #
# COARSE DETECTION #
# ------------------------------------------------------------ #
def _initial_search_range(self, img_down: np.ndarray) -> Tuple[int,int,int,int]:
"""Compute ROI and width range for current frame (downsampled)."""
h, w = img_down.shape
margin = h // 10 // 2
full = (margin, margin, w - 2*margin, h - 2*margin)
if not self.use_init_rect:
self.roi = full
return
# scale init_rect to down resolution
self._init_rect_down = tuple(int(x / self._ratio_down) for x in self.init_rect)
ix, iy, iw, ih = self._init_rect_down
# grow ROI adaptively near borders (imitates C++ code)
rx, ry, rw, rh = full
enlarge = 35
if ix < enlarge: rx, rw = 0, w
if iy < enlarge: ry, rh = 0, h
if ix+iw > w - enlarge: rx, rw = 0, w
if iy+ih > h - enlarge: ry, rh = 0, h
self.roi = (rx, ry, rw, rh)
# width search band tuned by first frame
self.width_min = max(int(iw*1.0), 24)
self.width_max = min(int(iw*1.5), 120)
def _coarse_detection(self, img_down: np.ndarray) -> None:
self._initial_search_range(img_down)
roi_x, roi_y, roi_w, roi_h = self.roi
# build integral image (cv2 adds +1 row/col)
ii = cv2.integral(img_down, sdepth=cv2.CV_32S)
best_f = -255
best_pupil = (0, 0, 0, 0)
best_outer = (0, 0, 0, 0)
best_mu_in, best_mu_out = 0, 0
for width in range(self.width_min, self.width_max+1, self.wh_step):
# height tied to width; rectangular pupils handled fine
for height in range(width, width+1, self.wh_step):
xmax = roi_x + roi_w - width
ymax = roi_y + roi_h - height
for x in range(roi_x, xmax+1, self.xy_step):
for y in range(roi_y, ymax+1, self.xy_step):
pupil = (x, y, width, height)
outer = _rect_scale(pupil, self.ratio_outer,
keep_center=True,
square_outer=self.use_square_haar)
outer = _clip_rect(outer, self._img_boundary)
mu_in, mu_out = 0.0, 0.0
mu_out = (_get_block_integral(ii, outer) -
_get_block_integral(ii, pupil)) / \
(outer[2]*outer[3] - width*height)
mu_in = _get_block_integral(ii, pupil) / (width*height)
f_val = mu_out - self.kf * mu_in
if f_val > best_f:
best_f = f_val
best_pupil = pupil
best_outer = outer
best_mu_in, best_mu_out = mu_in, mu_out
self.pupil_rect_coarse = best_pupil
self.outer_rect_coarse = best_outer
self.max_response_coarse = best_f
self.mu_inner, self.mu_outer = best_mu_in, best_mu_out
px, py, pw, ph = best_pupil
self.center_coarse = (px + pw / 2, py + ph / 2)
# ------------------------------------------------------------ #
# FINE DETECTION #
# ------------------------------------------------------------ #
def _fine_detection(self, img_down: np.ndarray) -> None:
px, py, pw, ph = self.pupil_rect_coarse
expand = 1.42
exp_rect = _clip_rect(_rect_scale(self.pupil_rect_coarse, expand, True),
self._img_boundary)
ex, ey, ew, eh = exp_rect
patch = img_down[ey:ey+eh, ex:ex+ew]
# threshold at µ_inner (same heuristic)
_, bw = cv2.threshold(patch, int(self.mu_inner), 255,
cv2.THRESH_BINARY_INV)
# dilate to merge gaps
bw = cv2.dilate(bw, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5,5)))
# connected components
n, labels, stats, centroids = cv2.connectedComponentsWithStats(bw)
if n <= 1:
# fallback: keep coarse rect
self.pupil_rect_fine = tuple(int(v) for v in
np.array(self.pupil_rect_coarse) *
self._ratio_down)
self.center_fine = tuple(v * self._ratio_down
for v in self.center_coarse)
return
# discard tiny blobs (<4% of patch)
areas = stats[1:, cv2.CC_STAT_AREA]
mask = areas > 0.04 * bw.size
if not np.any(mask):
mask = areas.argmax()[None] # keep largest if all tiny
# choose component through image centre, else darkest centroid
cx_local = patch.shape[1] // 2
cy_local = patch.shape[0] // 2
comp_idx = labels[cy_local, cx_local]
if comp_idx == 0 or not mask[comp_idx-1]:
# pick darkest of two largest blobs (C++ heuristic)
dark = 255
for idx in np.flatnonzero(mask) + 1:
cx_i, cy_i = centroids[idx]
val = patch[int(cy_i), int(cx_i)]
if val < dark:
dark = val
comp_idx = idx
# final bounding box in downscaled coords
x, y, w, h = stats[comp_idx, cv2.CC_STAT_LEFT : cv2.CC_STAT_HEIGHT+1]
x += ex
y += ey
self.pupil_rect_fine = (x, y, w, h)
self.center_fine = (x + w / 2, y + h / 2)
# ------------------------------------------------------------ #
# UPSAMPLE BACK #
# ------------------------------------------------------------ #
def _postprocess(self) -> None:
# scale coarse and fine rects + centres back to full resolution
scale = self._ratio_down
def _up(rect):
return tuple(int(round(v*scale)) for v in rect)
self.pupil_rect_coarse = _up(self.pupil_rect_coarse)
self.outer_rect_coarse = _up(self.outer_rect_coarse)
self.pupil_rect_fine = _up(self.pupil_rect_fine)
self.center_coarse = tuple(v*scale for v in self.center_coarse)
self.center_fine = tuple(v*scale for v in self.center_fine)

View File

@ -165,7 +165,7 @@ class EyeProcessor:
self.pupil_height = 0.0 self.pupil_height = 0.0
self.avg_velocity = 0.0 self.avg_velocity = 0.0
self.angle = 621 self.angle = 621
self.er_ahsf = None self.det = PupilDetectorHaar(ratio_outer=1.4, kf=1.4)
try: try:
@ -406,16 +406,14 @@ class EyeProcessor:
pass pass
self.hasrac_en = True self.hasrac_en = True
(
self.current_image_gray,
resize_img,
self.rawx,
self.rawy,
self.radius,
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
self.current_image_gray_clean = resize_img.copy()
self.thresh = resize_img self.current_image_gray_clean = self.current_image_gray.copy()
self.det.detect(self.current_image_gray)
cx, cy = map(int, self.det.center_fine)
cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1)
cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1)
self.thresh = self.current_image_gray_clean
( (
self.rawx, self.rawx,
self.rawy, self.rawy,
@ -522,13 +520,11 @@ class EyeProcessor:
) )
else: else:
pass pass
(
self.current_image_gray, self.det.detect(self.current_image_gray) # <- single call per frame
resize_img, cx, cy = map(int, self.det.center_fine) # fine centre (upsampled)
self.rawx, cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1)
self.rawy, cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1)
self.radius,
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
self.thresh = self.current_image_gray self.thresh = self.current_image_gray
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle) self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSF self.current_algorithm = EyeInfoOrigin.HSF
@ -603,13 +599,9 @@ class EyeProcessor:
# set algo priorities # set algo priorities
if self.settings.gui_AHSFRAC: if self.settings.gui_AHSFRAC:
if self.er_ahsf is None:
self.er_ahsf = AHSF(self.current_image_gray)
algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM
if self.settings.gui_AHSF: if self.settings.gui_AHSF:
if self.er_ahsf is None:
self.er_ahsf = AHSF(self.current_image_gray)
algolist[self.settings.gui_AHSFP] = self.AHSFM algolist[self.settings.gui_AHSFP] = self.AHSFM
if self.settings.gui_HSF: if self.settings.gui_HSF: