New AHSF code (fixed) ported from Summer
Some checks failed
App Builder / build (ubuntu-latest) (push) Has been cancelled
App Builder / build (windows-latest) (push) Has been cancelled
App Builder / Deploy (push) Has been cancelled
App Builder / Cleanup actions (push) Has been cancelled

This commit is contained in:
Prohurtz 2025-05-06 12:52:26 -05:00
parent 3fc82a43f2
commit ab441f2ce0
2 changed files with 294 additions and 652 deletions

View File

@ -19,7 +19,7 @@
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
Adaptive Haar Surround Feature: Summer, PallasNeko (Optimization)
Adaptive Haar Surround Feature: Summer
Algorithm App Implementations and Tweaks By: Prohurtz
Copyright (c) 2025 EyeTrackVR <3
@ -27,678 +27,328 @@ Copyright (c) 2025 EyeTrackVR <3
LICENSE: Summer Software Distribution License 1.0
------------------------------------------------------------------------------------------------------
"""
import functools
import math
import os
import sys
import time
import timeit
from logging import FileHandler, Formatter, INFO, StreamHandler, getLogger
from functools import lru_cache
from __future__ import annotations
from typing import Tuple, Optional
import cv2
import numpy as np
# from line_profiler_pycharm import profile
class AHSF:
def __init__(self, video_src, save_logfile=False, imshow_enable=False, save_video=False):
self.this_file_basename = os.path.basename(__file__)
self.this_file_name = self.this_file_basename.replace(".py", "")
self.alg_ver = "PallasNekoV3"
self.save_logfile = save_logfile
self.imshow_enable = imshow_enable
self.save_video = save_video
self.VideoCapture_SRC = video_src
self.input_is_webcam = False
self.benchmark_flag = True if not self.input_is_webcam and not self.imshow_enable and not self.save_video else False
self.loop_num = 1 if self.imshow_enable or self.save_video else 10
self.output_video_path = f"./{self.this_file_name}.mp4"
self.logfilename = f"./{self.this_file_name}.log"
self.print_enable = False
self.lru_maxsize_vvs = 16
self.lru_maxsize_vs = 64
self.lru_maxsize_s = 128
self.logger = getLogger(__name__)
self.logger.setLevel(INFO)
formatter = Formatter("%(message)s")
handler = StreamHandler()
handler.setLevel(INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
if self.save_logfile:
handler = FileHandler(self.logfilename, encoding="utf8", mode="w")
handler.setLevel(INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
# ------------------------- utility helpers ------------------------- #
def _rect_scale(rect: Tuple[int, int, int, int],
ratio: float,
keep_center: bool = True,
square_outer: bool = False) -> Tuple[int, int, int, int]:
"""Scale rectangle by *ratio* (optionally keep centre fixed)."""
x, y, w, h = rect
if square_outer:
w = h = int(max(w, h) * ratio)
else:
self.save_logfile = False
self.video_wr = cv2.VideoWriter if self.save_video else None
w = int(w * ratio)
h = int(h * ratio)
if keep_center:
cx, cy = x + rect[2] // 2, y + rect[3] // 2
x = int(cx - w / 2)
y = int(cy - h / 2)
return (x, y, w, h)
def _clip_rect(rect: Tuple[int, int, int, int],
boundary: Tuple[int, int, int, int]) -> Tuple[int, int, int, int]:
"""Clip *rect* to *boundary* = (x, y, w, h)."""
bx, by, bw, bh = boundary
x, y, w, h = rect
x = max(bx, x)
y = max(by, y)
w = min(x + w, bx + bw) - x
h = min(y + h, by + bh) - y
return (x, y, max(0, w), max(0, h))
def format_time(self, timespan, precision=3):
def _get_block_integral(ii: np.ndarray,
rect: Tuple[int, int, int, int]) -> int:
"""Integralimage sum over *rect*."""
x, y, w, h = rect
return (ii[y+h, x+w] - ii[y, x+w] - ii[y+h, x] + ii[y, x])
def _canny_pure(img: np.ndarray,
low: int = 64,
high_ratio: float = 2.0) -> np.ndarray:
"""Lightweight Canny wrapper (imitates canny_pure())."""
img_blur = cv2.GaussianBlur(img, (3, 3), 0)
return cv2.Canny(img_blur, low, int(low*high_ratio))
# --------------------------- main class ---------------------------- #
class PupilDetectorHaar:
"""
https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473
Formats the timespan in a human readable form
Haarbased coarsetofine pupil detector.
Parameters
----------
ratio_outer : float
Scaling factor for Haar outer rectangle.
kf : float
Weighting term in response function f = µ_outer kf*µ_inner.
use_square_haar : bool
If True, outer Haar window is square; else horizontal rectangle.
use_init_rect : bool
If True, provide an approximate pupil box in *init_rect*.
init_rect : Tuple[int,int,int,int] | None
Initial pupil location on the fullresolution frame.
target_resolution : Tuple[int,int]
Image is downsampled so the longer side ~320 px by default.
width_min / width_max / wh_step / xy_step
Searchgrid parameters for Haar scanning.
"""
if timespan >= 60.0:
# we have more than a minute, format that in a human readable form
# Idea from http://snipplr.com/view/5713/
parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)]
time = []
leftover = timespan
for suffix, length in parts:
value = int(leftover / length)
if value > 0:
leftover = leftover % length
time.append("%s%s" % (str(value), suffix))
if leftover < 1:
break
return " ".join(time)
# Unfortunately the unicode 'micro' symbol can cause problems in
# certain terminals.
# See bug: https://bugs.launchpad.net/ipython/+bug/348466
# Try to prevent crashes by being more secure than it needs to
# E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
units = ["s", "ms", "us", "ns"] # the save value
if hasattr(sys.stdout, "encoding") and sys.stdout.encoding:
try:
"\xb5".encode(sys.stdout.encoding)
units = ["s", "ms", "\xb5s", "ns"]
except:
pass
scaling = [1, 1e3, 1e6, 1e9]
if timespan > 0.0:
order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
else:
order = 3
return "%.*g %s" % (precision, timespan * scaling[order], units[order])
def filter_light(self, img_gray, img_blur, tau):
for i in range(img_gray.shape[1]):
for j in range(img_gray.shape[0]):
if img_gray[j, i] > tau:
img_blur[j, i] = tau
else:
img_blur[j, i] = img_gray[j, i]
return img_blur
def pupil_detector_haar(self, img_gray, params):
frame_num = 0
img_down = cv2.resize(
img_gray,
(
img_gray.shape[1] // params["ratio_downsample"],
img_gray.shape[0] // params["ratio_downsample"],
),
)
img_boundary = (0, 0, img_down.shape[1], img_down.shape[0])
if params["use_init_rect"]:
tau = max(params["mu_outer"], params["mu_inner"] + 30)
self.filter_light(img_down, img_down, tau)
# Coarse Detection
(
pupil_rect_coarse,
outer_rect_coarse,
max_response_coarse,
mu_inner,
mu_outer,
) = self.coarse_detection(img_down, params)
print(
"Coarse Detection: ",
pupil_rect_coarse,
outer_rect_coarse,
max_response_coarse,
mu_inner,
mu_outer,
)
if params["use_init_rect"] and frame_num == 0:
mu_inner0 = mu_inner
mu_outer0 = mu_outer
kf = 2 - 0.01 * mu_inner0
img_coarse = cv2.cvtColor(img_down, cv2.COLOR_GRAY2BGR)
# show image
# Fine Detection
if mu_outer - mu_inner >= 5:
pupil_rect_fine = self.fine_detection(img_down, pupil_rect_coarse)
else:
pupil_rect_fine = pupil_rect_coarse
# Postprocessing
pupil_rect_coarse = self.rect_scale(pupil_rect_coarse, params["ratio_downsample"], False)
outer_rect_coarse = self.rect_scale(outer_rect_coarse, params["ratio_downsample"], False)
pupil_rect_fine = self.rect_scale(pupil_rect_fine, params["ratio_downsample"], False)
center_coarse = (
pupil_rect_coarse[0] + pupil_rect_coarse[2] // 2,
pupil_rect_coarse[1] + pupil_rect_coarse[3] // 2,
)
center_fine = (
pupil_rect_fine[0] + pupil_rect_fine[2] // 2,
pupil_rect_fine[1] + pupil_rect_fine[3] // 2,
)
return (
pupil_rect_coarse,
outer_rect_coarse,
pupil_rect_fine,
center_coarse,
center_fine,
)
# @lru_cache(maxsize=self.lru_maxsize_vvs)
def get_empty_array(self, frame_shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer):
frame_int_dtype = np.intc
np_index_dtype = (
np.intc
) # memo: Better to use np.intp, but a little slower ref: https://numpy.org/doc/1.25/user/basics.indexing.html#detailed-notes
row, col = frame_shape
frame_int = np.empty((row + 1, col + 1), dtype=frame_int_dtype)
w_arr = np.arange(width_min, width_max + 1, wh_step, dtype=np_index_dtype)
h_arr = (w_arr / ratio_outer).astype(np.int16)
# memo: It is not smart code and needs to be changed.
y_out_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) for h in h_arr])
x_out_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) for w in w_arr])
y_out_h = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + h for h in h_arr])
x_out_w = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + w for w in w_arr])
out_h = y_out_h - y_out_n
out_w = x_out_w - x_out_n
y_in_n = np.hstack([np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) for h in h_arr])
x_in_n = np.hstack([np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) for w in w_arr])
y_in_h = np.hstack(
[np.arange(roi[1] + h, roi[3] - h, xy_step, dtype=np_index_dtype) + int(h / 4) + int(h / 2) for h in h_arr]
)
x_in_w = np.hstack(
[np.arange(roi[0] + w, roi[2] - w, xy_step, dtype=np_index_dtype) + int(w / 4) + int(w / 2) for w in w_arr]
)
in_h = y_in_h - y_in_n
in_w = x_in_w - x_in_n
# # memo: Unelegant code
# # memo: Non-transposed version
# wh_in_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,int(h/2),dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,int(w/2),dtype=np_index_dtype) for w in w_arr])[np.newaxis, :]
# wh_out_arr = np.hstack([np.full(((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,h,dtype=np_index_dtype) for h in h_arr])[:, np.newaxis] * np.hstack([np.full(((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,w,dtype=np_index_dtype) for w in w_arr])[np.newaxis, :]
# memo: Unelegant code
# memo: transposed version
wh_in_arr = (
np.hstack(
[
np.full(
((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,
int(w / 2),
dtype=np_index_dtype,
)
for w in w_arr
]
)[:, np.newaxis]
* np.hstack(
[
np.full(
((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,
int(h / 2),
dtype=np_index_dtype,
)
for h in h_arr
]
)[np.newaxis, :]
)
wh_out_arr = (
np.hstack(
[
np.full(
((roi[2] - w) - (roi[0] + w) - 1) // xy_step + 1,
w,
dtype=np_index_dtype,
)
for w in w_arr
]
)[:, np.newaxis]
* np.hstack(
[
np.full(
((roi[3] - h) - (roi[1] + h) - 1) // xy_step + 1,
h,
dtype=np_index_dtype,
)
for h in h_arr
]
)[np.newaxis, :]
)
mu_outer_rect = cv2.subtract(
wh_out_arr, wh_in_arr
) # ,dst=) # == (outer_rect[2] * outer_rect[3] - inner_rect[2] * inner_rect[3])
wh_in_arr = 1 / wh_in_arr # .astype(np.float32)
# wh_out_arr=wh_out_arr.astype(np.float64)
mu_outer_rect = 1 / mu_outer_rect # .astype(np.float32)
mu_outer_rect2 = -1.0 * mu_outer_rect # cv2.merge([mu_outer_rect,-1.0*mu_outer_rect])
# 1/wh_in_arr == wh_in_arr_mul
return (
frame_int,
y_out_n,
x_out_n,
y_out_h,
x_out_w,
out_h,
out_w,
y_in_n,
x_in_n,
y_in_h,
x_in_w,
in_h,
in_w,
wh_in_arr,
wh_out_arr,
mu_outer_rect,
mu_outer_rect2,
)
# @profile
def coarse_detection(self, img_gray, params):
ratio_outer = params["ratio_outer"]
kf = params["kf"]
width_min = params["width_min"]
width_max = params["width_max"]
wh_step = params["wh_step"]
xy_step = params["xy_step"]
roi = params["roi"]
init_rect_flag = params["init_rect_flag"]
init_rect = params["init_rect"]
mu_inner = params["mu_inner"]
mu_outer = params["mu_outer"]
max_response_coarse = -255
imgboundary = (0, 0, img_gray.shape[1], img_gray.shape[0])
img_blur = np.copy(img_gray)
rectlist = []
response = []
# Assign values to avoid unassigned errors
pupil_rect_coarse = (10, 10, 10, 10)
outer_rect_coarse = (5, 5, 5, 5)
if init_rect_flag:
init_rect_down = self.rect_scale(init_rect, params["ratio_downsample"], False)
init_rect_down = self.intersect_rect(init_rect_down, imgboundary)
img_blur = img_gray[
init_rect_down[1]: init_rect_down[1] + init_rect_down[3],
init_rect_down[0]: init_rect_down[0] + init_rect_down[2],
]
(
frame_int,
y_out_n,
x_out_n,
y_out_h,
x_out_w,
out_h,
out_w,
y_in_n,
x_in_n,
y_in_h,
x_in_w,
in_h,
in_w,
wh_in_arr,
wh_out_arr,
mu_outer_rect,
mu_outer_rect2,
) = self.get_empty_array(img_blur.shape, width_min, width_max, wh_step, xy_step, roi, ratio_outer)
cv2.integral(
img_blur, sum=frame_int, sdepth=cv2.CV_32S
)
out_p_temp = frame_int.take(y_out_n, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p00 = out_p_temp.take(x_out_n, axis=0, mode="clip")
out_p01 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p_temp = frame_int.take(y_out_h, axis=0, mode="clip")
out_p_temp = cv2.transpose(out_p_temp)
out_p11 = out_p_temp.take(x_out_w, axis=0, mode="clip")
out_p10 = out_p_temp.take(x_out_n, axis=0, mode="clip")
outer_sum = cv2.add(out_p00, out_p11)
cv2.subtract(outer_sum, out_p01, dst=outer_sum)
cv2.subtract(outer_sum, out_p10, dst=outer_sum)
in_p_temp = frame_int.take(y_in_n, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p00 = in_p_temp.take(x_in_n, axis=0, mode="clip")
in_p01 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p_temp = frame_int.take(y_in_h, axis=0, mode="clip")
in_p_temp = cv2.transpose(in_p_temp)
in_p11 = in_p_temp.take(x_in_w, axis=0, mode="clip")
in_p10 = in_p_temp.take(x_in_n, axis=0, mode="clip")
inner_sum = cv2.add(in_p00, in_p11)
cv2.subtract(inner_sum, in_p01, dst=inner_sum)
cv2.subtract(inner_sum, in_p10, dst=inner_sum)
inner_sum_f = inner_sum.astype(np.float64)
outer_sum_f = outer_sum.astype(np.float64)
response_value = np.empty(outer_sum.shape, dtype=np.float64)
inout_rect_sum = mu_outer_rect2.copy()
inout_rect_mul = mu_outer_rect.copy()
cv2.multiply(inner_sum_f, inout_rect_mul, inout_rect_mul)
cv2.multiply(outer_sum_f, inout_rect_sum, inout_rect_sum)
cv2.add(inout_rect_mul, inout_rect_sum, dst=inout_rect_sum)
cv2.multiply(inner_sum_f, wh_in_arr, inner_sum_f, kf)
cv2.add(inout_rect_sum, inner_sum_f, dst=response_value)
min_response, max_response, min_loc, max_loc = cv2.minMaxLoc(response_value)
rec_o = (
x_out_n[min_loc[1]],
y_out_n[min_loc[0]],
out_w[min_loc[1]],
out_h[min_loc[0]],
)
rec_in = (
x_in_n[min_loc[1]],
y_in_n[min_loc[0]],
in_w[min_loc[1]],
in_h[min_loc[0]],
)
max_response_coarse = -min_response
pupil_rect_coarse = rec_in
outer_rect_coarse = rec_o
return pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer
def fine_detection(self, img_gray, pupil_rect_coarse):
boundary = (0, 0, img_gray.shape[1], img_gray.shape[0])
valid_ratio = 1.2
valid_rect = self.intersect_rect(self.rect_scale(pupil_rect_coarse, valid_ratio), boundary)
img_pupil = img_gray[
valid_rect[1] : valid_rect[1] + valid_rect[3],
valid_rect[0] : valid_rect[0] + valid_rect[2],
]
img_pupil_blur = cv2.GaussianBlur(img_pupil, (5, 5), 0, 0)
edges_filter = self.detect_edges(img_pupil_blur)
# fit ellipse to edges
contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
# sort contours by area
contours = sorted(contours, key=lambda x: cv2.contourArea(x), reverse=True)
# fit ellipse to largest contour
try:
if len(contours) > 0 and len(contours[0]) >= 5:
pupil_contour = contours[0]
pupil_ellipse = cv2.fitEllipse(pupil_contour)
center_fitting = (
int(pupil_ellipse[0][0] + valid_rect[0]),
int(pupil_ellipse[0][1] + valid_rect[1]),
)
pupil_rect_fine = (
int(pupil_ellipse[0][0] - pupil_ellipse[1][0] / 2),
int(pupil_ellipse[0][1] - pupil_ellipse[1][1] / 2),
int(pupil_ellipse[1][0]),
int(pupil_ellipse[1][1]),
)
pupil_rect_fine = (
pupil_rect_fine[0] + valid_rect[0],
pupil_rect_fine[1] + valid_rect[1],
pupil_rect_fine[2],
pupil_rect_fine[3],
)
pupil_rect_fine = self.intersect_rect(pupil_rect_fine, boundary)
pupil_rect_fine = self.rect_scale(pupil_rect_fine, 1 / valid_ratio)
else:
pupil_rect_fine = pupil_rect_coarse
center_fitting = (
int(pupil_rect_fine[0] + pupil_rect_fine[2] / 2),
int(pupil_rect_fine[1] + pupil_rect_fine[3] / 2),
)
except:
pass
try:
return pupil_rect_fine, center_fitting
except:
pass
def detect_edges(self, img_pupil_blur):
tau1 = 1 - 20.0 / img_pupil_blur.shape[1]
edges = cv2.Canny(img_pupil_blur, 64, 128)
# img_bw = np.zeros_like(img_pupil_blur)
# img_bw[img_pupil_blur > 100] = 255
img_bw = cv2.compare(img_pupil_blur, 100, cv2.CMP_GT)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
img_bw = cv2.dilate(img_bw, kernel)
# edges_filter = edges & (~img_bw)
# or
edges_filter = cv2.bitwise_and(edges, cv2.bitwise_not(img_bw))
return edges_filter
def fit_pupil_ellipse_swirski(self, img_pupil, edges_filter):
contours, hierarchy = cv2.findContours(edges_filter, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
max_contour_area = 0
max_contour = None
#print("contours: ", contours)
for contour in contours:
area = cv2.contourArea(contour)
if area > max_contour_area:
max_contour_area = area
max_contour = contour
if max_contour is None:
return (0, 0, 0, 0), None
ellipse = cv2.fitEllipse(max_contour)
return ellipse
def rect_scale(self, rect, scale, round_up=True):
x, y, width, height = rect
new_width = int(width * scale)
new_height = int(height * scale)
if round_up:
new_width = int(np.ceil(width * scale))
new_height = int(np.ceil(height * scale))
new_x = x + int((width - new_width) / 2)
new_y = y + int((height - new_height) / 2)
return new_x, new_y, new_width, new_height
def intersect_rect(self, rect1, rect2):
x1, y1, w1, h1 = rect1
x2, y2, w2, h2 = rect2
x = max(x1, x2)
y = max(y1, y2)
w = min(x1 + w1, x2 + w2) - x
h = min(y1 + h1, y2 + h2) - y
return x, y, w, h
def rect_suppression(self, rectlist, response, rectlist_out, response_out):
for i in range(len(rectlist)):
flag_intersect = False
for j in range(len(rectlist_out)):
tmp = self.intersect_rect(rectlist[i], rectlist_out[j])
if tmp[2] > 0 and tmp[3] > 0:
flag_intersect = True
if response[i] > response_out[j]:
rectlist_out[j] = rectlist[i]
response_out[j] = response[i]
else:
continue
if not flag_intersect:
rectlist_out.append(rectlist[i])
response_out.append(response[i])
return rectlist_out, response_out
def put_number(self, img_bgr, number, position, color):
cv2.putText(
img_bgr,
str(number),
(int(position[0]) + 10, int(position[1]) - 10),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
color,
1,
cv2.LINE_AA,
)
def External_Run_AHSF(self, frame_gray):
average_color = np.mean(frame_gray)
height, width = frame_gray.shape
max_dimension = max(height, width)
square_background = np.full((max_dimension, max_dimension), average_color, dtype=np.uint8)
x_offset = (max_dimension - width) // 2
y_offset = (max_dimension - height) // 2
square_background[y_offset : y_offset + height, x_offset : x_offset + width] = frame_gray
frame_gray = cv2.resize(square_background, (100, 100))
frame_clear_resize = frame_gray.copy()
params = {
"ratio_downsample": 0.5,
"use_init_rect": False,
"mu_outer": 200,
"mu_inner": 50,
"ratio_outer": 1,
"kf": 1,
"width_min": 25,
"width_max": 50,
"wh_step": 1,
"xy_step": 5,
"roi": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
"init_rect_flag": False,
"init_rect": (0, 0, frame_gray.shape[1], frame_gray.shape[0]),
}
try:
pupil_rect_coarse, outer_rect_coarse, max_response_coarse, mu_inner, mu_outer = self.coarse_detection(frame_gray, params)
# ellipse_rect, center_fitting = self.fine_detection(frame_gray, pupil_rect_coarse)
except TypeError:
return frame_gray, frame_gray, 0, 0, 0
x_center = outer_rect_coarse[0] + outer_rect_coarse[2] / 2
y_center = outer_rect_coarse[1] + outer_rect_coarse[3] / 2
x, y, width, height = outer_rect_coarse
cv2.circle(frame_gray, (int(x_center), int(y_center)), 2, (255, 255, 255), -1)
thickness = 1
cv2.rectangle(frame_gray, (pupil_rect_coarse[0], pupil_rect_coarse[1]),
(pupil_rect_coarse[0] + pupil_rect_coarse[2], pupil_rect_coarse[1] + pupil_rect_coarse[3]),
(0, 255, 0), 2)
cv2.rectangle(frame_gray, (outer_rect_coarse[0], outer_rect_coarse[1]),
(outer_rect_coarse[0] + outer_rect_coarse[2], outer_rect_coarse[1] + outer_rect_coarse[3]),
(255, 0, 0), 2)
major_diameter = math.sqrt(width**2 + height**2)
minor_diameter = min(width, height)
average_diameter = (major_diameter + minor_diameter) / 2
return frame_gray, frame_clear_resize, x_center, y_center, abs(width - height)
class FPSResult(object):
# -------- initialisation -------- #
def __init__(self,
ratio_outer: float = 1.4,
kf: float = 1.5,
use_square_haar: bool = False,
use_init_rect: bool = False,
init_rect: Optional[Tuple[int, int, int, int]] = None,
target_resolution: Tuple[int, int] = (320, 240),
width_min: int = 31,
width_max: int = 120,
wh_step: int = 2,
xy_step: int = 2):
self.ratio_outer = ratio_outer
self.kf = kf
self.use_square_haar = use_square_haar
self.use_init_rect = use_init_rect
self.init_rect = (0, 0, 0, 0) if init_rect is None else init_rect
self.target_resolution = target_resolution
# searchgrid params (may be autotuned after first frame)
self.width_min = width_min
self.width_max = width_max
self.wh_step = wh_step
self.xy_step = xy_step
# dynamic state
self.frame_num = 0
self.mu_inner = 50
self.mu_outer = 200
self.mu_inner0 = 50 # first frame stats
self.mu_outer0 = 200
# outputs (public)
self.pupil_rect_coarse = (0, 0, 0, 0)
self.outer_rect_coarse = (0, 0, 0, 0)
self.max_response_coarse = -255
self.center_coarse = (0.0, 0.0)
self.pupil_rect_fine = (0, 0, 0, 0)
self.center_fine = (0.0, 0.0)
# private temp
self._ratio_down = 1.0
self._img_boundary = (0, 0, 0, 0)
self._init_rect_down = (0, 0, 0, 0)
# ---------------------------------------------------------------- #
# PUBLIC API #
# ---------------------------------------------------------------- #
def detect(self, img_gray: np.ndarray) -> Tuple[Tuple[int,int,int,int], Tuple[float,float]]:
"""
base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55
Run detector on a single *uint8* gray image.
Returns
-------
pupil_rect_fine : (x,y,w,h)
center_fine : (cx,cy) -- both on fullresolution image.
"""
if img_gray.dtype != np.uint8:
raise TypeError("img_gray must be uint8 [0,255]")
def __init__(self, loops, repeat, best, worst, all_runs, precision):
self.loops = loops
self.repeat = repeat
self.best = 1 / best
self.worst = 1 / worst
self.all_runs = all_runs
self._precision = precision
self.fps = [1 / dt for dt in all_runs]
self.unit = "fps"
self.frame_num += 1
img_down = self._preprocess(img_gray)
self._coarse_detection(img_down)
self._fine_detection(img_down)
self._postprocess()
@property
def average(self):
return math.fsum(self.fps) / len(self.fps)
return self.pupil_rect_fine, self.center_fine
@property
def stdev(self):
mean = self.average
return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5
# --------------- optional helper for visual debugging ------------ #
def draw_debug(self, bgr: np.ndarray) -> None:
"""Draw rectangular outputs on *bgr* inplace."""
cv2.rectangle(bgr, self.pupil_rect_fine, (0, 255, 0), 1)
cv2.rectangle(bgr, self.outer_rect_coarse, (255, 0, 0), 1)
cx, cy = map(int, self.center_fine)
cv2.drawMarker(bgr, (cx, cy), (0, 0, 255),
markerType=cv2.MARKER_CROSS, markerSize=10, thickness=1)
def __str__(self):
pm = "+-"
if hasattr(sys.stdout, "encoding") and sys.stdout.encoding:
try:
"\xb1".encode(sys.stdout.encoding)
pm = "\xb1"
except:
pass
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
pm=pm,
runs=self.repeat,
loops=self.loops,
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean="%.*g%s" % (self._precision, self.average, self.unit),
std="%.*g%s" % (self._precision, self.stdev, self.unit),
best="%.*g%s" % (self._precision, self.best, self.unit),
worst="%.*g%s" % (self._precision, self.worst, self.unit),
)
# ---------------------------------------------------------------- #
# INTERNAL STAGES #
# ---------------------------------------------------------------- #
def _preprocess(self, img_gray: np.ndarray) -> np.ndarray:
# downsample to target size (longer side ≈ target_resolution[0])
h, w = img_gray.shape
self._ratio_down = max(w / self.target_resolution[0],
h / self.target_resolution[1], 1.0)
new_w = int(round(w / self._ratio_down))
new_h = int(round(h / self._ratio_down))
img_down = cv2.resize(img_gray, (new_w, new_h),
interpolation=cv2.INTER_AREA)
def _repr_pretty_(self, p, cycle):
unic = self.__str__()
p.text("<FPSResult : " + unic + ">")
self._img_boundary = (0, 0, new_w, new_h)
# optional highintensity suppression on first frame
if self.use_init_rect and self.frame_num == 1:
# Estimate µ_inner0 / µ_outer0 inside init box
x, y, rw, rh = self.init_rect
region = img_gray[y:y+rh, x:x+rw]
self.mu_inner0 = np.percentile(region, 25)
self.mu_outer0 = np.percentile(region, 75)
# adjust kf like original code
if self.mu_outer0 - self.mu_inner0 > 30:
tau = self.mu_outer0
else:
tau = self.mu_inner0 + 30
img_down = np.minimum(img_down, tau).astype(np.uint8)
return img_down
# ------------------------------------------------------------ #
# COARSE DETECTION #
# ------------------------------------------------------------ #
def _initial_search_range(self, img_down: np.ndarray) -> Tuple[int,int,int,int]:
"""Compute ROI and width range for current frame (downsampled)."""
h, w = img_down.shape
margin = h // 10 // 2
full = (margin, margin, w - 2*margin, h - 2*margin)
if not self.use_init_rect:
self.roi = full
return
# scale init_rect to down resolution
self._init_rect_down = tuple(int(x / self._ratio_down) for x in self.init_rect)
ix, iy, iw, ih = self._init_rect_down
# grow ROI adaptively near borders (imitates C++ code)
rx, ry, rw, rh = full
enlarge = 35
if ix < enlarge: rx, rw = 0, w
if iy < enlarge: ry, rh = 0, h
if ix+iw > w - enlarge: rx, rw = 0, w
if iy+ih > h - enlarge: ry, rh = 0, h
self.roi = (rx, ry, rw, rh)
# width search band tuned by first frame
self.width_min = max(int(iw*1.0), 24)
self.width_max = min(int(iw*1.5), 120)
def _coarse_detection(self, img_down: np.ndarray) -> None:
self._initial_search_range(img_down)
roi_x, roi_y, roi_w, roi_h = self.roi
# build integral image (cv2 adds +1 row/col)
ii = cv2.integral(img_down, sdepth=cv2.CV_32S)
best_f = -255
best_pupil = (0, 0, 0, 0)
best_outer = (0, 0, 0, 0)
best_mu_in, best_mu_out = 0, 0
for width in range(self.width_min, self.width_max+1, self.wh_step):
# height tied to width; rectangular pupils handled fine
for height in range(width, width+1, self.wh_step):
xmax = roi_x + roi_w - width
ymax = roi_y + roi_h - height
for x in range(roi_x, xmax+1, self.xy_step):
for y in range(roi_y, ymax+1, self.xy_step):
pupil = (x, y, width, height)
outer = _rect_scale(pupil, self.ratio_outer,
keep_center=True,
square_outer=self.use_square_haar)
outer = _clip_rect(outer, self._img_boundary)
mu_in, mu_out = 0.0, 0.0
mu_out = (_get_block_integral(ii, outer) -
_get_block_integral(ii, pupil)) / \
(outer[2]*outer[3] - width*height)
mu_in = _get_block_integral(ii, pupil) / (width*height)
f_val = mu_out - self.kf * mu_in
if f_val > best_f:
best_f = f_val
best_pupil = pupil
best_outer = outer
best_mu_in, best_mu_out = mu_in, mu_out
self.pupil_rect_coarse = best_pupil
self.outer_rect_coarse = best_outer
self.max_response_coarse = best_f
self.mu_inner, self.mu_outer = best_mu_in, best_mu_out
px, py, pw, ph = best_pupil
self.center_coarse = (px + pw / 2, py + ph / 2)
# ------------------------------------------------------------ #
# FINE DETECTION #
# ------------------------------------------------------------ #
def _fine_detection(self, img_down: np.ndarray) -> None:
px, py, pw, ph = self.pupil_rect_coarse
expand = 1.42
exp_rect = _clip_rect(_rect_scale(self.pupil_rect_coarse, expand, True),
self._img_boundary)
ex, ey, ew, eh = exp_rect
patch = img_down[ey:ey+eh, ex:ex+ew]
# threshold at µ_inner (same heuristic)
_, bw = cv2.threshold(patch, int(self.mu_inner), 255,
cv2.THRESH_BINARY_INV)
# dilate to merge gaps
bw = cv2.dilate(bw, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5,5)))
# connected components
n, labels, stats, centroids = cv2.connectedComponentsWithStats(bw)
if n <= 1:
# fallback: keep coarse rect
self.pupil_rect_fine = tuple(int(v) for v in
np.array(self.pupil_rect_coarse) *
self._ratio_down)
self.center_fine = tuple(v * self._ratio_down
for v in self.center_coarse)
return
# discard tiny blobs (<4% of patch)
areas = stats[1:, cv2.CC_STAT_AREA]
mask = areas > 0.04 * bw.size
if not np.any(mask):
mask = areas.argmax()[None] # keep largest if all tiny
# choose component through image centre, else darkest centroid
cx_local = patch.shape[1] // 2
cy_local = patch.shape[0] // 2
comp_idx = labels[cy_local, cx_local]
if comp_idx == 0 or not mask[comp_idx-1]:
# pick darkest of two largest blobs (C++ heuristic)
dark = 255
for idx in np.flatnonzero(mask) + 1:
cx_i, cy_i = centroids[idx]
val = patch[int(cy_i), int(cx_i)]
if val < dark:
dark = val
comp_idx = idx
# final bounding box in downscaled coords
x, y, w, h = stats[comp_idx, cv2.CC_STAT_LEFT : cv2.CC_STAT_HEIGHT+1]
x += ex
y += ey
self.pupil_rect_fine = (x, y, w, h)
self.center_fine = (x + w / 2, y + h / 2)
# ------------------------------------------------------------ #
# UPSAMPLE BACK #
# ------------------------------------------------------------ #
def _postprocess(self) -> None:
# scale coarse and fine rects + centres back to full resolution
scale = self._ratio_down
def _up(rect):
return tuple(int(round(v*scale)) for v in rect)
self.pupil_rect_coarse = _up(self.pupil_rect_coarse)
self.outer_rect_coarse = _up(self.outer_rect_coarse)
self.pupil_rect_fine = _up(self.pupil_rect_fine)
self.center_coarse = tuple(v*scale for v in self.center_coarse)
self.center_fine = tuple(v*scale for v in self.center_fine)

View File

@ -165,7 +165,7 @@ class EyeProcessor:
self.pupil_height = 0.0
self.avg_velocity = 0.0
self.angle = 621
self.er_ahsf = None
self.det = PupilDetectorHaar(ratio_outer=1.4, kf=1.4)
try:
@ -406,16 +406,14 @@ class EyeProcessor:
pass
self.hasrac_en = True
(
self.current_image_gray,
resize_img,
self.rawx,
self.rawy,
self.radius,
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
self.current_image_gray_clean = resize_img.copy()
self.thresh = resize_img
self.current_image_gray_clean = self.current_image_gray.copy()
self.det.detect(self.current_image_gray)
cx, cy = map(int, self.det.center_fine)
cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1)
cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1)
self.thresh = self.current_image_gray_clean
(
self.rawx,
self.rawy,
@ -522,13 +520,11 @@ class EyeProcessor:
)
else:
pass
(
self.current_image_gray,
resize_img,
self.rawx,
self.rawy,
self.radius,
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
self.det.detect(self.current_image_gray) # <- single call per frame
cx, cy = map(int, self.det.center_fine) # fine centre (upsampled)
cv2.circle(self.current_image_gray, (cx, cy), 3, (0, 0, 255), -1)
cv2.rectangle(self.current_image_gray, self.det.pupil_rect_fine, (0, 255, 0), 1)
self.thresh = self.current_image_gray
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSF
@ -603,13 +599,9 @@ class EyeProcessor:
# set algo priorities
if self.settings.gui_AHSFRAC:
if self.er_ahsf is None:
self.er_ahsf = AHSF(self.current_image_gray)
algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM
if self.settings.gui_AHSF:
if self.er_ahsf is None:
self.er_ahsf = AHSF(self.current_image_gray)
algolist[self.settings.gui_AHSFP] = self.AHSFM
if self.settings.gui_HSF: