mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
new ellipse calibration
This commit is contained in:
parent
a0e74e6822
commit
ad8c86b54c
@ -158,6 +158,68 @@ class PupilDetectorHaar:
|
|||||||
self._img_boundary = (0, 0, 0, 0)
|
self._img_boundary = (0, 0, 0, 0)
|
||||||
self._init_rect_down = (0, 0, 0, 0)
|
self._init_rect_down = (0, 0, 0, 0)
|
||||||
|
|
||||||
|
def detect_etvr(self, img_gray) -> Tuple[np.ndarray, np.ndarray, float, float, float]:
|
||||||
|
"""
|
||||||
|
Runs the full detection and returns a visualized image and ETVR-specific data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
img_gray: The input grayscale image (uint8).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A tuple containing:
|
||||||
|
- vis_img (np.ndarray): The original image with visualizations drawn on it (BGR).
|
||||||
|
- resize_img (np.ndarray): The downscaled image used for processing.
|
||||||
|
- rawx (float): The final X coordinate of the pupil center.
|
||||||
|
- rawy (float): The final Y coordinate of the pupil center.
|
||||||
|
- radius (float): The calculated average radius of the final pupil rectangle.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# 1. Run the main detection.
|
||||||
|
# This populates all internal class attributes:
|
||||||
|
# self.pupil_rect_fine, self.center_fine,
|
||||||
|
# self.pupil_rect_coarse, self.outer_rect_coarse,
|
||||||
|
# and self._ratio_down. It also increments self.frame_num.
|
||||||
|
self.detect(img_gray)
|
||||||
|
|
||||||
|
# 2. Get the downscaled image.
|
||||||
|
# We call _preprocess again. This is slightly inefficient but
|
||||||
|
# avoids refactoring detect(). It will correctly use the
|
||||||
|
# self.frame_num that detect() just set.
|
||||||
|
resize_img = img_gray
|
||||||
|
|
||||||
|
# 3. Get the final data from class attributes
|
||||||
|
rawx, rawy = self.center_fine
|
||||||
|
px, py, pw, ph = self.pupil_rect_fine
|
||||||
|
|
||||||
|
# Calculate an average radius from the fine rect's width and height
|
||||||
|
radius = (pw + ph) / 4.0
|
||||||
|
|
||||||
|
# 4. Create the visualization image
|
||||||
|
# Convert the original grayscale image to BGR for color drawing
|
||||||
|
vis_img = cv2.cvtColor(img_gray, cv2.COLOR_GRAY2BGR)
|
||||||
|
|
||||||
|
# Draw coarse pupil rect (Green)
|
||||||
|
x, y, w, h = self.pupil_rect_coarse
|
||||||
|
if w > 0 and h > 0:
|
||||||
|
cv2.rectangle(vis_img, (x, y), (x + w, y + h), (0, 255, 0), 1)
|
||||||
|
|
||||||
|
# Draw coarse outer rect (Yellow)
|
||||||
|
x, y, w, h = self.outer_rect_coarse
|
||||||
|
if w > 0 and h > 0:
|
||||||
|
cv2.rectangle(vis_img, (x, y), (x + w, y + h), (0, 255, 255), 1)
|
||||||
|
|
||||||
|
# Draw fine pupil rect (Red)
|
||||||
|
x, y, w, h = self.pupil_rect_fine
|
||||||
|
if w > 0 and h > 0:
|
||||||
|
cv2.rectangle(vis_img, (x, y), (x + w, y + h), (0, 0, 255), 1)
|
||||||
|
|
||||||
|
# Draw fine center (Red)
|
||||||
|
cv2.circle(vis_img, (int(round(rawx)), int(round(rawy))), 3, (0, 0, 255), -1)
|
||||||
|
|
||||||
|
vis_img = cv2.cvtColor(vis_img, cv2.COLOR_BGR2GRAY)
|
||||||
|
# 5. Return the requested 5-tuple
|
||||||
|
return vis_img, resize_img, rawx, rawy, radius
|
||||||
|
|
||||||
def detect(self, img_gray: np.ndarray) -> Tuple[Tuple[int, int, int, int], Tuple[float, float]]:
|
def detect(self, img_gray: np.ndarray) -> Tuple[Tuple[int, int, int, int], Tuple[float, float]]:
|
||||||
if img_gray.dtype != np.uint8:
|
if img_gray.dtype != np.uint8:
|
||||||
raise TypeError("img_gray must be uint8 [0,255]")
|
raise TypeError("img_gray must be uint8 [0,255]")
|
||||||
|
|||||||
@ -2,7 +2,7 @@
|
|||||||
; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
|
; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
|
||||||
|
|
||||||
#define MyAppName "EyeTrackVR"
|
#define MyAppName "EyeTrackVR"
|
||||||
#define MyAppVersion "0.2.2"
|
#define MyAppVersion "0.2.4"
|
||||||
#define MyAppPublisher "EyeTrackVR"
|
#define MyAppPublisher "EyeTrackVR"
|
||||||
#define MyAppURL "https://redhawk989.github.io/EyeTrackVR/"
|
#define MyAppURL "https://redhawk989.github.io/EyeTrackVR/"
|
||||||
#define MyAppExeName "eyetrackapp.exe"
|
#define MyAppExeName "eyetrackapp.exe"
|
||||||
|
|||||||
@ -170,6 +170,7 @@ class EyeProcessor:
|
|||||||
self.angle = 621
|
self.angle = 621
|
||||||
self.er_ahsf = None
|
self.er_ahsf = None
|
||||||
self.cal = CalibrationEllipse()
|
self.cal = CalibrationEllipse()
|
||||||
|
self.AHSF = PupilDetectorHaar()
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -416,7 +417,7 @@ class EyeProcessor:
|
|||||||
self.rawx,
|
self.rawx,
|
||||||
self.rawy,
|
self.rawy,
|
||||||
self.radius,
|
self.radius,
|
||||||
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
|
) = self.er_ahsf.detect_etvr(self.current_image_gray)
|
||||||
self.current_image_gray_clean = resize_img.copy()
|
self.current_image_gray_clean = resize_img.copy()
|
||||||
|
|
||||||
self.thresh = resize_img
|
self.thresh = resize_img
|
||||||
@ -532,7 +533,7 @@ class EyeProcessor:
|
|||||||
self.rawx,
|
self.rawx,
|
||||||
self.rawy,
|
self.rawy,
|
||||||
self.radius,
|
self.radius,
|
||||||
) = self.er_ahsf.External_Run_AHSF(self.current_image_gray)
|
) = self.er_ahsf.detect_etvr(self.current_image_gray)
|
||||||
self.thresh = self.current_image_gray
|
self.thresh = self.current_image_gray
|
||||||
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
|
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
|
||||||
self.current_algorithm = EyeInfoOrigin.HSF
|
self.current_algorithm = EyeInfoOrigin.HSF
|
||||||
@ -608,12 +609,12 @@ class EyeProcessor:
|
|||||||
# set algo priorities
|
# set algo priorities
|
||||||
if self.settings.gui_AHSFRAC:
|
if self.settings.gui_AHSFRAC:
|
||||||
if self.er_ahsf is None:
|
if self.er_ahsf is None:
|
||||||
self.er_ahsf = AHSF(self.current_image_gray)
|
self.er_ahsf = self.AHSF
|
||||||
algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM
|
algolist[self.settings.gui_AHSFRACP] = self.AHSFRACM
|
||||||
|
|
||||||
if self.settings.gui_AHSF:
|
if self.settings.gui_AHSF:
|
||||||
if self.er_ahsf is None:
|
if self.er_ahsf is None:
|
||||||
self.er_ahsf = AHSF(self.current_image_gray)
|
self.er_ahsf = self.AHSF
|
||||||
algolist[self.settings.gui_AHSFP] = self.AHSFM
|
algolist[self.settings.gui_AHSFP] = self.AHSFM
|
||||||
|
|
||||||
if self.settings.gui_HSF:
|
if self.settings.gui_HSF:
|
||||||
|
|||||||
@ -194,7 +194,7 @@ class cal:
|
|||||||
self.config.calib_XOFF = cx
|
self.config.calib_XOFF = cx
|
||||||
self.config.calib_YOFF = cy
|
self.config.calib_YOFF = cy
|
||||||
self.baseconfig.save()
|
self.baseconfig.save()
|
||||||
self.cal.fit_and_visualize()
|
self.cal.fit_ellipse()
|
||||||
PlaySound(resource_path("Audio/completed.wav"), SND_FILENAME | SND_ASYNC)
|
PlaySound(resource_path("Audio/completed.wav"), SND_FILENAME | SND_ASYNC)
|
||||||
|
|
||||||
if self.calibration_frame_counter == self.settings.calibration_samples:
|
if self.calibration_frame_counter == self.settings.calibration_samples:
|
||||||
|
|||||||
@ -8,13 +8,16 @@ class CalibrationEllipse:
|
|||||||
self.n_std_devs = float(n_std_devs)
|
self.n_std_devs = float(n_std_devs)
|
||||||
self.fitted = False
|
self.fitted = False
|
||||||
|
|
||||||
self.scale_factor = 0.85 #TODO Test different values
|
self.scale_factor = 0.75
|
||||||
|
|
||||||
|
self.flip_y = False # Set to True if up/down are backwards
|
||||||
|
self.flip_x = False # Adjust if left/right are backwards
|
||||||
|
|
||||||
# Ellipse parameters
|
# Ellipse parameters
|
||||||
self.center = None # (x0,y0) - mean of the point cloud
|
self.center = None # Mean pupil position (ellipse center)
|
||||||
self.axes = None # (a, b) semi-axes (N*std_dev AT 100% SCALE)
|
self.axes = None # Semi-axes (std_dev based)
|
||||||
self.rotation = None # angle in radians (from PCA)
|
self.rotation = None # Rotation angle
|
||||||
self.evecs = None # Eigenvectors (principal axes directions)
|
self.evecs = None # Eigenvectors
|
||||||
|
|
||||||
def add_sample(self, x, y):
|
def add_sample(self, x, y):
|
||||||
self.xs.append(float(x))
|
self.xs.append(float(x))
|
||||||
@ -26,9 +29,7 @@ class CalibrationEllipse:
|
|||||||
self.scale_factor = 1.0 - (clamped_percent / 100.0)
|
self.scale_factor = 1.0 - (clamped_percent / 100.0)
|
||||||
print(f"Set inset to {clamped_percent}%. New scale_factor: {self.scale_factor}")
|
print(f"Set inset to {clamped_percent}%. New scale_factor: {self.scale_factor}")
|
||||||
|
|
||||||
|
|
||||||
def fit_ellipse(self):
|
def fit_ellipse(self):
|
||||||
|
|
||||||
N = len(self.xs)
|
N = len(self.xs)
|
||||||
if N < 2:
|
if N < 2:
|
||||||
print("Warning: Need >= 2 samples to fit PCA. Fit failed.")
|
print("Warning: Need >= 2 samples to fit PCA. Fit failed.")
|
||||||
@ -36,9 +37,7 @@ class CalibrationEllipse:
|
|||||||
return
|
return
|
||||||
|
|
||||||
points = np.column_stack([self.xs, self.ys])
|
points = np.column_stack([self.xs, self.ys])
|
||||||
|
|
||||||
self.center = np.mean(points, axis=0)
|
self.center = np.mean(points, axis=0)
|
||||||
|
|
||||||
centered_points = points - self.center
|
centered_points = points - self.center
|
||||||
|
|
||||||
cov = np.cov(centered_points, rowvar=False)
|
cov = np.cov(centered_points, rowvar=False)
|
||||||
@ -50,27 +49,43 @@ class CalibrationEllipse:
|
|||||||
self.fitted = False
|
self.fitted = False
|
||||||
return
|
return
|
||||||
|
|
||||||
self.evecs = evecs_cov
|
# Sort eigenvectors by alignment with screen axes (X, Y), not by magnitude
|
||||||
|
# evecs_cov[:, 0] is eigenvector for first eigenvalue, evecs_cov[:, 1] for second
|
||||||
|
# We want [0] to be X-axis aligned, [1] to be Y-axis aligned
|
||||||
|
|
||||||
|
# Determine which eigenvector is more X-aligned vs Y-aligned
|
||||||
|
x_alignment = np.abs(evecs_cov[0, :]) # How much each evec points in X direction
|
||||||
|
y_alignment = np.abs(evecs_cov[1, :]) # How much each evec points in Y direction
|
||||||
|
|
||||||
|
if x_alignment[0] > x_alignment[1]:
|
||||||
|
# evec 0 is more X-aligned, evec 1 is more Y-aligned - keep as is
|
||||||
|
self.evecs = evecs_cov
|
||||||
std_devs = np.sqrt(evals_cov)
|
std_devs = np.sqrt(evals_cov)
|
||||||
|
else:
|
||||||
|
# evec 1 is more X-aligned, evec 0 is more Y-aligned - swap them
|
||||||
|
self.evecs = evecs_cov[:, [1, 0]]
|
||||||
|
std_devs = np.sqrt(evals_cov[[1, 0]])
|
||||||
|
|
||||||
self.axes = std_devs * self.n_std_devs
|
self.axes = std_devs * self.n_std_devs
|
||||||
|
|
||||||
if self.axes[0] < 1e-12: self.axes[0] = 1e-12
|
if self.axes[0] < 1e-12: self.axes[0] = 1e-12
|
||||||
if self.axes[1] < 1e-12: self.axes[1] = 1e-12
|
if self.axes[1] < 1e-12: self.axes[1] = 1e-12
|
||||||
|
|
||||||
major_index = np.argmax(evals_cov)
|
major_index = np.argmax(std_devs)
|
||||||
major_vec = self.evecs[:, major_index]
|
major_vec = self.evecs[:, major_index]
|
||||||
self.rotation = np.arctan2(major_vec[1], major_vec[0])
|
self.rotation = np.arctan2(major_vec[1], major_vec[0])
|
||||||
|
|
||||||
self.fitted = True
|
self.fitted = True
|
||||||
|
print(f"Ellipse fitted: center={self.center}, axes={self.axes}, rotation={np.degrees(self.rotation):.1f}°")
|
||||||
|
|
||||||
def fit_and_visualize(self): # Helper function for debug
|
def fit_and_visualize(self):
|
||||||
|
"""Fit and plot the ellipse with calibration samples"""
|
||||||
plt.figure(figsize=(10, 8))
|
plt.figure(figsize=(10, 8))
|
||||||
plt.plot(self.xs, self.ys, 'k.', label='All Samples', alpha=0.3)
|
plt.plot(self.xs, self.ys, 'k.', label='Calibration Samples', alpha=0.5, markersize=8)
|
||||||
plt.axis('equal')
|
plt.axis('equal')
|
||||||
plt.grid(True)
|
plt.grid(True, alpha=0.3)
|
||||||
plt.xlabel('X')
|
plt.xlabel('Pupil X (pixels)')
|
||||||
plt.ylabel('Y')
|
plt.ylabel('Pupil Y (pixels)')
|
||||||
|
|
||||||
if not self.fitted:
|
if not self.fitted:
|
||||||
self.fit_ellipse()
|
self.fit_ellipse()
|
||||||
@ -83,34 +98,84 @@ class CalibrationEllipse:
|
|||||||
scaled_axes[1] * np.sin(t)])
|
scaled_axes[1] * np.sin(t)])
|
||||||
world_coords = (self.evecs @ local_coords.T).T + self.center
|
world_coords = (self.evecs @ local_coords.T).T + self.center
|
||||||
|
|
||||||
plt.plot(world_coords[:, 0], world_coords[:, 1], 'b-', linewidth=2, label=f'Fitted Ellipse ({self.scale_factor*100:.0f}% size)')
|
plt.plot(world_coords[:, 0], world_coords[:, 1], 'b-',
|
||||||
plt.plot(self.center[0], self.center[1], 'b+', markersize=15, label=f'Fitted Center (Mean)')
|
linewidth=2, label=f'Calibration Ellipse ({self.scale_factor * 100:.0f}% scale)')
|
||||||
plt.title(f'Successful Robust Fit (PCA, {self.n_std_devs} std devs)')
|
plt.plot(self.center[0], self.center[1], 'r+',
|
||||||
|
markersize=15, markeredgewidth=3, label='Ellipse Center (Mean)')
|
||||||
|
|
||||||
|
# Draw principal axes
|
||||||
|
for i, (axis_len, color, name) in enumerate([(scaled_axes[0], 'g', 'Major'),
|
||||||
|
(scaled_axes[1], 'm', 'Minor')]):
|
||||||
|
axis_vec = self.evecs[:, i] * axis_len
|
||||||
|
plt.arrow(self.center[0], self.center[1], axis_vec[0], axis_vec[1],
|
||||||
|
head_width=5, head_length=7, fc=color, ec=color, alpha=0.6,
|
||||||
|
label=f'{name} Axis')
|
||||||
|
|
||||||
|
plt.title(f'Eye Tracking Calibration Ellipse (PCA, {self.n_std_devs}σ)')
|
||||||
else:
|
else:
|
||||||
plt.title("Robust Fit FAILED (Not enough points)")
|
plt.title("Ellipse Fit FAILED (Not enough points)")
|
||||||
|
|
||||||
plt.legend()
|
plt.legend()
|
||||||
|
plt.tight_layout()
|
||||||
plt.show()
|
plt.show()
|
||||||
|
|
||||||
def normalize(self, point, center_point, clip=True):
|
def normalize(self, pupil_pos, target_pos=None, clip=True):
|
||||||
if not self.fitted:
|
if not self.fitted:
|
||||||
print("Ellipse not fitted yet. Call fit_ellipse() or fit_and_visualize().")
|
print("ERROR: Ellipse not fitted yet. Call fit_ellipse() first.")
|
||||||
return 0,0
|
return 0.0, 0.0
|
||||||
|
|
||||||
x, y = float(point[0]), float(point[1])
|
# Current pupil position
|
||||||
|
x, y = float(pupil_pos[0]), float(pupil_pos[1])
|
||||||
p = np.array([x, y], dtype=float)
|
p = np.array([x, y], dtype=float)
|
||||||
|
|
||||||
p_centered = p - np.asarray(center_point, dtype=float)
|
# Reference point (where we're measuring FROM)
|
||||||
|
# If no target specified, use ellipse center (neutral gaze position)
|
||||||
|
if target_pos is None:
|
||||||
|
reference = self.center
|
||||||
|
else:
|
||||||
|
reference = np.asarray(target_pos, dtype=float)
|
||||||
|
|
||||||
|
# Vector from reference to current pupil position
|
||||||
|
p_centered = p - reference
|
||||||
|
|
||||||
|
# Rotate into ellipse principal axes space
|
||||||
p_rot = self.evecs.T @ p_centered
|
p_rot = self.evecs.T @ p_centered
|
||||||
|
|
||||||
|
# Scale by ellipse axes (with scale factor for margins)
|
||||||
scaled_axes = self.axes * self.scale_factor
|
scaled_axes = self.axes * self.scale_factor
|
||||||
|
|
||||||
scaled_axes[scaled_axes < 1e-12] = 1e-12
|
scaled_axes[scaled_axes < 1e-12] = 1e-12
|
||||||
|
|
||||||
|
# Normalize: pupil offset / ellipse radius in that direction
|
||||||
norm = p_rot / scaled_axes
|
norm = p_rot / scaled_axes
|
||||||
|
|
||||||
if clip:
|
# Apply coordinate flips for eye tracking conventions
|
||||||
norm = np.clip(norm, -1.0, 1.0)
|
norm_x = -norm[0] if self.flip_x else norm[0]
|
||||||
|
norm_y = -norm[1] if self.flip_y else norm[1]
|
||||||
|
|
||||||
return float(norm[0]), float(norm[1])
|
if clip:
|
||||||
|
norm_x = np.clip(norm_x, -1.0, 1.0)
|
||||||
|
norm_y = np.clip(norm_y, -1.0, 1.0)
|
||||||
|
|
||||||
|
return float(norm_x), float(norm_y)
|
||||||
|
|
||||||
|
def denormalize(self, norm_x, norm_y, target_pos=None):
|
||||||
|
if not self.fitted:
|
||||||
|
print("ERROR: Ellipse not fitted yet.")
|
||||||
|
return 0.0, 0.0
|
||||||
|
|
||||||
|
# Apply inverse flips
|
||||||
|
nx = -norm_x if self.flip_x else norm_x
|
||||||
|
ny = -norm_y if self.flip_y else norm_y
|
||||||
|
|
||||||
|
# Scale by ellipse axes
|
||||||
|
scaled_axes = self.axes * self.scale_factor
|
||||||
|
p_rot = np.array([nx, ny]) * scaled_axes
|
||||||
|
|
||||||
|
# Rotate back to world space
|
||||||
|
p_centered = self.evecs @ p_rot
|
||||||
|
|
||||||
|
# Add reference point
|
||||||
|
reference = self.center if target_pos is None else np.asarray(target_pos, dtype=float)
|
||||||
|
p = p_centered + reference
|
||||||
|
|
||||||
|
return float(p[0]), float(p[1])
|
||||||
|
|||||||
1635
poetry.lock
generated
1635
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -11,7 +11,7 @@ python = "~3.11.0"
|
|||||||
python-osc = "^1.8.0"
|
python-osc = "^1.8.0"
|
||||||
requests = "^2.28.1"
|
requests = "^2.28.1"
|
||||||
opencv-python = "^4.6.0.66"
|
opencv-python = "^4.6.0.66"
|
||||||
numpy = "~1.23.5"
|
numpy = "~1.24.3"
|
||||||
pye3d = "^0.3.2"
|
pye3d = "^0.3.2"
|
||||||
pysimplegui-4-foss = "^4.6.4.1"
|
pysimplegui-4-foss = "^4.6.4.1"
|
||||||
pydantic = "^2.4.2"
|
pydantic = "^2.4.2"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user