some what working now

This commit is contained in:
Prohurtz 2022-12-20 15:24:31 -08:00
parent d87ab06415
commit a2377c722c
2 changed files with 199 additions and 245 deletions

View File

@ -242,6 +242,7 @@ class CameraWidget:
self.config.roi_window_h = abs(self.y0 - self.y1) self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save() self.main_config.save()
if event == self.gui_roi_selection: if event == self.gui_roi_selection:
# Event for mouse button down or mouse drag in ROI mode # Event for mouse button down or mouse drag in ROI mode
if self.is_mouse_up: if self.is_mouse_up:

View File

@ -30,16 +30,6 @@ Copyright (c) 2022 EyeTrackVR <3
------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------
''' '''
from operator import truth from operator import truth
from dataclasses import dataclass from dataclasses import dataclass
import sys import sys
@ -910,6 +900,8 @@ class EyeProcessor:
self.ts = 10 self.ts = 10
self.previous_rotation = self.config.rotation_angle self.previous_rotation = self.config.rotation_angle
self.calibration_frame_counter self.calibration_frame_counter
self.camera_model = None
self.detector_3d = None
#HSF #HSF
# CV param # CV param
@ -945,6 +937,7 @@ class EyeProcessor:
def capture_crop_rotate_image(self): def capture_crop_rotate_image(self):
# Get our current frame # Get our current frame
try: try:
# Get frame from capture source, crop to ROI # Get frame from capture source, crop to ROI
self.current_image = self.current_image[ self.current_image = self.current_image[
@ -955,6 +948,7 @@ class EyeProcessor:
self.config.roi_window_x + self.config.roi_window_w self.config.roi_window_x + self.config.roi_window_w
), ),
] ]
except: except:
# Failure to process frame, reuse previous frame. # Failure to process frame, reuse previous frame.
self.current_image = self.previous_image self.current_image = self.previous_image
@ -1251,7 +1245,9 @@ class EyeProcessor:
print(center_x, center_y) print(center_x, center_y)
out_x, out_y = cal_osc(self, center_x, center_y) out_x, out_y = cal_osc(self, center_x, center_y)
cv2.circle(frame, (center_x, center_y), 10, (0, 0, 255), -1) cv2.circle(frame, (center_x, center_y), 10, (0, 0, 255), -1)
self.output_images_and_update(frame,EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, False)) self.output_images_and_update(frame,EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, False))
if now_mode != cv_mode[0] and now_mode != cv_mode[1]: if now_mode != cv_mode[0] and now_mode != cv_mode[1]:
@ -1269,69 +1265,9 @@ class EyeProcessor:
#self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.HSF, 0, 0, 0, True)) #self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.HSF, 0, 0, 0, True))
def RANSAC3D(self): def RANSAC3D(self):
camera_model = None
detector_3d = None
f = 0
# If our ROI configuration has changed, reset our model and detector
if (camera_model is None
or detector_3d is None
or camera_model.resolution != (
self.config.roi_window_w,
self.config.roi_window_h,
)
):
camera_model = CameraModel(
focal_length=self.config.focal_length,
resolution=(self.config.roi_window_w, self.config.roi_window_h),
)
detector_3d = Detector3D(
camera=camera_model, long_term_mode=DetectorMode.blocking
)
# Check to make sure we haven't been requested to close f = False
try: self.capture_crop_rotate_image()
if self.cancellation_event.is_set():
print("Exiting RANSAC thread")
return
if self.config.roi_window_w <= 0 or self.config.roi_window_h <= 0:
# At this point, we're waiting for the user to set up the ROI window in the GUI.
# Sleep a bit while we wait.
if self.cancellation_event.wait(0.1):
return
pass
# If our ROI configuration has changed, reset our model and detector
if (camera_model is None
or detector_3d is None
or camera_model.resolution != (
self.config.roi_window_w,
self.config.roi_window_h,
)
):
camera_model = CameraModel(
focal_length=self.config.focal_length,
resolution=(self.config.roi_window_w, self.config.roi_window_h),
)
detector_3d = Detector3D(
camera=camera_model, long_term_mode=DetectorMode.blocking
)
try:
if self.capture_queue_incoming.empty():
self.capture_event.set()
# Wait a bit for images here. If we don't get one, just try again.
(
self.current_image,
self.current_frame_number,
self.current_fps,
) = self.capture_queue_incoming.get(block=True, timeout=0.2)
except queue.Empty:
# print("No image available")
pass
if not self.capture_crop_rotate_image():
pass
# Convert the image to grayscale, and set up thresholding. Thresholds here are basically a # Convert the image to grayscale, and set up thresholding. Thresholds here are basically a
# low-pass filter that will set any pixel < the threshold value to 0. Thresholding is user # low-pass filter that will set any pixel < the threshold value to 0. Thresholding is user
@ -1341,9 +1277,6 @@ class EyeProcessor:
# The goal of thresholding settings is to make sure we can ONLY see the pupil. This is why we # The goal of thresholding settings is to make sure we can ONLY see the pupil. This is why we
# crop the image earlier; it gives us less possible dark area to get confused about in the # crop the image earlier; it gives us less possible dark area to get confused about in the
# next step. # next step.
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
)
if self.config.gui_circular_crop == True: if self.config.gui_circular_crop == True:
if self.cct == 0: if self.cct == 0:
@ -1398,11 +1331,6 @@ class EyeProcessor:
# If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to # If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to
# using blob tracking. # using blob tracking.
if len(convex_hulls) == 0: if len(convex_hulls) == 0:
if self.settings.gui_BLOB:
self.BLOB()
else:
print("[INFO] Blob fallback disabled. Assuming blink.")
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, 0, 0, 0, True))
pass pass
# Find our largest hull, which we expect will probably be the ellipse that represents the 2d # Find our largest hull, which we expect will probably be the ellipse that represents the 2d
@ -1416,8 +1344,7 @@ class EyeProcessor:
cx, cy, w, h, theta = fit_rotated_ellipse_ransac( cx, cy, w, h, theta = fit_rotated_ellipse_ransac(
largest_hull.reshape(-1, 2) largest_hull.reshape(-1, 2)
) )
except:
f = True
# Get axis and angle of the ellipse, using pupil labs 2d algos. The next bit of code ranges # Get axis and angle of the ellipse, using pupil labs 2d algos. The next bit of code ranges
# from somewhat to completely magic, as most of it happens in native libraries (hence passing # from somewhat to completely magic, as most of it happens in native libraries (hence passing
@ -1426,6 +1353,7 @@ class EyeProcessor:
result_2d_final = {} result_2d_final = {}
result_2d["center"] = (cx, cy) result_2d["center"] = (cx, cy)
result_2d["axes"] = (w, h) result_2d["axes"] = (w, h)
result_2d["angle"] = theta * 180.0 / np.pi result_2d["angle"] = theta * 180.0 / np.pi
result_2d_final["ellipse"] = result_2d result_2d_final["ellipse"] = result_2d
@ -1435,7 +1363,8 @@ class EyeProcessor:
result_2d_final["timestamp"] = self.current_frame_number / self.current_fps result_2d_final["timestamp"] = self.current_frame_number / self.current_fps
# Black magic happens here, but after this we have our reprojected pupil/eye, and all we had # Black magic happens here, but after this we have our reprojected pupil/eye, and all we had
# to do was sell our soul to satan and/or C++. # to do was sell our soul to satan and/or C++.
result_3d = detector_3d.update_and_detect(
result_3d = self.detector_3d.update_and_detect(
result_2d_final, self.current_image_gray result_2d_final, self.current_image_gray
) )
@ -1451,8 +1380,9 @@ class EyeProcessor:
d = result_3d["diameter_3d"] d = result_3d["diameter_3d"]
out_x, out_y = cal_osc(self, cx, cy) #filter and calibrate values out_x, out_y = cal_osc(self, cx, cy) #filter and calibrate values
output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 1, False) self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, False))
except:
f = True
# Draw our image and stack it for visual output # Draw our image and stack it for visual output
try: try:
cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1)
@ -1486,8 +1416,7 @@ class EyeProcessor:
360, # start/end angle for drawing 360, # start/end angle for drawing
(0, 255, 0), # color (BGR): red (0, 255, 0), # color (BGR): red
) )
except:
pass
# draw line from center of eyeball to center of pupil # draw line from center of eyeball to center of pupil
cv2.line( cv2.line(
@ -1496,18 +1425,23 @@ class EyeProcessor:
tuple(int(v) for v in ellipse_3d["center"]), tuple(int(v) for v in ellipse_3d["center"]),
(0, 255, 0), # color (BGR): red (0, 255, 0), # color (BGR): red
) )
# Shove a concatenated image out to the main GUI thread for rendering
self.output_images_and_update(thresh, output_info)
except: except:
f = True pass
# Shove a concatenated image out to the main GUI thread for rendering
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.FAILURE, 0 ,0, 0, False))
#self.output_images_and_update(thresh, output_info)
#except:
# f = True
return f return f
def run(self): def run(self):
self.camera_model = None
self.detector_3d = None
f = False
while True: while True:
f = False f = True
# Check to make sure we haven't been requested to close # Check to make sure we haven't been requested to close
if self.cancellation_event.is_set(): if self.cancellation_event.is_set():
print("Exiting Tracking thread") print("Exiting Tracking thread")
@ -1519,6 +1453,25 @@ class EyeProcessor:
if self.cancellation_event.wait(0.1): if self.cancellation_event.wait(0.1):
return return
continue continue
# If our ROI configuration has changed, reset our model and detector
if (self.camera_model is None
or self.detector_3d is None
or self.camera_model.resolution != (
self.config.roi_window_w,
self.config.roi_window_h,
)
):
self.camera_model = CameraModel(
focal_length=self.config.focal_length,
resolution=(self.config.roi_window_w, self.config.roi_window_h),
)
self.detector_3d = Detector3D(
camera=self.camera_model, long_term_mode=DetectorMode.blocking
)
try: try:
if self.capture_queue_incoming.empty(): if self.capture_queue_incoming.empty():
self.capture_event.set() self.capture_event.set()
@ -1531,19 +1484,19 @@ class EyeProcessor:
except queue.Empty: except queue.Empty:
# print("No image available") # print("No image available")
continue continue
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
)
if not self.capture_crop_rotate_image(): if not self.capture_crop_rotate_image():
continue continue
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
)
try: try:
if self.settings.gui_RANSAC3D: #for now ransac goes first if self.settings.gui_RANSAC3D: #for now ransac goes first
f == self.RANSAC3D() f == self.RANSAC3D
if f and self.settings.gui_HSF: #if a fail has been reported and other algo is enabled, use it. if f and self.settings.gui_HSF: #if a fail has been reported and other algo is enabled, use it.
f == self.HSF() f == self.HSF()
if f and self.settings.gui_blob_fallback: if f and self.settings.gui_BLOB:
f == self.BLOB() f == self.BLOB()
except: except:
print("[WARN] ALL ALGORITHIMS HAVE FAILED OR ARE DISABLED.") print("[WARN] ALL ALGORITHIMS HAVE FAILED OR ARE DISABLED.")