Merge pull request #42 from ShyAssassin/Cleanup

Update derecated dependency + small code cleanup
This commit is contained in:
Prohurtz 2022-11-02 13:11:03 -05:00 committed by GitHub
commit efa969fe44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 114 additions and 215 deletions

View File

@ -9,6 +9,8 @@ from camera import Camera, CameraState
from osc import EyeId from osc import EyeId
import cv2 import cv2
from winsound import PlaySound, SND_FILENAME, SND_ASYNC from winsound import PlaySound, SND_FILENAME, SND_ASYNC
import traceback
class CameraWidget: class CameraWidget:
def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue): def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue: Queue):
@ -27,7 +29,6 @@ class CameraWidget:
self.gui_recenter_eyes = f"-RECENTEREYES{widget_id}-" self.gui_recenter_eyes = f"-RECENTEREYES{widget_id}-"
self.gui_mode_readout = f"-APPMODE{widget_id}-" self.gui_mode_readout = f"-APPMODE{widget_id}-"
self.gui_circular_crop = f"-CIRCLECROP{widget_id}-" self.gui_circular_crop = f"-CIRCLECROP{widget_id}-"
# self.gui_show_color_image = f"-SHOWCOLORIMAGE{widget_id}-"
self.gui_roi_message = f"-ROIMESSAGE{widget_id}-" self.gui_roi_message = f"-ROIMESSAGE{widget_id}-"
self.osc_queue = osc_queue self.osc_queue = osc_queue
@ -41,7 +42,6 @@ class CameraWidget:
self.config = main_config.right_eye self.config = main_config.right_eye
elif self.eye_id == EyeId.LEFT: elif self.eye_id == EyeId.LEFT:
self.config = main_config.left_eye self.config = main_config.left_eye
else: else:
raise RuntimeError("Cannot have a camera widget represent both eyes!") raise RuntimeError("Cannot have a camera widget represent both eyes!")
@ -82,11 +82,13 @@ class CameraWidget:
), ),
], ],
[ [
sg.Button("Restart Calibration", key=self.gui_restart_calibration, button_color = '#6f4ca1'), sg.Button("Restart Calibration", key=self.gui_restart_calibration, button_color='#6f4ca1'),
sg.Button("Recenter Eyes", key=self.gui_recenter_eyes, button_color = '#6f4ca1'), sg.Button("Recenter Eyes", key=self.gui_recenter_eyes, button_color='#6f4ca1'),
], ],
[sg.Text("Mode:", background_color='#424042'), sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#424042'), [
sg.Text("Mode:", background_color='#424042'),
sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#424042'),
sg.Checkbox( sg.Checkbox(
"Circle crop:", "Circle crop:",
default=self.config.gui_circular_crop, default=self.config.gui_circular_crop,
@ -115,16 +117,14 @@ class CameraWidget:
sg.InputText(self.config.capture_source, key=self.gui_camera_addr), sg.InputText(self.config.capture_source, key=self.gui_camera_addr),
], ],
[ [
sg.Button( sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#6f4ca1'),
"Save and Restart Tracking", key=self.gui_save_tracking_button, button_color = '#6f4ca1'
),
], ],
[ [
sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color = '#6f4ca1'), sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color='#6f4ca1'),
sg.Button("Cropping Mode", key=self.gui_roi_button, button_color = '#6f4ca1'), sg.Button("Cropping Mode", key=self.gui_roi_button, button_color='#6f4ca1'),
], ],
[ [
sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042' ), sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042'),
sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False), sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False),
], ],
] ]
@ -211,9 +211,9 @@ class CameraWidget:
self.config.rotation_angle = int(values[self.gui_rotation_slider]) self.config.rotation_angle = int(values[self.gui_rotation_slider])
changed = True changed = True
# if self.config.show_color_image != values[self.gui_show_color_image]: if self.config.gui_circular_crop != values[self.gui_circular_crop]:
# self.config.show_color_image = values[self.gui_show_color_image] self.config.gui_circular_crop = values[self.gui_circular_crop]
# changed = True changed = True
if changed: if changed:
self.main_config.save() self.main_config.save()
@ -224,13 +224,15 @@ class CameraWidget:
self.camera.set_output_queue(self.capture_queue) self.camera.set_output_queue(self.capture_queue)
window[self.gui_roi_layout].update(visible=False) window[self.gui_roi_layout].update(visible=False)
window[self.gui_tracking_layout].update(visible=True) window[self.gui_tracking_layout].update(visible=True)
elif event == self.gui_roi_button:
if event == self.gui_roi_button:
print("Move to roi mode") print("Move to roi mode")
self.in_roi_mode = True self.in_roi_mode = True
self.camera.set_output_queue(self.roi_queue) self.camera.set_output_queue(self.roi_queue)
window[self.gui_roi_layout].update(visible=True) window[self.gui_roi_layout].update(visible=True)
window[self.gui_tracking_layout].update(visible=False) window[self.gui_tracking_layout].update(visible=False)
elif event == "{}+UP".format(self.gui_roi_selection):
if event == "{}+UP".format(self.gui_roi_selection):
# Event for mouse button up in ROI mode # Event for mouse button up in ROI mode
self.is_mouse_up = True self.is_mouse_up = True
if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0: if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0:
@ -239,24 +241,20 @@ class CameraWidget:
self.config.roi_window_w = abs(self.x0 - self.x1) self.config.roi_window_w = abs(self.x0 - self.x1)
self.config.roi_window_h = abs(self.y0 - self.y1) self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save() self.main_config.save()
elif event == self.gui_roi_selection:
if event == self.gui_roi_selection:
# Event for mouse button down or mouse drag in ROI mode # Event for mouse button down or mouse drag in ROI mode
if self.is_mouse_up: if self.is_mouse_up:
self.is_mouse_up = False self.is_mouse_up = False
self.x0, self.y0 = values[self.gui_roi_selection] self.x0, self.y0 = values[self.gui_roi_selection]
self.x1, self.y1 = values[self.gui_roi_selection] self.x1, self.y1 = values[self.gui_roi_selection]
elif event == self.gui_restart_calibration:
if event == self.gui_restart_calibration:
self.ransac.calibration_frame_counter = 300 self.ransac.calibration_frame_counter = 300
PlaySound('Audio/start.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
if event == self.gui_recenter_eyes:
elif event == self.gui_recenter_eyes:
self.settings.gui_recenter_eyes = True self.settings.gui_recenter_eyes = True
if self.config.gui_circular_crop != values[self.gui_circular_crop]:
self.config.gui_circular_crop = values[self.gui_circular_crop]
changed = True
#self.ransac.recenter_eye = True
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
@ -311,14 +309,10 @@ class CameraWidget:
graph = window[self.gui_output_graph] graph = window[self.gui_output_graph]
graph.erase() graph.erase()
if ( if eye_info.info_type != InformationOrigin.FAILURE and not eye_info.blink:
eye_info.info_type != InformationOrigin.FAILURE
and not eye_info.blink
):
graph.update(background_color="white") graph.update(background_color="white")
try: try:
graph.draw_circle( graph.draw_circle(
(eye_info.x * -100, eye_info.y * -100), (eye_info.x * -100, eye_info.y * -100),
25, 25,
@ -333,7 +327,6 @@ class CameraWidget:
graph.update(background_color="red") graph.update(background_color="red")
# Relay information to OSC # Relay information to OSC
if eye_info.info_type != InformationOrigin.FAILURE: if eye_info.info_type != InformationOrigin.FAILURE:
self.osc_queue.put((self.eye_id, eye_info)) self.osc_queue.put((self.eye_id, eye_info))
except Empty: except Empty:
return pass

View File

@ -2,6 +2,7 @@ from operator import truth
from dataclasses import dataclass from dataclasses import dataclass
import sys import sys
import asyncio import asyncio
sys.path.append(".") sys.path.append(".")
from config import EyeTrackCameraConfig from config import EyeTrackCameraConfig
from config import EyeTrackSettingsConfig from config import EyeTrackSettingsConfig
@ -17,6 +18,8 @@ from one_euro_filter import OneEuroFilter
from sympy import symbols, Eq, solve from sympy import symbols, Eq, solve
from winsound import PlaySound, SND_FILENAME, SND_ASYNC from winsound import PlaySound, SND_FILENAME, SND_ASYNC
import scipy.signal as sp import scipy.signal as sp
class InformationOrigin(Enum): class InformationOrigin(Enum):
RANSAC = 1 RANSAC = 1
BLOB = 2 BLOB = 2
@ -30,13 +33,17 @@ class EyeInformation:
y: float y: float
pupil_dialation: int pupil_dialation: int
blink: bool blink: bool
lowb = np.array(0) lowb = np.array(0)
def run_once(f): def run_once(f):
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
if not wrapper.has_run: if not wrapper.has_run:
wrapper.has_run = True wrapper.has_run = True
return f(*args, **kwargs) return f(*args, **kwargs)
wrapper.has_run = False wrapper.has_run = False
return wrapper return wrapper
@ -44,7 +51,8 @@ def run_once(f):
async def delayed_setting_change(setting, value): async def delayed_setting_change(setting, value):
await asyncio.sleep(5) await asyncio.sleep(5)
setting = value setting = value
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC)
def fit_rotated_ellipse_ransac( def fit_rotated_ellipse_ransac(
data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80 data, iter=5, sample_num=10, offset=80 # 80.0, 10, 80
@ -116,17 +124,17 @@ def fit_rotated_ellipse(data):
w = np.sqrt( w = np.sqrt(
cu cu
/ ( / (
a * np.cos(theta) ** 2 a * np.cos(theta)**2
+ b * np.cos(theta) * np.sin(theta) + b * np.cos(theta) * np.sin(theta)
+ c * np.sin(theta) ** 2 + c * np.sin(theta)**2
) )
) )
h = np.sqrt( h = np.sqrt(
cu cu
/ ( / (
a * np.sin(theta) ** 2 a * np.sin(theta)**2
- b * np.cos(theta) * np.sin(theta) - b * np.cos(theta) * np.sin(theta)
+ c * np.cos(theta) ** 2 + c * np.cos(theta)**2
) )
) )
@ -151,7 +159,6 @@ class EyeProcessor:
self.config = config self.config = config
self.settings = settings self.settings = settings
# Cross-thread communication management # Cross-thread communication management
self.capture_queue_incoming = capture_queue_incoming self.capture_queue_incoming = capture_queue_incoming
self.image_queue_outgoing = image_queue_outgoing self.image_queue_outgoing = image_queue_outgoing
@ -160,7 +167,6 @@ class EyeProcessor:
self.eye_id = eye_id self.eye_id = eye_id
# Cross algo state # Cross algo state
self.lkg_projected_sphere = None self.lkg_projected_sphere = None
self.xc = None self.xc = None
self.yc = None self.yc = None
@ -191,45 +197,20 @@ class EyeProcessor:
self.calibration_frame_counter self.calibration_frame_counter
try: try:
min_cutoff = float(self.settings.gui_min_cutoff) #0.0004 min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004
beta = float(self.settings.gui_speed_coefficient) #0.9 beta = float(self.settings.gui_speed_coefficient) # 0.9
except: except:
print('[WARN] OneEuroFilter values must be a legal number.') print('[WARN] OneEuroFilter values must be a legal number.')
min_cutoff = 0.0004 min_cutoff = 0.0004
beta = 0.9 beta = 0.9
noisy_point = np.array([1, 1]) noisy_point = np.array([1, 1])
self.one_euro_filter = OneEuroFilter( self.one_euro_filter = OneEuroFilter(
noisy_point, noisy_point,
min_cutoff=min_cutoff, min_cutoff=min_cutoff,
beta=beta beta=beta
) )
def output_images_and_update(self, threshold_image, output_information: EyeInformation):
def output_images_and_update(
self, threshold_image, output_information: EyeInformation
):
# if self.config.show_color_image:
# image_stack = np.concatenate(
# (
# self.current_image,
# cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR),
# cv2.cvtColor(threshold_image, cv2.COLOR_GRAY2BGR),
# ),
# axis=1,
# )
# else:
image_stack = np.concatenate( image_stack = np.concatenate(
( (
cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR),
@ -246,10 +227,10 @@ class EyeProcessor:
try: try:
# Get frame from capture source, crop to ROI # Get frame from capture source, crop to ROI
self.current_image = self.current_image[ self.current_image = self.current_image[
int(self.config.roi_window_y) : int( int(self.config.roi_window_y): int(
self.config.roi_window_y + self.config.roi_window_h self.config.roi_window_y + self.config.roi_window_h
), ),
int(self.config.roi_window_x) : int( int(self.config.roi_window_x): int(
self.config.roi_window_x + self.config.roi_window_w self.config.roi_window_x + self.config.roi_window_w
), ),
] ]
@ -275,11 +256,8 @@ class EyeProcessor:
return True return True
def blob_tracking_fallback(self): def blob_tracking_fallback(self):
# define circle
if self.config.gui_circular_crop:
# define circle
if self.config.gui_circular_crop == True:
if self.cct == 0: if self.cct == 0:
try: try:
ht, wd = self.current_image_gray.shape[:2] ht, wd = self.current_image_gray.shape[:2]
@ -287,8 +265,8 @@ class EyeProcessor:
radius = int(float(self.lkg_projected_sphere["axes"][0])) radius = int(float(self.lkg_projected_sphere["axes"][0]))
# draw filled circle in white on black background as mask # draw filled circle in white on black background as mask
mask = np.zeros((ht,wd), dtype=np.uint8) mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background # create white colored background
color = np.full_like(self.current_image_gray, (255)) color = np.full_like(self.current_image_gray, (255))
@ -297,7 +275,7 @@ class EyeProcessor:
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image # apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255-mask) masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images # combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color) self.current_image_gray = cv2.add(masked_img, masked_color)
@ -306,15 +284,14 @@ class EyeProcessor:
else: else:
self.cct = self.cct - 1 self.cct = self.cct - 1
# Increase our threshold value slightly, in order to have a better possibility of getting back # Increase our threshold value slightly, in order to have a better possibility of getting back
# something to do blob tracking on. # something to do blob tracking on.
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
histr = hist.ravel() histr = hist.ravel()
peaks, properties = sp.find_peaks(histr, distance=5) peaks, properties = sp.find_peaks(histr, distance=5)
minpeak = np.min(peaks) minpeak = np.min(peaks)
thresholdoptics = np.array(minpeak + int(self.config.threshold + 12)) thresholdoptics = np.array(minpeak + int(self.config.threshold + 12))
larger_threshold = cv2.inRange(self.current_image_gray,lowb,thresholdoptics) #faster than cv2.threshold larger_threshold = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
larger_threshold = cv2.bitwise_not(larger_threshold) larger_threshold = cv2.bitwise_not(larger_threshold)
# Blob tracking requires that we have a vague idea of where the eye may be at the moment. This # Blob tracking requires that we have a vague idea of where the eye may be at the moment. This
# means we need to have had at least one successful runthrough of the Pupil Labs algorithm in # means we need to have had at least one successful runthrough of the Pupil Labs algorithm in
@ -325,11 +302,6 @@ class EyeProcessor:
) )
return return
try: try:
# Try rebuilding our contours # Try rebuilding our contours
contours, _ = cv2.findContours( contours, _ = cv2.findContours(
@ -354,19 +326,14 @@ class EyeProcessor:
# TODO This should be scaled based on camera resolution. # TODO This should be scaled based on camera resolution.
if not self.settings.gui_blob_minsize <= h <= self.settings.gui_blob_maxsize or not self.settings.gui_blob_minsize <= w <= self.settings.gui_blob_maxsize: if not self.settings.gui_blob_minsize <= h <= self.settings.gui_blob_maxsize or not self.settings.gui_blob_minsize <= w <= self.settings.gui_blob_maxsize:
continue continue
cx = x + int(w / 2) cx = x + int(w / 2)
cy = y + int(h / 2) cy = y + int(h / 2)
xrlb = ( xrlb = (cx - self.lkg_projected_sphere["center"][0]) / self.lkg_projected_sphere["axes"][0]
cx - self.lkg_projected_sphere["center"][0] eyeyb = (cy - self.lkg_projected_sphere["center"][1]) / self.lkg_projected_sphere["axes"][1]
) / self.lkg_projected_sphere["axes"][0]
eyeyb = (
cy - self.lkg_projected_sphere["center"][1]
) / self.lkg_projected_sphere["axes"][1]
cv2.line( cv2.line(
self.current_image_gray, self.current_image_gray,
(x + int(w / 2), 0), (x + int(w / 2), 0),
@ -390,14 +357,14 @@ class EyeProcessor:
self.calibration_frame_counter = None self.calibration_frame_counter = None
self.xoff = cx self.xoff = cx
self.yoff = cy self.yoff = cy
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC)
elif self.calibration_frame_counter != None: elif self.calibration_frame_counter != None:
self.settings.gui_recenter_eyes = False self.settings.gui_recenter_eyes = False
if cx > self.xmax: if cx > self.xmax:
self.xmax = cx self.xmax = cx
if cx < self.xmin: if cx < self.xmin:
self.xmin = cx self.xmin = cx
if cy> self.ymax: if cy > self.ymax:
self.ymax = cy self.ymax = cy
if cy < self.ymin: if cy < self.ymin:
self.ymin = cy self.ymin = cy
@ -407,33 +374,28 @@ class EyeProcessor:
self.yoff = cy self.yoff = cy
if self.ts == 0: if self.ts == 0:
self.settings.gui_recenter_eyes = False self.settings.gui_recenter_eyes = False
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC)
else: else:
self.ts = self.ts - 1 self.ts = self.ts - 1
else: else:
self.ts = 10 self.ts = 10
xl = float( xl = float(
((cx - self.xoff)) / (self.xmax - self.xoff) (cx - self.xoff) / (self.xmax - self.xoff)
) )
xr = float( xr = float(
((cx - self.xoff)) / (self.xmin - self.xoff) (cx - self.xoff) / (self.xmin - self.xoff)
) )
yu = float( yu = float(
((cy - self.yoff)) / (self.ymin - self.yoff) (cy - self.yoff) / (self.ymin - self.yoff)
) )
yd = float( yd = float(
((cy - self.yoff)) / (self.ymax - self.yoff) (cy - self.yoff) / (self.ymax - self.yoff)
) )
# print(self.)
out_x = 0 out_x = 0
out_y = 0 out_y = 0
if self.settings.gui_flip_y_axis == True: #check config on flipped values settings and apply accordingly if self.settings.gui_flip_y_axis: # check config on flipped values settings and apply accordingly
if yd > 0: if yd > 0:
out_y = max(0.0, min(1.0, yd)) out_y = max(0.0, min(1.0, yd))
if yu > 0: if yu > 0:
@ -444,7 +406,7 @@ class EyeProcessor:
if yu > 0: if yu > 0:
out_y = max(0.0, min(1.0, yu)) out_y = max(0.0, min(1.0, yu))
if self.settings.gui_flip_x_axis_right == True: if self.settings.gui_flip_x_axis_right:
if xr > 0: if xr > 0:
out_x = -abs(max(0.0, min(1.0, xr))) out_x = -abs(max(0.0, min(1.0, xr)))
if xl > 0: if xl > 0:
@ -456,16 +418,13 @@ class EyeProcessor:
out_x = -abs(max(0.0, min(1.0, xl))) out_x = -abs(max(0.0, min(1.0, xl)))
try: try:
noisy_point = np.array([out_x, out_y]) #fliter our values with a One Euro Filter noisy_point = np.array([out_x, out_y]) # fliter our values with a One Euro Filter
point_hat = self.one_euro_filter(noisy_point) point_hat = self.one_euro_filter(noisy_point)
out_x = point_hat[0] out_x = point_hat[0]
out_y = point_hat[1] out_y = point_hat[1]
except: except:
pass pass
self.output_images_and_update( self.output_images_and_update(
larger_threshold, larger_threshold,
EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False), EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False),
@ -479,18 +438,14 @@ class EyeProcessor:
def run(self): def run(self):
camera_model = None camera_model = None
detector_3d = None detector_3d = None
out_pupil_dialation = 1 out_pupil_dialation = 1
if self.eye_id == "EyeId.RIGHT": if self.eye_id == "EyeId.RIGHT":
flipx = self.settings.gui_flip_x_axis_right flipx = self.settings.gui_flip_x_axis_right
#elif self.eye_id == "EyeId.LEFT":
# flipx = self.config.gui_flip_x_axis_left
else: else:
flipx = self.settings.gui_flip_x_axis_left flipx = self.settings.gui_flip_x_axis_left
while True:
# oef = init_filter()
while True:
# Check to make sure we haven't been requested to close # Check to make sure we haven't been requested to close
if self.cancellation_event.is_set(): if self.cancellation_event.is_set():
print("Exiting RANSAC thread") print("Exiting RANSAC thread")
@ -504,11 +459,9 @@ class EyeProcessor:
continue continue
# If our ROI configuration has changed, reset our model and detector # If our ROI configuration has changed, reset our model and detector
if ( if (camera_model is None
camera_model is None
or detector_3d is None or detector_3d is None
or camera_model.resolution or camera_model.resolution != (
!= (
self.config.roi_window_w, self.config.roi_window_w,
self.config.roi_window_h, self.config.roi_window_h,
) )
@ -549,20 +502,17 @@ class EyeProcessor:
self.current_image, cv2.COLOR_BGR2GRAY self.current_image, cv2.COLOR_BGR2GRAY
) )
#print(self.config.gui_circular_crop)
# print(self.cct)
if self.config.gui_circular_crop == True: if self.config.gui_circular_crop == True:
if self.cct == 0: if self.cct == 0:
try: try:
ht, wd = self.current_image_gray.shape[:2] ht, wd = self.current_image_gray.shape[:2]
radius = int(float(self.lkg_projected_sphere["axes"][0])) radius = int(float(self.lkg_projected_sphere["axes"][0]))
self.xc = int(float(self.lkg_projected_sphere["center"][0])) self.xc = int(float(self.lkg_projected_sphere["center"][0]))
self.yc = int(float(self.lkg_projected_sphere["center"][1])) self.yc = int(float(self.lkg_projected_sphere["center"][1]))
# draw filled circle in white on black background as mask # draw filled circle in white on black background as mask
mask = np.zeros((ht,wd), dtype=np.uint8) mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc,self.yc), radius, 255, -1) mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background # create white colored background
color = np.full_like(self.current_image_gray, (255)) color = np.full_like(self.current_image_gray, (255))
@ -571,7 +521,7 @@ class EyeProcessor:
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask) masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image # apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255-mask) masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images # combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color) self.current_image_gray = cv2.add(masked_img, masked_color)
@ -582,22 +532,15 @@ class EyeProcessor:
else: else:
self.cct = 300 self.cct = 300
# Using Histogram based thresholding. Improves robustness insanely
#Using Histogram based thresholding. Improves robustness insanely
hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256]) hist = cv2.calcHist([self.current_image_gray], [0], None, [256], [0, 256])
histr = hist.ravel() histr = hist.ravel()
peaks, properties = sp.find_peaks(histr, distance=5) peaks, properties = sp.find_peaks(histr, distance=5)
minpeak = np.min(peaks) minpeak = np.min(peaks)
thresholdoptics = np.array(minpeak + int(self.config.threshold)) thresholdoptics = np.array(minpeak + int(self.config.threshold))
thresh = cv2.inRange(self.current_image_gray,lowb,thresholdoptics) #faster than cv2.threshold thresh = cv2.inRange(self.current_image_gray, lowb, thresholdoptics) # faster than cv2.threshold
thresh = cv2.bitwise_not(thresh) thresh = cv2.bitwise_not(thresh)
# Set up morphological transforms, for smoothing and clearing the image we get out of the # Set up morphological transforms, for smoothing and clearing the image we get out of the
# thresholding operation. After this, we'd really like to just have a black blob in the middle # thresholding operation. After this, we'd really like to just have a black blob in the middle
# of a bunch of white area. # of a bunch of white area.
@ -618,9 +561,8 @@ class EyeProcessor:
# If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to # If we have no convex maidens, we have no pupil, and can't progress from here. Dump back to
# using blob tracking. # using blob tracking.
#
if len(convex_hulls) == 0: if len(convex_hulls) == 0:
if self.settings.gui_blob_fallback == True: if self.settings.gui_blob_fallback:
self.blob_tracking_fallback() self.blob_tracking_fallback()
else: else:
print("[INFO] Blob fallback disabled. Assuming blink.") print("[INFO] Blob fallback disabled. Assuming blink.")
@ -639,7 +581,7 @@ class EyeProcessor:
largest_hull.reshape(-1, 2) largest_hull.reshape(-1, 2)
) )
except: except:
if self.settings.gui_blob_fallback == True: if self.settings.gui_blob_fallback:
self.blob_tracking_fallback() self.blob_tracking_fallback()
else: else:
print("[INFO] Blob fallback disabled. Assuming blink.") print("[INFO] Blob fallback disabled. Assuming blink.")
@ -675,16 +617,13 @@ class EyeProcessor:
exm = ellipse_3d["center"][0] exm = ellipse_3d["center"][0]
eym = ellipse_3d["center"][1] eym = ellipse_3d["center"][1]
d = result_3d["diameter_3d"] d = result_3d["diameter_3d"]
if self.calibration_frame_counter == 0: if self.calibration_frame_counter == 0:
self.calibration_frame_counter = None self.calibration_frame_counter = None
self.xoff = cx self.xoff = cx
self.yoff = cy self.yoff = cy
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC)
elif self.calibration_frame_counter != None: # TODO reset calibration values on button press elif self.calibration_frame_counter != None: # TODO reset calibration values on button press
if exm > self.xmax: if exm > self.xmax:
self.xmax = exm self.xmax = exm
@ -695,46 +634,34 @@ class EyeProcessor:
if eym < self.ymin: if eym < self.ymin:
self.ymin = eym self.ymin = eym
self.calibration_frame_counter -= 1 self.calibration_frame_counter -= 1
if self.settings.gui_recenter_eyes == True: if self.settings.gui_recenter_eyes:
self.xoff = cx self.xoff = cx
self.yoff = cy self.yoff = cy
if self.ts == 0: if self.ts == 0:
self.settings.gui_recenter_eyes = False self.settings.gui_recenter_eyes = False
PlaySound('Audio/compleated.wav', SND_FILENAME|SND_ASYNC) PlaySound('Audio/compleated.wav', SND_FILENAME | SND_ASYNC)
else: else:
self.ts = self.ts - 1 self.ts = self.ts - 1
else: else:
self.ts = 20 self.ts = 20
#print(self.yoff)
# noisy_point = np.array([cx, cy]) #fliter our values with a One Euro Filter
# point_hat = self.one_euro_filter(noisy_point)
# cx = point_hat[0]
# cy = point_hat[1]
xl = float( xl = float(
((cx - self.xoff)) / (self.xmax - self.xoff) (cx - self.xoff) / (self.xmax - self.xoff)
) )
xr = float( xr = float(
((cx - self.xoff)) / (self.xmin - self.xoff) (cx - self.xoff) / (self.xmin - self.xoff)
) )
yu = float( yu = float(
((cy - self.yoff)) / (self.ymin - self.yoff) (cy - self.yoff) / (self.ymin - self.yoff)
) )
yd = float( yd = float(
((cy - self.yoff)) / (self.ymax - self.yoff) (cy - self.yoff) / (self.ymax - self.yoff)
) )
out_x = 0 out_x = 0
out_y = 0 out_y = 0
if self.settings.gui_flip_y_axis == True: if self.settings.gui_flip_y_axis:
if yd > 0: if yd > 0:
out_y = max(0.0, min(1.0, yd)) out_y = max(0.0, min(1.0, yd))
if yu > 0: if yu > 0:
@ -745,7 +672,7 @@ class EyeProcessor:
if yu > 0: if yu > 0:
out_y = max(0.0, min(1.0, yu)) out_y = max(0.0, min(1.0, yu))
if flipx == True: if flipx:
if xr > 0: if xr > 0:
out_x = -abs(max(0.0, min(1.0, xr))) out_x = -abs(max(0.0, min(1.0, xr)))
if xl > 0: if xl > 0:
@ -756,30 +683,23 @@ class EyeProcessor:
if xl > 0: if xl > 0:
out_x = -abs(max(0.0, min(1.0, xl))) out_x = -abs(max(0.0, min(1.0, xl)))
try: try:
noisy_point = np.array([out_x, out_y]) #fliter our values with a One Euro Filter noisy_point = np.array([out_x, out_y]) # fliter our values with a One Euro Filter
point_hat = self.one_euro_filter(noisy_point) point_hat = self.one_euro_filter(noisy_point)
out_x = point_hat[0] out_x = point_hat[0]
out_y = point_hat[1] out_y = point_hat[1]
except: except:
pass pass
# print(cy, self.yoff, self.ymin, self.ymax, out_y)
# print(out_y, yu, yd)
output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, out_pupil_dialation, False) output_info = EyeInformation(InformationOrigin.RANSAC, out_x, out_y, out_pupil_dialation, False)
# Draw our image and stack it for visual output # Draw our image and stack it for visual output
try: try:
cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1)
cv2.circle(self.current_image_gray, (int(cx), int(cy)), 2, (0, 0, 255), -1) cv2.circle(self.current_image_gray, (int(cx), int(cy)), 2, (0, 0, 255), -1)
# draw pupil
except: except:
pass pass
try: try:
cv2.ellipse( cv2.ellipse(
self.current_image_gray, self.current_image_gray,
@ -795,9 +715,8 @@ class EyeProcessor:
# validity beforehand, but for now just pass. It usually fixes itself on the next frame. # validity beforehand, but for now just pass. It usually fixes itself on the next frame.
pass pass
try: try:
# print(self.lkg_projected_sphere["angle"], self.lkg_projected_sphere["axes"], self.lkg_projected_sphere["center"]) # print(self.lkg_projected_sphere["angle"], self.lkg_projected_sphere["axes"], self.lkg_projected_sphere["center"])
cv2.ellipse( cv2.ellipse(
self.current_image_gray, self.current_image_gray,
tuple(int(v) for v in self.lkg_projected_sphere["center"]), tuple(int(v) for v in self.lkg_projected_sphere["center"]),
@ -807,13 +726,9 @@ class EyeProcessor:
360, # start/end angle for drawing 360, # start/end angle for drawing
(0, 255, 0), # color (BGR): red (0, 255, 0), # color (BGR): red
) )
except: except:
pass pass
# draw line from center of eyeball to center of pupil # draw line from center of eyeball to center of pupil
cv2.line( cv2.line(
self.current_image_gray, self.current_image_gray,
@ -824,4 +739,3 @@ class EyeProcessor:
# Shove a concatenated image out to the main GUI thread for rendering # Shove a concatenated image out to the main GUI thread for rendering
self.output_images_and_update(thresh, output_info) self.output_images_and_update(thresh, output_info)

View File

@ -21,6 +21,7 @@ RIGHT_EYE_RADIO_NAME = "-RIGHTEYERADIO-"
BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-" BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-"
SETTINGS_RADIO_NAME = '-SETTINGSRADIO-' SETTINGS_RADIO_NAME = '-SETTINGSRADIO-'
def main(): def main():
# Get Configuration # Get Configuration
config: EyeTrackConfig = EyeTrackConfig.load() config: EyeTrackConfig = EyeTrackConfig.load()
@ -40,16 +41,9 @@ def main():
# start worker threads # start worker threads
osc_thread.start() osc_thread.start()
# t2s_queue: "queue.Queue[str | None]" = queue.Queue()
# t2s_engine = SpeechEngine(t2s_queue)
# t2s_thread = threading.Thread(target=t2s_engine.run)
# t2s_thread.start()
# t2s_queue.put("App Starting")
eyes = [ eyes = [
CameraWidget(EyeId.RIGHT, config, osc_queue), CameraWidget(EyeId.RIGHT, config, osc_queue),
CameraWidget(EyeId.LEFT, config, osc_queue), CameraWidget(EyeId.LEFT, config, osc_queue),
# CameraWidget(EyeId.SETTINGS, config, osc_queue),
] ]
settings = [ settings = [
@ -136,21 +130,18 @@ def main():
# If we're in either mode and someone hits q, quit immediately # If we're in either mode and someone hits q, quit immediately
if event == "Exit" or event == sg.WIN_CLOSED: if event == "Exit" or event == sg.WIN_CLOSED:
# eyes[2].stop() # eyes[2].stop()
for eye in eyes: for eye in eyes:
eye.stop() eye.stop()
cancellation_event.set() cancellation_event.set()
# shut down worker threads # shut down worker threads
osc_thread.join() osc_thread.join()
# TODO: find a way to have this function run on join maybe?? # TODO: find a way to have this function run on join maybe??
# threading.Event() wont work because pythonosc spawns its own thread. # threading.Event() wont work because pythonosc spawns its own thread.
# only way i can see to get around this is an ugly while loop that only checks if a threading event is trigggered # only way i can see to get around this is an ugly while loop that only checks if a threading event is trigggered
# and then call the pythonosc shutdown function # and then call the pythonosc shutdown function
osc_receiver.shutdown() osc_receiver.shutdown()
osc_receiver_thread.join() osc_receiver_thread.join()
# t2s_engine.force_stop()
# t2s_queue.put(None)
# t2s_thread.join()
print("Exiting EyeTrackApp") print("Exiting EyeTrackApp")
return return
@ -206,3 +197,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -9,7 +9,7 @@ future==0.18.2
idna==3.3 idna==3.3
importlib-metadata==4.8.3 importlib-metadata==4.8.3
joblib==1.1.0 joblib==1.1.0
msgpack-python==0.5.6 msgpack==1.0.4
numpy==1.19.5 numpy==1.19.5
opencv-python==4.5.3.56 opencv-python==4.5.3.56
pefile==2022.5.30 pefile==2022.5.30